From c7d9f4fe9e7b02249ca481c1e2df1fcf94a61c70 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 21 May 2019 14:41:27 +0100 Subject: [PATCH 001/180] Renaming of 'shaderset' to 'shader family' (#499) * Some renaming of 'shaderset' to 'shader family'. --- .../com/graphicsfuzz/server/webui/WebUi.java | 163 +++++++++--------- .../server/webui/WebUiConstants.java | 2 +- 2 files changed, 83 insertions(+), 82 deletions(-) diff --git a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java index 30d61d26f..957c895a9 100755 --- a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java +++ b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java @@ -29,7 +29,6 @@ import com.graphicsfuzz.server.thrift.CommandResult; import com.graphicsfuzz.server.thrift.FuzzerServiceManager; import com.graphicsfuzz.server.thrift.WorkerInfo; -import com.graphicsfuzz.server.thrift.WorkerNameNotFoundException; import com.graphicsfuzz.util.Constants; import java.io.File; import java.io.FileNotFoundException; @@ -72,7 +71,7 @@ * / : homepage: list workers, and options to start experiments. * Should also display the server queue state. * worker/"workername" : worker page, with overview of results - * worker/"workername"/"shadersetname" : result of this worker on this shaderset. + * worker/"workername"/"shadersetname" : result of this worker on this shader family. * Can start reduction on a single variant * startExperiment/ * @@ -112,16 +111,16 @@ public WebUi(FuzzerServiceManager.Iface fuzzerServiceManager, ShaderJobFileOpera this.fuzzerServiceManagerProxy = fuzzerServiceManager; } - private static final class Shaderset { + private static final class ShaderFamily { final String name; final File dir; final File preview; final int nbVariants; final boolean isComp; - public Shaderset(String name) { + public ShaderFamily(String name) { this.name = name; - this.dir = new File(WebUiConstants.SHADERSET_DIR, name); + this.dir = new File(WebUiConstants.SHADER_FAMILIES_DIR, name); this.preview = new File(dir, "thumb.png"); this.isComp = new File(dir, "reference.comp").isFile(); this.nbVariants = getNbVariants(); @@ -140,8 +139,8 @@ private int getNbVariants() { } } - private static final class ShadersetExp { - final Shaderset shaderset; + private static final class ShaderFamilyResult { + final ShaderFamily shaderFamily; final String name; final String worker; final File dir; @@ -153,13 +152,13 @@ private static final class ShadersetExp { int nbMetricsDisagree; int nbWrongImage; - public ShadersetExp(String name, String worker, AccessFileInfo accessFileInfo) + public ShaderFamilyResult(String name, String worker, AccessFileInfo accessFileInfo) throws FileNotFoundException { this.name = name; this.worker = worker; this.dir = new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + name); - this.shaderset = new Shaderset(name); - this.nbVariants = shaderset.nbVariants; + this.shaderFamily = new ShaderFamily(name); + this.nbVariants = shaderFamily.nbVariants; // Set variant counters for (File file : dir.listFiles()) { @@ -325,23 +324,23 @@ private List getLiveWorkers(boolean includeInactive) throws TExcepti return workers; } - // Get list of all shaderset directories - private List getAllShadersets(HttpServletRequest request, HttpServletResponse response) + // Get list of all shader family directories + private List getAllShaderFamilies(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - List shadersets = new ArrayList<>(); - File shadersetDir = new File(WebUiConstants.SHADERSET_DIR); - if (!shadersetDir.isDirectory()) { + List shaderFamilies = new ArrayList<>(); + File shaderFamiliesDir = new File(WebUiConstants.SHADER_FAMILIES_DIR); + if (!shaderFamiliesDir.isDirectory()) { err404(request, response); - return shadersets; + return shaderFamilies; } - File[] shadersetFiles = shadersetDir.listFiles(); - for (File shaderset : shadersetFiles) { - if (shaderset.isDirectory()) { - shadersets.add(shaderset); + File[] shaderFamilyFiles = shaderFamiliesDir.listFiles(); + for (File shaderFamily : shaderFamilyFiles) { + if (shaderFamily.isDirectory()) { + shaderFamilies.add(shaderFamily); } } - shadersets.sort(Comparator.naturalOrder()); - return shadersets; + shaderFamilies.sort(Comparator.naturalOrder()); + return shaderFamilies; } // Homepage: /webui @@ -409,21 +408,20 @@ private void homepage(HttpServletRequest request, HttpServletResponse response) } htmlAppendLn(""); - //List of shadersets + // List of shader families htmlAppendLn( "
\n", "

Shader Families

\n", "
\n"); - //StringBuilder shadersetListHTML = new StringBuilder(); - List shadersets = getAllShadersets(request, response); - if (shadersets.size() > 0) { - for (File file : shadersets) { - Shaderset shaderset = new Shaderset(file.getName()); - htmlAppendLn("", + List shaderFamilies = getAllShaderFamilies(request, response); + if (shaderFamilies.size() > 0) { + for (File file : shaderFamilies) { + ShaderFamily shaderFamily = new ShaderFamily(file.getName()); + htmlAppendLn("", "Reference image preview", - "
", shaderset.name, - "
#variants: ", Integer.toString(shaderset.nbVariants), + shaderFamily.preview.getPath(), "' onerror=\"this.style.display='none'\">", + "
", shaderFamily.name, + "
#variants: ", Integer.toString(shaderFamily.nbVariants), "
"); } } @@ -564,11 +562,12 @@ private void worker(HttpServletRequest request, HttpServletResponse response) for (File shaderFamilyFile : shaderFamilies) { final String shaderFamily = shaderFamilyFile.getName(); - ShadersetExp shadersetExp = new ShadersetExp(shaderFamily, workerName, accessFileInfo); + ShaderFamilyResult shaderFamilyResult = new ShaderFamilyResult(shaderFamily, workerName, + accessFileInfo); // TODO(360): Show results for compute shaders. For now, just indicate that some results // exists, and point to documentation. - if (shadersetExp.shaderset.isComp) { + if (shaderFamilyResult.shaderFamily.isComp) { htmlAppendLn( "", @@ -585,12 +584,13 @@ private void worker(HttpServletRequest request, HttpServletResponse response) "", "
", shaderFamily, "
", - "Variant done: ", Integer.toString(shadersetExp.nbVariantDone), - " / ", Integer.toString(shadersetExp.nbVariants), - " | Wrong images: ", Integer.toString(shadersetExp.nbWrongImage), - " | Slightly different images: ", Integer.toString(shadersetExp.nbSlightlyDifferentImage), - " | Errors: ", Integer.toString(shadersetExp.nbErrors), - " | Metrics disagree: ", Integer.toString(shadersetExp.nbMetricsDisagree), + "Variant done: ", Integer.toString(shaderFamilyResult.nbVariantDone), + " / ", Integer.toString(shaderFamilyResult.nbVariants), + " | Wrong images: ", Integer.toString(shaderFamilyResult.nbWrongImage), + " | Slightly different images: ", Integer.toString( + shaderFamilyResult.nbSlightlyDifferentImage), + " | Errors: ", Integer.toString(shaderFamilyResult.nbErrors), + " | Metrics disagree: ", Integer.toString(shaderFamilyResult.nbMetricsDisagree), "
"); } htmlAppendLn("
"); @@ -725,10 +725,10 @@ private void experimentSetup(HttpServletRequest request, HttpServletResponse res } } - List shadersets = getAllShadersets(request, response); + List shaderFamilies = getAllShaderFamilies(request, response); htmlAppendLn("

Shader families

"); - if (shadersets.size() == 0) { + if (shaderFamilies.size() == 0) { htmlAppendLn("

No shader families detected

"); } else { htmlAppendLn("
"); int dataNum = 0; - for (File f : shadersets) { + for (File f : shaderFamilies) { htmlAppendLn("
", "
", " - private void shadersetResults(HttpServletRequest request, HttpServletResponse response) + // Results page for a shader family showing results by all workers - + // /webui/shaderset/ + private void shaderFamilyResults(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { response.setContentType("text/html"); @@ -810,7 +811,7 @@ private void viewShader(HttpServletRequest request, HttpServletResponse response shaderPath.append("/").append(path[i]); } File shader = new File(shaderPath.toString()); - String shaderset = shader.getParentFile().getName(); + String shaderFamily = shader.getParentFile().getName(); String shaderName = FilenameUtils.removeExtension(shader.getName()); htmlHeader(shaderName); @@ -1001,7 +1002,7 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response final ReductionStatus referenceReductionStatus = getReductionStatus(worker, shaderFamily, "reference"); - File referenceShader = new File(WebUiConstants.SHADERSET_DIR + "/" + File referenceShader = new File(WebUiConstants.SHADER_FAMILIES_DIR + "/" + shaderFamily, "reference.frag"); if (referenceReductionStatus == ReductionStatus.FINISHED) { referenceShader = new File( @@ -1147,8 +1148,8 @@ private void produceDiff(String shader, File reductionDir, File referenceShader) } } - private ReductionStatus getReductionStatus(String worker, String shaderSet, String shader) { - final File reductionDir = ReductionFilesHelper.getReductionDir(worker, shaderSet, shader); + private ReductionStatus getReductionStatus(String worker, String shaderFamily, String shader) { + final File reductionDir = ReductionFilesHelper.getReductionDir(worker, shaderFamily, shader); if (! reductionDir.exists()) { return ReductionStatus.NOREDUCTION; } @@ -1241,17 +1242,17 @@ private void startExperiment(HttpServletRequest request, HttpServletResponse res StringBuilder msg = new StringBuilder(); String[] workers = request.getParameterValues("workercheck"); - String[] shadersets = request.getParameterValues("shadersetcheck"); + String[] shaderFamilies = request.getParameterValues("shadersetcheck"); if (workers == null || workers.length == 0) { msg.append("Select at least one worker"); - } else if (shadersets == null || shadersets.length == 0) { - msg.append("Select at least one shaderset"); + } else if (shaderFamilies == null || shaderFamilies.length == 0) { + msg.append("Select at least one shader family"); } else { //Start experiments for each worker/shader combination, display result in alert for (String worker : workers) { - for (String shaderset : shadersets) { - msg.append("Experiment ").append(shaderset).append(" on worker ").append(worker); + for (String shaderFamily : shaderFamilies) { + msg.append("Experiment ").append(shaderFamily).append(" on worker ").append(worker); List commands = new ArrayList<>(); commands.add("run_shader_family"); commands.add("--server"); @@ -1259,10 +1260,10 @@ private void startExperiment(HttpServletRequest request, HttpServletResponse res commands.add("--worker"); commands.add(worker); commands.add("--output"); - commands.add("processing/" + worker + "/" + shaderset); - commands.add(WebUiConstants.SHADERSET_DIR + "/" + shaderset); - fuzzerServiceManagerProxy.queueCommand("run_shader_family: " + shaderset, commands, - worker,"processing/" + worker + "/" + shaderset + "/command.log"); + commands.add("processing/" + worker + "/" + shaderFamily); + commands.add(WebUiConstants.SHADER_FAMILIES_DIR + "/" + shaderFamily); + fuzzerServiceManagerProxy.queueCommand("run_shader_family: " + shaderFamily, commands, + worker,"processing/" + worker + "/" + shaderFamily + "/command.log"); msg.append(" started successfully!\\n"); } } @@ -1277,7 +1278,7 @@ private void startExperiment(HttpServletRequest request, HttpServletResponse res response.getWriter().println(html); } - // Page for selecting workers/shadersets to compare results - /webui/compareResults + // Page for selecting workers/shader families to compare results - /webui/compareResults private void compareResults(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { @@ -1302,7 +1303,7 @@ private void compareResults(HttpServletRequest request, HttpServletResponse resp response.getWriter().println(html); } - // Page for selecting workers/shadersets to compare results - /webui/compare + // Page for selecting workers/shader families to compare results - /webui/compare private void compareWorkers(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { @@ -1358,11 +1359,11 @@ private void compareWorkers(HttpServletRequest request, HttpServletResponse resp ""); dataNum = 0; - for (File shaderFamily: getAllShadersets(request, response)) { + for (File shaderFamilyFile: getAllShaderFamilies(request, response)) { // TODO(360): Handle compute shaders - Shaderset shaderset = new Shaderset(shaderFamily.getName()); - if (shaderset.isComp) { + ShaderFamily shaderFamily = new ShaderFamily(shaderFamilyFile.getName()); + if (shaderFamily.isComp) { continue; } @@ -1370,9 +1371,9 @@ private void compareWorkers(HttpServletRequest request, HttpServletResponse resp "
", "", + " value='", shaderFamilyFile.getName(), "'>", "", + shaderFamilyFile.getName(), "", "
"); dataNum += 1; } @@ -1520,15 +1521,15 @@ private void reduceReference(String shaderJobFilePath, String worker) throws TEx } else { referenceShaderJobFile = shaderJobFile; } - String shaderset = referenceShaderJobFile.getParentFile().getName(); + String shaderFamily = referenceShaderJobFile.getParentFile().getName(); File reductionDir = - new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderset + "/reductions", + new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderFamily + "/reductions", "reference"); if (reductionDir.isDirectory()) { return; } File referenceResult = - new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderset, + new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderFamily, "reference.info.json"); List args = new ArrayList<>(); args.add("glsl-reduce"); @@ -1545,7 +1546,7 @@ private void reduceReference(String shaderJobFilePath, String worker) throws TEx args.add("--server"); args.add("http://localhost:8080"); fuzzerServiceManagerProxy.queueCommand( - "Reference Reduction: " + shaderset, + "Reference Reduction: " + shaderFamily, args, worker, new File(reductionDir, "command.log").toString()); @@ -1659,7 +1660,7 @@ private void runShader(HttpServletRequest request, HttpServletResponse response) // e.g. shaderfamilies/family1/variant2.json File shader = new File(shaderPath); // e.g. family1 - String shaderset = shader.getParentFile().getName(); + String shaderFamily = shader.getParentFile().getName(); String[] workers = request.getParameterValues("workercheck"); String javascript = getResourceContent("goBack.js"); @@ -1674,11 +1675,11 @@ private void runShader(HttpServletRequest request, HttpServletResponse response) commands.add("--worker"); commands.add(worker); commands.add("--output"); - commands.add("processing/" + worker + "/" + shaderset + "/"); + commands.add("processing/" + worker + "/" + shaderFamily + "/"); try { fuzzerServiceManagerProxy .queueCommand("run_shader_family: " + shaderPath, commands, worker, - "processing/" + worker + "/" + shaderset + "/command.log"); + "processing/" + worker + "/" + shaderFamily + "/command.log"); } catch (TException exception) { err404(request, response, exception.getMessage()); return; @@ -1782,7 +1783,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) } else if (actions[1].equals("experiment")) { experimentSetup(request, response); } else if (actions[1].equals("shaderset")) { - shadersetResults(request, response); + shaderFamilyResults(request, response); } else if (actions[1].equals("shader")) { viewShader(request, response); } else if (actions[1].equals("result")) { @@ -2139,14 +2140,14 @@ private void htmlReductionForm( ""); } - private void htmlComparativeTable(String shaderFamily, String[] workers) + private void htmlComparativeTable(String shaderFamilyFilename, String[] workers) throws FileNotFoundException { - Shaderset shaderset = new Shaderset(shaderFamily); - if (shaderset.isComp) { + ShaderFamily shaderFamily = new ShaderFamily(shaderFamilyFilename); + if (shaderFamily.isComp) { // TODO(360): Handle compute shaders htmlAppendLn("

Compute shader family: ", - shaderFamily, ". ", + shaderFamilyFilename, ". ", "The UI does not display results for compute shaders yet.", " ", @@ -2157,7 +2158,7 @@ private void htmlComparativeTable(String shaderFamily, String[] workers) htmlAppendLn("\n", ""); - File variantsDir = new File(WebUiConstants.SHADERSET_DIR + "/" + shaderFamily); + File variantsDir = new File(WebUiConstants.SHADER_FAMILIES_DIR + "/" + shaderFamily); File[] variantFragFiles = variantsDir.listFiles(variantFragFilter); Arrays.sort(variantFragFiles, (f1, f2) -> new AlphanumComparator().compare(f1.getName(), f2.getName())); @@ -2170,9 +2171,9 @@ private void htmlComparativeTable(String shaderFamily, String[] workers) } htmlAppendLn("
", "", "reference", @@ -2225,7 +2226,7 @@ private void htmlComparativeTable(String shaderFamily, String[] workers) + "/" + shaderFamily + "/" + f.getName().replace(".frag", ".info.json")); if (infoFile.isFile()) { - ReductionStatus reductionStatus = getReductionStatus(worker, shaderFamily, + ReductionStatus reductionStatus = getReductionStatus(worker, shaderFamilyFilename, infoFile.getName().replace(".info.json", "")); htmlVariantResultTableCell(infoFile, refPngPath, reductionStatus); diff --git a/server/src/main/java/com/graphicsfuzz/server/webui/WebUiConstants.java b/server/src/main/java/com/graphicsfuzz/server/webui/WebUiConstants.java index fc9d796ca..d5fe97504 100644 --- a/server/src/main/java/com/graphicsfuzz/server/webui/WebUiConstants.java +++ b/server/src/main/java/com/graphicsfuzz/server/webui/WebUiConstants.java @@ -24,7 +24,7 @@ private WebUiConstants() { static final String FILE_ROUTE = "file"; static final String WORKER_DIR = "processing"; - static final String SHADERSET_DIR = "shaderfamilies"; + static final String SHADER_FAMILIES_DIR = "shaderfamilies"; static final String WORKER_INFO_FILE = "client.json"; static final String COMPUTE_SHADER_DOC_URL = "https://github.com/google/graphicsfuzz/blob/master/docs/glsl-fuzz-walkthrough" From 5091c31c8ec2897cb7eb509200071e3b329b2c37 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Tue, 21 May 2019 18:18:42 +0200 Subject: [PATCH 002/180] Add macro wrapper and fix space before comma (#500) --- .../com/graphicsfuzz/reducer/util/SimplifyTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java index 8289411d7..0fcf1bed1 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java @@ -44,7 +44,7 @@ public void testIfParenthesesRemoved() throws Exception { @Test public void testWhileParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" - + " while (_GLF_DEAD(_GLF_FALSE(false, false))) {" + + " while(_GLF_WRAPPED_LOOP(_GLF_FALSE(false ,_GLF_DEAD(_GLF_FALSE(false, false))))) {" + " }" + "}"); final String expected = "void main() {" @@ -60,14 +60,14 @@ public void testWhileParenthesesRemoved() throws Exception { public void testForParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" + " for (" - + " int i = _GLF_ONE(1, _GLF_IDENTITY(1, 1));" + + " int i = int(_GLF_ONE(1.0, _GLF_IDENTITY(1.0, 1.0)));" + " i > _GLF_IDENTITY(1, _GLF_FUZZED(1));" + " i++)" + " { }" + " }" + "}"); final String expected = "void main() {" - + " for (int i = 1; i > 1; i++)" + + " for (int i = int(1.0); i > 1; i++)" + " { }" + "}"; final TranslationUnit simplifiedTu = Simplify.simplify(tu); @@ -78,7 +78,7 @@ public void testForParenthesesRemoved() throws Exception { @Test public void testSwitchParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" - + " switch(_GLF_ZERO(0, _GLF_IDENTITY(0, 0)))" + + " switch(_GLF_SWITCH(_GLF_ZERO(0, _GLF_IDENTITY(0, 0))))" + " {" + " }" + "}"); @@ -95,7 +95,7 @@ public void testSwitchParenthesesRemoved() throws Exception { @Test public void testDoWhileParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" - + " do { } while (_GLF_DEAD(_GLF_FALSE(false, false)));" + + " do { } while (_GLF_WRAPPED_LOOP(_GLF_DEAD(_GLF_FALSE(false, false))));" + "}"); final String expected = "void main() {" + " do { } while (false);" @@ -113,7 +113,7 @@ public void testMacroBlockRemoved() throws Exception { + " int a = _GLF_IDENTITY(_GLF_FUZZED(1), 1);" + "}" ); - final String expected = "void() {" + final String expected = "void main() {" + " 1;" + " 1;" + " int a = 1;" From 712a4a20bd8a58817124f1092b12dd950b902b10 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 23 May 2019 03:20:55 +0100 Subject: [PATCH 003/180] Removed code to check for shader translator throwing a 'memory exhausted' error, since the issue that was causing this has been fixed in shader translator. (#501) --- .../common/util/ShaderJobFileOperations.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java index 2ec304b17..a11dbf889 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java @@ -1123,22 +1123,12 @@ private boolean shaderIsValidShaderTranslator( shaderFile, ShaderTranslatorShadingLanguageVersionSupport .getShaderTranslatorArgument(shadingLanguageVersion)); - if (isMemoryExhaustedError(shaderTranslatorResult)) { - return true; - } return checkValidationResult( shaderTranslatorResult, shaderFile.getName(), throwExceptionOnValidationError); } - // TODO(171): This is a workaround for an issue where shader_translator reports memory exhaustion. - // If the issue in shader_translator can be fixed, we should get rid of this check. - private boolean isMemoryExhaustedError(ExecResult shaderTranslatorResult) { - return shaderTranslatorResult.res != 0 - && shaderTranslatorResult.stdout.toString().contains("memory exhausted"); - } - private boolean checkValidationResult(ExecResult res, String filename, boolean throwExceptionOnValidationError) { if (res.res != 0) { From d8f63eb3eb53a74e4adee79731902b85b48e6b82 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 23 May 2019 09:35:12 -0700 Subject: [PATCH 004/180] Exclude .pdb files from graphicsfuzz.zip (#496) --- graphicsfuzz/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphicsfuzz/pom.xml b/graphicsfuzz/pom.xml index a61db5ab4..06b1bf236 100644 --- a/graphicsfuzz/pom.xml +++ b/graphicsfuzz/pom.xml @@ -247,7 +247,7 @@ limitations under the License. - + From 421a4b48186a43f5c2690e10d8d94837ba304b0d Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 23 May 2019 09:37:31 -0700 Subject: [PATCH 005/180] Make releases be draft releases (#497) --- build/travis/install-github-release-tool.sh | 4 ++-- build/travis/release.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/travis/install-github-release-tool.sh b/build/travis/install-github-release-tool.sh index 20e42a24d..0ca7513ec 100755 --- a/build/travis/install-github-release-tool.sh +++ b/build/travis/install-github-release-tool.sh @@ -20,8 +20,8 @@ set -u echo "Installing github-release ${GITHUB_RELEASE_TOOL_PLATFORM} (linux_amd64, darwin_amd64, windows_amd64, etc.) tool to $(pwd)." -GITHUB_RELEASE_TOOL_USER="c4milo" -GITHUB_RELEASE_TOOL_VERSION="v1.1.0" +GITHUB_RELEASE_TOOL_USER="paulthomson" +GITHUB_RELEASE_TOOL_VERSION="v1.1.0.1" GITHUB_RELEASE_TOOL_FILE="github-release_${GITHUB_RELEASE_TOOL_VERSION}_${GITHUB_RELEASE_TOOL_PLATFORM}.tar.gz" if test ! -f "${GITHUB_RELEASE_TOOL_FILE}.touch"; then diff --git a/build/travis/release.py b/build/travis/release.py index 51d9e0c74..5ca5e5cfa 100644 --- a/build/travis/release.py +++ b/build/travis/release.py @@ -51,7 +51,7 @@ def go(): subprocess.check_call([ "github-release", - "-prerelease", + "-draft", repo_name, tag_name, commit_hash, From 73392dc5512fda09f3e515ef0b154d4daaa5d240 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 23 May 2019 19:49:09 +0100 Subject: [PATCH 006/180] Added more tests for removing uniforms (#504) The tests check the cases where (a) a uniform is truly unused, but there is a use of a shadowing variable with the same name, and (b) a uniform is used in a fragment shader but not vertex shader, and vice versa. --- ...ormMetadataReductionOpportunitiesTest.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java index cabfda1d4..cffbad872 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java @@ -24,6 +24,7 @@ import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderKind; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -66,6 +67,41 @@ public void testRemoveUnused() throws Exception { assertEquals(0, shaderJob.getPipelineInfo().getNumUniforms()); } + // TODO(478): enable this test once the issue is addressed. + @Ignore + @Test + public void testRemoveUnusedNameShadowing() throws Exception { + // Checks for the case where a uniform declared in the pipeline state is not used, but another + // variable shadows its name. + final String minimalShader = "void main() { int shadow; }"; + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("shadow", BasicType.FLOAT, Optional.empty(), + Collections.singletonList(10.0)); + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + pipelineInfo, ParseHelper.parse(minimalShader)); + // The pipeline info has an unused uniform named 'shadow', and the shader declares an unrelated + // variable called 'shadow'. We would like the uniform to be removed from the pipeline info, + // as it is not used. + assertEquals(1, shaderJob.getPipelineInfo().getNumUniforms()); + + // There should be exactly one opportunity to remove a piece of unused pipeline state. + // TODO(478): remove the Assert.fail(), and un-comment the lines that follow it. + Assert.fail(); + //List ops = + // RemoveRedundantUniformMetadataReductionOpportunities + // .findOpportunities(shaderJob, + // new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + // null, + // true)); + //assertEquals(1, ops.size()); + //ops.get(0).applyReduction(); + + // Check that after applying the reduction opportunity there are no uniforms in the pipeline + // state and that the shader has not changed. + CompareAsts.assertEqualAsts(minimalShader, shaderJob.getFragmentShader().get()); + assertEquals(0, shaderJob.getPipelineInfo().getNumUniforms()); + } + // TODO(478): enable this test once the issue is addressed. @Ignore @Test @@ -92,4 +128,38 @@ public void testDoNotRemoveUsed() throws Exception { //assertEquals(0, ops.size()); } + // TODO(478): enable this test once the issue is addressed. + @Ignore + @Test + public void testDoNotRemoveUsedMultipleShaders() throws Exception { + // A shader job with a vertex shader and fragment shader that each use a different + // uniform (but that do not use a uniform in common). Neither uniform should be removed + // from the pipeline state. + final String minimalVertexShader = "uniform float used_in_vertex_only; void main() { " + + "used_in_vertex_only; }"; + final String minimalFragmentShader = "uniform float used_in_fragment_only; void main() { " + + "used_in_fragment_only; }"; + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("used_in_vertex_only", BasicType.FLOAT, Optional.empty(), + Collections.singletonList(10.0)); + pipelineInfo.addUniform("used_in_fragment_only", BasicType.FLOAT, Optional.empty(), + Collections.singletonList(20.0)); + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + pipelineInfo, ParseHelper.parse(minimalVertexShader, ShaderKind.VERTEX), + ParseHelper.parse(minimalVertexShader, ShaderKind.FRAGMENT)); + // Check that initially there are indeed two uniforms in the pipeline state. + assertEquals(2, shaderJob.getPipelineInfo().getNumUniforms()); + + // There should be no opportunities to remove a piece of unused pipeline state, since both + // uniforms are referenced. + // TODO(478): remove the Assert.fail(), and un-comment the lines that follow it. + Assert.fail(); + //List ops = + // RemoveRedundantUniformMetadataReductionOpportunities + // .findOpportunities(shaderJob, + // new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + // null, true)); + //assertEquals(0, ops.size()); + } + } From 19848d937abf0107bf7503100fe4448d154d5daa Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 23 May 2019 14:14:38 -0700 Subject: [PATCH 007/180] Add testIdentityNotNestedRemoved (#492) --- .../reducer/util/SimplifyTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java index 0fcf1bed1..ac3ea0d5f 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java @@ -122,4 +122,24 @@ public void testMacroBlockRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } + // TODO(491): Enable once issue 491 is fixed. + @Ignore + @Test + public void testIdentityNotNestedRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() {" + + " if(_GLF_IDENTITY(true, true)) {}" + + " int x = _GLF_IDENTITY(1, 1);" + + " x = _GLF_IDENTITY(1, 1);" + + " _GLF_IDENTITY(1, 1);" + + "}" + ); + final String expected = "void main() {" + + " if(true) {}" + + " int x = 1;" + + " x = 1;" + + " 1;" + + "}"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } } From fb2d54eb7f7e1b0d93dbad348875e49dc8a8a871 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 23 May 2019 14:40:59 -0700 Subject: [PATCH 008/180] Update a few maven plug-in versions and fix some warnings (#494) --- .../main/resources/graphicsfuzz/checkstyle.xml | 4 ++-- .../com/graphicsfuzz/generator/tool/Generate.java | 1 + parent-all/pom.xml | 8 ++++---- parent-checkstyle/pom.xml | 4 ++++ .../com/graphicsfuzz/reducer/ReductionDriver.java | 12 ++++++------ .../glslreducers/SystematicReductionPass.java | 15 ++++++++++----- .../com/graphicsfuzz/reducer/tool/GlslReduce.java | 2 +- .../common/util/CheckUtilityClass.java | 3 +-- 8 files changed, 29 insertions(+), 20 deletions(-) diff --git a/checkstyle-config/src/main/resources/graphicsfuzz/checkstyle.xml b/checkstyle-config/src/main/resources/graphicsfuzz/checkstyle.xml index 01f65e2d4..7bd1f641d 100644 --- a/checkstyle-config/src/main/resources/graphicsfuzz/checkstyle.xml +++ b/checkstyle-config/src/main/resources/graphicsfuzz/checkstyle.xml @@ -74,8 +74,8 @@ limitations under the License. - - + + diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java index dce9cc76c..5b2ed2131 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java @@ -219,6 +219,7 @@ public static StringBuilder generateVariant(ShaderJob shaderJob, * preprocessor or validator. * @throws GlslParserException if a shader in the job fails to parse. */ + @SuppressWarnings("deprecation") public static void generateVariant(ShaderJobFileOperations fileOps, File referenceShaderJobFile, File outputShaderJobFile, diff --git a/parent-all/pom.xml b/parent-all/pom.xml index f740038eb..02f0fd0b6 100644 --- a/parent-all/pom.xml +++ b/parent-all/pom.xml @@ -42,7 +42,7 @@ limitations under the License. 1.8 1.8 - 9.4.12.v20180830 + 9.4.18.v20190429 @@ -482,7 +482,7 @@ limitations under the License. org.apache.maven.plugins maven-compiler-plugin - 3.6.1 + 3.8.1 @@ -500,7 +500,7 @@ limitations under the License. org.apache.maven.plugins maven-jar-plugin - 3.0.2 + 3.1.2 org.apache.maven.plugins @@ -530,7 +530,7 @@ limitations under the License. com.puppycrawl.tools checkstyle - 8.14 + 8.20 diff --git a/parent-checkstyle/pom.xml b/parent-checkstyle/pom.xml index 7b8a165c2..ca77ffb17 100644 --- a/parent-checkstyle/pom.xml +++ b/parent-checkstyle/pom.xml @@ -39,6 +39,10 @@ limitations under the License. + + ${project.build.sourceDirectory} + + graphicsfuzz/checkstyle.xml UTF-8 true diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java index fb24f8b7a..7f7bbfabc 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java @@ -23,6 +23,7 @@ import com.graphicsfuzz.reducer.glslreducers.IReductionPassManager; import com.graphicsfuzz.reducer.glslreducers.SystematicReductionPass; import com.graphicsfuzz.reducer.glslreducers.SystematicReductionPassManager; +import com.graphicsfuzz.reducer.reductionopportunities.IReductionOpportunity; import com.graphicsfuzz.reducer.reductionopportunities.IReductionOpportunityFinder; import com.graphicsfuzz.reducer.reductionopportunities.ReducerContext; import com.graphicsfuzz.reducer.util.Simplify; @@ -31,6 +32,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -84,20 +86,19 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.largestFunctionsFinder(5), 1)); final List cleanupPasses = new ArrayList<>(); - for (IReductionOpportunityFinder finder : new IReductionOpportunityFinder[]{ + for (IReductionOpportunityFinder finder : Arrays.asList( IReductionOpportunityFinder.inlineUniformFinder(), IReductionOpportunityFinder.inlineInitializerFinder(), IReductionOpportunityFinder.inlineFunctionFinder(), IReductionOpportunityFinder.unusedParamFinder(), - IReductionOpportunityFinder.foldConstantFinder(), - }) { + IReductionOpportunityFinder.foldConstantFinder())) { cleanupPasses.add(new SystematicReductionPass(context, verbose, finder)); } final List corePasses = new ArrayList<>(); - for (IReductionOpportunityFinder finder : new IReductionOpportunityFinder[]{ + for (IReductionOpportunityFinder finder : Arrays.asList( IReductionOpportunityFinder.vectorizationFinder(), IReductionOpportunityFinder.unswitchifyFinder(), IReductionOpportunityFinder.stmtFinder(), @@ -117,8 +118,7 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.liveFragColorWriteFinder(), IReductionOpportunityFinder.functionFinder(), IReductionOpportunityFinder.variableDeclFinder(), - IReductionOpportunityFinder.globalVariablesDeclarationFinder(), - }) { + IReductionOpportunityFinder.globalVariablesDeclarationFinder())) { final SystematicReductionPass pass = new SystematicReductionPass(context, verbose, finder); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/SystematicReductionPass.java b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/SystematicReductionPass.java index 68bf10639..b71a5905e 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/SystematicReductionPass.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/SystematicReductionPass.java @@ -31,17 +31,22 @@ public class SystematicReductionPass extends AbstractReductionPass { private int granularity; private final int maximumGranularity; - public SystematicReductionPass(ReducerContext reducerContext, - boolean verbose, IReductionOpportunityFinder finder, - int maximumGranularity) { + public SystematicReductionPass( + ReducerContext reducerContext, + boolean verbose, + IReductionOpportunityFinder finder, + int maximumGranularity + ) { // Ignore verbose argument for now. super(reducerContext, finder); this.isInitialized = false; this.maximumGranularity = maximumGranularity; } - public SystematicReductionPass(ReducerContext reducerContext, - boolean verbose, IReductionOpportunityFinder finder) { + public SystematicReductionPass( + ReducerContext reducerContext, + boolean verbose, + IReductionOpportunityFinder finder) { this(reducerContext, verbose, finder, Integer.MAX_VALUE); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java index 3c5a21b8d..6e3815991 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java @@ -275,7 +275,7 @@ public static void mainHelper( final double threshold = ns.get("threshold"); // TODO: integrate timeout into reducer - @SuppressWarnings("UnusedAssignment") Integer timeout = ns.get("timeout"); + @SuppressWarnings("unused") Integer timeout = ns.get("timeout"); final Integer maxSteps = ns.get("max_steps"); final Integer retryLimit = ns.get("retry_limit"); final Boolean verbose = ns.get("verbose"); diff --git a/test-util/src/main/java/com/graphicsfuzz/common/util/CheckUtilityClass.java b/test-util/src/main/java/com/graphicsfuzz/common/util/CheckUtilityClass.java index 7e7272a72..38b8fbe82 100644 --- a/test-util/src/main/java/com/graphicsfuzz/common/util/CheckUtilityClass.java +++ b/test-util/src/main/java/com/graphicsfuzz/common/util/CheckUtilityClass.java @@ -41,8 +41,7 @@ public static void assertUtilityClassWellDefined(final Class utilityClass) assertEquals("Utility class can only have one constructor", 1, utilityClass.getDeclaredConstructors().length); final Constructor constructor = utilityClass.getDeclaredConstructor(); - if (constructor.isAccessible() - || !Modifier.isPrivate(constructor.getModifiers())) { + if (!Modifier.isPrivate(constructor.getModifiers())) { fail("Utility class constructor must be private"); } constructor.setAccessible(true); From da53c6e39bfd6ac7e27f50275ddf2ac93f8ec3ef Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 24 May 2019 09:47:33 -0400 Subject: [PATCH 009/180] Fix to WebUI to avoid null pointer exception (#506) A previous refactoring had led to confusion between two variables, one a file, the other a string, such that the result of calling toString() on the file was appearing in URLs. --- .../main/java/com/graphicsfuzz/server/webui/WebUi.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java index 957c895a9..1a827bbda 100755 --- a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java +++ b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java @@ -2143,8 +2143,7 @@ private void htmlReductionForm( private void htmlComparativeTable(String shaderFamilyFilename, String[] workers) throws FileNotFoundException { - ShaderFamily shaderFamily = new ShaderFamily(shaderFamilyFilename); - if (shaderFamily.isComp) { + if (new ShaderFamily(shaderFamilyFilename).isComp) { // TODO(360): Handle compute shaders htmlAppendLn("

Compute shader family: ", shaderFamilyFilename, ". ", @@ -2158,7 +2157,7 @@ private void htmlComparativeTable(String shaderFamilyFilename, String[] workers) htmlAppendLn("\n", ""); - File variantsDir = new File(WebUiConstants.SHADER_FAMILIES_DIR + "/" + shaderFamily); + File variantsDir = new File(WebUiConstants.SHADER_FAMILIES_DIR + "/" + shaderFamilyFilename); File[] variantFragFiles = variantsDir.listFiles(variantFragFilter); Arrays.sort(variantFragFiles, (f1, f2) -> new AlphanumComparator().compare(f1.getName(), f2.getName())); @@ -2196,7 +2195,7 @@ private void htmlComparativeTable(String shaderFamilyFilename, String[] workers) // Reference result is separate as it doesn't contain a "identical" field, etc // FIXME: make sure reference result has same format as variants to be able to refactor String refHref = WebUiConstants.WORKER_DIR + "/" + worker + "/" - + shaderFamily + "/reference"; + + shaderFamilyFilename + "/reference"; File refInfoFile = new File(refHref + ".info.json"); String refPngPath = refHref + ".png"; @@ -2223,7 +2222,7 @@ private void htmlComparativeTable(String shaderFamilyFilename, String[] workers) for (File f : variantFragFiles) { File infoFile = new File(WebUiConstants.WORKER_DIR + "/" + worker - + "/" + shaderFamily + "/" + f.getName().replace(".frag", ".info.json")); + + "/" + shaderFamilyFilename + "/" + f.getName().replace(".frag", ".info.json")); if (infoFile.isFile()) { ReductionStatus reductionStatus = getReductionStatus(worker, shaderFamilyFilename, From 8b203015c1413eb7246e2fbd658d186d32bf2b18 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 24 May 2019 15:20:34 -0400 Subject: [PATCH 010/180] Fixed issue where attempts to make array access in donated code in-bounds was failing. (#505) Fixes #503. The issue was that the AST was being cloned between donating statements and fixing up the array accesses in said statements, and the fixup was being attemped on the pre-cloning versions of statements. This change (a) adds a test to expose the issue, and (b) fixes it. A small amount of cleanup of related code is also included. --- .../DonateCodeTransformation.java | 21 ++-- .../DonateLiveCodeTransformationTest.java | 112 ++++++++++++++++++ 2 files changed, 121 insertions(+), 12 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java index f25a18bca..5a1c1a00d 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java @@ -33,7 +33,6 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.type.ArrayType; import com.graphicsfuzz.common.ast.type.BasicType; -import com.graphicsfuzz.common.ast.type.LayoutQualifier; import com.graphicsfuzz.common.ast.type.LayoutQualifierSequence; import com.graphicsfuzz.common.ast.type.QualifiedType; import com.graphicsfuzz.common.ast.type.StructDefinitionType; @@ -79,7 +78,7 @@ public abstract class DonateCodeTransformation implements ITransformation { - final Function probabilityOfDonation; + private final Function probabilityOfDonation; private Map donorsToTranslationUnits; private final List functionPrototypes; @@ -413,14 +412,8 @@ private void donateFunctionsAndGlobals(TranslationUnit recipient) { newRecipientTopLevelDeclarations.addAll(recipient.getTopLevelDeclarations().stream() .filter(d -> d instanceof FunctionDefinition).collect(Collectors.toList())); - // Make sure we clone the top-level FunctionPrototypes that are added, otherwise each will - // exist twice in the AST: first as a top-level declaration, second as part of the - // FunctionDefinition. newRecipientTopLevelDeclarations = - addNecessaryForwardDeclarations(newRecipientTopLevelDeclarations) - .stream() - .map(Declaration::clone) - .collect(Collectors.toList()); + addNecessaryForwardDeclarations(newRecipientTopLevelDeclarations); recipient.setTopLevelDeclarations(newRecipientTopLevelDeclarations); @@ -480,12 +473,16 @@ private List addNecessaryForwardDeclarations(List decl Set calledFunctionNames = getCalledFunctions(decls.get(i)); - Set toDeclare = + List toDeclare = functionsDefinedAfterDecl.get(i).stream().filter(item -> calledFunctionNames.contains(item.getName())).filter(item -> !declared.stream().anyMatch(alreadyDeclared -> alreadyDeclared.matches(item))) - .collect(Collectors.toSet()); - result.addAll(toDeclare); + .collect(Collectors.toList()); + + // Make sure we clone the FunctionPrototypes that are added, otherwise each will exist + // twice in the AST: first as a top-level declaration, second as part of the + // FunctionDefinition. + result.addAll(toDeclare.stream().map(Declaration::clone).collect(Collectors.toList())); declared.addAll(toDeclare); } if (decls.get(i) instanceof FunctionDefinition) { diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java index ebe1ff952..00ac78081 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java @@ -23,8 +23,10 @@ import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; @@ -36,6 +38,7 @@ import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.typing.ScopeEntry; import com.graphicsfuzz.common.typing.ScopeTreeBuilder; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.ParseHelper; @@ -57,6 +60,8 @@ import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public class DonateLiveCodeTransformationTest { @@ -328,4 +333,111 @@ public void verySimpleDonorAndSourceNoPrecision() throws Exception { IParentMap.createParentMap(referenceShaderJob.getFragmentShader().get()); } + @Test + public void testArrayAccessesAreInBounds() throws Exception { + // This checks that array accesses are correctly made in-bounds when injecting live code. + + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + final File donors = testFolder.newFolder("donors"); + final File referenceFile = testFolder.newFile("reference.json"); + + { + // This donor is designed to have a high chance of leading to an array access getting injected + // such that the array indexing expression will be a free variable for which a fuzzed initial + // value will be created. + final String donorSource = + "#version 300 es\n" + + "void main() {\n" + + " int x = 0;" + + " {" + + " int A[1];" + + " A[x] = 42;" + + " {" + + " int B[1];" + + " B[x] = 42;" + + " {" + + " int C[1];" + + " C[x] = 42;" + + " }" + + " }" + + " }" + + "}\n"; + + fileOps.writeShaderJobFileFromImageJob( + new ImageJob() + .setFragmentSource(donorSource) + .setUniformsInfo("{}"), + new File(donors, "donor.json") + ); + } + + { + final String referenceSource = "#version 300 es\n" + + "void main() {" + + " " + + "}"; + + fileOps.writeShaderJobFileFromImageJob( + new ImageJob() + .setFragmentSource(referenceSource) + .setUniformsInfo("{}"), + referenceFile + + ); + } + + // Try the following a few times, so that there is a good chance of triggering the issue + // this test was used to catch, should it return: + for (int seed = 0; seed < 5; seed++) { + + final ShaderJob referenceShaderJob = fileOps.readShaderJobFile(referenceFile); + + // Do live code donation. + DonateLiveCodeTransformation transformation = + new DonateLiveCodeTransformation(IRandom::nextBoolean, donors, + GenerationParams.normal(ShaderKind.FRAGMENT, true), false); + + assert referenceShaderJob.getFragmentShader().isPresent(); + + boolean result = transformation.apply( + referenceShaderJob.getFragmentShader().get(), + TransformationProbabilities.onlyLiveCodeAlwaysSubstitute(), + new RandomWrapper(seed), + GenerationParams.normal(ShaderKind.FRAGMENT, true) + ); + + Assert.assertTrue(result); + + // An array access injected into the shader must either be (1) already in bounds, or + // (2) made in bounds. Only in the former case can the array index be a variable identifier + // expression, and in that case the expression cannot realistically be statically in bounds + // if the initializer for that expression is under a _GLF_FUZZED macro. (There is a tiny + // chance that the fuzzed expression might statically evaluate to 0, but currently a + // _GLF_FUZZED macro will be treated as not statically in bounds, so the access would be + // made in bounds in that case.) + // + // The following thus checks that if an array is indexed directly by a variable reference, + // the initializer for that variable is not a function call expression. + new ScopeTreeBuilder() { + + @Override + public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { + super.visitArrayIndexExpr(arrayIndexExpr); + if (arrayIndexExpr.getIndex() instanceof VariableIdentifierExpr) { + final ScopeEntry scopeEntry = currentScope.lookupScopeEntry( + ((VariableIdentifierExpr) arrayIndexExpr.getIndex()).getName()); + assertTrue(scopeEntry.hasVariableDeclInfo()); + assertNotNull(scopeEntry.getVariableDeclInfo().getInitializer()); + assertFalse(((ScalarInitializer) scopeEntry.getVariableDeclInfo().getInitializer()) + .getExpr() instanceof FunctionCallExpr); + } + } + + }.visit(referenceShaderJob.getFragmentShader().get()); + + } + + } + } From a84dbc26f32a0773980a45f218f5f38806ac0f27 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Sun, 26 May 2019 12:10:43 +0200 Subject: [PATCH 011/180] Add a test for function call parentheses removal (#508) --- .../graphicsfuzz/reducer/util/SimplifyTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java index ac3ea0d5f..fa91ebc73 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java @@ -142,4 +142,18 @@ public void testIdentityNotNestedRemoved() throws Exception { final TranslationUnit simplifiedTu = Simplify.simplify(tu); CompareAsts.assertEqualAsts(expected, simplifiedTu); } + + @Ignore + @Test + public void testFunctionCallParenthesesRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() {" + + "foo(_GLF_IDENTITY(1 + 2, 1 + 2));" + + "}" + ); + final String expected = "void main() {" + + " foo(1 + 2);" + + "}"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } } From 79dd9a9ce53343ccc8349abfb6a60d0dd06302b8 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Sun, 26 May 2019 05:14:43 -0500 Subject: [PATCH 012/180] Added additional tests for empty for statements (#507) Fixes #205. --- .../common/ast/stmt/ForStmtTest.java | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/ast/src/test/java/com/graphicsfuzz/common/ast/stmt/ForStmtTest.java b/ast/src/test/java/com/graphicsfuzz/common/ast/stmt/ForStmtTest.java index 3a021ee6b..7ec1adfbf 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/ast/stmt/ForStmtTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/ast/stmt/ForStmtTest.java @@ -17,7 +17,6 @@ package com.graphicsfuzz.common.ast.stmt; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import com.graphicsfuzz.common.ast.CompareAstsDuplicate; @@ -103,4 +102,69 @@ public void testCloneEmptyFor() throws Exception { CompareAstsDuplicate.assertEqualAsts(tu, tu2); } + @Test + public void testEmptyFor() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (;;) ; }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertTrue(stmt.getInit() instanceof NullStmt); + assertFalse(stmt + .hasCondition()); + assertFalse(stmt + .hasIncrement()); + } + + @Test + public void testOnlyHasInit() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (1; ; ) { } }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertFalse(stmt.getInit() instanceof NullStmt); + assertFalse(stmt + .hasCondition()); + assertFalse(stmt + .hasIncrement()); + } + + @Test + public void testOnlyHasCondition() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (; 1; ) { } }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertTrue(stmt.getInit() instanceof NullStmt); + assertTrue(stmt + .hasCondition()); + assertFalse(stmt + .hasIncrement()); + } + + @Test + public void testOnlyHasIncrement() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (; ; 1) { } }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertTrue(stmt.getInit() instanceof NullStmt); + assertFalse(stmt + .hasCondition()); + assertTrue(stmt + .hasIncrement()); + } + + @Test + public void testHasAllFields() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (1; 1; 1) { } }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertFalse(stmt.getInit() instanceof NullStmt); + assertTrue(stmt + .hasCondition()); + assertTrue(stmt + .hasIncrement()); + } + } From 0c9c376f3465c843a94f66d368d32e9938c4c1a8 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Mon, 27 May 2019 01:01:06 +0200 Subject: [PATCH 013/180] Reducer opportunity to remove redundant uniform metadata (#490) Fixes #478. --- .../common/util/ShaderJobFileOperations.java | 14 +++- .../graphicsfuzz/reducer/ReductionDriver.java | 5 +- .../IReductionOpportunityFinder.java | 19 +++++ .../ReductionOpportunities.java | 3 +- ...UniformMetadataReductionOpportunities.java | 66 ++++++++++++++++ ...ntUniformMetadataReductionOpportunity.java | 46 +++++++++++ .../reducer/ReductionDriverTest.java | 2 - ...ormMetadataReductionOpportunitiesTest.java | 76 +++++++------------ 8 files changed, 178 insertions(+), 53 deletions(-) create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunity.java diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java index a11dbf889..5d47d05a2 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java @@ -1011,7 +1011,13 @@ private String getMD5(File shaderJobFile) throws IOException { byte[] computeData = isFile(computeShaderFile) ? readFileToByteArray(computeShaderFile) : new byte[0]; - byte[] combinedData = new byte[vertexData.length + fragmentData.length + computeData.length]; + // This metadata is required in order to distinguish between shader jobs + // with identical shaders but different pipeline information. + byte[] metaData = isFile(shaderJobFile) + ? readFileToByteArray(shaderJobFile) + : new byte[0]; + byte[] combinedData = + new byte[vertexData.length + fragmentData.length + computeData.length + metaData.length ]; System.arraycopy( vertexData, 0, @@ -1030,6 +1036,12 @@ private String getMD5(File shaderJobFile) throws IOException { combinedData, vertexData.length + fragmentData.length, computeData.length); + System.arraycopy( + metaData, + 0, + combinedData, + vertexData.length + fragmentData.length + computeData.length, + metaData.length); return DigestUtils.md5Hex(combinedData); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java index 7f7bbfabc..deee715e3 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java @@ -91,7 +91,8 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.inlineInitializerFinder(), IReductionOpportunityFinder.inlineFunctionFinder(), IReductionOpportunityFinder.unusedParamFinder(), - IReductionOpportunityFinder.foldConstantFinder())) { + IReductionOpportunityFinder.foldConstantFinder(), + IReductionOpportunityFinder.redundantUniformMetadataFinder())) { cleanupPasses.add(new SystematicReductionPass(context, verbose, finder)); @@ -296,7 +297,7 @@ private void writeState(ShaderJob state, File shaderJobFileOutput, context.getEmitGraphicsFuzzDefines() ); if (requiresUniformBindings) { - assert state.hasUniformBindings(); + assert state.getPipelineInfo().getNumUniforms() == 0 || state.hasUniformBindings(); state.removeUniformBindings(); } } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java index c2ae7a351..8d8ec8e94 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java @@ -462,4 +462,23 @@ public String getName() { }; } + static IReductionOpportunityFinder + redundantUniformMetadataFinder() { + return new IReductionOpportunityFinder() { + @Override + public List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return RemoveRedundantUniformMetadataReductionOpportunities.findOpportunities( + shaderJob, + context); + } + + @Override + public String getName() { + return "redundantUniformMetadata"; + } + }; + } + } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java index 669e5680e..6f9d8e862 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java @@ -65,7 +65,8 @@ public static List getReductionOpportunities( IReductionOpportunityFinder.liveFragColorWriteFinder(), IReductionOpportunityFinder.unusedParamFinder(), IReductionOpportunityFinder.foldConstantFinder(), - IReductionOpportunityFinder.inlineUniformFinder())) { + IReductionOpportunityFinder.inlineUniformFinder(), + IReductionOpportunityFinder.redundantUniformMetadataFinder())) { final List currentOpportunities = ros .findOpportunities(shaderJob, context); if (ReductionDriver.DEBUG_REDUCER) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java new file mode 100644 index 000000000..f025d6166 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class RemoveRedundantUniformMetadataReductionOpportunities { + + static List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + + // We initially grab names of all uniforms in the pipeline info. + final List canBeRemoved = + new ArrayList<>(shaderJob.getPipelineInfo().getUniformNames()); + + for (TranslationUnit tu : shaderJob.getShaders()) { + new StandardVisitor() { + @Override + public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { + super.visitVariablesDeclaration(variablesDeclaration); + // A uniform in the pipeline info that its declaration is found in shaders + // will not be removed by the reducer. + if (variablesDeclaration.getBaseType().hasQualifier(TypeQualifier.UNIFORM)) { + for (VariableDeclInfo vdi : variablesDeclaration.getDeclInfos()) { + canBeRemoved.remove(vdi.getName()); + } + } + } + }.visit(tu); + } + + // Since we are going to remove redundant uniforms in the pipeline info that do not + // belong to any shader, the visitation depth should be zero. + return canBeRemoved + .stream() + .map(item -> new RemoveRedundantUniformMetadataReductionOpportunity(item, + shaderJob.getPipelineInfo(), + new VisitationDepth(0)) + ).collect(Collectors.toList()); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunity.java new file mode 100644 index 000000000..aaa98914e --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunity.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import com.graphicsfuzz.common.util.PipelineInfo; + +public class RemoveRedundantUniformMetadataReductionOpportunity + extends AbstractReductionOpportunity { + + private final PipelineInfo pipelineInfo; + private final String uniformName; + + RemoveRedundantUniformMetadataReductionOpportunity(String uniformName, + PipelineInfo pipelineInfo, + VisitationDepth depth) { + super(depth); + this.pipelineInfo = pipelineInfo; + this.uniformName = uniformName; + } + + @Override + void applyReductionImpl() { + pipelineInfo.removeUniform(uniformName); + } + + @Override + public boolean preconditionHolds() { + return pipelineInfo.getUniformNames().contains(uniformName); + } + +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java index 5a7b8b213..7ac4d6c59 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java @@ -563,8 +563,6 @@ public void testReductionWithUniformBindings() throws Exception { } - // TODO(478): Enable this test once the issue is fixed. - @Ignore @Test public void testReductionOfUnreferencedUniform() throws Exception { diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java index cffbad872..4bbe08355 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java @@ -28,16 +28,12 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; public class RemoveRedundantUniformMetadataReductionOpportunitiesTest { - // TODO(478): enable this test once the issue is addressed. - @Ignore @Test public void testRemoveUnused() throws Exception { // Make a shader job with an empty fragment shader, and declare one uniform in the pipeline @@ -51,15 +47,13 @@ public void testRemoveUnused() throws Exception { assertEquals(1, shaderJob.getPipelineInfo().getNumUniforms()); // There should be exactly one opportunity to remove a piece of unused pipeline state. - // TODO(478): remove the Assert.fail(), and un-comment the lines that follow it. - Assert.fail(); - //List ops = - // RemoveRedundantUniformMetadatReductionOpportunities - // .findOpportunities(shaderJob, - // new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, - // true)); - //assertEquals(1, ops.size()); - //ops.get(0).applyReduction(); + List ops = + RemoveRedundantUniformMetadataReductionOpportunities + .findOpportunities(shaderJob, + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, + true)); + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); // Check that after applying the reduction opportunity there are no uniforms in the pipeline // state and that the shader has not changed. @@ -67,8 +61,6 @@ public void testRemoveUnused() throws Exception { assertEquals(0, shaderJob.getPipelineInfo().getNumUniforms()); } - // TODO(478): enable this test once the issue is addressed. - @Ignore @Test public void testRemoveUnusedNameShadowing() throws Exception { // Checks for the case where a uniform declared in the pipeline state is not used, but another @@ -85,16 +77,14 @@ public void testRemoveUnusedNameShadowing() throws Exception { assertEquals(1, shaderJob.getPipelineInfo().getNumUniforms()); // There should be exactly one opportunity to remove a piece of unused pipeline state. - // TODO(478): remove the Assert.fail(), and un-comment the lines that follow it. - Assert.fail(); - //List ops = - // RemoveRedundantUniformMetadataReductionOpportunities - // .findOpportunities(shaderJob, - // new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - // null, - // true)); - //assertEquals(1, ops.size()); - //ops.get(0).applyReduction(); + List ops = + RemoveRedundantUniformMetadataReductionOpportunities + .findOpportunities(shaderJob, + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + null, + true)); + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); // Check that after applying the reduction opportunity there are no uniforms in the pipeline // state and that the shader has not changed. @@ -102,8 +92,6 @@ public void testRemoveUnusedNameShadowing() throws Exception { assertEquals(0, shaderJob.getPipelineInfo().getNumUniforms()); } - // TODO(478): enable this test once the issue is addressed. - @Ignore @Test public void testDoNotRemoveUsed() throws Exception { // Make a shader job with a simple fragment shader that declares (but does not use) @@ -117,19 +105,15 @@ public void testDoNotRemoveUsed() throws Exception { // Check that initially there is indeed one uniform in the pipeline state. assertEquals(1, shaderJob.getPipelineInfo().getNumUniforms()); - // There should be exactly one opportunity to remove a piece of unused pipeline state. - // TODO(478): remove the Assert.fail(), and un-comment the lines that follow it. - Assert.fail(); - //List ops = - // RemoveRedundantUniformMetadatReductionOpportunities - // .findOpportunities(shaderJob, - // new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, - // true)); - //assertEquals(0, ops.size()); + // There should be no opportunities to remove a piece of unused pipeline state. + List ops = + RemoveRedundantUniformMetadataReductionOpportunities + .findOpportunities(shaderJob, + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, + true)); + assertEquals(0, ops.size()); } - // TODO(478): enable this test once the issue is addressed. - @Ignore @Test public void testDoNotRemoveUsedMultipleShaders() throws Exception { // A shader job with a vertex shader and fragment shader that each use a different @@ -146,20 +130,18 @@ public void testDoNotRemoveUsedMultipleShaders() throws Exception { Collections.singletonList(20.0)); final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), pipelineInfo, ParseHelper.parse(minimalVertexShader, ShaderKind.VERTEX), - ParseHelper.parse(minimalVertexShader, ShaderKind.FRAGMENT)); + ParseHelper.parse(minimalFragmentShader, ShaderKind.FRAGMENT)); // Check that initially there are indeed two uniforms in the pipeline state. assertEquals(2, shaderJob.getPipelineInfo().getNumUniforms()); // There should be no opportunities to remove a piece of unused pipeline state, since both // uniforms are referenced. - // TODO(478): remove the Assert.fail(), and un-comment the lines that follow it. - Assert.fail(); - //List ops = - // RemoveRedundantUniformMetadataReductionOpportunities - // .findOpportunities(shaderJob, - // new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - // null, true)); - //assertEquals(0, ops.size()); + List ops = + RemoveRedundantUniformMetadataReductionOpportunities + .findOpportunities(shaderJob, + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + null, true)); + assertEquals(0, ops.size()); } } From 585cc98b909a73102ddd9243e8f36ca192e2d45b Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Mon, 27 May 2019 15:47:53 +0200 Subject: [PATCH 014/180] Remove parentheses after eliminating macros (#489) Fixes #110. --- .../EliminateInjectionMacrosVisitor.java | 154 ++++++------------ .../reducer/util/SimplifyTest.java | 11 -- 2 files changed, 52 insertions(+), 113 deletions(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java index b027214b2..869fdcf7b 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java @@ -16,128 +16,78 @@ package com.graphicsfuzz.reducer.glslreducers; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; -import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.ConstantExpr; import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; -import com.graphicsfuzz.common.ast.expr.MemberLookupExpr; import com.graphicsfuzz.common.ast.expr.ParenExpr; -import com.graphicsfuzz.common.ast.expr.TernaryExpr; -import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; -import com.graphicsfuzz.common.ast.expr.UnaryExpr; -import com.graphicsfuzz.common.ast.stmt.DoStmt; -import com.graphicsfuzz.common.ast.stmt.ForStmt; -import com.graphicsfuzz.common.ast.stmt.IfStmt; -import com.graphicsfuzz.common.ast.stmt.LoopStmt; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.reducer.reductionopportunities.MacroNames; public class EliminateInjectionMacrosVisitor extends StandardVisitor { @Override - public void visitBinaryExpr(BinaryExpr binaryExpr) { - super.visitBinaryExpr(binaryExpr); - cleanUpMacros(binaryExpr); - } - - @Override - public void visitUnaryExpr(UnaryExpr unaryExpr) { - super.visitUnaryExpr(unaryExpr); - cleanUpMacros(unaryExpr); - } - - @Override - public void visitParenExpr(ParenExpr parenExpr) { - super.visitParenExpr(parenExpr); - cleanUpMacros(parenExpr); - } - - @Override - public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { - super.visitFunctionCallExpr(functionCallExpr); - cleanUpMacros(functionCallExpr); - } - - public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { - super.visitArrayIndexExpr(arrayIndexExpr); - cleanUpMacros(arrayIndexExpr); - } - - public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { - super.visitMemberLookupExpr(memberLookupExpr); - cleanUpMacros(memberLookupExpr); - } - - public void visitTernaryExpr(TernaryExpr ternaryExpr) { - super.visitTernaryExpr(ternaryExpr); - cleanUpMacros(ternaryExpr); - } - - @Override - public void visitTypeConstructorExpr(TypeConstructorExpr typeConstructorExpr) { - super.visitTypeConstructorExpr(typeConstructorExpr); - cleanUpMacros(typeConstructorExpr); - } - - @Override - public void visitIfStmt(IfStmt ifStmt) { - super.visitIfStmt(ifStmt); - if (MacroNames.isDeadByConstruction(ifStmt.getCondition()) - || MacroNames.isIfWrapperFalse(ifStmt.getCondition()) - || MacroNames.isIfWrapperTrue(ifStmt.getCondition())) { - ifStmt.setCondition(ifStmt.getCondition().getChild(0)); + protected void visitChildFromParent(IAstNode child, IAstNode parent) { + super.visitChildFromParent(child, parent); + if (child instanceof FunctionCallExpr) { + final FunctionCallExpr functionCallExpr = (FunctionCallExpr) child; + if (MacroNames.isIdentity(functionCallExpr) + || MacroNames.isZero(functionCallExpr) + || MacroNames.isOne(functionCallExpr) + || MacroNames.isFalse(functionCallExpr) + || MacroNames.isTrue(functionCallExpr)) { + parent.replaceChild(child, + addParenthesesIfNecessary(parent, functionCallExpr.getChild(1))); + } else if (MacroNames.isFuzzed(functionCallExpr) + || MacroNames.isDeadByConstruction(functionCallExpr) + || MacroNames.isSwitch(functionCallExpr) + || MacroNames.isLoopWrapper(functionCallExpr) + || MacroNames.isIfWrapperFalse(functionCallExpr) + || MacroNames.isIfWrapperTrue(functionCallExpr) + ) { + parent.replaceChild(child, + addParenthesesIfNecessary(parent, functionCallExpr.getChild(0))); + } } } - private void cleanupLoop(LoopStmt loopStmt) { - if (MacroNames.isLoopWrapper(loopStmt.getCondition())) { - loopStmt.setCondition(loopStmt.getCondition().getChild(0)); + private IAstNode addParenthesesIfNecessary(IAstNode parent, Expr child) { + if (child instanceof ConstantExpr + || child instanceof ParenExpr + || child instanceof VariableIdentifierExpr + || child instanceof FunctionCallExpr) { + // Parentheses is unnecessary in cases such as _GLF_FUNCTION(1), + // _GLF_FUNCTION((1)), _GLF_FUNCTION(a), _GLF_FUNCTION(sin(a)). + return child; } - } - - @Override - public void visitDoStmt(DoStmt doStmt) { - super.visitDoStmt(doStmt); - cleanupLoop(doStmt); - } - @Override - public void visitForStmt(ForStmt forStmt) { - super.visitForStmt(forStmt); - cleanupLoop(forStmt); - } + if (!(parent instanceof Expr)) { + // No parentheses needed if the parent is not an expression, + // for example, int x = _GLF_FUNCTION(a + b). + return child; + } - private void cleanUpMacros(Expr parent) { - for (int i = 0; i < parent.getNumChildren(); i++) { - Expr child = parent.getChild(i); - // Note: it would be redundant to have a special function reduction opportunity - // be classed also as a fuzzed expression reduction opportunity - if (MacroNames.isIdentity(child) - || MacroNames.isZero(child) - || MacroNames.isOne(child) - || MacroNames.isFalse(child) - || MacroNames.isTrue(child)) { - replaceChildWithGrandchild(parent, i, 1); - } else if (MacroNames.isFuzzed(child) - || MacroNames.isDeadByConstruction(child)) { - replaceChildWithGrandchild(parent, i, 0); - } + if (parent instanceof ParenExpr) { + // If parent is parentheses, adding a new parentheses would be redundant, + // e.g. (_GLF_FUNCTION(a + b)). + return child; } - } - @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (MacroNames.isFuzzed(scalarInitializer.getExpr())) { - scalarInitializer.setExpr(scalarInitializer.getExpr().getChild(0)); + if (parent instanceof FunctionCallExpr) { + // Parentheses is unnecessary if parent is a function call expression. + // For example, foo(_GLF_FUNCTION(a)). + + // This asserts that the binary expression inside the function call is not a comma operator + // as it is invalid to have a comma appear directly here, e.g. _GLF_IDENTITY(expr, a, b) is + // not valid since a and b are treated as function arguments instead. + assert (!(child instanceof BinaryExpr) || ((BinaryExpr) child).getOp() != BinOp.COMMA); + return child; } - } - private void replaceChildWithGrandchild(Expr parent, int childIndex, int grandchildIndex) { - assert parent.getChild(childIndex).getNumChildren() > grandchildIndex; - parent - .setChild(childIndex, new ParenExpr(parent.getChild(childIndex).getChild(grandchildIndex))); + return new ParenExpr(child); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java index fa91ebc73..1af838a27 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java @@ -19,13 +19,10 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.ParseHelper; -import org.junit.Ignore; import org.junit.Test; public class SimplifyTest { - // TODO(110) - this test fails due to issue 110, and should be enabled once that issue is fixed. - @Ignore @Test public void testIfParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" @@ -40,7 +37,6 @@ public void testIfParenthesesRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testWhileParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" @@ -55,7 +51,6 @@ public void testWhileParenthesesRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testForParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" @@ -74,7 +69,6 @@ public void testForParenthesesRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testSwitchParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" @@ -91,7 +85,6 @@ public void testSwitchParenthesesRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testDoWhileParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" @@ -104,7 +97,6 @@ public void testDoWhileParenthesesRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testMacroBlockRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" @@ -122,8 +114,6 @@ public void testMacroBlockRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - // TODO(491): Enable once issue 491 is fixed. - @Ignore @Test public void testIdentityNotNestedRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" @@ -143,7 +133,6 @@ public void testIdentityNotNestedRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testFunctionCallParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" From 3a0a244ca98f87f115f5f40f71b1d40e50041181 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Tue, 4 Jun 2019 12:24:38 +0200 Subject: [PATCH 015/180] Allow GLF_DEAD conditionals to be removed by compound-to-block (#514) Fixes #482. --- ...CompoundToBlockReductionOpportunities.java | 3 -- .../reducer/ReductionDriverTest.java | 2 +- ...oundToBlockReductionOpportunitiesTest.java | 44 ++++++++++++++----- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java index 79a13854a..84f998d6a 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java @@ -90,9 +90,6 @@ private void handleLoopStmt(LoopStmt loopStmt) { @Override public void visitIfStmt(IfStmt ifStmt) { super.visitIfStmt(ifStmt); - if (MacroNames.isDeadByConstruction(ifStmt.getCondition())) { - return; - } addOpportunity(ifStmt, ifStmt.getThenStmt()); if (ifStmt.hasElseStmt()) { addOpportunity(ifStmt, ifStmt.getElseStmt()); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java index 7ac4d6c59..9b25d422b 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java @@ -113,7 +113,7 @@ public boolean isInteresting( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, version, generator, new IdGenerator(), true), fileOps); - assertEquals(2, ops.size()); + assertEquals(3, ops.size()); new ReductionDriver( new ReducerContext( diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java index 7c7337a1e..18f4a1fc1 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java @@ -31,7 +31,6 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -126,7 +125,7 @@ public void testUnderUnreachableSwitch() throws Exception { } @Test - public void testDoNotRemoveDeadIf() throws Exception { + public void testDoRemoveDeadIf() throws Exception { final String original = "" + "void main() {" + " if (" + Constants.GLF_DEAD + "(false)) {" @@ -134,9 +133,14 @@ public void testDoNotRemoveDeadIf() throws Exception { + " a = a + 1;" + " }" + "}"; - final TranslationUnit tu = ParseHelper.parse(original); - assertTrue(CompoundToBlockReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new RandomWrapper(0), null, true)).isEmpty()); + final String expected = "" + + "void main() {" + + " {" + + " int a = 2;" + + " a = a + 1;" + + " }" + + "}"; + check(false, original, expected); } @Test @@ -150,7 +154,7 @@ public void testInDeadCode() throws Exception { + " }" + " }" + "}"; - final String expected = "" + final String expected1 = "" + "void main() {" + " if (" + Constants.GLF_DEAD + "(false)) {" + " {" @@ -159,11 +163,18 @@ public void testInDeadCode() throws Exception { + " }" + " }" + "}"; - check(false, original, expected); + final String expected2 = "" + + "void main() {" + + " {" + + " if (false) {" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + check(false, original, expected1, expected2); } - // TODO(482): Enable this test once the issue is fixed. - @Ignore @Test public void testDeadIfReduceEverywhere() throws Exception { final String original = "" @@ -245,7 +256,20 @@ public void testInDeadCode2() throws Exception { + " }" + " }" + "}"; - check(false, original, expected1, expected2, expected3); + final String expected4 = "" + + "void main() {" + + " int a = 4;" + + " {" + + " if (a) {" + + " if (a > 0) {" + + " int a = 2;" + + " a = a + 1;" + + " } else" + + " a++;" + + " }" + + " }" + + "}"; + check(false, original, expected1, expected2, expected3, expected4); } @Test From 23f85b312d78155db1acb8cc26b0800a5a2ee82d Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 4 Jun 2019 09:20:28 -0500 Subject: [PATCH 016/180] Generate ternary identities for nonscalar types (#509) Fixes #254 --- .../generator/fuzzer/OpaqueExpressionGenerator.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index e7aabd060..bac7ca896 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -663,7 +663,7 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, private class IdentityTernary extends AbstractIdentityTransformation { private IdentityTernary() { - super(BasicType.allScalarTypes(), false); + super(BasicType.allNumericTypes(), false); } @Override @@ -672,8 +672,7 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, // (false ? whatever : expr) // or // (true ? expr : whatever) - assert BasicType.allScalarTypes().contains(type); - // Only generate ternary expressions for scalars; the vector case causes massive blow-up + assert BasicType.allNumericTypes().contains(type); Expr exprWithIdentityApplied = applyIdentityFunction(expr, type, constContext, depth, fuzzer); Expr something = fuzzedConstructor(fuzzer.fuzzExpr(type, false, constContext, depth)); if (generator.nextBoolean()) { From eef536b24f8e3367058857e1deea0b56fe850623 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 4 Jun 2019 09:24:09 -0500 Subject: [PATCH 017/180] Add new transformation for rewriting nonscalars as their constructors (#512) Fixes #510 --- .../fuzzer/OpaqueExpressionGenerator.java | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index bac7ca896..72dc1b895 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -79,6 +79,8 @@ public OpaqueExpressionGenerator(IRandom generator, GenerationParams generationP expressionIdentities.add(new IdentityMax()); expressionIdentities.add(new IdentityClamp()); + expressionIdentities.add(new IdentityRewriteComposite()); + if (shadingLanguageVersion.supportedMixNonfloatBool()) { expressionIdentities.add(new IdentityMixBvec()); } @@ -530,7 +532,7 @@ private abstract class AbstractIdentityTransformation implements ExpressionIdent } @Override - public final boolean preconditionHolds(Expr expr, BasicType basicType) { + public boolean preconditionHolds(Expr expr, BasicType basicType) { if (!acceptableTypes.contains(basicType)) { return false; } @@ -801,4 +803,47 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, } + private class IdentityRewriteComposite extends AbstractIdentityTransformation { + private IdentityRewriteComposite() { + // all non-boolean vector/matrix types + super(BasicType.allNumericTypes().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()), + false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + // v -> (true ? vecX(..., identity(v[Y]), ...) : _) + // v -> (false ? _ : vecX(..., identity(v[Y]), ...)) + // where v is a vector of size X, y is the random entry in v we want to apply + // identities to, and ... is the other entries in v that we don't change. + // Similarly for matrices. + + assert BasicType.allVectorTypes().contains(type) + || BasicType.allMatrixTypes().contains(type); + assert expr instanceof VariableIdentifierExpr; + + final int numIndices = + (BasicType.allVectorTypes().contains(type) + ? type.getNumElements() : type.getNumColumns()); + final int indexToFurtherTransform = generator.nextInt(numIndices); + final List typeConstructorArguments = new ArrayList<>(); + for (int i = 0; i < numIndices; i++) { + Expr argument = new ArrayIndexExpr(expr.clone(), new IntConstantExpr(String.valueOf(i))); + if (i == indexToFurtherTransform) { + argument = applyIdentityFunction(argument, type.getElementType(), constContext, depth, + fuzzer); + } + typeConstructorArguments.add(argument); + } + return identityConstructor(expr, + new TypeConstructorExpr(type.toString(), typeConstructorArguments)); + } + + @Override + public boolean preconditionHolds(Expr expr, BasicType basicType) { + return super.preconditionHolds(expr, basicType) && expr instanceof VariableIdentifierExpr; + } + } } From 11e9b8062658f0931b259be12e1d477df8be7e5d Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 5 Jun 2019 15:57:15 -0500 Subject: [PATCH 018/180] Fix matrix constructors generating identities for the wrong base type (#525) * Fixed applying identities to wrong type in matrix constructors * Use isVector() and isMatrix() --- .../generator/fuzzer/OpaqueExpressionGenerator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 72dc1b895..ed9763be5 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -820,8 +820,7 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, // identities to, and ... is the other entries in v that we don't change. // Similarly for matrices. - assert BasicType.allVectorTypes().contains(type) - || BasicType.allMatrixTypes().contains(type); + assert type.isVector() || type.isMatrix(); assert expr instanceof VariableIdentifierExpr; final int numIndices = @@ -832,8 +831,9 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, for (int i = 0; i < numIndices; i++) { Expr argument = new ArrayIndexExpr(expr.clone(), new IntConstantExpr(String.valueOf(i))); if (i == indexToFurtherTransform) { - argument = applyIdentityFunction(argument, type.getElementType(), constContext, depth, - fuzzer); + argument = applyIdentityFunction(argument, + (type.isVector() ? type.getElementType() : type.getColumnType()), + constContext, depth, fuzzer); } typeConstructorArguments.add(argument); } From 9151414b5d9209e9d479f95603f1d084d00becc0 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Thu, 6 Jun 2019 10:33:51 +0200 Subject: [PATCH 019/180] Add class and tests : reduce VariablesDeclaration with expression (#513) * Add empty class and tests --- ...iableDeclToExprReductionOpportunities.java | 56 ++++++++ ...ariableDeclToExprReductionOpportunity.java | 35 +++++ ...eDeclToExprReductionOpportunitiesTest.java | 133 ++++++++++++++++++ 3 files changed, 224 insertions(+) create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java create mode 100644 reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java new file mode 100644 index 000000000..46f6b11bc --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.ListConcat; +import java.util.Arrays; +import java.util.List; + +public class VariableDeclToExprReductionOpportunities + extends ReductionOpportunitiesBase { + + public VariableDeclToExprReductionOpportunities(TranslationUnit tu, ReducerContext context) { + super(tu, context); + } + + /** + * Find all initialized variable declaration opportunities for the given translation unit. + * + * @param shaderJob The shader job to be searched. + * @param context Includes info such as whether we reduce everywhere or only reduce injections + * @return The opportunities that can be reduced + */ + static List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return shaderJob.getShaders() + .stream() + .map(item -> findOpportunitiesForShader(item, context)) + .reduce(Arrays.asList(), ListConcat::concatenate); + } + + private static List findOpportunitiesForShader( + TranslationUnit tu, + ReducerContext context) { + VariableDeclToExprReductionOpportunities finder = + new VariableDeclToExprReductionOpportunities(tu, context); + finder.visit(tu); + return finder.getOpportunities(); + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java new file mode 100644 index 000000000..c0c236ceb --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; + +public class VariableDeclToExprReductionOpportunity extends AbstractReductionOpportunity { + + VariableDeclToExprReductionOpportunity(VisitationDepth depth) { + super(depth); + } + + @Override + void applyReductionImpl() { + } + + @Override + public boolean preconditionHolds() { + return false; + } +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java new file mode 100644 index 000000000..bea155693 --- /dev/null +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java @@ -0,0 +1,133 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; +import java.util.List; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class VariableDeclToExprReductionOpportunitiesTest { + + // TODO(480): Enable this test once the issue is fixed. + @Ignore + @Test + public void testDoNotReplace() throws Exception { + final String original = "void main() { int a = 1, b = 2; int c = 3; }"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + // There should be no opportunities as the preserve semantics is enabled. + assertTrue(ops.isEmpty()); + } + + @Ignore + @Test + public void testMultipleDeclarations() throws Exception { + final String program = "void main() {" + + "int a = 1;" // Initialized variable declaration. + + "int b = foo();" // Initialized variable declaration. + + "int c;" // Uninitialized variable declaration. + + "}"; + final String expected = "void main() {" + + " int a;" + + " a = 1;" + + " int b;" + + " b = foo();" + + " int c;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + List ops = VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + // Only variable declarations a and b have the initializer. + // Thus, we expect the reducer to find only 2 opportunities. + assertEquals(2, ops.size()); + ops.forEach(VariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Ignore + @Test + public void testMultipleLineDeclarationsOneLine() throws Exception { + final String program = "void main() {" + + "int a;" + + "int b = 1, c, d = foo(), e, f = bar(); " // This variable declaration has many + // declaration infos but we consider only + // the one that has initializer (b, d, and f). + + "int g;" + + "}"; + final String expected = "void main() {" + + " int a;" + + " int b, c, d, e, f;" + + " b = 1;" + + " d = foo();" + + " f = bar();" + + " int g;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + List ops = VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + assertEquals(3, ops.size()); + ops.forEach(VariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Ignore + @Test + public void testAssignVariableIdentifier() throws Exception { + final String program = "void main() {" + + "int a = 1;" + + "int b = a, c = b;" // b depends on a and c depends on b. + + "int d = c;" // d depends on c. + + "}"; + // As here we have the variable identifier as the initializer, we need to + // make sure that new expressions generated by the reducer are added + // in the correct order. + final String expected = "void main() {" + + " int a;" + + " a = 1;" + + " int b, c;" + + " b = a;" + + " c = b;" + + " int d;" + + " d = c;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + List ops = VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + assertEquals(4, ops.size()); + ops.forEach(VariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + +} From 3ccfdc9a32011cbfb5740b23d8195d8ea1f55cc9 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Thu, 6 Jun 2019 16:12:55 -0500 Subject: [PATCH 020/180] Add built-in function support for GLSL Vector Relational Functions (#528) Added built-in function support for GLSL Vector Relational Functions --- .../common/typing/TyperHelper.java | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index d47575141..aa71cfe8f 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; /** * This helper class factors out some context-independent parts of typechecking, @@ -838,6 +839,8 @@ private static Map> getBuiltinsForGlslVersion( // 8.7: Vector Relational Functions + getBuiltinsForGlslVersionVectorRelational(builtinsForVersion, shadingLanguageVersion); + // 8.8: Integer Functions // 8.13: Fragment Processing Functions (only available in fragment shaders) @@ -847,6 +850,131 @@ private static Map> getBuiltinsForGlslVersion( return builtinsForVersion; } + private static void getBuiltinsForGlslVersionVectorRelational( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + // We need these for every function, so instead of constantly calling the functions, + // we'll just cache them to reduce cruft. + final List genVectors = genType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final List igenVectors = igenType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final List ugenVectors = ugenType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final List bgenVectors = bgenType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final boolean supportsUnsigned = shadingLanguageVersion.supportedUnsigned(); + + { + final String name = "lessThan"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + } + } + + { + final String name = "lessThanEqual"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + } + } + + { + final String name = "greaterThan"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + } + } + + { + final String name = "greaterThanEqual"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + } + } + + { + final String name = "equal"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), bgenVectors.get(i), + bgenVectors.get(i)); + } + } + + { + final String name = "notEqual"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), bgenVectors.get(i), + bgenVectors.get(i)); + } + } + + { + final String name = "any"; + for (Type t : bgenVectors) { + addBuiltin(builtinsForVersion, name, BasicType.BOOL, t); + } + } + + { + final String name = "all"; + for (Type t : bgenVectors) { + addBuiltin(builtinsForVersion, name, BasicType.BOOL, t); + } + } + + { + final String name = "not"; + for (Type t : bgenVectors) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + private static void addBuiltin(Map> builtinsForVersion, String name, Type resultType, Type... args) { if (!builtinsForVersion.containsKey(name)) { From 49e5a03c48db9d2b5dede08000ea2ae81a4b097f Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Fri, 7 Jun 2019 04:47:25 -0500 Subject: [PATCH 021/180] Enhance matrix constructor identity to potentially use all elements of matrix (#527) Allow matrix constructor identity to use all elements. --- .../common/ast/type/BasicType.java | 18 ++++++++ .../fuzzer/OpaqueExpressionGenerator.java | 43 +++++++++++++++---- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java index 78f912fec..4a55897d8 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java @@ -456,4 +456,22 @@ public int getNumColumns() { assert Arrays.asList(BasicType.MAT4X2, BasicType.MAT4X3, BasicType.MAT4X4).contains(this); return 4; } + + /** + * Finds the number of rows in a matrix type. Will cause an assertion failure if + * used with a non-matrix type. + * + * @return the number of rows that the matrix type has. + */ + public int getNumRows() { + assert allMatrixTypes().contains(this); + if (Arrays.asList(BasicType.MAT2X2, BasicType.MAT3X2, BasicType.MAT4X2).contains(this)) { + return 2; + } + if (Arrays.asList(BasicType.MAT2X3, BasicType.MAT3X3, BasicType.MAT4X3).contains(this)) { + return 3; + } + assert Arrays.asList(BasicType.MAT2X4, BasicType.MAT3X4, BasicType.MAT4X4).contains(this); + return 4; + } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index ed9763be5..238efff72 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -823,19 +823,44 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, assert type.isVector() || type.isMatrix(); assert expr instanceof VariableIdentifierExpr; - final int numIndices = + final int numColumns = (BasicType.allVectorTypes().contains(type) ? type.getNumElements() : type.getNumColumns()); - final int indexToFurtherTransform = generator.nextInt(numIndices); + final int columnToFurtherTransform = generator.nextInt(numColumns); final List typeConstructorArguments = new ArrayList<>(); - for (int i = 0; i < numIndices; i++) { - Expr argument = new ArrayIndexExpr(expr.clone(), new IntConstantExpr(String.valueOf(i))); - if (i == indexToFurtherTransform) { - argument = applyIdentityFunction(argument, - (type.isVector() ? type.getElementType() : type.getColumnType()), - constContext, depth, fuzzer); + + if (type.isMatrix() && generator.nextBoolean()) { + // Further break down matrix constructor into single components, to increase diversity. + // For example, a 2x2 matrix may become mat2(m[0][0], m[0][1], IDENTITY(m[1][0]), m[1][1]). + // GLSL's matrix notation follows column-major indexing rules. + final int rowToFurtherTransform = generator.nextInt(type.getNumRows()); + for (int i = 0; i < numColumns; i++) { + for (int j = 0; j < type.getNumRows(); j++) { + // The inner ArrayIndexExpr is the column (first) index, while the outer ArrayIndexExpr + // is the row (second) index. + Expr argument = new ArrayIndexExpr(new ArrayIndexExpr(expr.clone(), + new IntConstantExpr(String.valueOf(i)).clone()), + new IntConstantExpr(String.valueOf(j))); + if (i == columnToFurtherTransform && j == rowToFurtherTransform) { + argument = applyIdentityFunction(argument, type.getElementType(), constContext, + depth, fuzzer); + } + typeConstructorArguments.add(argument); + } + } + } else { + // Create a constructor with columns only (or single entries, in the case of vectors). + // A 2x2 matrix may become mat2(m[0], IDENTITY(m[1])). + // A vec4 may become vec4(v[0], v[1], IDENTITY(v[2]), v[3]). + for (int i = 0; i < numColumns; i++) { + Expr argument = new ArrayIndexExpr(expr.clone(), new IntConstantExpr(String.valueOf(i))); + if (i == columnToFurtherTransform) { + argument = applyIdentityFunction(argument, + type.isVector() ? type.getElementType() : type.getColumnType(), + constContext, depth, fuzzer); + } + typeConstructorArguments.add(argument); } - typeConstructorArguments.add(argument); } return identityConstructor(expr, new TypeConstructorExpr(type.toString(), typeConstructorArguments)); From 4316319dbf96b629c242b7d145a8a8203dc4e8d9 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Fri, 7 Jun 2019 14:47:22 -0500 Subject: [PATCH 022/180] Refactored BasicType with comments and UnsupportedOperationException (#532) --- .../common/ast/type/BasicType.java | 64 +++++++++++++++---- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java index 4a55897d8..eac8ed00e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java @@ -344,26 +344,44 @@ private static List allBoolTypes() { BVEC4)); } + /** + * Static version of getNumColumns(). Finds the number of columns in a matrix. Can only be + * invoked on a matrix type. + * + * @return the number of columns that the matrix type has. + * @throws UnsupportedOperationException if the type is not a matrix. + */ public static int numColumns(BasicType basicType) { - assert allMatrixTypes().contains(basicType); + if (!allMatrixTypes().contains(basicType)) { + throw new UnsupportedOperationException( + "Type" + basicType.toString() + " does not have columns"); + } if (Arrays.asList(MAT2X2, MAT2X3, MAT2X4).contains(basicType)) { return 2; } if (Arrays.asList(MAT3X2, MAT3X3, MAT3X4).contains(basicType)) { return 3; } - if (Arrays.asList(MAT4X2, MAT4X3, MAT4X4).contains(basicType)) { - return 4; - } - throw new RuntimeException("Camnnot get number of columns for " + basicType); + assert Arrays.asList(MAT4X2, MAT4X3, MAT4X4).contains(basicType); + return 4; } + /** + * Creates a vector type of a specified size from a scalar type. + * + * @return a vector type of numElements dimension, or the scalar type if numElements is 1. + * @throws UnsupportedOperationException if the specified base type is not a scalar, or + * if numElements is outside the bounds of possible dimensions + * (numElements < 0, numElements > 4) + */ public static BasicType makeVectorType(BasicType elementType, int numElements) { if (!allScalarTypes().contains(elementType)) { - throw new RuntimeException("Cannot make vector type from element type " + elementType); + throw new UnsupportedOperationException( + "Cannot make vector type from element type " + elementType); } if (numElements < 0 || numElements > 4) { - throw new RuntimeException("Cannot make vector type with " + numElements + " elements"); + throw new UnsupportedOperationException( + "Cannot make vector type with " + numElements + " elements"); } if (elementType == FLOAT) { switch (numElements) { @@ -433,8 +451,18 @@ public boolean isMatrix() { return allMatrixTypes().contains(this); } + /** + * Determines the vector type of the columns in the matrix. For example, accessing a column of a + * mat2x2 would give you a variable of type vec2. Can only be invoked on a matrix type. + * + * @return the type that represents that the matrix type has. + * @throws UnsupportedOperationException if the type is not a matrix. + */ public BasicType getColumnType() { - assert allMatrixTypes().contains(this); + if (!this.isMatrix()) { + throw new UnsupportedOperationException( + "Type" + this.toString() + " does not have a column type"); + } if (Arrays.asList(BasicType.MAT2X2, BasicType.MAT3X2, BasicType.MAT4X2).contains(this)) { return VEC2; } @@ -445,8 +473,17 @@ public BasicType getColumnType() { return VEC4; } + /** + * Finds the number of columns in a matrix. Can only be invoked on a matrix type. + * + * @return the number of columns that the matrix type has. + * @throws UnsupportedOperationException if the type is not a matrix. + */ public int getNumColumns() { - assert allMatrixTypes().contains(this); + if (!this.isMatrix()) { + throw new UnsupportedOperationException( + "Type" + this.toString() + " does not have columns"); + } if (Arrays.asList(BasicType.MAT2X2, BasicType.MAT2X3, BasicType.MAT2X4).contains(this)) { return 2; } @@ -458,13 +495,16 @@ public int getNumColumns() { } /** - * Finds the number of rows in a matrix type. Will cause an assertion failure if - * used with a non-matrix type. + * Finds the number of rows in a matrix. Can only be invoked on a matrix type. * * @return the number of rows that the matrix type has. + * @throws UnsupportedOperationException if the type is not a matrix. */ public int getNumRows() { - assert allMatrixTypes().contains(this); + if (!this.isMatrix()) { + throw new UnsupportedOperationException( + "Type" + this.toString() + " does not have rows"); + } if (Arrays.asList(BasicType.MAT2X2, BasicType.MAT3X2, BasicType.MAT4X2).contains(this)) { return 2; } From 425c1dd6a95de3425d9f881b8357589d93162619 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Mon, 10 Jun 2019 02:24:40 -0500 Subject: [PATCH 023/180] Remove unused static numColumns function (#535) --- .../common/ast/type/BasicType.java | 22 ------------------- .../VectorMatrixIndexExprTemplate.java | 2 +- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java index eac8ed00e..88251a8b0 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java @@ -344,28 +344,6 @@ private static List allBoolTypes() { BVEC4)); } - /** - * Static version of getNumColumns(). Finds the number of columns in a matrix. Can only be - * invoked on a matrix type. - * - * @return the number of columns that the matrix type has. - * @throws UnsupportedOperationException if the type is not a matrix. - */ - public static int numColumns(BasicType basicType) { - if (!allMatrixTypes().contains(basicType)) { - throw new UnsupportedOperationException( - "Type" + basicType.toString() + " does not have columns"); - } - if (Arrays.asList(MAT2X2, MAT2X3, MAT2X4).contains(basicType)) { - return 2; - } - if (Arrays.asList(MAT3X2, MAT3X3, MAT3X4).contains(basicType)) { - return 3; - } - assert Arrays.asList(MAT4X2, MAT4X3, MAT4X4).contains(basicType); - return 4; - } - /** * Creates a vector type of a specified size from a scalar type. * diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java index 314e0fe2d..882241bb4 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java @@ -48,7 +48,7 @@ public Expr generateExpr(IRandom generator, Expr... args) { index = generator.nextInt(argType.getNumElements()); } else { assert BasicType.allMatrixTypes().contains(argType); - index = generator.nextInt(BasicType.numColumns(argType)); + index = generator.nextInt(argType.getNumColumns()); } return new ArrayIndexExpr(args[0], new IntConstantExpr(String.valueOf(index))); From bb782c6de150fe3188d08fc9c34fbf9d17eb941f Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Tue, 11 Jun 2019 14:17:42 +0200 Subject: [PATCH 024/180] Add test for const variable declaration (#538) --- ...iableDeclToExprReductionOpportunitiesTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java index bea155693..ea565eac7 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java @@ -45,6 +45,21 @@ public void testDoNotReplace() throws Exception { assertTrue(ops.isEmpty()); } + @Ignore + @Test + public void testDoNotReplaceConst() throws Exception { + final String original = "void main() { const int a = 1;}"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + // There should be no opportunities as it is invalid to declare constant variable + // without an initial value. + assertTrue(ops.isEmpty()); + } + @Ignore @Test public void testMultipleDeclarations() throws Exception { From b31a28290482d8a19351936f1e558da9184ab7a3 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 11 Jun 2019 07:18:28 -0500 Subject: [PATCH 025/180] Refactor code to use isType functions when possible (#537) --- .../java/com/graphicsfuzz/common/ast/type/BasicType.java | 2 +- .../com/graphicsfuzz/common/typing/SupportedTypes.java | 3 +-- .../generator/fuzzer/OpaqueExpressionGenerator.java | 6 ++---- .../generator/fuzzer/templates/SwizzleExprTemplate.java | 5 ++--- .../fuzzer/templates/VectorMatrixIndexExprTemplate.java | 7 +++---- .../semanticspreserving/IdentityMutationFinder.java | 3 +-- .../FoldConstantReductionOpportunities.java | 2 +- 7 files changed, 11 insertions(+), 17 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java index 88251a8b0..07492a34a 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java @@ -353,7 +353,7 @@ private static List allBoolTypes() { * (numElements < 0, numElements > 4) */ public static BasicType makeVectorType(BasicType elementType, int numElements) { - if (!allScalarTypes().contains(elementType)) { + if (!elementType.isScalar()) { throw new UnsupportedOperationException( "Cannot make vector type from element type " + elementType); } diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/SupportedTypes.java b/ast/src/main/java/com/graphicsfuzz/common/typing/SupportedTypes.java index 67a34e0f1..3441b6d17 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/SupportedTypes.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/SupportedTypes.java @@ -26,8 +26,7 @@ public static boolean supported(BasicType type, ShadingLanguageVersion shadingLa && !shadingLanguageVersion.supportedUnsigned()) { return false; } - if (BasicType.allMatrixTypes().contains(type) - && !BasicType.allSquareMatrixTypes().contains(type) + if (type.isMatrix() && !BasicType.allSquareMatrixTypes().contains(type) && !shadingLanguageVersion.supportedNonSquareMatrices()) { return false; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 238efff72..63cbac139 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -807,7 +807,7 @@ private class IdentityRewriteComposite extends AbstractIdentityTransformation { private IdentityRewriteComposite() { // all non-boolean vector/matrix types super(BasicType.allNumericTypes().stream().filter( - item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()), + item -> !item.isScalar()).collect(Collectors.toList()), false); } @@ -823,9 +823,7 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, assert type.isVector() || type.isMatrix(); assert expr instanceof VariableIdentifierExpr; - final int numColumns = - (BasicType.allVectorTypes().contains(type) - ? type.getNumElements() : type.getNumColumns()); + final int numColumns = type.isVector() ? type.getNumElements() : type.getNumColumns(); final int columnToFurtherTransform = generator.nextInt(numColumns); final List typeConstructorArguments = new ArrayList<>(); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/SwizzleExprTemplate.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/SwizzleExprTemplate.java index 82c29dc60..cab95ee32 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/SwizzleExprTemplate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/SwizzleExprTemplate.java @@ -32,9 +32,8 @@ public class SwizzleExprTemplate extends AbstractExprTemplate { private final boolean isLValue; public SwizzleExprTemplate(BasicType argType, BasicType resultType, boolean isLValue) { - assert BasicType.allVectorTypes().contains(argType); - assert BasicType.allVectorTypes().contains(resultType) || BasicType.allScalarTypes() - .contains(resultType); + assert argType.isVector(); + assert resultType.isVector() || resultType.isScalar(); if (isLValue) { assert resultType.getNumElements() <= argType.getNumElements(); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java index 882241bb4..427884913 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java @@ -33,8 +33,7 @@ public class VectorMatrixIndexExprTemplate extends AbstractExprTemplate { private final boolean isLValue; public VectorMatrixIndexExprTemplate(BasicType argType, boolean isLValue) { - assert BasicType.allVectorTypes().contains(argType) || BasicType.allMatrixTypes() - .contains(argType); + assert argType.isVector() || argType.isMatrix(); this.argType = argType; this.isLValue = isLValue; } @@ -44,10 +43,10 @@ public Expr generateExpr(IRandom generator, Expr... args) { assert args.length == getNumArguments(); int index; - if (BasicType.allVectorTypes().contains(argType)) { + if (argType.isVector()) { index = generator.nextInt(argType.getNumElements()); } else { - assert BasicType.allMatrixTypes().contains(argType); + assert argType.isMatrix(); index = generator.nextInt(argType.getNumColumns()); } return new ArrayIndexExpr(args[0], new IntConstantExpr(String.valueOf(index))); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java index ff48e76ff..dd2a44b08 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java @@ -83,8 +83,7 @@ protected void visitExpr(Expr expr) { iterators.forEach(clonedScope::remove); } } - if (BasicType.allScalarTypes().contains(basicType) - || BasicType.allVectorTypes().contains(basicType) + if (basicType.isScalar() || basicType.isVector() || BasicType.allSquareMatrixTypes().contains(basicType)) { // TODO: add support for non-square matrices. addMutation(new Expr2ExprMutation(parentMap.getParent(expr), diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java index 392bcc679..a3fe7d1de 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java @@ -223,7 +223,7 @@ private void findReplaceTypeConstructorWithElementOpportunities( return; } final BasicType basicType = (BasicType) structureType; - if (!BasicType.allVectorTypes().contains(basicType)) { + if (!basicType.isVector()) { return; } if (basicType.getNumElements() != tce.getNumArgs()) { From a6a3d0a6fba8295ebb5976c942f2a8cad51ca1e1 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 11 Jun 2019 08:29:37 -0500 Subject: [PATCH 026/180] Check for side effects with lvalue parameters of built-in functions (#533) --- .../common/util/SideEffectChecker.java | 16 +++++++++ .../common/util/SideEffectCheckerTest.java | 36 +++++++++++++++++++ .../templates/FunctionCallExprTemplate.java | 4 ++- 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java b/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java index 9cac37cba..5dc406a43 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java @@ -17,6 +17,8 @@ package com.graphicsfuzz.common.util; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; @@ -28,6 +30,7 @@ import com.graphicsfuzz.common.ast.stmt.ExprCaseLabel; import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.TyperHelper; @@ -45,6 +48,19 @@ public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { // Assume that any call to a non-builtin might have a side-effect. predicateHolds(); } + for (FunctionPrototype p : + TyperHelper.getBuiltins(shadingLanguageVersion).get(functionCallExpr.getCallee())) { + // We check each argument of the built-in's prototypes to see if they require lvalues - + // if so, they can cause side effects. + // We could be more precise here by finding the specific overload of the function rather + // than checking every possible prototype for lvalue parameters. + for (ParameterDecl param : p.getParameters()) { + if (param.getType().hasQualifier(TypeQualifier.OUT_PARAM) + || param.getType().hasQualifier(TypeQualifier.INOUT_PARAM)) { + predicateHolds(); + } + } + } super.visitFunctionCallExpr(functionCallExpr); } diff --git a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java index 00b5fe45e..729a29f4b 100644 --- a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java +++ b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java @@ -16,8 +16,10 @@ package com.graphicsfuzz.common.util; +import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.UnOp; import com.graphicsfuzz.common.ast.expr.UnaryExpr; @@ -25,14 +27,22 @@ import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import java.util.Collections; +import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import static org.junit.Assert.*; public class SideEffectCheckerTest { + @Rule + public TemporaryFolder testFolder = new TemporaryFolder(); + @Test public void testAssignmentHasSideEffects() throws Exception { assertFalse(SideEffectChecker.isSideEffectFree(new ExprStmt( @@ -62,4 +72,30 @@ public void testCountingLoopHasSideEffects() throws Exception { ShadingLanguageVersion.ESSL_310)); } + // TODO(521): Unignore when #521 is ready to merge. + @Ignore + @Test + public void testOutParamHasSideEffects() throws Exception { + final String shader = "void main { " + + " uint out1;" + + " uvec2 out2;" + + " uvec3 out3;" + + " uvec4 out4;" + + " uaddCarry(uint(0), uint(0), out1);" + + " uaddCarry(uvec2(0, 0), uvec2(0, 0), out2);" + + " uaddCarry(uvec3(0, 0, 0), uvec3(0, 0, 0), out3);" + + " uaddCarry(uvec4(0, 0, 0, 0), uvec4(0, 0, 0, 0), out4);" + + "}"; + + final TranslationUnit tu = ParseHelper.parse(shader, ShaderKind.FRAGMENT); + tu.setShadingLanguageVersion(ShadingLanguageVersion.ESSL_310); + + new StandardVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr expr) { + assertTrue(expr.getCallee().equals("uaddCarry")); + assertFalse(SideEffectChecker.isSideEffectFree(expr, ShadingLanguageVersion.ESSL_310)); + } + }.visit(tu); + } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/FunctionCallExprTemplate.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/FunctionCallExprTemplate.java index f4952d1f3..6a5ee200b 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/FunctionCallExprTemplate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/FunctionCallExprTemplate.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.util.IRandom; import java.util.ArrayList; import java.util.Arrays; @@ -62,7 +63,8 @@ public List> getArgumentTypes() { @Override public boolean requiresLValueForArgument(int index) { assert index >= 0 && index < getNumArguments(); - return false; + return argTypes.get(index).hasQualifier(TypeQualifier.OUT_PARAM) + || argTypes.get(index).hasQualifier(TypeQualifier.INOUT_PARAM); } @Override From dd80208feb2a6e6f151b04f038bba5664962689c Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 11 Jun 2019 08:48:28 -0500 Subject: [PATCH 027/180] Add built-in support for GLSL Integer Functions (#521) * Implemented version-dependent integer function check * Added void return type check to TyperTest * Added built-in function support for GLSL Integer Functions --- .../CompositeShadingLanguageVersion.java | 5 + .../common/glslversion/Essl100.java | 5 + .../common/glslversion/Essl310.java | 5 + .../common/glslversion/Glsl110.java | 5 + .../common/glslversion/Glsl400.java | 5 + .../glslversion/ShadingLanguageVersion.java | 10 ++ .../common/typing/TyperHelper.java | 100 ++++++++++++++++++ .../graphicsfuzz/common/typing/TyperTest.java | 7 +- .../common/util/SideEffectCheckerTest.java | 4 +- 9 files changed, 142 insertions(+), 4 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java index a9135e4da..61dcbfbc2 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java @@ -108,6 +108,11 @@ public boolean supportedIntBitsToFloat() { return prototype.supportedIntBitsToFloat(); } + @Override + public boolean supportedIntegerFunctions() { + return prototype.supportedIntegerFunctions(); + } + @Override public boolean supportedInverse() { return prototype.supportedInverse(); diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java index 799ccdd6b..1a893ab51 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java @@ -109,6 +109,11 @@ public boolean supportedIntBitsToFloat() { return false; } + @Override + public boolean supportedIntegerFunctions() { + return false; + } + @Override public boolean supportedInverse() { return false; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java index ab457c7f9..8fe4fa368 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java @@ -30,6 +30,11 @@ public String getVersionString() { return "310 es"; } + @Override + public boolean supportedIntegerFunctions() { + return true; + } + @Override public boolean supportedMixNonfloatBool() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java index a30a7540c..57e9860c6 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java @@ -109,6 +109,11 @@ public boolean supportedIntBitsToFloat() { return false; } + @Override + public boolean supportedIntegerFunctions() { + return false; + } + @Override public boolean supportedInverse() { return false; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java index c703e29d7..841248661 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java @@ -35,6 +35,11 @@ public boolean supportedFma() { return true; } + @Override + public boolean supportedIntegerFunctions() { + return true; + } + @Override public boolean supportedPackSnorm4x8() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java index 924b49b30..ce5543179 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java @@ -177,6 +177,16 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedIntBitsToFloat(); + /** + * Integer Functions are a set of built-in functions that allow manipulation of integers and + * their corresponding vectors in ways difficult or impossible with normal GLSL syntax - for + * example, summing two unsigned integers where the result causes an overflow. + * GLSL versions 4.0+ and ESSL versions 3.1+ support these functions. + * + * @return true if Integer Functions are supported - false otherwise. + */ + boolean supportedIntegerFunctions(); + boolean supportedInverse(); boolean supportedIsinf(); diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index aa71cfe8f..9f43bcc8e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -18,7 +18,10 @@ import com.graphicsfuzz.common.ast.decl.FunctionPrototype; import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.QualifiedType; import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import java.util.ArrayList; import java.util.Arrays; @@ -843,6 +846,10 @@ private static Map> getBuiltinsForGlslVersion( // 8.8: Integer Functions + if (shadingLanguageVersion.supportedIntegerFunctions()) { + getBuiltinsForGlslVersionInteger(builtinsForVersion); + } + // 8.13: Fragment Processing Functions (only available in fragment shaders) // 8.14: Noise Functions - deprecated, so we do not consider them @@ -975,6 +982,99 @@ private static void getBuiltinsForGlslVersionVectorRelational( } } + private static void getBuiltinsForGlslVersionInteger( + Map> builtinsForVersion) { + { + final String name = "uaddCarry"; + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, new QualifiedType(t, + Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } + + { + final String name = "usubBorrow"; + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, new QualifiedType(t, + Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } + + { + final String name = "umulExtended"; + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, VoidType.VOID, t, t, + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM)), + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } + + { + final String name = "imulExtended"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, VoidType.VOID, t, t, + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM)), + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } + + { + final String name = "bitfieldExtract"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); + } + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); + } + } + + { + final String name = "bitfieldInsert"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, BasicType.INT, BasicType.INT); + } + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, BasicType.INT, BasicType.INT); + } + } + + { + final String name = "bitfieldReverse"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + // We need to use both igen and ugen types of the same size for these builtins, so we need a + // counting loop instead of an iterator to access both lists at the same time. + { + final String name = "bitCount"; + for (int i = 0; i < igenType().size(); i++) { + addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); + addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + } + } + + { + final String name = "findLSB"; + for (int i = 0; i < igenType().size(); i++) { + addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); + addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + } + } + + { + final String name = "findMSB"; + for (int i = 0; i < igenType().size(); i++) { + addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); + addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + } + } + } + private static void addBuiltin(Map> builtinsForVersion, String name, Type resultType, Type... args) { if (!builtinsForVersion.containsKey(name)) { diff --git a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java index 754b57b6d..02791dae3 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java @@ -44,6 +44,7 @@ import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.QualifiedType; import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; @@ -712,7 +713,11 @@ private StringBuilder makeBuiltinsProgram(ShadingLanguageVersion shadingLanguage result.append(decl.getType() + " " + decl.getName()); } result.append(") {\n"); - result.append(" return " + fp.getName() + "("); + result.append(" "); + if (fp.getReturnType() != VoidType.VOID) { + result.append("return "); + } + result.append(fp.getName() + "("); first = true; for (ParameterDecl decl : fp.getParameters()) { if (!first) { diff --git a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java index 729a29f4b..c5dc5dee8 100644 --- a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java +++ b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java @@ -72,11 +72,9 @@ public void testCountingLoopHasSideEffects() throws Exception { ShadingLanguageVersion.ESSL_310)); } - // TODO(521): Unignore when #521 is ready to merge. - @Ignore @Test public void testOutParamHasSideEffects() throws Exception { - final String shader = "void main { " + final String shader = "void main() { " + " uint out1;" + " uvec2 out2;" + " uvec3 out3;" From aed57ca27c66a85c4ef6ccc29bf6ca54da976b91 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 11 Jun 2019 15:34:21 +0100 Subject: [PATCH 028/180] Small fix to Generate.main (#546) --- .../main/java/com/graphicsfuzz/generator/tool/Generate.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java index 5b2ed2131..1b9504bd8 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java @@ -311,6 +311,9 @@ private static StringBuilder transformShader(TranslationUnit shaderToTransform, public static void main(String[] args) { try { mainHelper(args); + } catch (ArgumentParserException exception) { + exception.getParser().handleError(exception); + System.exit(1); } catch (Throwable exception) { LOGGER.error("", exception); System.exit(1); From eeae331c03b65a8aa5cab880b8459a1f77d7e3aa Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 11 Jun 2019 15:34:52 +0100 Subject: [PATCH 029/180] Update SPIRV-Tools and glslang (#542) --- assembly-binaries/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assembly-binaries/pom.xml b/assembly-binaries/pom.xml index 2f78a1432..5491452e9 100644 --- a/assembly-binaries/pom.xml +++ b/assembly-binaries/pom.xml @@ -58,10 +58,10 @@ limitations under the License. - ac0884b2c8dfd214f0d293c12cf3b5931b1eb345 + 422f2fe0f0f32494fa687a12ba343d24863b330a 9d3202492d78aae5ced08ff14679b81c98d71c15 803082da4f73920729938905cb1727e11ff79530 - b938f8045de9556bcd763eca362c2443d25c0b77 + 1586e566f4949b1957e7c32454cbf27e501ed632 d987dc0e5f8aeae0dc2f5aac2013e9a3edd54465 From 9dca8273e9cd16865db8181378d20c0cb5fc86b2 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 11 Jun 2019 16:44:17 +0100 Subject: [PATCH 030/180] Add graphicsfuzz-tool for running an arbitrary Java class (#544) --- .../src/main/python/drivers/graphicsfuzz-tool | 25 +++++++++++ .../main/python/drivers/graphicsfuzz-tool.bat | 29 +++++++++++++ .../main/python/drivers/graphicsfuzz-tool.py | 43 +++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 python/src/main/python/drivers/graphicsfuzz-tool create mode 100644 python/src/main/python/drivers/graphicsfuzz-tool.bat create mode 100644 python/src/main/python/drivers/graphicsfuzz-tool.py diff --git a/python/src/main/python/drivers/graphicsfuzz-tool b/python/src/main/python/drivers/graphicsfuzz-tool new file mode 100644 index 000000000..dbafbbaf2 --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-tool @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2018 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if test -n "${PYTHON_GF}"; then + "${PYTHON_GF}" ${BASH_SOURCE}.py "$@" +elif type -P python3 >/dev/null; then + python3 ${BASH_SOURCE}.py "$@" +elif type -P py >/dev/null; then + py -3 ${BASH_SOURCE}.py "$@" +else + python ${BASH_SOURCE}.py "$@" +fi diff --git a/python/src/main/python/drivers/graphicsfuzz-tool.bat b/python/src/main/python/drivers/graphicsfuzz-tool.bat new file mode 100644 index 000000000..cafd86cce --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-tool.bat @@ -0,0 +1,29 @@ +@echo off + +@REM +@REM Copyright 2018 The GraphicsFuzz Project Authors +@REM +@REM Licensed under the Apache License, Version 2.0 (the "License"); +@REM you may not use this file except in compliance with the License. +@REM You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. +@REM + +where /q py +IF ERRORLEVEL 0 ( + py -3 "%~dpn0.py" %* +) ELSE ( + where /q python3 + IF ERRORLEVEL 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) +) diff --git a/python/src/main/python/drivers/graphicsfuzz-tool.py b/python/src/main/python/drivers/graphicsfuzz-tool.py new file mode 100644 index 000000000..2ca507658 --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-tool.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +# Copyright 2018 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +HERE = os.path.abspath(__file__) + +sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) + +import cmd_helpers + + +def go(argv): + java_tool_path = cmd_helpers.get_tool_path() + print(java_tool_path) + + # Run the tool + + cmd = ["java", "-ea", "-cp", java_tool_path] + argv + print(cmd) + + generate_proc = subprocess.Popen(cmd) + generate_proc.communicate() + return generate_proc.returncode + + +if __name__ == "__main__": + sys.exit(go(sys.argv[1:])) From 0fda530b5e4f0722ee963705ce426f44bebc9283 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Wed, 12 Jun 2019 18:20:30 +0200 Subject: [PATCH 031/180] Implementation: Reducer Opportunity to replace variable declaration with an expression (#529) --- .../graphicsfuzz/reducer/ReductionDriver.java | 3 +- .../IReductionOpportunityFinder.java | 19 +++++++ .../ReductionOpportunities.java | 3 +- ...iableDeclToExprReductionOpportunities.java | 51 +++++++++++++++++++ ...ariableDeclToExprReductionOpportunity.java | 35 ++++++++++++- ...eDeclToExprReductionOpportunitiesTest.java | 7 --- 6 files changed, 107 insertions(+), 11 deletions(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java index deee715e3..3b0944952 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java @@ -92,7 +92,8 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.inlineFunctionFinder(), IReductionOpportunityFinder.unusedParamFinder(), IReductionOpportunityFinder.foldConstantFinder(), - IReductionOpportunityFinder.redundantUniformMetadataFinder())) { + IReductionOpportunityFinder.redundantUniformMetadataFinder(), + IReductionOpportunityFinder.variableDeclToExprFinder())) { cleanupPasses.add(new SystematicReductionPass(context, verbose, finder)); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java index 8d8ec8e94..24443c760 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java @@ -481,4 +481,23 @@ public String getName() { }; } + static IReductionOpportunityFinder + variableDeclToExprFinder() { + return new IReductionOpportunityFinder() { + @Override + public List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return VariableDeclToExprReductionOpportunities.findOpportunities( + shaderJob, + context); + } + + @Override + public String getName() { + return "variableDeclToExpr"; + } + }; + } + } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java index 6f9d8e862..6597aba5a 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java @@ -66,7 +66,8 @@ public static List getReductionOpportunities( IReductionOpportunityFinder.unusedParamFinder(), IReductionOpportunityFinder.foldConstantFinder(), IReductionOpportunityFinder.inlineUniformFinder(), - IReductionOpportunityFinder.redundantUniformMetadataFinder())) { + IReductionOpportunityFinder.redundantUniformMetadataFinder(), + IReductionOpportunityFinder.variableDeclToExprFinder())) { final List currentOpportunities = ros .findOpportunities(shaderJob, context); if (ReductionDriver.DEBUG_REDUCER) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java index 46f6b11bc..6db2ba13b 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java @@ -17,11 +17,31 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; +import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; import java.util.Arrays; import java.util.List; +/* + * This class finds opportunities to remove initializers from variable declarations and + * replace them with assignment statements following the declaration. For example, in: + * int a = 1; + * int b = foo(); + * int c; + * + *

Because each of a and b has an initializer, we can transform the code fragment + * into the following: + * int a; + * a = 1; + * int b; + * b = foo(); + * int c; + */ + public class VariableDeclToExprReductionOpportunities extends ReductionOpportunitiesBase { @@ -53,4 +73,35 @@ private static List findOpportunitiesFor finder.visit(tu); return finder.getOpportunities(); } + + @Override + public void visitDeclarationStmt(DeclarationStmt declarationStmt) { + super.visitDeclarationStmt(declarationStmt); + if (!context.reduceEverywhere()) { + // Replacing initializers with a new assignment statement might have a side-effect, + // do not consider these reduction opportunities if we are not reducing everywhere. + return; + } + + if (declarationStmt.getVariablesDeclaration().getBaseType().hasQualifier(TypeQualifier.CONST)) { + // A constant must always be initialized, so do not consider variable declarations that + // have const qualifier (i.e., const int a = 1). + return; + } + final List declInfos = + declarationStmt.getVariablesDeclaration().getDeclInfos(); + // We iterate backwards so that when applying reduction the new expression is inserted at the + // correct order with respect to its original order in the variable declaration info list. + for (int i = declInfos.size() - 1; i >= 0; i--) { + final VariableDeclInfo variableDeclInfo = declInfos.get(i); + if (variableDeclInfo.hasInitializer() + && variableDeclInfo.getInitializer() instanceof ScalarInitializer) { + addOpportunity(new VariableDeclToExprReductionOpportunity( + variableDeclInfo, + currentBlock(), + declarationStmt, + getVistitationDepth())); + } + } + } } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java index c0c236ceb..9a2f17098 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java @@ -16,20 +16,51 @@ package com.graphicsfuzz.reducer.reductionopportunities; +import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; public class VariableDeclToExprReductionOpportunity extends AbstractReductionOpportunity { - VariableDeclToExprReductionOpportunity(VisitationDepth depth) { + // The initialized variable declaration info. + private final VariableDeclInfo variableDeclInfo; + // The parent of variableDeclInfo. + private final DeclarationStmt declarationStmt; + // The block in which the declaration statement resides. + private final BlockStmt enclosingBlock; + + VariableDeclToExprReductionOpportunity(VariableDeclInfo variableDeclInfo, + BlockStmt enclosingBlock, + DeclarationStmt declarationStmt, VisitationDepth depth) { super(depth); + this.variableDeclInfo = variableDeclInfo; + this.enclosingBlock = enclosingBlock; + this.declarationStmt = declarationStmt; } @Override void applyReductionImpl() { + // Given the variable declaration info, we unset its initializer and derive a new assignment + // statement which will be inserted right after the declaration in the block statement. + assert variableDeclInfo.getInitializer() instanceof ScalarInitializer; + final BinaryExpr binaryExpr = new BinaryExpr( + new VariableIdentifierExpr(variableDeclInfo.getName()), + ((ScalarInitializer) variableDeclInfo.getInitializer()).getExpr(), + BinOp.ASSIGN + ); + enclosingBlock.insertAfter(declarationStmt, new ExprStmt(binaryExpr)); + variableDeclInfo.setInitializer(null); } @Override public boolean preconditionHolds() { - return false; + return enclosingBlock.hasChild(declarationStmt) + && variableDeclInfo.hasInitializer(); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java index ea565eac7..9b0c36c7a 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java @@ -22,7 +22,6 @@ import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; -import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -30,8 +29,6 @@ public class VariableDeclToExprReductionOpportunitiesTest { - // TODO(480): Enable this test once the issue is fixed. - @Ignore @Test public void testDoNotReplace() throws Exception { final String original = "void main() { int a = 1, b = 2; int c = 3; }"; @@ -45,7 +42,6 @@ public void testDoNotReplace() throws Exception { assertTrue(ops.isEmpty()); } - @Ignore @Test public void testDoNotReplaceConst() throws Exception { final String original = "void main() { const int a = 1;}"; @@ -60,7 +56,6 @@ public void testDoNotReplaceConst() throws Exception { assertTrue(ops.isEmpty()); } - @Ignore @Test public void testMultipleDeclarations() throws Exception { final String program = "void main() {" @@ -87,7 +82,6 @@ public void testMultipleDeclarations() throws Exception { CompareAsts.assertEqualAsts(expected, tu); } - @Ignore @Test public void testMultipleLineDeclarationsOneLine() throws Exception { final String program = "void main() {" @@ -115,7 +109,6 @@ public void testMultipleLineDeclarationsOneLine() throws Exception { CompareAsts.assertEqualAsts(expected, tu); } - @Ignore @Test public void testAssignVariableIdentifier() throws Exception { final String program = "void main() {" From a4e3c6883badd0245fd9c78bd1acfd4d81922075 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 12 Jun 2019 11:24:50 -0500 Subject: [PATCH 032/180] Add built-in function support for GLES 3.20 Fragment Processing Functions (#549) --- .../CompositeShadingLanguageVersion.java | 15 ++++ .../common/glslversion/Essl100.java | 15 ++++ .../common/glslversion/Essl300.java | 5 ++ .../common/glslversion/Essl320.java | 5 ++ .../common/glslversion/Glsl110.java | 15 ++++ .../common/glslversion/Glsl400.java | 5 ++ .../common/glslversion/Glsl450.java | 5 ++ .../glslversion/ShadingLanguageVersion.java | 28 +++++++ .../common/typing/TyperHelper.java | 84 +++++++++++++++++++ 9 files changed, 177 insertions(+) diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java index 61dcbfbc2..fac57e20d 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java @@ -73,6 +73,11 @@ public boolean supportedClampUint() { return prototype.supportedClampUint(); } + @Override + public boolean supportedDerivativeFunctions() { + return prototype.supportedDerivativeFunctions(); + } + @Override public boolean supportedDeterminant() { return prototype.supportedDeterminant(); @@ -83,6 +88,11 @@ public boolean supportedDoStmt() { return prototype.supportedDoStmt(); } + @Override + public boolean supportedExplicitDerivativeFunctions() { + return prototype.supportedExplicitDerivativeFunctions(); + } + @Override public boolean supportedFloatBitsToInt() { return prototype.supportedFloatBitsToInt(); @@ -113,6 +123,11 @@ public boolean supportedIntegerFunctions() { return prototype.supportedIntegerFunctions(); } + @Override + public boolean supportedInterpolationFunctions() { + return prototype.supportedInterpolationFunctions(); + } + @Override public boolean supportedInverse() { return prototype.supportedInverse(); diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java index 1a893ab51..3c8d07c14 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java @@ -74,6 +74,11 @@ public boolean supportedClampUint() { return false; } + @Override + public boolean supportedDerivativeFunctions() { + return false; + } + @Override public boolean supportedDeterminant() { return false; @@ -84,6 +89,11 @@ public boolean supportedDoStmt() { return true; } + @Override + public boolean supportedExplicitDerivativeFunctions() { + return false; + } + @Override public boolean supportedFloatBitsToInt() { return false; @@ -114,6 +124,11 @@ public boolean supportedIntegerFunctions() { return false; } + @Override + public boolean supportedInterpolationFunctions() { + return false; + } + @Override public boolean supportedInverse() { return false; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java index 6f27f8dc2..61a11687d 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java @@ -55,6 +55,11 @@ public boolean supportedClampUint() { return true; } + @Override + public boolean supportedDerivativeFunctions() { + return true; + } + @Override public boolean supportedDeterminant() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl320.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl320.java index 8c0426d3b..776f78023 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl320.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl320.java @@ -30,4 +30,9 @@ public String getVersionString() { return "320 es"; } + @Override + public boolean supportedInterpolationFunctions() { + return true; + } + } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java index 57e9860c6..249450df2 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java @@ -74,6 +74,11 @@ public boolean supportedClampUint() { return false; } + @Override + public boolean supportedDerivativeFunctions() { + return true; + } + @Override public boolean supportedDeterminant() { return false; @@ -84,6 +89,11 @@ public boolean supportedDoStmt() { return true; } + @Override + public boolean supportedExplicitDerivativeFunctions() { + return false; + } + @Override public boolean supportedFloatBitsToInt() { return false; @@ -114,6 +124,11 @@ public boolean supportedIntegerFunctions() { return false; } + @Override + public boolean supportedInterpolationFunctions() { + return false; + } + @Override public boolean supportedInverse() { return false; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java index 841248661..3bb98a75e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java @@ -40,6 +40,11 @@ public boolean supportedIntegerFunctions() { return true; } + @Override + public boolean supportedInterpolationFunctions() { + return true; + } + @Override public boolean supportedPackSnorm4x8() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl450.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl450.java index 8aacf7415..6ae02e3f7 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl450.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl450.java @@ -30,6 +30,11 @@ public String getVersionString() { return "450"; } + @Override + public boolean supportedExplicitDerivativeFunctions() { + return true; + } + @Override public boolean supportedMixNonfloatBool() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java index ce5543179..70b72d3d0 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java @@ -163,10 +163,29 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedClampUint(); + /** + * Derivative Functions are a subset of fragment processing functions that compute + * the rate of change between pixels in a given fragment. + * GLSL versions 1.1+ and ESSL versions 3.0+ support these functions. + * + * @return true if explicit derivative functions are supported - false otherwise. + */ + boolean supportedDerivativeFunctions(); + boolean supportedDeterminant(); boolean supportedDoStmt(); + /** + * In recent GLSL specifications, new derivative functions were added that allow a user to + * specify how much precision the user wants in the calculation, instead of leaving the choice + * to the compiler. + * GLSL versions 4.5+ support these explicit derivative functions. + * + * @return true if explicit derivative functions are supported - false otherwise. + */ + boolean supportedExplicitDerivativeFunctions(); + boolean supportedFloatBitsToInt(); boolean supportedFloatBitsToUint(); @@ -187,6 +206,15 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { */ boolean supportedIntegerFunctions(); + /** + * Interpolation Functions are a subset of fragment processing functions that + * compute an interpolated value of a fragment shader input variable at a specific location. + * GLSL versions 4.0+ and ESSL versions 3.2+ support these functions. + * + * @return true if Interpolation Functions are supported - false otherwise. + */ + boolean supportedInterpolationFunctions(); + boolean supportedInverse(); boolean supportedIsinf(); diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index 9f43bcc8e..1c237fc45 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -852,6 +852,10 @@ private static Map> getBuiltinsForGlslVersion( // 8.13: Fragment Processing Functions (only available in fragment shaders) + if (shadingLanguageVersion.supportedDerivativeFunctions()) { + getBuiltinsForGlslVersionFragmentProcessing(builtinsForVersion, shadingLanguageVersion); + } + // 8.14: Noise Functions - deprecated, so we do not consider them return builtinsForVersion; @@ -1075,6 +1079,86 @@ private static void getBuiltinsForGlslVersionInteger( } } + /** + * Helper function to register built-in function prototypes for Fragment Processing Functions, + * as specified in section 8.14 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionFragmentProcessing( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + // 8.14.1 Derivative Functions + { + final String name = "dFdx"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "dFdy"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "fwidth"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + if (shadingLanguageVersion.supportedExplicitDerivativeFunctions()) { + { + final String name = "dFdxFine"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "dFdxCoarse"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "dFdyFine"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "dFdyCoarse"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "fwidthFine"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "fwidthCoarse"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + + // 8.14.2 Interpolation Functions + // TODO(550): Support functions that take non-uniform shader input variables as parameters. + } + private static void addBuiltin(Map> builtinsForVersion, String name, Type resultType, Type... args) { if (!builtinsForVersion.containsKey(name)) { From 50fd45f933d682016e863e81d58654a101adf254 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Thu, 13 Jun 2019 10:27:36 -0500 Subject: [PATCH 033/180] Add new identities for integer types that use bitwise operations (#553) --- .../common/ast/type/BasicType.java | 18 +++ .../fuzzer/OpaqueExpressionGenerator.java | 135 +++++++++++++++++- 2 files changed, 148 insertions(+), 5 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java index 07492a34a..5b37c2759 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java @@ -324,6 +324,24 @@ public static List allNumericTypes() { return result; } + /** + * Helper function to return a list of every basic type that is considered an integer. + * + * @return a list of all basic integer types. + */ + public static List allIntegerTypes() { + return new ArrayList(Arrays.asList( + UINT, + UVEC2, + UVEC3, + UVEC4, + INT, + IVEC2, + IVEC3, + IVEC4 + )); + } + public static List allNumericTypesExceptNonSquareMatrices() { final List result = allNumericTypes(); result.removeAll(allNonSquareMatrixTypes()); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 63cbac139..016a55279 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -63,18 +63,25 @@ public OpaqueExpressionGenerator(IRandom generator, GenerationParams generationP ShadingLanguageVersion shadingLanguageVersion) { this.generator = generator; this.generationParams = generationParams; - // TODO: there are many more identities that we can easily play with here, e.g. bitwise and 1 - // for integer types this.expressionIdentities = new ArrayList<>(); this.shadingLanguageVersion = shadingLanguageVersion; + // TODO: there are many more identities that we can easily play with here expressionIdentities.add(new IdentityAddSubZero()); expressionIdentities.add(new IdentityMulDivOne()); expressionIdentities.add(new IdentityAndTrue()); expressionIdentities.add(new IdentityOrFalse()); - expressionIdentities.add(new IdentityNotNot()); + expressionIdentities.add(new IdentityLogicalNotNot()); expressionIdentities.add(new IdentityTernary()); + if (shadingLanguageVersion.supportedBitwiseOperations()) { + expressionIdentities.add(new IdentityBitwiseNotNot()); + expressionIdentities.add(new IdentityBitwiseOrSelf()); + expressionIdentities.add(new IdentityBitwiseOrZero()); + expressionIdentities.add(new IdentityBitwiseXorZero()); + expressionIdentities.add(new IdentityBitwiseShiftZero()); + } + expressionIdentities.add(new IdentityMin()); expressionIdentities.add(new IdentityMax()); expressionIdentities.add(new IdentityClamp()); @@ -638,9 +645,9 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, } - private class IdentityNotNot extends AbstractIdentityTransformation { + private class IdentityLogicalNotNot extends AbstractIdentityTransformation { - private IdentityNotNot() { + private IdentityLogicalNotNot() { super(Arrays.asList(BasicType.BOOL), false); } @@ -691,6 +698,124 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, } + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * double bitwise inverts an integer, producing the same integer as output. When performed, + * transforms an expression, e, such that: + * e -> ~(~(expr)). + */ + private class IdentityBitwiseNotNot extends AbstractIdentityTransformation { + private IdentityBitwiseNotNot() { + super(BasicType.allIntegerTypes(), false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + // Invert once + Expr result = new UnaryExpr(new ParenExpr(applyIdentityFunction(expr, type, + constContext, + depth, fuzzer)), UnOp.BNEG); + // Invert again + result = new UnaryExpr(new ParenExpr(applyIdentityFunction(result, type, constContext, + depth, fuzzer)), UnOp.BNEG); + return identityConstructor(expr, result); + } + } + + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * ORs an integer with itself, producing the same integer as output. This identity requires + * expressions to be side effect free because the same mutated expression is evaluated twice. + * When performed, transforms an expression, e, such that: + * e -> (e) | (e). + */ + private class IdentityBitwiseOrSelf extends AbstractIdentityTransformation { + private IdentityBitwiseOrSelf() { + super(BasicType.allIntegerTypes(), true); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + // We use parentheses to prevent issues with order of operations in ternary expressions. + return identityConstructor( + expr, + new BinaryExpr( + new ParenExpr(applyIdentityFunction(expr.clone(), type, constContext, depth, fuzzer)), + new ParenExpr(applyIdentityFunction(expr.clone(), type, constContext, depth, fuzzer)), + BinOp.BOR)); + } + } + + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * ORs an integer with zero, producing the same integer as output. + * When performed, transforms an expression, e, such that: + * e -> (e) | (opaque 0) or e -> (opaque 0) | (e). + */ + private class IdentityBitwiseOrZero extends AbstractIdentityTransformation { + private IdentityBitwiseOrZero() { + super(BasicType.allIntegerTypes(), false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + return applyBinaryIdentityFunction( + expr.clone(), + makeOpaqueZero(type, constContext, depth, fuzzer), + BinOp.BOR, true, type, constContext, depth, fuzzer); + } + } + + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * XORs an integer with zero, producing the same integer as output. When performed, transforms an + * expression, e, such that: + * e -> (e) ^ (opaque 0) or e -> (opaque 0) ^ (e). + */ + private class IdentityBitwiseXorZero extends AbstractIdentityTransformation { + private IdentityBitwiseXorZero() { + super(BasicType.allIntegerTypes(), false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + return applyBinaryIdentityFunction( + expr.clone(), + makeOpaqueZero(type, constContext, depth, fuzzer), + BinOp.BXOR, true, type, constContext, depth, fuzzer); + } + } + + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * shifts an integer by zero. When performed, transforms an expression, e, such that: + * e -> (e) >> (opaque 0) or e -> (e) << (opaque 0) + */ + private class IdentityBitwiseShiftZero extends AbstractIdentityTransformation { + private IdentityBitwiseShiftZero() { + super(BasicType.allIntegerTypes(), false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + final BinOp operator = generator.nextBoolean() ? BinOp.SHL : BinOp.SHR; + return applyBinaryIdentityFunction( + expr.clone(), + makeOpaqueZero(type, constContext, depth, fuzzer), + operator, false, type, constContext, depth, fuzzer); + } + } + private class IdentityMin extends AbstractIdentityTransformation { private IdentityMin() { From c052314956ac21dff92bfdee16efd8902fb3ca25 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Fri, 14 Jun 2019 08:09:25 +0200 Subject: [PATCH 034/180] Add support for angle and trigonometric built-in functions (#554) * Add angle and trigonometric built-in functions * Handle GLSL support check inside helper functions --- .../CompositeShadingLanguageVersion.java | 9 + .../common/glslversion/Essl100.java | 10 + .../common/glslversion/Essl300.java | 4 + .../common/glslversion/Glsl110.java | 10 + .../common/glslversion/Glsl130.java | 4 + .../glslversion/ShadingLanguageVersion.java | 18 + .../common/typing/TyperHelper.java | 350 +++++++++++------- 7 files changed, 272 insertions(+), 133 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java index fac57e20d..e984245d1 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java @@ -278,4 +278,13 @@ public boolean supportedUnsigned() { return prototype.supportedUnsigned(); } + @Override + public boolean supportedAngleAndTrigonometricFunctions() { + return prototype.supportedAngleAndTrigonometricFunctions(); + } + + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return prototype.supportedHyperbolicAngleAndTrigonometricFunctions(); + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java index 3c8d07c14..c4e35a4ad 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java @@ -278,4 +278,14 @@ public boolean supportedUnpackUnorm4x8() { public boolean supportedUnsigned() { return false; } + + @Override + public boolean supportedAngleAndTrigonometricFunctions() { + return true; + } + + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return false; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java index 61a11687d..4596f0661 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java @@ -215,4 +215,8 @@ public boolean supportedUnsigned() { return true; } + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java index 249450df2..ddaff22e7 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java @@ -281,4 +281,14 @@ public boolean supportedUnpackUnorm4x8() { public boolean supportedUnsigned() { return false; } + + @Override + public boolean supportedAngleAndTrigonometricFunctions() { + return true; + } + + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return false; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java index 5a349711a..c390e4d7a 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java @@ -115,4 +115,8 @@ public boolean supportedUnsigned() { return true; } + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java index 70b72d3d0..3b2767010 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java @@ -275,4 +275,22 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedUnsigned(); + /** + * Angle and Trigonometric Functions are a set of built-in functions related to the calculation + * of an angle. For example, sin(angle) - computes the sine value of the angle provided. + * GLSL versions 1.1+ and ESSL versions 1.0+ support these functions. + * + * @return true if Angle and Trigonometric Functions are supported - false otherwise. + */ + boolean supportedAngleAndTrigonometricFunctions(); + + /** + * Hyperbolic Angle and Trigonometric Functions are a set of built-in functions that + * computes the hyperbolic trigonometric functions. For example, sinh() - calculate the + * hyperbolic sine function of the given value. + * GLSL versions 1.3+ and ESSL versions 3.0+ support these functions. + * + * @return true if Hyperbolic Angle and Trigonometric Functions are supported - false otherwise. + */ + boolean supportedHyperbolicAngleAndTrigonometricFunctions(); } diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index 1c237fc45..6310b664b 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -297,49 +297,10 @@ private static Map> getBuiltinsForGlslVersion( ShadingLanguageVersion shadingLanguageVersion) { Map> builtinsForVersion = new HashMap<>(); - // Trigonometric Functions - { - final String name = "sin"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "cos"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "asin"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "acos"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "tan"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "atan"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "atan"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } + // 8.1: Angle and Trigonometric Functions + + getBuiltinsForGlslVersionAngleAndTrigonometric(shadingLanguageVersion, + builtinsForVersion); //Exponential Functions @@ -846,21 +807,131 @@ private static Map> getBuiltinsForGlslVersion( // 8.8: Integer Functions - if (shadingLanguageVersion.supportedIntegerFunctions()) { - getBuiltinsForGlslVersionInteger(builtinsForVersion); - } + getBuiltinsForGlslVersionInteger(builtinsForVersion, shadingLanguageVersion); // 8.13: Fragment Processing Functions (only available in fragment shaders) - if (shadingLanguageVersion.supportedDerivativeFunctions()) { - getBuiltinsForGlslVersionFragmentProcessing(builtinsForVersion, shadingLanguageVersion); - } + getBuiltinsForGlslVersionFragmentProcessing(builtinsForVersion, shadingLanguageVersion); // 8.14: Noise Functions - deprecated, so we do not consider them return builtinsForVersion; } + /** + * Helper function to register built-in function prototypes for Angle and Trigonometric + * Functions, as specified in section 8.1 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionAngleAndTrigonometric( + ShadingLanguageVersion shadingLanguageVersion, + Map> builtinsForVersion) { + if (shadingLanguageVersion.supportedAngleAndTrigonometricFunctions()) { + { + final String name = "radians"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "degrees"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "sin"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "cos"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "tan"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "asin"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "acos"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "atan"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + } + + if (shadingLanguageVersion.supportedHyperbolicAngleAndTrigonometricFunctions()) { + { + final String name = "sinh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "cosh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "tanh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "asinh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "acosh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "atanh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + } + private static void getBuiltinsForGlslVersionVectorRelational( Map> builtinsForVersion, ShadingLanguageVersion shadingLanguageVersion) { @@ -986,95 +1057,106 @@ private static void getBuiltinsForGlslVersionVectorRelational( } } + + /** + * Helper function to register built-in function prototypes for Integer Functions, + * as specified in section 8.8 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ private static void getBuiltinsForGlslVersionInteger( - Map> builtinsForVersion) { - { - final String name = "uaddCarry"; - for (Type t : ugenType()) { - addBuiltin(builtinsForVersion, name, t, t, t, new QualifiedType(t, - Arrays.asList(TypeQualifier.OUT_PARAM))); + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + if (shadingLanguageVersion.supportedIntegerFunctions()) { + { + final String name = "uaddCarry"; + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, new QualifiedType(t, + Arrays.asList(TypeQualifier.OUT_PARAM))); + } } - } - { - final String name = "usubBorrow"; - for (Type t : ugenType()) { - addBuiltin(builtinsForVersion, name, t, t, t, new QualifiedType(t, - Arrays.asList(TypeQualifier.OUT_PARAM))); + { + final String name = "usubBorrow"; + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, new QualifiedType(t, + Arrays.asList(TypeQualifier.OUT_PARAM))); + } } - } - { - final String name = "umulExtended"; - for (Type t : ugenType()) { - addBuiltin(builtinsForVersion, name, VoidType.VOID, t, t, - new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM)), - new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM))); + { + final String name = "umulExtended"; + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, VoidType.VOID, t, t, + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM)), + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM))); + } } - } - { - final String name = "imulExtended"; - for (Type t : igenType()) { - addBuiltin(builtinsForVersion, name, VoidType.VOID, t, t, - new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM)), - new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM))); + { + final String name = "imulExtended"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, VoidType.VOID, t, t, + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM)), + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM))); + } } - } - { - final String name = "bitfieldExtract"; - for (Type t : igenType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); - } - for (Type t : ugenType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); + { + final String name = "bitfieldExtract"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); + } + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); + } } - } - { - final String name = "bitfieldInsert"; - for (Type t : igenType()) { - addBuiltin(builtinsForVersion, name, t, t, t, BasicType.INT, BasicType.INT); - } - for (Type t : ugenType()) { - addBuiltin(builtinsForVersion, name, t, t, t, BasicType.INT, BasicType.INT); + { + final String name = "bitfieldInsert"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, BasicType.INT, BasicType.INT); + } + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, BasicType.INT, BasicType.INT); + } } - } - { - final String name = "bitfieldReverse"; - for (Type t : igenType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - for (Type t : ugenType()) { - addBuiltin(builtinsForVersion, name, t, t); + { + final String name = "bitfieldReverse"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t); + } } - } - // We need to use both igen and ugen types of the same size for these builtins, so we need a - // counting loop instead of an iterator to access both lists at the same time. - { - final String name = "bitCount"; - for (int i = 0; i < igenType().size(); i++) { - addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); - addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + // We need to use both igen and ugen types of the same size for these builtins, so we need a + // counting loop instead of an iterator to access both lists at the same time. + { + final String name = "bitCount"; + for (int i = 0; i < igenType().size(); i++) { + addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); + addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + } } - } - { - final String name = "findLSB"; - for (int i = 0; i < igenType().size(); i++) { - addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); - addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + { + final String name = "findLSB"; + for (int i = 0; i < igenType().size(); i++) { + addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); + addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + } } - } - { - final String name = "findMSB"; - for (int i = 0; i < igenType().size(); i++) { - addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); - addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + { + final String name = "findMSB"; + for (int i = 0; i < igenType().size(); i++) { + addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); + addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + } } } } @@ -1090,24 +1172,26 @@ private static void getBuiltinsForGlslVersionFragmentProcessing( Map> builtinsForVersion, ShadingLanguageVersion shadingLanguageVersion) { // 8.14.1 Derivative Functions - { - final String name = "dFdx"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + if (shadingLanguageVersion.supportedDerivativeFunctions()) { + { + final String name = "dFdx"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } } - } - { - final String name = "dFdy"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + { + final String name = "dFdy"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } } - } - { - final String name = "fwidth"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + { + final String name = "fwidth"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } } } From 10fd07c00fe7ee8c913269f9d5b48854b2b4e948 Mon Sep 17 00:00:00 2001 From: Hugues Evrard Date: Tue, 18 Jun 2019 12:10:14 +0100 Subject: [PATCH 035/180] Clean up for Amber limitation of a single type per SSBO (#565) * Work around Amber limitation by using a single type per buffer * Mark some tests as skipped * Clarify how to use pytest from the command line * Check only one type is used per buffer * Add test: reject compute shader with several types in SSBO * Fix typo --- python/src/main/python/drivers/runspv.py | 17 ++++++++++------- .../python/test_scripts/test_runspv/README.md | 17 ++++++++++++----- .../test_scripts/test_runspv/runspv_tests.py | 16 +++++++++++++++- .../compute/310es/comp-0004-koggestone.comp | 8 +++++--- .../compute/310es/comp-0004-koggestone.json | 2 +- .../compute/310es/comp-0005-sklansky.comp | 12 +++++++----- .../compute/310es/comp-0005-sklansky.json | 2 +- 7 files changed, 51 insertions(+), 23 deletions(-) diff --git a/python/src/main/python/drivers/runspv.py b/python/src/main/python/drivers/runspv.py index 937e0d72b..0c15907f0 100755 --- a/python/src/main/python/drivers/runspv.py +++ b/python/src/main/python/drivers/runspv.py @@ -1067,12 +1067,6 @@ def run_image_amber( # Amber worker: compute test -def translate_type_for_amber(type_name: str) -> str: - if type_name == 'bool': - return 'uint' - return type_name - - def comp_json_to_amberscript(comp_json): """ Returns the string representing VkScript version of compute shader setup, @@ -1109,12 +1103,21 @@ def comp_json_to_amberscript(comp_json): binding = j['buffer']['binding'] offset = 0 + + # Amber only supports one type per buffer, check this limitation. + field_type = None + for field_info in j['buffer']['fields']: + if not field_type: + field_type = field_info['type'] + elif field_type != field_info['type']: + raise ValueError('Amber only supports one type per buffer') + for field_info in j['buffer']['fields']: result += ( 'ssbo ' + str(binding) + ' subdata ' - + translate_type_for_amber(field_info['type']) + + field_info['type'] + ' ' + str(offset) ) diff --git a/python/src/main/python/test_scripts/test_runspv/README.md b/python/src/main/python/test_scripts/test_runspv/README.md index 247451ab9..84f3f8714 100644 --- a/python/src/main/python/test_scripts/test_runspv/README.md +++ b/python/src/main/python/test_scripts/test_runspv/README.md @@ -5,16 +5,23 @@ The tests assume that you are working under Linux and that an Android device is They require Python 3, pathlib2, Pillow, and PyTest. +## Using PyCharm + If you are using PyCharm these can all be installed by pressing Alt+Return on the errors about unknown imports and following the instructions thereafter. -Alternatively, pytest can be installed as follows: +If using PyCharm you need to [configure it to use PyTest for tests](https://www.jetbrains.com/help/pycharm/pytest.html). + +## Using the command line + +Alternatively, pytest and dependencies can be installed as follows: ``` -sudo apt-get install python3-pip -pip3 install pytest pathlib2 Pillow +python3 -m pip install pytest pathlib2 Pillow ``` -(The other dependencies should be similar). +To run the tests from the command line: -If using PyCharm you need to [configure it to use PyTest for tests](https://www.jetbrains.com/help/pycharm/pytest.html). +``` +python3 -m pytest ./runspv_tests.py +``` diff --git a/python/src/main/python/test_scripts/test_runspv/runspv_tests.py b/python/src/main/python/test_scripts/test_runspv/runspv_tests.py index 41842d7a4..d3121d916 100755 --- a/python/src/main/python/test_scripts/test_runspv/runspv_tests.py +++ b/python/src/main/python/test_scripts/test_runspv/runspv_tests.py @@ -494,6 +494,13 @@ def test_spirv_opt_fails_given_silly_argument_compute(tmp_path: pathlib2.Path): assert '-ODOESNOTEXIST is not a valid flag' in str(called_process_error) +def test_runspv_fails_if_comp_buffer_has_several_types(tmp_path: pathlib2.Path): + with pytest.raises(ValueError) as value_error: + runspv.main_helper(['host', get_compute_test('buffer-with-several-types.json'), + str(tmp_path / 'out')]) + assert 'Amber only supports one type per buffer' in str(value_error) + + def test_skip_render_image_amber_host(tmp_path: pathlib2.Path): check_no_image_skip_render(tmp_path, is_android=False, is_legacy_worker=False, json_filename='image_test_0007.json') @@ -636,7 +643,8 @@ def test_image_0002_host_vs_android_amber(tmp_path: pathlib2.Path): def test_image_0003_host_vs_android_amber(tmp_path: pathlib2.Path): check_images_match_host_vs_android_amber(tmp_path, 'image_test_0003.json') - +@pytest.mark.skip( + reason="Slight image difference probably due to floating point sensitivity.") def test_image_0004_host_vs_android_amber(tmp_path: pathlib2.Path): check_images_match_host_vs_android_amber(tmp_path, 'image_test_0004.json') @@ -645,6 +653,8 @@ def test_image_0005_host_vs_android_amber(tmp_path: pathlib2.Path): check_images_match_host_vs_android_amber(tmp_path, 'image_test_0005.json') +@pytest.mark.skip( + reason="Slight image difference probably due to floating point sensitivity.") def test_image_0006_host_vs_android_amber(tmp_path: pathlib2.Path): check_images_match_host_vs_android_amber(tmp_path, 'image_test_0006.json') @@ -709,6 +719,8 @@ def test_compute_0002_spirvopt_host_amber(tmp_path: pathlib2.Path): ################################# # spirv-opt vs. normal (android, amber) +@pytest.mark.skip( + reason="May trigger a bug on some device, see github issue #564.") def test_image_0003_spirvopt_android_amber(tmp_path: pathlib2.Path): check_images_match_spirvopt(tmp_path, 'image_test_0003.json', options='-O', @@ -716,6 +728,8 @@ def test_image_0003_spirvopt_android_amber(tmp_path: pathlib2.Path): is_amber=True) +@pytest.mark.skip( + reason="Slight image difference probably due to floating point sensitivity.") def test_image_0004_spirvopt_android_amber(tmp_path: pathlib2.Path): check_images_match_spirvopt(tmp_path, 'image_test_0004.json', options='-Os', diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.comp b/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.comp index 611b2b632..31e2b52d2 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.comp +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.comp @@ -23,7 +23,9 @@ layout(std430, binding = 0) buffer theSSBO { float data_in[N]; float data_out[N]; - bool is_exclusive; + // Amber only supports one type in a given buffer, hence the 'is_exclusive' + // bool field is declared as a float + float is_exclusive; }; layout(local_size_x=N, local_size_y=1, local_size_z=1) in; @@ -38,7 +40,7 @@ void main() { barrier(); - for (uint offset = 1u; offset < uint(N); offset <<= 1u) + for (uint offset = 1u; offset < uint(N); offset <<= 1u) { if (tid >= offset) { @@ -55,7 +57,7 @@ void main() { barrier(); } - if (is_exclusive) { + if (bool(is_exclusive)) { data_out[tid] = (tid == 0u) ? 0.0 : result[tid - 1u]; } else { data_out[tid] = result[tid]; diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.json b/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.json index 99c8752ab..f8fb16916 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.json +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.json @@ -40,7 +40,7 @@ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] }, - { "type": "bool", "data": [ 0 ] } + { "type": "float", "data": [ 0 ] } ] } } diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.comp b/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.comp index 27474d827..81e304338 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.comp +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.comp @@ -23,7 +23,9 @@ #define INVOCATIONS 100 layout(std430, binding = 0) buffer theSSBO { - bool is_exclusive; + // Amber only supports one type in a given buffer, hence the 'is_exclusive' + // bool field is declared as a uint + uint is_exclusive; uint data_out[DATA_SIZE]; uint data_in[DATA_SIZE]; }; @@ -36,12 +38,12 @@ void main() { uint tid = gl_LocalInvocationIndex; uint gid = gl_GlobalInvocationID.x; - + if (tid != gid) { // They should be the same. Use them both and do this check just to shake things up a bit. return; } - + result[2u * tid] = data_in[2u * gid]; result[2u * tid + 1u] = data_in[2u * gid + 1u]; @@ -65,10 +67,10 @@ void main() { result[me] = max(result[spine], result[me]); d = d * 2u; } - + barrier(); - if (is_exclusive) { + if (bool(is_exclusive)) { data_out[2u * gid] = (tid == 0u) ? 0u : result[2u * tid - 1u]; data_out[2u * gid + 1u] = result[2u * tid]; } else { diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.json b/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.json index 3dcec9fd4..78bf8e149 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.json +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.json @@ -5,7 +5,7 @@ "binding": 0, "fields": [ - { "type": "bool", "data": [ 1 ] }, + { "type": "uint", "data": [ 1 ] }, { "type": "uint", "data": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, From c790ba6a3fd2cfce04a5daf327c0e1f5f45650b2 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 18 Jun 2019 12:33:18 +0100 Subject: [PATCH 036/180] Use try-with-resources to ensure file is closed. (#558) Fixes #556, whereby some tests would fail under Windows with OpenJDK due to an attempt to move a file that was still open. --- .../java/com/graphicsfuzz/common/util/PipelineInfo.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java b/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java index cc7c2cf39..0e63ae2bc 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java @@ -33,6 +33,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -52,9 +53,11 @@ public PipelineInfo() { this(new JsonObject()); } - public PipelineInfo(File file) throws FileNotFoundException { - dictionary = new Gson().fromJson(new FileReader(file), + public PipelineInfo(File file) throws IOException { + try (FileReader fr = new FileReader(file)) { + dictionary = new Gson().fromJson(fr, JsonObject.class); + } } public PipelineInfo(String string) { From 654598bfa3482bc767a8d86bbcf1c27618a3333c Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 18 Jun 2019 06:35:38 -0500 Subject: [PATCH 037/180] Refactor exponential builtins into separate function (#563) --- .../common/typing/TyperHelper.java | 104 ++++++++++-------- 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index 6310b664b..9aba3a6da 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -302,51 +302,9 @@ private static Map> getBuiltinsForGlslVersion( getBuiltinsForGlslVersionAngleAndTrigonometric(shadingLanguageVersion, builtinsForVersion); - //Exponential Functions - - { - final String name = "pow"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - { - final String name = "exp"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "log"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "exp2"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "log2"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "sqrt"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - { - final String name = "inversesqrt"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } + // 8.2: Exponential Functions + getBuiltinsForGlslVersionExponential(builtinsForVersion); // 8.3: Common Functions @@ -932,6 +890,64 @@ private static void getBuiltinsForGlslVersionAngleAndTrigonometric( } } + /** + * Helper function to register built-in function prototypes for Exponential Functions, as + * specified in section 8.2 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + */ + private static void getBuiltinsForGlslVersionExponential( + Map> builtinsForVersion) { + { + final String name = "pow"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + + { + final String name = "exp"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "log"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "exp2"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "log2"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "sqrt"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "inversesqrt"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + private static void getBuiltinsForGlslVersionVectorRelational( Map> builtinsForVersion, ShadingLanguageVersion shadingLanguageVersion) { From 2336e5ac496f4866f2b7481fa2ad6c3506d60082 Mon Sep 17 00:00:00 2001 From: Hugues Evrard Date: Wed, 19 Jun 2019 07:20:34 +0100 Subject: [PATCH 038/180] Add AmberScript generation for image and compute (--use-amberscript), and relevant runspv tests (#539) Add --use-amberscript flag to runspv --- python/src/main/python/drivers/runspv.py | 481 ++++++++++++++++-- .../python/test_scripts/test_runspv/README.md | 3 + .../test_scripts/test_runspv/runspv_tests.py | 219 +++++++- .../compute/310es/comp-0001-findmax.json | 4 +- vulkan-worker/samples/shader.vert | 1 - vulkan-worker/samples/shader.vert.spv | Bin 940 -> 888 bytes 6 files changed, 649 insertions(+), 59 deletions(-) diff --git a/python/src/main/python/drivers/runspv.py b/python/src/main/python/drivers/runspv.py index 0c15907f0..3d92c7268 100755 --- a/python/src/main/python/drivers/runspv.py +++ b/python/src/main/python/drivers/runspv.py @@ -53,6 +53,8 @@ TARGET_HELP = 'One of \'host\' (run on host machine) or \'android\' (run on Android device).' +USE_AMBERSCRIPT_OPTION_HELP = 'Use AmberScript (not VkScript) test file format.' + ################################################################################ # Constants @@ -794,7 +796,7 @@ def spv_get_disassembly(shader_filename): return subprocess_helper(cmd).stdout -def uniform_json_to_amberscript(uniform_json): +def uniform_json_to_vkscript(uniform_json): """ Returns the string representing VkScript version of uniform declarations. Skips the special '$compute' key, if present. @@ -871,29 +873,19 @@ def get_spirv_opt_args_comment(spirv_args: Optional[List[str]]) -> str: else: return '' - -def amberscriptify_image( - vert, - frag, - uniform_json, - vert_original, - frag_original, - spirv_args +def get_header_comment_original_source( + vert_original: str, + frag_original: str, + comp_original: str, + spirv_args: str ): - """ - Generates Amberscript representation of an image test - """ - - result = '# Generated\n\n' - - result += '# A test for a bug found by GraphicsFuzz.\n\n' - has_frag_glsl = frag_original and filename_extension_suggests_glsl(frag_original) has_vert_glsl = vert_original and filename_extension_suggests_glsl(vert_original) + has_comp_glsl = comp_original and filename_extension_suggests_glsl(comp_original) - result += get_spirv_opt_args_comment(spirv_args) + result = get_spirv_opt_args_comment(spirv_args) - if has_frag_glsl or has_vert_glsl: + if has_frag_glsl or has_vert_glsl or has_comp_glsl: result += '# Derived from the following GLSL.\n\n' if has_vert_glsl: @@ -906,6 +898,47 @@ def amberscriptify_image( result += get_shader_as_comment(frag_original) result += '\n\n' + if has_comp_glsl: + result += '# Compute shader GLSL:\n' + result += get_shader_as_comment(comp_original) + result += '\n\n' + + return result + + +def get_header_comment_original_source_image( + vert_original: str, + frag_original: str, + spirv_args: str +): + return get_header_comment_original_source(vert_original, frag_original, None, spirv_args) + + +def get_header_comment_original_source_comp( + comp_original: str, + spirv_args: str +): + return get_header_comment_original_source(None, None, comp_original, spirv_args) + + +def vkscriptify_image( + vert, + frag, + uniform_json, + vert_original, + frag_original, + spirv_args +): + """ + Generates VkScript representation of an image test + """ + + result = '# Generated\n\n' + + result += '# A test for a bug found by GraphicsFuzz.\n\n' + + result += get_header_comment_original_source_image(vert_original, frag_original, spirv_args) + result += '[require]\n' result += 'fbsize 256 256\n\n' @@ -925,13 +958,158 @@ def amberscriptify_image( result += '[test]\n' result += '## Uniforms\n' - result += uniform_json_to_amberscript(uniform_json) + result += uniform_json_to_vkscript(uniform_json) result += '\n' result += 'draw rect -1 -1 2 2\n' return result +def amberscript_uniform_buffer_decl(uniform_json): + ''' + Returns the string representing AmberScript version of uniform declarations. + Skips the special '$compute' key, if present. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + "$compute": { ... will be ignored ... } + } + + becomes: + + # myuniform + BUFFER myuniform DATA_TYPE float DATA + 42.0 + END + + ''' + + UNIFORM_TYPE = { + 'glUniform1i': 'int32', + 'glUniform2i': 'vec2', + 'glUniform3i': 'vec3', + 'glUniform4i': 'vec4', + 'glUniform1f': 'float', + 'glUniform2f': 'vec2', + 'glUniform3f': 'vec3', + 'glUniform4f': 'vec4', + } + + result = '' + with open(uniform_json, 'r') as f: + j = json.load(f) + for name, entry in j.items(): + + if name == '$compute': + continue + + func = entry['func'] + if func not in UNIFORM_TYPE.keys(): + raise ValueError('Error: unknown uniform type for function: ' + func) + uniform_type = UNIFORM_TYPE[func] + + result += '# ' + name + '\n' + result += 'BUFFER ' + name + ' DATA_TYPE ' + uniform_type + ' DATA\n' + for arg in entry['args']: + result += ' {}'.format(arg) + result += '\n' + result += 'END\n' + + return result + + +def amberscript_uniform_buffer_bind(uniform_json): + ''' + Returns the string representing AmberScript version of uniform binding. + Skips the special '$compute' key, if present. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + "$compute": { ... will be ignored ... } + } + + becomes: + + BIND BUFFER myuniform AS uniform DESCRIPTOR_SET 0 BINDING 3 + ''' + + result = '' + with open(uniform_json, 'r') as f: + j = json.load(f) + for name, entry in j.items(): + + if name == '$compute': + continue + + result += 'BIND BUFFER ' + name + ' AS uniform ' + result += 'DESCRIPTOR_SET 0 BINDING {}\n'.format(entry['binding']) + + return result + + +def amberscriptify_image( + vert, + frag, + uniform_json, + vert_original, + frag_original, + spirv_args +): + ''' + Generates AmberScript representation of an image test + ''' + + has_frag_glsl = frag_original and filename_extension_suggests_glsl(frag_original) + has_vert_glsl = vert_original and filename_extension_suggests_glsl(vert_original) + + result = '#!amber\n' + result += '# Generated AmberScript for a bug found by GraphicsFuzz\n\n' + + result += get_header_comment_original_source_image(vert_original, frag_original, spirv_args) + + result += 'SET ENGINE_DATA fence_timeout_ms ' + str(AMBER_FENCE_TIMEOUT_MS) + '\n\n' + + if vert: + result += 'SHADER vertex gfz_vert SPIRV-ASM\n' + result += spv_get_disassembly(vert) + result += 'END\n\n' + else: + result += 'SHADER vertex gfz_vert PASSTHROUGH\n\n' + + result += 'SHADER fragment gfz_frag SPIRV-ASM\n' + result += spv_get_disassembly(frag) + result += 'END\n\n' + + # This buffer MUST be named framebuffer to be able to retrieve the image + # Format MUST be B8G8R8A8_UNORM (other options may become available once + # Amber supports more formats for image extraction) + result += 'BUFFER framebuffer FORMAT B8G8R8A8_UNORM\n' + result += amberscript_uniform_buffer_decl(uniform_json) + result += '\n' + + result += 'PIPELINE graphics gfz_pipeline\n' + result += ' ATTACH gfz_vert\n' + result += ' ATTACH gfz_frag\n' + result += ' FRAMEBUFFER_SIZE 256 256\n' + result += ' BIND BUFFER framebuffer AS color LOCATION 0\n' + result += amberscript_uniform_buffer_bind(uniform_json) + result += 'END\n\n' + + result += 'CLEAR_COLOR gfz_pipeline 0 0 0 255\n' + result += 'CLEAR gfz_pipeline\n' + result += 'RUN gfz_pipeline DRAW_RECT POS 0 0 SIZE 256 256\n' + + return result + + def run_image_amber( vert_original: str, frag_original: str, @@ -941,6 +1119,7 @@ def run_image_amber( is_android: bool, skip_render: bool, spirv_opt_args: Optional[List[str]], + use_amberscript: bool, ): # The vertex shader is optional; passthrough will be used if it is not present assert not vert_original or os.path.isfile(vert_original) @@ -950,28 +1129,40 @@ def run_image_amber( frag = prepare_shader(output_dir, frag_original, spirv_opt_args) vert = prepare_shader(output_dir, vert_original, spirv_opt_args) if vert_original else None - amberscript_file = os.path.join(output_dir, 'tmpscript.shader_test') status_file = os.path.join(output_dir, 'STATUS') png_image = os.path.join(output_dir, 'image_0.png') device_image = ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/image_0.png' - with open_helper(amberscript_file, 'w') as f: - f.write(amberscriptify_image( - vert, - frag, - json_file, - vert_original, - frag_original, - spirv_opt_args, - )) + if use_amberscript: + shader_test_file = os.path.join(output_dir, 'tmp_shader_test.amber') + with open_helper(shader_test_file, 'w') as f: + f.write(amberscriptify_image( + vert, + frag, + json_file, + vert_original, + frag_original, + spirv_opt_args, + )) + else: + shader_test_file = os.path.join(output_dir, 'tmp_shader_test.vkscript') + with open_helper(shader_test_file, 'w') as f: + f.write(vkscriptify_image( + vert, + frag, + json_file, + vert_original, + frag_original, + spirv_opt_args, + )) if is_android: prepare_device(force, using_legacy_worker=False) adb_check([ 'push', - amberscript_file, + shader_test_file, ANDROID_DEVICE_GRAPHICSFUZZ_DIR ]) @@ -996,7 +1187,7 @@ def run_image_amber( 'cd ' + ANDROID_DEVICE_DIR + ' && ' './' + ANDROID_AMBER_NDK + flags - + ' -d ' + ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/' + os.path.basename(amberscript_file) + + ' -d ' + ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/' + os.path.basename(shader_test_file) ] adb_check(['logcat', '-c']) @@ -1041,7 +1232,7 @@ def run_image_amber( # -i tells amber to dump the framebuffer cmd.append('-i') cmd.append(png_image) - cmd.append(amberscript_file) + cmd.append(shader_test_file) status = 'SUCCESS' if added_catchsegv: @@ -1066,8 +1257,20 @@ def run_image_amber( ################################################################################ # Amber worker: compute test +def amber_check_buffer_single_type(json_filename): + with open_helper(json_filename, 'r') as f: + j = json.load(f) -def comp_json_to_amberscript(comp_json): + # Amber only supports one type per buffer, check this limitation. + field_type = None + for field_info in j['$compute']['buffer']['fields']: + if not field_type: + field_type = field_info['type'] + elif field_type != field_info['type']: + raise ValueError('Amber only supports one type per buffer') + + +def comp_json_to_vkscript(comp_json): """ Returns the string representing VkScript version of compute shader setup, found under the special "$compute" key in JSON @@ -1104,14 +1307,6 @@ def comp_json_to_amberscript(comp_json): binding = j['buffer']['binding'] offset = 0 - # Amber only supports one type per buffer, check this limitation. - field_type = None - for field_info in j['buffer']['fields']: - if not field_type: - field_type = field_info['type'] - elif field_type != field_info['type']: - raise ValueError('Amber only supports one type per buffer') - for field_info in j['buffer']['fields']: result += ( 'ssbo ' @@ -1136,14 +1331,14 @@ def comp_json_to_amberscript(comp_json): return result -def amberscriptify_comp( +def vkscriptify_comp( comp_spv: str, comp_json: str, comp_original: Optional[str], spirv_args: Optional[List[str]] ): """ - Generates an AmberScript representation of a compute test + Generates an Vkscript representation of a compute test """ result = '# Generated\n\n' @@ -1167,10 +1362,170 @@ def amberscriptify_comp( result += '[test]\n' result += '## Uniforms\n' - result += uniform_json_to_amberscript(comp_json) + result += uniform_json_to_vkscript(comp_json) result += '## SSBO\n' - result += comp_json_to_amberscript(comp_json) + result += comp_json_to_vkscript(comp_json) + result += '\n' + + return result + + +def amberscript_comp_buff_decl(comp_json): + """ + Returns the string representing AmberScript declaration of buffers for a + compute shader test. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + + "$compute": { + "num_groups": [12, 13, 14]; + "buffer": { + "binding": 123, + "fields": + [ + { "type": "int", "data": [ 0 ] }, + { "type": "int", "data": [ 1, 2 ] }, + ] + } + } + + } + + becomes: + + # myuniform + BUFFER myuniform DATA_TYPE float DATA + 42.0 + END + + BUFFER gfz_ssbo DATA_TYPE int DATA + 0 1 2 + END + """ + + SSBO_TYPES = { + 'int': 'int32', + 'ivec2': 'vec2', + 'ivec3': 'vec3', + 'ivec4': 'vec4', + 'uint': 'uint32', + 'float': 'float', + 'vec2': 'vec2', + 'vec3': 'vec3', + 'vec4': 'vec4', + } + + # regular uniforms + result = amberscript_uniform_buffer_decl(comp_json) + + with open_helper(comp_json, 'r') as f: + j = json.load(f) + + assert '$compute' in j.keys(), 'Cannot find "$compute" key in JSON file' + j = j['$compute'] + + binding = j['buffer']['binding'] + offset = 0 + + # A single type for all fields is assumed here + assert len(j['buffer']['fields']) > 0, 'Compute shader test with empty SSBO' + json_datum_type = j['buffer']['fields'][0]['type'] + if json_datum_type not in SSBO_TYPES.keys(): + raise ValueError('Unsupported SSBO datum type: ' + json_datum_type) + datum_type = SSBO_TYPES[json_datum_type] + + result += 'BUFFER gfz_ssbo DATA_TYPE ' + datum_type + ' DATA\n' + for field_info in j['buffer']['fields']: + for datum in field_info['data']: + result += ' ' + str(datum) result += '\n' + result += 'END\n\n' + + return result + + +def amberscript_comp_buff_bind(comp_json): + """ + Returns the string representing AmberScript binding of buffers for a + compute shader test. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + + "$compute": { + "num_groups": [12, 13, 14]; + "buffer": { + "binding": 123, + "fields": + [ + { "type": "int", "data": [ 0 ] }, + { "type": "int", "data": [ 1, 2 ] }, + ] + } + } + + } + + becomes: + + BIND BUFFER myuniform AS uniform DESCRIPTOR_SET 0 BINDING 3 + BIND BUFFER gfz_ssbo AS storage DESCRIPTOR_SET 0 BINDING 123 + """ + + # regular uniforms + result = amberscript_uniform_buffer_bind(comp_json) + + result += 'BIND BUFFER gfz_ssbo AS storage DESCRIPTOR_SET 0 BINDING ' + str(get_ssbo_binding(comp_json)) + '\n\n' + + return result + + +def amberscriptify_comp( + comp_spv: str, + comp_json: str, + comp_original: Optional[str], + spirv_args: Optional[List[str]] +): + """ + Generates an AmberScript representation of a compute test + """ + + has_comp_glsl = comp_original and filename_extension_suggests_glsl(comp_original) + + result = '#!amber\n' + result += '# Generated AmberScript for a bug found by GraphicsFuzz\n\n' + + result += get_header_comment_original_source_comp(comp_original, spirv_args) + + result += 'SET ENGINE_DATA fence_timeout_ms ' + str(AMBER_FENCE_TIMEOUT_MS) + '\n\n' + + result += 'SHADER compute gfz_comp SPIRV-ASM\n' + result += spv_get_disassembly(comp_spv) + result += 'END\n\n' + + result += amberscript_comp_buff_decl(comp_json) + + result += 'PIPELINE compute gfz_pipeline\n' + result += ' ATTACH gfz_comp\n' + result += amberscript_comp_buff_bind(comp_json) + result += 'END\n\n' + + result += 'RUN gfz_pipeline' + with open_helper(comp_json, 'r') as f: + j = json.load(f) + num_groups = j['$compute']['num_groups'] + for dimension in num_groups: + result += ' ' + str(dimension) + result += '\n\n' return result @@ -1228,20 +1583,37 @@ def run_compute_amber( is_android: bool, skip_render: bool, spirv_opt_args: Optional[List[str]], + use_amberscript: bool, ) -> None: assert os.path.isfile(comp_original) assert os.path.isfile(json_file) + amber_check_buffer_single_type(json_file) - amberscript_file = os.path.join(output_dir, 'tmpscript.shader_test') ssbo_output = os.path.join(output_dir, 'ssbo') ssbo_json = os.path.join(output_dir, SSBO_JSON_FILENAME) status_file = os.path.join(output_dir, 'STATUS') comp = prepare_shader(output_dir, comp_original, spirv_opt_args) - with open_helper(amberscript_file, 'w') as f: - f.write(amberscriptify_comp(comp, json_file, comp_original, spirv_opt_args)) + if use_amberscript: + shader_test_file = os.path.join(output_dir, 'tmp_shader_test.amber') + with open_helper(shader_test_file, 'w') as f: + f.write(amberscriptify_comp( + comp, + json_file, + comp_original, + spirv_opt_args, + )) + else: + shader_test_file = os.path.join(output_dir, 'tmp_shader_test.vkscript') + with open_helper(shader_test_file, 'w') as f: + f.write(vkscriptify_comp( + comp, + json_file, + comp_original, + spirv_opt_args, + )) # FIXME: in case of multiple SSBOs, we should pass the binding of the ones to be dumped ssbo_binding = str(get_ssbo_binding(json_file)) @@ -1252,7 +1624,7 @@ def run_compute_amber( # Prepare files on device. adb_check([ 'push', - amberscript_file, + shader_test_file, ANDROID_DEVICE_GRAPHICSFUZZ_DIR, ]) @@ -1278,7 +1650,7 @@ def run_compute_amber( 'cd ' + ANDROID_DEVICE_DIR + ' && ' './' + ANDROID_AMBER_NDK + flags - + ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/' + os.path.basename(amberscript_file) + + ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/' + os.path.basename(shader_test_file) ] adb_check(['logcat', '-c']) @@ -1323,7 +1695,7 @@ def run_compute_amber( cmd.append(ssbo_output) cmd.append('-B') cmd.append(ssbo_binding) - cmd.append(amberscript_file) + cmd.append(shader_test_file) status = 'SUCCESS' @@ -1408,6 +1780,7 @@ def main_helper(args): # Optional arguments parser.add_argument('--serial', help=SERIAL_OPTION_HELP) parser.add_argument('--legacy-worker', action='store_true', help=LEGACY_OPTION_HELP) + parser.add_argument('--use-amberscript', action='store_true', help=USE_AMBERSCRIPT_OPTION_HELP) parser.add_argument('--skip-render', action='store_true', help=SKIP_RENDER_OPTION_HELP) parser.add_argument('--spirvopt', help=SPIRV_OPT_OPTION_HELP) parser.add_argument('--force', action='store_true', help=FORCE_OPTION_HELP) @@ -1498,7 +1871,8 @@ def main_helper(args): force=args.force, is_android=(args.target == 'android'), skip_render=args.skip_render, - spirv_opt_args=spirv_args + spirv_opt_args=spirv_args, + use_amberscript=args.use_amberscript, ) return @@ -1523,6 +1897,7 @@ def main_helper(args): is_android=(args.target == 'android'), skip_render=args.skip_render, spirv_opt_args=spirv_args, + use_amberscript=args.use_amberscript, ) finally: log_to_file = None diff --git a/python/src/main/python/test_scripts/test_runspv/README.md b/python/src/main/python/test_scripts/test_runspv/README.md index 84f3f8714..b927aaa5e 100644 --- a/python/src/main/python/test_scripts/test_runspv/README.md +++ b/python/src/main/python/test_scripts/test_runspv/README.md @@ -24,4 +24,7 @@ To run the tests from the command line: ``` python3 -m pytest ./runspv_tests.py + +# Filter to test names containing "foo" but not "bar": +python3 -m pytest ./runspv_tests.py -k 'foo and not bar' ``` diff --git a/python/src/main/python/test_scripts/test_runspv/runspv_tests.py b/python/src/main/python/test_scripts/test_runspv/runspv_tests.py index d3121d916..3dfd7ce2a 100755 --- a/python/src/main/python/test_scripts/test_runspv/runspv_tests.py +++ b/python/src/main/python/test_scripts/test_runspv/runspv_tests.py @@ -111,22 +111,33 @@ def get_ssbo_json(output_dir: pathlib2.Path) -> {}: return json.load(open(str(ssbo_json), 'r')) -def check_images_match(tmp_path: pathlib2.Path, json_filename: str, is_android_1: bool, +def check_images_match(tmp_path: pathlib2.Path, + json_filename: str, + is_android_1: bool, is_android_2: bool, - is_amber_1: bool, is_amber_2: bool, fuzzy_image_comparison: bool, - spirvopt_options_1: str=None, spirvopt_options_2: str=None): + is_amber_1: bool, + is_amber_2: bool, + fuzzy_image_comparison: bool, + use_amberscript_1: bool=None, + use_amberscript_2: bool=None, + spirvopt_options_1: str=None, + spirvopt_options_2: str=None): out_dir_1 = tmp_path / 'out_1' out_dir_2 = tmp_path / 'out_2' args_1 = ['android' if is_android_1 else 'host', get_image_test(json_filename), str(out_dir_1)] if not is_amber_1: args_1.append('--legacy-worker') + elif use_amberscript_1: + args_1.append('--use-amberscript') if spirvopt_options_1: args_1.append('--spirvopt=' + spirvopt_options_1) args_2 = ['android' if is_android_2 else 'host', get_image_test(json_filename), str(out_dir_2)] if not is_amber_2: args_2.append('--legacy-worker') + elif use_amberscript_2: + args_2.append('--use-amberscript') if spirvopt_options_2: args_2.append('--spirvopt=' + spirvopt_options_2) @@ -170,9 +181,26 @@ def check_images_match_spirvopt(tmp_path: pathlib2.Path, json_filename: str, opt spirvopt_options_2=None) +def check_images_match_vkscript_amberscript(tmp_path: pathlib2.Path, json_filename: str, + is_android: bool) -> None: + check_images_match(tmp_path, + json_filename=json_filename, + is_android_1=is_android, + is_android_2=is_android, + is_amber_1=True, + is_amber_2=True, + fuzzy_image_comparison=False, + use_amberscript_1=False, + use_amberscript_2=True, + spirvopt_options_1=None, + spirvopt_options_2=None) + + def check_compute_matches(tmp_path: pathlib2.Path, json_filename: str, is_android_1: bool, is_android_2: bool, + use_amberscript_1: bool=None, + use_amberscript_2: bool=None, spirvopt_options_1: str=None, spirvopt_options_2: str=None): out_dir_1 = tmp_path / 'out_1' @@ -180,6 +208,10 @@ def check_compute_matches(tmp_path: pathlib2.Path, json_filename: str, args_1 = ['android' if is_android_1 else 'host', get_compute_samples_dir() + os.sep + json_filename, str(out_dir_1)] args_2 = ['android' if is_android_2 else 'host', get_compute_samples_dir() + os.sep + json_filename, str(out_dir_2)] + if use_amberscript_1: + args_1.append('--use-amberscript') + if use_amberscript_2: + args_2.append('--use-amberscript') if spirvopt_options_1: args_1.append('--spirvopt=' + spirvopt_options_1) if spirvopt_options_2: @@ -211,6 +243,17 @@ def check_host_and_android_match_compute(tmp_path: pathlib2.Path, json_filename: is_android_2=True) +def check_compute_matches_vkscript_amberscript(tmp_path: pathlib2.Path, json_filename: str, is_android: bool): + check_compute_matches(tmp_path, + json_filename=json_filename, + is_android_1=is_android, + is_android_2=is_android, + use_amberscript_1=False, + use_amberscript_2=True, + spirvopt_options_1=None, + spirvopt_options_2=None) + + def check_no_image_skip_render(tmp_path: pathlib2.Path, is_android: bool, is_legacy_worker: bool, json_filename: str): out_dir = tmp_path / 'out' @@ -766,3 +809,173 @@ def test_image_0006_spirvopt_android_legacy(tmp_path: pathlib2.Path): options='-Os', is_android=True, is_amber=False) + +################################# +# VkScript vs AmberScript + +def test_image_0000_vksript_amberscript_host(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0000.json', + is_android=False) + + +def test_image_0000_vksript_amberscript_android(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0000.json', + is_android=True) + + +def test_image_0001_vksript_amberscript_host(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0001.json', + is_android=False) + + +def test_image_0001_vksript_amberscript_android(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0001.json', + is_android=True) + + +def test_image_0002_vksript_amberscript_host(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0002.json', + is_android=False) + + +def test_image_0002_vksript_amberscript_android(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0002.json', + is_android=True) + + +def test_image_0003_vksript_amberscript_host(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0003.json', + is_android=False) + + +def test_image_0003_vksript_amberscript_android(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0003.json', + is_android=True) + + +def test_image_0004_vksript_amberscript_host(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0004.json', + is_android=False) + + +def test_image_0004_vksript_amberscript_android(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0004.json', + is_android=True) + + +def test_image_0005_vksript_amberscript_host(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0005.json', + is_android=False) + + +def test_image_0005_vksript_amberscript_android(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0005.json', + is_android=True) + + +def test_image_0006_vksript_amberscript_host(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0006.json', + is_android=False) + + +def test_image_0006_vksript_amberscript_android(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0006.json', + is_android=True) + + +def test_image_0007_vksript_amberscript_host(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0007.json', + is_android=False) + + +def test_image_0007_vksript_amberscript_android(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0007.json', + is_android=True) + + +def test_image_0008_vksript_amberscript_host(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0008.json', + is_android=False) + + +def test_image_0008_vksript_amberscript_android(tmp_path: pathlib2.Path): + check_images_match_vkscript_amberscript(tmp_path, + 'image_test_0008.json', + is_android=True) + + +def test_compute_0001_vkscript_amberscript_host(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0001-findmax.json', + is_android=False) + + +def test_compute_0001_vkscript_amberscript_android(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0001-findmax.json', + is_android=True) + + +def test_compute_0002_vkscript_amberscript_host(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0002-smooth-mean.json', + is_android=False) + + +def test_compute_0002_vkscript_amberscript_android(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0002-smooth-mean.json', + is_android=True) + + +def test_compute_0003_vkscript_amberscript_host(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0003-random-middle-square.json', + is_android=False) + + +def test_compute_0003_vkscript_amberscript_android(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0003-random-middle-square.json', + is_android=True) + + +def test_compute_0004_vkscript_amberscript_host(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0004-koggestone.json', + is_android=False) + + +def test_compute_0004_vkscript_amberscript_android(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0004-koggestone.json', + is_android=True) + + +def test_compute_0005_vkscript_amberscript_host(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0005-sklansky.json', + is_android=False) + + +def test_compute_0005_vkscript_amberscript_android(tmp_path: pathlib2.Path): + check_compute_matches_vkscript_amberscript(tmp_path, + 'comp-0005-sklansky.json', + is_android=True) diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0001-findmax.json b/shaders/src/main/glsl/samples/compute/310es/comp-0001-findmax.json index 3c4e1dcc0..af1fb5c35 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0001-findmax.json +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0001-findmax.json @@ -6,8 +6,8 @@ "fields": [ { "type": "int", "data": [ 0 ] }, - { "type": "int", "data": - [ 11, 12, 13, 14, 15, 16, 17, 18, + { "type": "int", "data": + [ 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 41, 42, 43, 44, 45, 46, 47, 48, diff --git a/vulkan-worker/samples/shader.vert b/vulkan-worker/samples/shader.vert index 28cbd521e..143937fbf 100644 --- a/vulkan-worker/samples/shader.vert +++ b/vulkan-worker/samples/shader.vert @@ -20,7 +20,6 @@ layout (std140, binding = 0) uniform bufferVals { float f; } myBufferVals; layout (location = 0) in vec4 pos; -layout (location = 1) in vec4 inColor; void main() { gl_Position = pos; } diff --git a/vulkan-worker/samples/shader.vert.spv b/vulkan-worker/samples/shader.vert.spv index 3ce34d2eec599ab0a25dd75ea63ca7281b1c2cbb..298b0718bd2d8fa759969cb9233e772fa955bda0 100644 GIT binary patch delta 62 zcmZ3({)3H|nMs+Qfq{{Moq>ZvVj{1pIFJVfEI^!_n3)G+^8&Hp##m*>$rBi7Y_?z$ KV4NJm90C9dw+Z|J delta 110 zcmeytwuYUTnMs+Qfq{{Moq>Zvaw4y(GzS9+umEvxVrCwQ%?reWKrFE_QkjvT6(}bG r Date: Wed, 19 Jun 2019 14:37:24 +0200 Subject: [PATCH 039/180] Add class skeleton and tests for reduction opportunity to move a global variable initializer into main (#561) --- ...iableDeclToExprReductionOpportunities.java | 49 +++++ ...ariableDeclToExprReductionOpportunity.java | 35 +++ ...eDeclToExprReductionOpportunitiesTest.java | 208 ++++++++++++++++++ 3 files changed, 292 insertions(+) create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java create mode 100644 reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java new file mode 100644 index 000000000..08d864ca6 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.ListConcat; +import java.util.Arrays; +import java.util.List; + +public class GlobalVariableDeclToExprReductionOpportunities + extends ReductionOpportunitiesBase { + public GlobalVariableDeclToExprReductionOpportunities(TranslationUnit tu, + ReducerContext context) { + super(tu, context); + } + + static List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return shaderJob.getShaders() + .stream() + .map(item -> findOpportunitiesForShader(item, context)) + .reduce(Arrays.asList(), ListConcat::concatenate); + } + + private static List findOpportunitiesForShader( + TranslationUnit tu, + ReducerContext context) { + GlobalVariableDeclToExprReductionOpportunities finder = + new GlobalVariableDeclToExprReductionOpportunities(tu, context); + finder.visit(tu); + return finder.getOpportunities(); + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java new file mode 100644 index 000000000..6f56ce6be --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; + +public class GlobalVariableDeclToExprReductionOpportunity extends AbstractReductionOpportunity { + GlobalVariableDeclToExprReductionOpportunity(VisitationDepth depth) { + super(depth); + } + + @Override + void applyReductionImpl() { + throw new RuntimeException("Not implemented yet."); + } + + @Override + public boolean preconditionHolds() { + throw new RuntimeException("Not implemented yet."); + } +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java new file mode 100644 index 000000000..f9b019606 --- /dev/null +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java @@ -0,0 +1,208 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; +import java.util.List; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class GlobalVariableDeclToExprReductionOpportunitiesTest { + + // TODO(519): Enable this test when the issue is solved. + @Ignore + @Test + public void testDoNotReplace() throws Exception { + final String original = "int a = 1; void main() { }"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + // There should be no opportunities as the preserve semantics is enabled. + assertTrue(ops.isEmpty()); + } + + @Ignore + @Test + public void testZeroMethod() throws Exception { + final String original = "int a = 1;"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + // Since the new assignment statement must be only inserted as the first statement of + // the main function, thus we have to ensure that main function exists. + // In this case, there is no method at all. + assertTrue(ops.isEmpty()); + } + + @Ignore + @Test + public void testNoMainMethod() throws Exception { + final String original = "int a = 1; void foo() { }"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + // Since the new assignment statement must be only inserted as the first statement of + // the main function, thus we have to ensure that main function exists. + // In this case, there is one method but it is not main though so there should be no + // opportunities. + assertTrue(ops.isEmpty()); + } + + @Ignore + @Test + public void testInsertAsFirstStatement() throws Exception { + final String program = "" + + "int a = 1;" + + "void main() {" + + " int b = a; " + + "}"; + // We have to ensure that the new assignment statements is inserted + // as the first statement in main() function. + final String expected = "" + + "int a;" + + "void main() {" + + " a = 1;" + + " int b = a;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + assertEquals(1, ops.size()); + ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Ignore + @Test + public void testMultipleDeclarations() throws Exception { + final String program = "" + + "int a = 1;" // Initialized variable declaration. + + "int b = foo();" // Initialized variable declaration. + + "int c;" // Uninitialized variable declaration. + + "void main() { }"; + final String expected = "" + + "int a;" + + "int b;" + + "int c;" // Uninitialized variable declaration. + + "void main() {" + + " a = 1;" + + " b = foo();" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + // Only variable declarations a and b have the initializer. + // Thus, we expect the reducer to find only 2 opportunities. + assertEquals(2, ops.size()); + ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Ignore + @Test + public void testDoNotReplaceConst() throws Exception { + final String original = "const int a = 1; void main() { }"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + // There should be no opportunities as it is invalid to declare constant variable + // without an initial value. + assertTrue(ops.isEmpty()); + } + + @Ignore + @Test + public void testMultipleLineDeclarationsOneLine() throws Exception { + final String program = "" + + "int b = 1, c, d = foo(), e, f = bar();" // This variable declaration has many + // declaration infos but we consider only + // the one that has initializer (b, d, and f). + + "void main() { }"; + final String expected = "" + + "int b, c, d, e, f;" + + "void main() {" + + " b = 1;" + + " d = foo();" + + " f = bar();" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + assertEquals(3, ops.size()); + ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Ignore + @Test + public void testAssignVariableIdentifier() throws Exception { + final String program = "" + + "int a = 1;" + + "int b = a, c = b;" // b depends on a and c depends on b. + + "int d = c;" // d depends on c. + + "void main() { }"; + // As here we have the variable identifier as the initializer, we need to + // make sure that new expressions generated by the reducer are added + // in the correct order. + final String expected = "" + + "int a;" + + "int b, c;" + + "int d;" + + "void main() {" + + " a = 1;" + + " b = a;" + + " c = b;" + + " d = c;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + assertEquals(4, ops.size()); + ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } +} From b9f193175952079df0aaf36602595afcd4fe09e5 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 20 Jun 2019 12:22:38 +0100 Subject: [PATCH 040/180] Improve custom reduction support. (#574) Fix #572. Fix #573. * Improve glsl-reduce help text. * Allow custom interestingness command to take arguments. * Run custom interestingness command from current directory. * Don't pass second argument (an undocumented output file path) to interestingness test. --- .../reducer/filejudge/CustomFileJudge.java | 17 +++---- .../graphicsfuzz/reducer/tool/GlslReduce.java | 45 ++++++++++++------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/filejudge/CustomFileJudge.java b/reducer/src/main/java/com/graphicsfuzz/reducer/filejudge/CustomFileJudge.java index fb5a6a7f5..3e9efbf86 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/filejudge/CustomFileJudge.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/filejudge/CustomFileJudge.java @@ -22,6 +22,8 @@ import com.graphicsfuzz.util.ExecResult; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,25 +31,24 @@ public class CustomFileJudge implements IFileJudge { private static final Logger LOGGER = LoggerFactory.getLogger(CustomFileJudge.class); - private final File judgeScript; - private final File directory; + private final List judgeScript; - public CustomFileJudge(File judgeScript, File directory) { + public CustomFileJudge(List judgeScript) { this.judgeScript = judgeScript; - this.directory = directory; } @Override public boolean isInteresting(File shaderJobFile, File shaderResultFileOutput) throws FileJudgeException { + List scriptPlusShaderArg = new ArrayList<>(judgeScript); + scriptPlusShaderArg.add(shaderJobFile.toString()); try { final ExecResult execResult = new ExecHelper().exec( ExecHelper.RedirectType.TO_LOG, - directory, + null, true, - judgeScript.getAbsolutePath(), - shaderJobFile.getAbsolutePath(), - shaderResultFileOutput.getAbsolutePath()); + scriptPlusShaderArg.toArray(new String[0]) + ); LOGGER.info("Custom file judge result: " + execResult.res); return execResult.res == 0; } catch (IOException | InterruptedException exception) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java index 6e3815991..68b929025 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java @@ -46,6 +46,7 @@ import com.graphicsfuzz.util.Constants; import java.io.File; import java.io.IOException; +import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import net.sourceforge.argparse4j.ArgumentParsers; @@ -78,9 +79,9 @@ private static ArgumentParser getParser() { .description("glsl-reduce takes a shader job `SHADER_JOB.json` " + "(a .json file alongside shader files with the same name, such as SHADER_JOB.frag " + "and/or SHADER_JOB.vert or SHADER_JOB.comp), " - + "as well as further arguments or options to specify the interestingness test." - + "glsl-reduce will try to simplify the given shader to a smaller," - + "simpler shader that is still deemed \"interesting\"."); + + "as well as further arguments or options to specify the interestingness test. " + + "glsl-reduce will try to simplify the shaders to smaller, simpler shaders that are " + + "still deemed \"interesting\"."); // Required arguments parser.addArgument("shader-job") @@ -89,11 +90,17 @@ private static ArgumentParser getParser() { // Optional positional argument parser.addArgument("interestingness-test") - .help("Path to an executable shell script that should decide whether a shader job is " - + "interesting. Only allowed (and then also required) when performing a custom " - + "reduction, which is the default.") - .nargs("?") - .type(File.class); + .help("A command to execute (plus any fixed arguments) to determine " + + "whether a shader job is interesting. The command will typically compile and/or " + + "run the shader job and check some property. An exit status of 0 indicates that the " + + "shader job is interesting. The shader job will be passed as an argument " + + "(after any fixed arguments). Only allowed (and then also required) when performing " + + "a custom reduction, which is the default. Use -- to ensure all command line " + + "arguments that follow are parsed as positional arguments. " + + "E.g.\n" + + "glsl-reduce --preserve-semantics -- shader_job.json is-interesting --run-on-android") + .nargs("*") + .type(String.class); parser.addArgument("--reduction-kind") .help("Kind of reduction to be performed. Options are:\n" @@ -112,10 +119,10 @@ private static ArgumentParser getParser() { + " Reduces while histogram difference between produced image and " + "reference is above a threshold.\n" + " " + ReductionKind.VALIDATOR_ERROR - + " Reduces while validator gives a particular error\n" + + " Reduces while validator gives a particular error.\n" + " " + ReductionKind.ALWAYS_REDUCE - + " Always reduces (useful for testing)\n") - .setDefault("CUSTOM") + + " Always reduces (useful for testing).\n") + .setDefault(ReductionKind.CUSTOM.toString()) .type(String.class); parser.addArgument("--output") @@ -310,7 +317,7 @@ public static void mainHelper( final File referenceResultFile = ns.get("reference"); - final File customJudgeScript = ns.get("interestingness_test"); + final List customJudgeScript = ns.get("interestingness_test"); if (reductionKind == ReductionKind.CUSTOM) { if (server != null) { @@ -325,15 +332,21 @@ public static void mainHelper( if (referenceResultFile != null) { throwExceptionForCustomReduction("reference"); } - if (customJudgeScript == null) { + if (customJudgeScript.isEmpty()) { throw new RuntimeException("A custom reduction requires an interestingness test to be " + "specified."); } - if (!customJudgeScript.canExecute()) { + + // Sanity check that the custom judge script is executable. However, don't fail if the + // script can't be found, as the script/tool could be on the PATH. + File scriptFile = new File(customJudgeScript.get(0)); + if (scriptFile.exists() && !scriptFile.canExecute()) { throw new RuntimeException("Custom judge script must be executable."); } } else { - if (customJudgeScript != null) { + // Not a custom reduction. + + if (!customJudgeScript.isEmpty()) { throw new RuntimeException("An interestingness test is only supported when a custom " + "reduction is used."); } @@ -380,7 +393,7 @@ public static void mainHelper( switch (reductionKind) { case CUSTOM: fileJudge = - new CustomFileJudge(customJudgeScript, workDir); + new CustomFileJudge(customJudgeScript); break; case NO_IMAGE: fileJudge = From 2d50635834ddfb7c14aa084a722a94b95553341e Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 20 Jun 2019 12:33:45 +0100 Subject: [PATCH 041/180] Script to convert GraphicsFuzz uniform information into spirv-fuzz facts (#569) --- ...shader-job-uniforms-to-spirv-fuzz-facts.py | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 python/src/main/python/drivers/shader-job-uniforms-to-spirv-fuzz-facts.py diff --git a/python/src/main/python/drivers/shader-job-uniforms-to-spirv-fuzz-facts.py b/python/src/main/python/drivers/shader-job-uniforms-to-spirv-fuzz-facts.py new file mode 100644 index 000000000..4862a3bfc --- /dev/null +++ b/python/src/main/python/drivers/shader-job-uniforms-to-spirv-fuzz-facts.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +import struct +from typing import Any, Dict, List + +import runspv + + +# Turns a GraphicsFuzz .json file into a spirv-fuzz .facts file, with one fact per word of uniform +# data. +def main() -> None: + parser = argparse.ArgumentParser() + + # Required arguments + parser.add_argument( + 'shader_job', + help='Shader job (.json) file.') + + args = parser.parse_args() + + # Generate uniform facts from .json file + fact_list = [] # type: List[dict] + + # Turn the information about uniform values into facts for spirv-fuzz. + with runspv.open_helper(args.shader_job, 'r') as f: + j = json.load(f) # type: Dict[str, Any] + for name, entry in j.items(): + + # Skip compute shader data + if name == "$compute": + continue + + # Make a separate fact for every component value of each uniform. + for i in range(0, len(entry["args"])): + # Every uniform is in its own struct, so we index into the first element of that struct + # using index 0. If the uniform is a vector, we need to further index into the vector. + if entry["func"] in ['glUniform1i', 'glUniform1f']: + # The uniform is a scalar. + assert i == 0 + index_list = [0] + elif entry["func"] in ['glUniform2i', 'glUniform3i', 'glUniform4i', 'glUniform2f', + 'glUniform3f', 'glUniform4f']: + # The uniform is a vector, so we have two levels of indexing. + index_list = [0, i] + else: + # We can deal with other uniforms in due course. + index_list = [] + print("Unsupported uniform function " + entry["func"]) + exit(1) + + # We need to pass the component value as an integer. If it is a float, we need to + # reinterpret its bits as an integer. + int_representation = entry["args"][i] + if isinstance(int_representation, float): + int_representation = struct.unpack(' Date: Thu, 20 Jun 2019 12:35:36 +0100 Subject: [PATCH 042/180] Compute shaders in WebUI (#502) Provide some basic support for viewing compute shader results in the WebUI. Reductions are still not supported, but you can browse results to see which variants passed/failed and to see reports on where SSBOs differ between reference and variant. --- .../common/util/ShaderJobFileOperations.java | 108 +++- .../main/python/drivers/glsl-to-spv-worker.py | 2 + .../com/graphicsfuzz/server/webui/WebUi.java | 570 ++++++++++++------ .../main/resources/public/graphicsfuzz.css | 4 +- 4 files changed, 472 insertions(+), 212 deletions(-) diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java index 5d47d05a2..64d5ec8b6 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java @@ -37,6 +37,7 @@ import com.graphicsfuzz.util.ExecHelper; import com.graphicsfuzz.util.ExecResult; import com.graphicsfuzz.util.ToolHelper; +import com.graphicsfuzz.util.ToolPaths; import java.awt.image.BufferedImage; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -50,6 +51,7 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.NoSuchFileException; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; @@ -786,7 +788,7 @@ public void writeShaderJobFileFromImageJob( public void writeShaderResultToFile( ImageJobResult shaderResult, File shaderResultFile, - Optional referenceShaderResultFile) throws IOException { + Optional referenceShaderResultFile) throws InterruptedException, IOException { writeShaderResultToFileHelper( shaderResult, @@ -1165,17 +1167,30 @@ protected static void writeShaderResultToFileHelper( ImageJobResult shaderResult, File shaderJobResultFile, ShaderJobFileOperations fileOps, - Optional referenceShaderResultFile) throws IOException { + Optional referenceShaderResultFile) throws InterruptedException, IOException { assertIsShaderJobResultFile(shaderJobResultFile); String shaderJobResultNoExtension = FileHelper.removeEnd(shaderJobResultFile.toString(), ".info.json"); - // Special case: compute shader job. + // Write the log component of the result to a text file, for easy viewing. + if (shaderResult.isSetLog()) { + fileOps.writeStringToFile( + new File(shaderJobResultNoExtension + ".txt"), + shaderResult.getLog()); + } + // Special case: compute shader job. if (shaderResult.isSetComputeOutputs()) { + // In addition to a status and log, results for a compute shader job have: + // - an "outputs" property, which maps to a dictionary representing the results + // that were obtained by running the shader; + // - (if reference result is present) a "comparison_with_reference" property + // describing whether or not the computed results exactly or nearly match those + // for the reference. + JsonObject infoJson = new JsonObject(); if (shaderResult.isSetStatus()) { infoJson.addProperty("status", shaderResult.getStatus().toString()); @@ -1188,8 +1203,64 @@ protected static void writeShaderResultToFileHelper( "outputs", new Gson().fromJson(shaderResult.getComputeOutputs(), JsonObject.class)); } + // We write out the .info.json file now, so that the Python tooling for diffing compute + // shader results can be invoked on it if needed. + fileOps.writeStringToFile( + shaderJobResultFile, + infoJson.toString()); + + if (referenceShaderResultFile.isPresent()) { + + // We have reference results, so can populate the "comparison_with_reference" property. + + // This maps to a dictionary with up to 4 keys: + // - "exact_match", true if and only if the results are identical + // - "exactdiff_output", populated only if "exact_match" is false, with the result of + // exact diffing + // - "fuzzy_match", present only if "exact_match" is false, and then true if and only if + // the results are similar + // - "fuzzydiff_output", present only if "fuzzy_diff" is set, with the result of + // fuzzy diffing. + + final JsonObject computeShaderComparisonWithReference = new JsonObject(); + + // Check whether the results exactly match those of the reference. + + final ExecResult exactDiffResult = + fileOps.runPythonDriver(ExecHelper.RedirectType.TO_BUFFER, + null, + "inspect-compute-results", + "exactdiff", + referenceShaderResultFile.get().getAbsolutePath(), + shaderJobResultFile.getAbsolutePath()); + computeShaderComparisonWithReference.addProperty("exact_match", + exactDiffResult.res == 0); + + if (exactDiffResult.res != 0) { + + // In the case that we do not have an exact match, store the output obtained by exact + // diffing (as it may be useful to inspect). + computeShaderComparisonWithReference.addProperty("exactdiff_output", + exactDiffResult.stderr.toString()); + + // Now perform a fuzzy diff. + final ExecResult fuzzyDiffResult = + fileOps.runPythonDriver(ExecHelper.RedirectType.TO_BUFFER, + null, + "inspect-compute-results", + "fuzzydiff", + referenceShaderResultFile.get().getAbsolutePath(), + shaderJobResultFile.getAbsolutePath()); + computeShaderComparisonWithReference.addProperty("fuzzy_match", + fuzzyDiffResult.res == 0); + computeShaderComparisonWithReference.addProperty("fuzzydiff_output", + fuzzyDiffResult.stderr.toString()); + } + infoJson.add("comparison_with_reference", computeShaderComparisonWithReference); + } + fileOps.writeStringToFile( - new File(shaderJobResultNoExtension + ".info.json"), + shaderJobResultFile, infoJson.toString()); return; @@ -1197,12 +1268,6 @@ protected static void writeShaderResultToFileHelper( final File outputImage = new File(shaderJobResultNoExtension + ".png"); - if (shaderResult.isSetLog()) { - fileOps.writeStringToFile( - new File(shaderJobResultNoExtension + ".txt"), - shaderResult.getLog()); - } - if (shaderResult.isSetPNG()) { fileOps.writeByteArrayToFile(outputImage, shaderResult.getPNG()); } @@ -1242,7 +1307,6 @@ protected static void writeShaderResultToFileHelper( gifOutput.close(); } catch (Exception err) { LOGGER.error("Error while creating GIF for nondet"); - err.printStackTrace(); } } @@ -1267,6 +1331,28 @@ protected static void writeShaderResultToFileHelper( JsonHelper.jsonToString(infoObject)); } + /** + * Runs a GraphicsFuzz Python driver script, from the python/drivers directory. + * @param redirectType Determines where output is redirected to. + * @param directory Working directory; set to null if current directory is fine. + * @param driverName Name of the Python driver, with no extension. + * @param driverArgs Arguments to be passed to the Python driver. + * @return the result of executing the Python driver. + * @throws IOException if something IO-related goes wrong. + * @throws InterruptedException if something goes wrong running the driver command. + */ + public ExecResult runPythonDriver(ExecHelper.RedirectType redirectType, File directory, + String driverName, String... driverArgs) throws IOException, + InterruptedException { + final String[] execArgs = new String[driverArgs.length + 1]; + execArgs[0] = Paths.get(ToolPaths.getPythonDriversDir(), driverName).toString() + + (System.getProperty("os.name").startsWith("Windows") ? ".bat" : ""); + System.arraycopy(driverArgs, 0, execArgs, 1, driverArgs.length); + return new ExecHelper().exec(redirectType, + directory, false, + execArgs); + } + /** * Stores data about an image; right now its file and histogram. * Could be extended in due course with e.g. PSNR diff --git a/python/src/main/python/drivers/glsl-to-spv-worker.py b/python/src/main/python/drivers/glsl-to-spv-worker.py index 4e93bce2a..422729654 100755 --- a/python/src/main/python/drivers/glsl-to-spv-worker.py +++ b/python/src/main/python/drivers/glsl-to-spv-worker.py @@ -235,6 +235,7 @@ def do_image_job( is_android=(args.target == 'android'), skip_render=skip_render, spirv_opt_args=resolved_spirvopt_args, + use_amberscript=True ) except Exception as ex: runspv.log('Exception: ' + str(ex)) @@ -330,6 +331,7 @@ def do_compute_job( is_android=(args.target == 'android'), skip_render=comp_job.skipRender, spirv_opt_args=spirv_opt_args, + use_amberscript=True ) except Exception as ex: runspv.log('Exception: ' + str(ex)) diff --git a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java index 1a827bbda..31c5931ba 100755 --- a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java +++ b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java @@ -38,6 +38,7 @@ import java.io.PrintWriter; import java.nio.charset.Charset; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -84,8 +85,9 @@ public class WebUi extends HttpServlet { private long startTime; private final AccessFileInfo accessFileInfo; - private final FilenameFilter variantFragFilter = - (dir, name) -> name.startsWith("variant_") && name.endsWith(".frag"); + private static final String WARNING_CLASS_WRONG_RESULT = "wrongresult"; + private static final String WARNING_CLASS_WARN_RESULT = "warnresult"; + private static final String WARNING_CLASS_METRICS_DISAGREE = "metricsdisimg"; private enum ImageDifferenceResult { IDENTICAL, @@ -101,6 +103,12 @@ private static final class ImageDifferenceResultSet { public double histogramDistance = 0.0; } + private enum ComputeDifferenceResult { + IDENTICAL, + SIMILAR, + DIFFERENT + } + private final ShaderJobFileOperations fileOps; private final FuzzerServiceManager.Iface fuzzerServiceManagerProxy; @@ -116,19 +124,19 @@ private static final class ShaderFamily { final File dir; final File preview; final int nbVariants; - final boolean isComp; + final boolean isCompute; public ShaderFamily(String name) { this.name = name; this.dir = new File(WebUiConstants.SHADER_FAMILIES_DIR, name); this.preview = new File(dir, "thumb.png"); - this.isComp = new File(dir, "reference.comp").isFile(); + this.isCompute = new File(dir, "reference.comp").isFile(); this.nbVariants = getNbVariants(); } private int getNbVariants() { final File[] variants; - if (this.isComp) { + if (this.isCompute) { variants = dir.listFiles(item -> item.getName().startsWith("variant") && item.getName().endsWith(".comp")); } else { @@ -147,11 +155,18 @@ private static final class ShaderFamilyResult { int nbVariants; int nbVariantDone; int nbErrors; + + // For image shader families: int nbSameImage; int nbSlightlyDifferentImage; int nbMetricsDisagree; int nbWrongImage; + // For compute shader families: + int nbSameComputeResult; + int nbSlightlyDifferentComputeResult; + int nbWrongComputeResult; + public ShaderFamilyResult(String name, String worker, AccessFileInfo accessFileInfo) throws FileNotFoundException { this.name = name; @@ -167,23 +182,40 @@ public ShaderFamilyResult(String name, String worker, AccessFileInfo accessFileI JsonObject info = accessFileInfo.getResultInfo(file); String status = info.get("status").getAsString(); if (status.contentEquals("SUCCESS")) { - ImageDifferenceResult result = getImageDiffResult(info).summary; - switch (result) { - case IDENTICAL: - ++nbSameImage; - break; - case SIMILAR: - ++nbSlightlyDifferentImage; - break; - case DIFFERENT: - ++nbWrongImage; - break; - case METRICS_DISAGREE: - ++nbWrongImage; - ++nbMetricsDisagree; - break; - default: - LOGGER.error("Unrecognized image difference result: " + result); + if (shaderFamily.isCompute) { + ComputeDifferenceResult result = getComputeDiffResult(info); + switch (result) { + case IDENTICAL: + ++nbSameComputeResult; + break; + case SIMILAR: + ++nbSlightlyDifferentComputeResult; + break; + case DIFFERENT: + ++nbWrongComputeResult; + break; + default: + LOGGER.error("Unrecognized compute difference result: " + result); + } + } else { + ImageDifferenceResult result = getImageDiffResult(info).summary; + switch (result) { + case IDENTICAL: + ++nbSameImage; + break; + case SIMILAR: + ++nbSlightlyDifferentImage; + break; + case DIFFERENT: + ++nbWrongImage; + break; + case METRICS_DISAGREE: + ++nbWrongImage; + ++nbMetricsDisagree; + break; + default: + LOGGER.error("Unrecognized image difference result: " + result); + } } } else { nbErrors++; @@ -208,10 +240,10 @@ private static ImageDifferenceResultSet getImageDiffResult(JsonObject info) { return result; } - if (!info.has("metrics")) { + if (!info.has(metricsKey)) { return result; } - final JsonObject metricsJson = info.get("metrics").getAsJsonObject(); + final JsonObject metricsJson = info.get(metricsKey).getAsJsonObject(); // Check if fuzzy diff metric thinks the images are different. @@ -255,6 +287,28 @@ private static ImageDifferenceResultSet getImageDiffResult(JsonObject info) { return result; } + private static ComputeDifferenceResult getComputeDiffResult(JsonObject info) { + + final String comparisonWithReferenceKey = "comparison_with_reference"; + + if (info == null || !info.has(comparisonWithReferenceKey)) { + return ComputeDifferenceResult.DIFFERENT; + } + + final JsonObject comparisonJson = info.get(comparisonWithReferenceKey).getAsJsonObject(); + + if (comparisonJson.has("exact_match") && comparisonJson.get("exact_match").getAsBoolean()) { + return ComputeDifferenceResult.IDENTICAL; + } + + if (comparisonJson.has("fuzzy_match") && comparisonJson.get("fuzzy_match").getAsBoolean()) { + return ComputeDifferenceResult.SIMILAR; + } + + return ComputeDifferenceResult.DIFFERENT; + + } + private enum ReductionStatus { NOREDUCTION, ONGOING, FINISHED, NOTINTERESTING, EXCEPTION, INCOMPLETE } @@ -565,33 +619,33 @@ private void worker(HttpServletRequest request, HttpServletResponse response) ShaderFamilyResult shaderFamilyResult = new ShaderFamilyResult(shaderFamily, workerName, accessFileInfo); - // TODO(360): Show results for compute shaders. For now, just indicate that some results - // exists, and point to documentation. - if (shaderFamilyResult.shaderFamily.isComp) { + if (shaderFamilyResult.shaderFamily.isCompute) { htmlAppendLn( - "", - "

", - "
Compute shader family: ", shaderFamily, "
", - "The Web interface does not support showing results for compute shaders", - " yet, click to open documentation.", + "", + "COMPUTE", + "
", shaderFamily, "
", + "Variant done: ", Integer.toString(shaderFamilyResult.nbVariantDone), + " / ", Integer.toString(shaderFamilyResult.nbVariants), + " | Wrong results: ", Integer.toString(shaderFamilyResult.nbWrongComputeResult), + " | Slightly different results: ", + Integer.toString(shaderFamilyResult.nbSlightlyDifferentComputeResult), + " | Errors: ", Integer.toString(shaderFamilyResult.nbErrors), + "
"); + } else { + htmlAppendLn( + "", + "", + "
", shaderFamily, "
", + "Variant done: ", Integer.toString(shaderFamilyResult.nbVariantDone), + " / ", Integer.toString(shaderFamilyResult.nbVariants), + " | Wrong images: ", Integer.toString(shaderFamilyResult.nbWrongImage), + " | Slightly different images: ", Integer.toString( + shaderFamilyResult.nbSlightlyDifferentImage), + " | Errors: ", Integer.toString(shaderFamilyResult.nbErrors), + " | Metrics disagree: ", Integer.toString(shaderFamilyResult.nbMetricsDisagree), "
"); - continue; } - - htmlAppendLn( - "", - "", - "
", shaderFamily, "
", - "Variant done: ", Integer.toString(shaderFamilyResult.nbVariantDone), - " / ", Integer.toString(shaderFamilyResult.nbVariants), - " | Wrong images: ", Integer.toString(shaderFamilyResult.nbWrongImage), - " | Slightly different images: ", Integer.toString( - shaderFamilyResult.nbSlightlyDifferentImage), - " | Errors: ", Integer.toString(shaderFamilyResult.nbErrors), - " | Metrics disagree: ", Integer.toString(shaderFamilyResult.nbMetricsDisagree), - "
"); } htmlAppendLn("
"); @@ -865,19 +919,24 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response final String worker = path[3]; final String shaderFamily = path[4]; final String variant = path[5]; - final String variantDir = - WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderFamily + "/"; - final String variantFullPathNoExtension = variantDir + variant; + final Path variantDir = + Paths.get(WebUiConstants.WORKER_DIR, worker, shaderFamily); + final Path variantFullPathNoExtension = Paths.get(variantDir.toString(), variant); - File infoFile = new File(variantDir, variant + ".info.json"); + File infoFile = new File(variantDir.toString(), variant + ".info.json"); if (!infoFile.isFile()) { err404(request, response, "Invalid result path: cannot find corresponding info file"); return; } + final boolean isCompute = + new File(Paths.get("shaderfamilies", shaderFamily, variant + ".comp").toString()) + .isFile(); + JsonObject info = accessFileInfo.getResultInfo(infoFile); final String status = info.get("status").getAsString(); - String shaderPath = "shaderfamilies/" + shaderFamily + "/" + variant + ".frag"; + final String shaderPath = "shaderfamilies/" + shaderFamily + "/" + variant + "." + + (isCompute ? "comp" : "frag"); htmlHeader("Single result"); htmlAppendLn("

Single result

", @@ -893,82 +952,139 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response "\n", "
", "Delete this result
\n", "", "
"); - String referencePngPath = - variantFullPathNoExtension.replace(variant, "reference.png"); + if (isCompute) { - htmlAppendLn("

Reference image:

", - ""); + ComputeDifferenceResult computeDiffResult = getComputeDiffResult(info); - String pngPath = variantDir + variant + ".png"; - File pngFile = new File(pngPath); + if (computeDiffResult == ComputeDifferenceResult.IDENTICAL + && status.equals("SUCCESS") + && !variant.equals("reference")) { + htmlAppendLn("

Compute results are identical.

"); + } + + if (computeDiffResult != ComputeDifferenceResult.IDENTICAL + && status.equals("SUCCESS") + && !variant.equals("reference")) { + + String exactDiffOutput = null; + String fuzzyDiffOutput = null; + + final JsonElement comparisonWithReference = + info.get("comparison_with_reference"); + if (comparisonWithReference != null && comparisonWithReference.isJsonObject()) { + { + JsonElement exactDiffElement = comparisonWithReference.getAsJsonObject().get( + "exactdiff_output"); + if (exactDiffElement != null && exactDiffElement.isJsonPrimitive() + && exactDiffElement.getAsJsonPrimitive().isString()) { + exactDiffOutput = exactDiffElement.getAsJsonPrimitive().getAsString(); + } + } + { + JsonElement fuzzyDiffElement = comparisonWithReference.getAsJsonObject().get( + "fuzzydiff_output"); + if (fuzzyDiffElement != null && fuzzyDiffElement.isJsonPrimitive() + && fuzzyDiffElement.getAsJsonPrimitive().isString()) { + fuzzyDiffOutput = fuzzyDiffElement.getAsJsonPrimitive().getAsString(); + } + } + } + + htmlAppendLn( + "Compute output comparison results:", + "
    ", + "
  • ", + "Exact comparison: ", + exactDiffOutput != null ? exactDiffOutput : "No results", + "
    • ", + "
  • ", + "Fuzzy comparison: ", + fuzzyDiffOutput != null ? fuzzyDiffOutput : "No results", + "
    • ", + "
" + ); - if (!variant.equals("reference")) { - if (pngFile.exists()) { - htmlAppendLn("

Result image:

", - ""); } - } - String gifPath = variantDir + variant + ".gif"; - File gifFile = new File(gifPath); - if (gifFile.exists()) { - htmlAppendLn("

Results non-deterministic animation:

", - "", - "

Here are the second-to-last and last renderings:

\n", - " ", - " "); - } - - if (!(pngFile.exists()) && !(gifFile.exists())) { - htmlAppendLn("

No image to display for this result status

"); - } - - htmlAppendLn("
"); - - ImageDifferenceResultSet metricResults = getImageDiffResult(info); - - if (metricResults.summary == ImageDifferenceResult.IDENTICAL - && status.equals("SUCCESS") - && !variant.equals("reference")) { - htmlAppendLn("

Images are identical.

"); - } - - if (metricResults.summary != ImageDifferenceResult.IDENTICAL - && status.equals("SUCCESS") - && !variant.equals("reference")) { - - htmlAppendLn( - "Image comparison metrics:", - "
    ", - "
  • ", - "Summary: ", - metricResults.summary.toString(), - "
    • ", - "
  • ", - "Fuzzy comparison: ", - metricResults.fuzzy != null ? metricResults.fuzzy.toString() : "No results", - "
    • ", - "
  • ", - "Histogram comparison: ", - metricResults.histogram != null - ? ( - metricResults.histogram.toString() - + " (distance: " + metricResults.histogramDistance + ")" - ) - : "No results", - "
    • ", - "
" - ); + } else { + final String referencePngPath = + Paths.get(variantDir.toString(), "reference.png").toString(); + + htmlAppendLn("

Reference image:

", + ""); + + String pngPath = variantDir + variant + ".png"; + File pngFile = new File(pngPath); + + if (!variant.equals("reference")) { + if (pngFile.exists()) { + htmlAppendLn("

Result image:

", + ""); + } + } + + String gifPath = variantDir + variant + ".gif"; + File gifFile = new File(gifPath); + if (gifFile.exists()) { + htmlAppendLn("

Results non-deterministic animation:

", + "", + "

Here are the second-to-last and last renderings:

\n", + " ", + " "); + } + + if (!(pngFile.exists()) && !(gifFile.exists())) { + htmlAppendLn("

No image to display for this result status

"); + } + + htmlAppendLn("
"); + + ImageDifferenceResultSet metricResults = getImageDiffResult(info); + + if (metricResults.summary == ImageDifferenceResult.IDENTICAL + && status.equals("SUCCESS") + && !variant.equals("reference")) { + htmlAppendLn("

Images are identical.

"); + } + + if (metricResults.summary != ImageDifferenceResult.IDENTICAL + && status.equals("SUCCESS") + && !variant.equals("reference")) { + + htmlAppendLn( + "Image comparison metrics:", + "
    ", + "
  • ", + "Summary: ", + metricResults.summary.toString(), + "
    • ", + "
  • ", + "Fuzzy comparison: ", + metricResults.fuzzy != null ? metricResults.fuzzy.toString() : "No results", + "
    • ", + "
  • ", + "Histogram comparison: ", + metricResults.histogram != null + ? ( + metricResults.histogram.toString() + + " (distance: " + metricResults.histogramDistance + ")" + ) + : "No results", + "
    • ", + "
" + ); + } + } @@ -982,12 +1098,13 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response "
\n", "

Run log

\n", "\n", "
"); // Get result file - File result = new File(variantFullPathNoExtension); + File result = variantFullPathNoExtension.toFile(); // Information/links for result File referenceRes = new File(result.getParentFile(), "reference.info.json"); @@ -1011,24 +1128,28 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response ); } - - String reductionHtml = ""; final ReductionStatus reductionStatus = getReductionStatus(worker, shaderFamily, variant); htmlAppendLn("

Reduction status: ", reductionStatus.toString(), "

"); if (reductionStatus == ReductionStatus.NOREDUCTION) { - htmlAppendLn("", - ""); + + if (isCompute) { + htmlAppendLn("

Reductions for compute shaders are not currently supported via the" + + " web UI.

"); + } else { + htmlAppendLn("", + ""); + } } else { htmlAppendLn("

\n", "\n", @@ -1363,7 +1484,7 @@ private void compareWorkers(HttpServletRequest request, HttpServletResponse resp // TODO(360): Handle compute shaders ShaderFamily shaderFamily = new ShaderFamily(shaderFamilyFile.getName()); - if (shaderFamily.isComp) { + if (shaderFamily.isCompute) { continue; } @@ -1921,7 +2042,7 @@ private String reductionLabelColor(ReductionStatus reductionStatus) { } private void htmlVariantResultTableCell(File variantInfoFile, String referencePngPath, - ReductionStatus reductionStatus) throws FileNotFoundException { + ReductionStatus reductionStatus, boolean isCompute) throws FileNotFoundException { JsonObject info = accessFileInfo.getResultInfo(variantInfoFile); String status = info.get("status").getAsString(); @@ -1929,44 +2050,87 @@ private void htmlVariantResultTableCell(File variantInfoFile, String referencePn if (status.contentEquals("SUCCESS")) { - ImageDifferenceResult result = getImageDiffResult(info).summary; + if (isCompute) { + + final ComputeDifferenceResult result = getComputeDiffResult(info); - if (result == ImageDifferenceResult.IDENTICAL) { - htmlAppendLn("

", - ""); + if (result == ComputeDifferenceResult.IDENTICAL) { + htmlAppendLn("", + "MATCH", + ""); + } else { + String warningClass = WARNING_CLASS_WRONG_RESULT; + switch (result) { + case SIMILAR: + warningClass = WARNING_CLASS_WARN_RESULT; + break; + case DIFFERENT: + warningClass = WARNING_CLASS_WRONG_RESULT; + break; + default: + LOGGER.error("Unrecognized compute difference result: " + result); + } + htmlAppendLn("", + "", + "DIFFERENCE", + "\n", + "
", + reductionStatus.toString(), + "
"); + } } else { - String warningClass = "wrongimg"; - switch (result) { - case SIMILAR: - warningClass = "warnimg"; - break; - case DIFFERENT: - warningClass = "wrongimg"; - break; - case METRICS_DISAGREE: - warningClass = "metricsdisimg"; - break; - default: - LOGGER.error("Unrecognized image difference result: " + result); + + final ImageDifferenceResult result = getImageDiffResult(info).summary; + + if (result == ImageDifferenceResult.IDENTICAL) { + htmlAppendLn("		", + ""); + } else { + String warningClass = WARNING_CLASS_WRONG_RESULT; + switch (result) { + case SIMILAR: + warningClass = WARNING_CLASS_WARN_RESULT; + break; + case DIFFERENT: + warningClass = WARNING_CLASS_WRONG_RESULT; + break; + case METRICS_DISAGREE: + warningClass = WARNING_CLASS_METRICS_DISAGREE; + break; + default: + LOGGER.error("Unrecognized image difference result: " + result); + } + htmlAppendLn("", + "", + "\n", + "
", + reductionStatus.toString(), + "
"); } - htmlAppendLn("		", - "", - "\n", - "
", - reductionStatus.toString(), - "
"); + } } else if (status.contentEquals("NONDET")) { + // This code applies to image results only. That is OK at present, as we do not check + // nondeterminism for compute results. Should that change, this could will have to be + // re-worked. + htmlAppendLn("		", @@ -2143,27 +2307,29 @@ private void htmlReductionForm( private void htmlComparativeTable(String shaderFamilyFilename, String[] workers) throws FileNotFoundException { - if (new ShaderFamily(shaderFamilyFilename).isComp) { - // TODO(360): Handle compute shaders - htmlAppendLn("

Compute shader family: ", - shaderFamilyFilename, ". ", - "The UI does not display results for compute shaders yet.", - " ", - "See the compute shader documentation for more information.", - "

"); - return; - } + final ShaderFamily shaderFamily = new ShaderFamily(shaderFamilyFilename); + + // A filter for all variant shader jobs + final FilenameFilter variantShaderJobFilter = + (dir, name) -> name.startsWith("variant_") && name.endsWith(".json"); htmlAppendLn("\n", ""); - File variantsDir = new File(WebUiConstants.SHADER_FAMILIES_DIR + "/" + shaderFamilyFilename); - File[] variantFragFiles = variantsDir.listFiles(variantFragFilter); - Arrays.sort(variantFragFiles, (f1, f2) -> + final File variantsDir = new File(WebUiConstants.SHADER_FAMILIES_DIR, shaderFamilyFilename); + File[] variantShaderJobFiles = variantsDir.listFiles(variantShaderJobFilter); + if (variantShaderJobFiles == null) { + // If no variant shader job files are found, the array reference will be null; set it to an + // empty array so that we can render a table with references only. + variantShaderJobFiles = new File[0]; + } + + Arrays.sort(variantShaderJobFiles, (f1, f2) -> new AlphanumComparator().compare(f1.getName(), f2.getName())); boolean showWorkerNames = workers.length > 1; + final String extension = shaderFamily.isCompute ? "comp" : "frag"; + // First row: variant names if (showWorkerNames) { htmlAppendLn(""); @@ -2173,14 +2339,14 @@ private void htmlComparativeTable(String shaderFamilyFilename, String[] workers) WebUiConstants.SHADER_FAMILIES_DIR, "/", shaderFamilyFilename, - "/reference.frag", + "/reference." + extension, "'>", "reference", ""); - for (File f: variantFragFiles) { + for (File f: variantShaderJobFiles) { htmlAppendLn(""); + FilenameUtils.removeExtension(f.getName()), ""); } htmlAppendLn("\n", ""); @@ -2192,12 +2358,10 @@ private void htmlComparativeTable(String shaderFamilyFilename, String[] workers) htmlAppendLn(""); } - // Reference result is separate as it doesn't contain a "identical" field, etc - // FIXME: make sure reference result has same format as variants to be able to refactor - String refHref = WebUiConstants.WORKER_DIR + "/" + worker + "/" + final String refHref = WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderFamilyFilename + "/reference"; - File refInfoFile = new File(refHref + ".info.json"); - String refPngPath = refHref + ".png"; + final File refInfoFile = new File(refHref + ".info.json"); + final String refPngPath = refHref + ".png"; htmlAppendLn(""); } @@ -2246,15 +2418,15 @@ private void htmlResultColorLegendTable() { "", "", "", - "", + "", "", - "", + "", "", "", "", "", "", - "", + "", "", "", diff --git a/server/src/main/resources/public/graphicsfuzz.css b/server/src/main/resources/public/graphicsfuzz.css index a2a129496..8cba5d483 100644 --- a/server/src/main/resources/public/graphicsfuzz.css +++ b/server/src/main/resources/public/graphicsfuzz.css @@ -34,11 +34,11 @@ td.gfz-error { background-color: #ffaaaa; } -td.wrongimg { +td.wrongresult { background-color: #ffff44; } -td.warnimg { +td.warnresult { background-color: #ccccbb; } From f7b4124d94b96b8ba0a301dcbbff4c11bce763db Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Thu, 20 Jun 2019 16:23:01 +0200 Subject: [PATCH 043/180] Add support for built-in Matrix functions (#555) --- .../common/glslversion/Glsl150.java | 11 +- .../common/glslversion/Glsl330.java | 16 --- .../glslversion/ShadingLanguageVersion.java | 24 ++++ .../common/typing/TyperHelper.java | 110 ++++++++++-------- 4 files changed, 96 insertions(+), 65 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl150.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl150.java index ef28609c5..c89b6939e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl150.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl150.java @@ -27,7 +27,16 @@ private Glsl150(ShadingLanguageVersion prototype) { @Override public String getVersionString() { - return "140"; + return "150"; } + @Override + public boolean supportedDeterminant() { + return true; + } + + @Override + public boolean supportedInverse() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl330.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl330.java index f15e7ca33..4f188e67b 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl330.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl330.java @@ -30,14 +30,6 @@ public String getVersionString() { return "330"; } - @Override - public boolean supportedDeterminant() { - // According to this page: - // https://www.khronos.org/registry/OpenGL-Refpages/gl4/html/determinant.xhtml - // determinant should be supported from GLSL 1.40, but glslangValidator disagrees. - return true; - } - @Override public boolean supportedFloatBitsToInt() { return true; @@ -53,14 +45,6 @@ public boolean supportedIntBitsToFloat() { return true; } - @Override - public boolean supportedInverse() { - // According to this page: - // https://www.khronos.org/registry/OpenGL-Refpages/gl4/html/inverse.xhtml - // inverse should be supported from GLSL 1.50, but glslangValidator disagrees. - return true; - } - @Override public boolean supportedUintBitsToFloat() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java index 3b2767010..ce402864d 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java @@ -172,6 +172,12 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { */ boolean supportedDerivativeFunctions(); + /** + * Determinant Function calculates the determinant of a given square matrix. + * GLSL versions 1.5+ and ESSL versions 3.0+ support this function. + * + * @return true if Determinant Function is supported - false otherwise. + */ boolean supportedDeterminant(); boolean supportedDoStmt(); @@ -215,6 +221,12 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { */ boolean supportedInterpolationFunctions(); + /** + * Inverse Function returns the matrix that is the inverse of the given square matrix. + * GLSL versions 1.5+ and ESSL versions 3.0+ support this function. + * + * @return true if Inverse Function is supported - false otherwise. + */ boolean supportedInverse(); boolean supportedIsinf(); @@ -237,6 +249,12 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedNonSquareMatrices(); + /** + * OuterProduct Function does a linear algebraic matrix multiplication of two given vectors. + * GLSL versions 1.2+ and ESSL versions 3.0+ support this function. + * + * @return true if OuterProduct Function is supported - false otherwise. + */ boolean supportedOuterProduct(); boolean supportedPackHalf2x16(); @@ -257,6 +275,12 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedSwitchStmt(); + /** + * Transpose Function returns the transposed matrix of the given matrix. + * GLSL versions 1.2+ and ESSL versions 3.0+ support this function. + * + * @return true if Transpose Function is supported - false otherwise. + */ boolean supportedTranspose(); boolean supportedTrunc(); diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index 9aba3a6da..555727a93 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -709,55 +709,8 @@ private static Map> getBuiltinsForGlslVersion( } // 8.6: Matrix Functions - { - final String name = "matrixCompMult"; - for (Type t : BasicType.allMatrixTypes()) { - if (BasicType.allSquareMatrixTypes().contains(t) - || shadingLanguageVersion.supportedMatrixCompMultNonSquare()) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - } - if (shadingLanguageVersion.supportedOuterProduct()) { - final String name = "outerProduct"; - addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.VEC2, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.VEC3, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.VEC4, BasicType.VEC4); - addBuiltin(builtinsForVersion, name, BasicType.MAT2X3, BasicType.VEC3, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X2, BasicType.VEC2, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.MAT2X4, BasicType.VEC4, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X2, BasicType.VEC2, BasicType.VEC4); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X4, BasicType.VEC4, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X3, BasicType.VEC3, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedTranspose()) { - final String name = "transpose"; - addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.MAT2X2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X2, BasicType.MAT2X3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X2, BasicType.MAT2X4); - addBuiltin(builtinsForVersion, name, BasicType.MAT2X3, BasicType.MAT3X2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.MAT3X3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X3, BasicType.MAT3X4); - addBuiltin(builtinsForVersion, name, BasicType.MAT2X4, BasicType.MAT4X2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X4, BasicType.MAT4X3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.MAT4X4); - } - - if (shadingLanguageVersion.supportedDeterminant()) { - final String name = "determinant"; - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT2X2); - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT3X3); - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT4X4); - } - - if (shadingLanguageVersion.supportedInverse()) { - final String name = "inverse"; - addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.MAT2X2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.MAT3X3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.MAT4X4); - } + getBuiltinsForGlslVersionMatrix(builtinsForVersion, shadingLanguageVersion); // 8.7: Vector Relational Functions @@ -1259,6 +1212,67 @@ private static void getBuiltinsForGlslVersionFragmentProcessing( // TODO(550): Support functions that take non-uniform shader input variables as parameters. } + /** + * Helper function to register built-in function prototypes for Matrix Functions, + * as specified in section 8.6 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionMatrix( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + { + final String name = "matrixCompMult"; + for (Type t : BasicType.allMatrixTypes()) { + if (BasicType.allSquareMatrixTypes().contains(t) + || shadingLanguageVersion.supportedMatrixCompMultNonSquare()) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + } + + if (shadingLanguageVersion.supportedOuterProduct()) { + final String name = "outerProduct"; + addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.VEC2, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.VEC3, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.VEC4, BasicType.VEC4); + addBuiltin(builtinsForVersion, name, BasicType.MAT2X3, BasicType.VEC3, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X2, BasicType.VEC2, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.MAT2X4, BasicType.VEC4, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X2, BasicType.VEC2, BasicType.VEC4); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X4, BasicType.VEC4, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X3, BasicType.VEC3, BasicType.VEC4); + } + + if (shadingLanguageVersion.supportedTranspose()) { + final String name = "transpose"; + addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.MAT2X2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X2, BasicType.MAT2X3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X2, BasicType.MAT2X4); + addBuiltin(builtinsForVersion, name, BasicType.MAT2X3, BasicType.MAT3X2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.MAT3X3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X3, BasicType.MAT3X4); + addBuiltin(builtinsForVersion, name, BasicType.MAT2X4, BasicType.MAT4X2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X4, BasicType.MAT4X3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.MAT4X4); + } + + if (shadingLanguageVersion.supportedDeterminant()) { + final String name = "determinant"; + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT2X2); + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT3X3); + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT4X4); + } + + if (shadingLanguageVersion.supportedInverse()) { + final String name = "inverse"; + addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.MAT2X2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.MAT3X3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.MAT4X4); + } + } + private static void addBuiltin(Map> builtinsForVersion, String name, Type resultType, Type... args) { if (!builtinsForVersion.containsKey(name)) { From 50a576e52d2adb9d5cf144f55be510ecae83d858 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 20 Jun 2019 16:41:46 +0100 Subject: [PATCH 044/180] Do not generate GraphicsFuzz header when preparing a reference shader. (#575) The header is not needed, so it is pointless and misleading to generate it. --- .../java/com/graphicsfuzz/generator/tool/PrepareReference.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java index edf877f9b..9bb7e56cb 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java @@ -119,7 +119,7 @@ public static void prepareReference( maxUniforms, generateUniformBindings); - fileOps.writeShaderJobFile(shaderJob, outputShaderJobFile); + fileOps.writeShaderJobFile(shaderJob, outputShaderJobFile, false); } public static void prepareReference(ShaderJob shaderJob, From 2a7f8041f04d2e604d2f8494e730bde34416eabc Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Thu, 20 Jun 2019 10:55:21 -0500 Subject: [PATCH 045/180] Add new ways to generate opaque zero/one from bitwise operations (#562) --- .../fuzzer/OpaqueExpressionGenerator.java | 204 +++++++++++++++++- 1 file changed, 203 insertions(+), 1 deletion(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 016a55279..db1b75704 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -99,7 +99,9 @@ private List waysToMakeZeroOrOne() { this::opaqueZeroOrOneFromIdentityFunction, this::opaqueZeroOrOneFromInjectionSwitch, this::opaqueZeroOrOneSquareRoot, - this::opaqueZeroOrOneAbsolute + this::opaqueZeroOrOneAbsolute, + this::opaqueZeroOrOneBitwiseShift, + this::opaqueZeroOrOneBitwiseOp ); } @@ -164,6 +166,206 @@ private Optional opaqueZeroOrOneAbsolute(BasicType type, boolean constCont depth, fuzzer))); } + /** + * Function to generate an opaque zero or one by bitwise shifting left or right by an amount of + * bits dependent on which opaque is being generated. + * The minimum precision for a lowp integer in GLSL is 9 bits, with one of those bits reserved as + * a sign bit if the integer is signed. The OpenGL specification does not define behavior for + * shifting an integer beyond its maximum size in bits. + * This influences the process of generating opaque values such that: + * If we're generating an opaque zero, we don't have to worry about losing bits, so our + * maximum shift value is 8 bits. + * If we're generating an opaque one, we have to make sure that we don't lose our 1 bit. + * This limits our maximum shift value to 7 bits (more on how we shift an opaque one below). + * Possibilities for generating an opaque zero include: + * Shifting an opaque zero by m bits: (opaque zero) >> n or (opaque zero) << n, where n is + * an expression of a clamped value between 0 and m, inclusive: + * n = clamp(fuzzedexpr, (opaque zero), identity(typeconstructor(m)), and m is an integer + * between 0 and 8, inclusive. + * Possibilities for generating an opaque zero include: + * Shifting an opaque one to the left by n bits, then shifting it to the right by n bits: + * ((opaque one) << n) >> n, where n is an expression of a clamped value between 0 and m, + * inclusive: n = clamp(fuzzedexpr, (opaque zero), identity(typeconstructor(m)), and m is an + * integer between 0 and 7, inclusive. + * + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque value + * made from bitwise shifting. + */ + private Optional opaqueZeroOrOneBitwiseShift(BasicType type, boolean constContext, + final int depth, Fuzzer fuzzer, + boolean isZero) { + if (!BasicType.allIntegerTypes().contains(type)) { + return Optional.empty(); + } + if (!shadingLanguageVersion.supportedBitwiseOperations()) { + return Optional.empty(); + } + // The minimum precision for a lowp integer in GLSL is 9 bits (with one reserved for a sign + // bit if the integer is signed) - we don't have to worry about losing information if we're + // shifting zero, but shifting more than 8 bits may result in undefined behavior. + final int minBitsForLowpInt = 9; + // While we still have a hard maximum bits to shift of 8 bits, if we're shifting one, then + // we can potentially lose information because GLSL bit shifting is not circular. To remedy + // that, we make sure not to shift our 1 bit out of the integer by limiting our maximum shift + // to 7 bits. + final int minBitsForLowpUnsignedInt = minBitsForLowpInt - 1; + final int maxValue = generator.nextInt(isZero ? minBitsForLowpInt : minBitsForLowpUnsignedInt); + // We pass true as constContext when fuzzing here because the expression will be evaluated, + // so we don't want any side effects. + final Expr shiftValue = makeClampedFuzzedExpr(type, constContext, depth, fuzzer, maxValue); + if (isZero) { + final BinOp operator = generator.nextBoolean() ? BinOp.SHL : BinOp.SHR; + return Optional.of( + new ParenExpr( + new BinaryExpr( + makeOpaqueZero(type, constContext, depth, fuzzer), + shiftValue, + operator))); + } else { + // We're going to shift twice in opposite directions by the same value. + final Expr shiftBackValue = generator.nextBoolean() ? shiftValue.clone() + : makeClampedFuzzedExpr(type, constContext, depth, fuzzer, maxValue); + return Optional.of( + new ParenExpr( + new BinaryExpr( + new ParenExpr( + new BinaryExpr( + makeOpaqueOne(type, constContext, depth, fuzzer), + shiftValue, + BinOp.SHL)), + shiftBackValue, + BinOp.SHR))); + } + } + + /** + * Utility function to clamp a fuzzed expression between an opaque zero and the identity of a + * type constructor of the given type, with the value of the bound argument. Note that this + * function only supports integer types currently - it could be extended to support floating + * point numbers as well if needed. Another note is that this function does not check its bound + * for validity - specifying a bound larger than 256 or a negative value could cause invalid GLSL + * to be generated depending on the precision and type of the integer. + * + * @param type - the type to make a clamped expression from. + * @param bound - the upper bound for the clamped expression. + * @return an expression of a clamped value between 0 and bound, inclusive: + * clamp(fuzzedexpr, (opaque zero), identity(typeconstructor(bound)), + */ + private Expr makeClampedFuzzedExpr(BasicType type, boolean constContext, + final int depth, Fuzzer fuzzer, int bound) { + assert BasicType.allIntegerTypes().contains(type); + return new FunctionCallExpr( + "clamp", + fuzzer.fuzzExpr(type, false, true, depth), + makeOpaqueZero(type, constContext, depth, fuzzer), + applyIdentityFunction( + new TypeConstructorExpr( + type.toString(), + type.getElementType() == BasicType.INT + ? new IntConstantExpr(String.valueOf(bound)) + : new UIntConstantExpr(bound + "u")), + type, constContext, depth, fuzzer)); + } + + /** + * Function to generate an opaque value by performing a bitwise operation on an opaque zero or + * an opaque one. + * Possibilities for generating an opaque zero include: + * Bitwise ANDing a fuzzed expression with an opaque zero: (fuzzedexpr) & (opaque zero) + * Bitwise ORing an opaque zero with an opaque zero: (opaque zero) | (opaque zero) + * Bitwise XORing an opaque zero with an opaque zero or an opaque one with an opaque one: + * (opaque zero) ^ (opaque zero) or (opaque one) ^ (opaque one) + * Possibilities for generating an opaque one include: + * Bitwise ANDing an opaque one with an opaque one: (opaque one) & (opaque one) + * Bitwise ORing an opaque one with an opaque zero or one: (opaque one) | (opaque zero or one) + * Bitwise XORing an opaque zero with an opaque one or an opaque one with an opaque zero: + * (opaque zero) ^ (opaque one) or (opaque one) ^ (opaque zero) + * + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque value + * made from performing a bitwise operation on an opaque zero or opaque one. + */ + private Optional opaqueZeroOrOneBitwiseOp(BasicType type, boolean constContext, + final int depth, Fuzzer fuzzer, boolean isZero) { + if (!BasicType.allIntegerTypes().contains(type)) { + return Optional.empty(); + } + if (!shadingLanguageVersion.supportedBitwiseOperations()) { + return Optional.empty(); + } + final int numPossibleOperators = 3; + final int operator = generator.nextInt(numPossibleOperators); + Optional opaqueExpr; + switch (operator) { + case 0: + // Bitwise AND + if (isZero) { + // We pass true as constContext when fuzzing here because the expression will be + // evaluated, so we don't want any side effects. + final Expr fuzzedExpr = fuzzer.fuzzExpr(type, false, true, depth); + final Expr opaqueZero = makeOpaqueZero(type, constContext, depth, fuzzer); + opaqueExpr = Optional.of( + new ParenExpr( + generator.nextBoolean() + ? new BinaryExpr(fuzzedExpr, opaqueZero, BinOp.BAND) + : new BinaryExpr(opaqueZero, fuzzedExpr, BinOp.BAND))); + } else { + opaqueExpr = Optional.of( + new ParenExpr( + new BinaryExpr( + makeOpaqueOne(type, constContext, depth, fuzzer), + makeOpaqueOne(type, constContext, depth, fuzzer), + BinOp.BAND))); + } + break; + case 1: + // Bitwise OR + if (isZero) { + opaqueExpr = Optional.of( + new ParenExpr( + new BinaryExpr( + makeOpaqueZero(type, constContext, depth, fuzzer), + makeOpaqueZero(type, constContext, depth, fuzzer), + BinOp.BOR))); + } else { + final Expr opaqueOne = makeOpaqueOne(type, constContext, depth, fuzzer); + final Expr opaqueZeroOrOne = makeOpaqueZeroOrOne(generator.nextBoolean(), type, + constContext, depth, fuzzer); + opaqueExpr = Optional.of( + new ParenExpr( + generator.nextBoolean() + ? new BinaryExpr(opaqueOne, opaqueZeroOrOne, BinOp.BOR) + : new BinaryExpr(opaqueZeroOrOne, opaqueOne, BinOp.BOR))); + } + break; + default: + // Bitwise XOR + assert operator == numPossibleOperators - 1; + boolean useZeroOrOne = generator.nextBoolean(); + opaqueExpr = Optional.of( + new ParenExpr( + isZero + ? new BinaryExpr( + makeOpaqueZeroOrOne(useZeroOrOne, type, constContext, depth, fuzzer), + makeOpaqueZeroOrOne(useZeroOrOne, type, constContext, depth, fuzzer), + BinOp.BXOR) + : new BinaryExpr( + makeOpaqueZeroOrOne(useZeroOrOne, type, constContext, depth, fuzzer), + makeOpaqueZeroOrOne(!useZeroOrOne, type, constContext, depth, fuzzer), + BinOp.BXOR))); + } + return opaqueExpr; + } + private Optional opaqueZeroSin(BasicType type, boolean constContext, final int depth, Fuzzer fuzzer, boolean isZero) { // represent 0 as sin(opaqueZero) function, e.g. sin(0.0) From 2239d55e80fce5edbd91717b14d1d0032542fc42 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 25 Jun 2019 12:21:18 -0500 Subject: [PATCH 046/180] Add script to convert a GraphicsFuzz shader job to a piglit shader test (#577) --- .../drivers/graphicsfuzz-piglit-converter | 25 ++ .../drivers/graphicsfuzz-piglit-converter.bat | 29 +++ .../drivers/graphicsfuzz-piglit-converter.py | 24 ++ .../drivers/graphicsfuzz_piglit_converter.py | 224 ++++++++++++++++++ 4 files changed, 302 insertions(+) create mode 100755 python/src/main/python/drivers/graphicsfuzz-piglit-converter create mode 100644 python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat create mode 100644 python/src/main/python/drivers/graphicsfuzz-piglit-converter.py create mode 100644 python/src/main/python/drivers/graphicsfuzz_piglit_converter.py diff --git a/python/src/main/python/drivers/graphicsfuzz-piglit-converter b/python/src/main/python/drivers/graphicsfuzz-piglit-converter new file mode 100755 index 000000000..941317d4e --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-piglit-converter @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if test -n "${PYTHON_GF}"; then + "${PYTHON_GF}" ${BASH_SOURCE}.py "$@" +elif type -P python3 >/dev/null; then + python3 ${BASH_SOURCE}.py "$@" +elif type -P py >/dev/null; then + py -3 ${BASH_SOURCE}.py "$@" +else + python ${BASH_SOURCE}.py "$@" +fi \ No newline at end of file diff --git a/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat new file mode 100644 index 000000000..619bd62d5 --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat @@ -0,0 +1,29 @@ +@echo off + +@REM +@REM Copyright 2019 The GraphicsFuzz Project Authors +@REM +@REM Licensed under the Apache License, Version 2.0 (the "License"); +@REM you may not use this file except in compliance with the License. +@REM You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. +@REM + +where /q py +IF ERRORLEVEL 0 ( + py -3 "%~dpn0.py" %* +) ELSE ( + where /q python3 + IF ERRORLEVEL 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) +) \ No newline at end of file diff --git a/python/src/main/python/drivers/graphicsfuzz-piglit-converter.py b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.py new file mode 100644 index 000000000..61b081486 --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import graphicsfuzz_piglit_converter +import sys + +try: + graphicsfuzz_piglit_converter.main_helper(sys.argv[1:]) +except BaseException as error: + sys.stderr.write('Error: ' + str(error) + '\n') + sys.exit(1) diff --git a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py new file mode 100644 index 000000000..a4512c7ce --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import runspv +import argparse +import json +from typing import List + +# Note: We define a 'shader job' as the JSON file of a GraphicsFuzz shader. +# Each shader job has a corresponding shader file that has the same base name +# as the shader job - for example, a shader job file named 'variant_005.json' will +# have a corresponding shader file 'variant_005.frag' or 'variant_005.comp' in +# the same directory. + +# Piglit test section headers +REQUIRE_HEADER = '[require]' +VERTEX_HEADER = '[vertex shader passthrough]' +FRAGMENT_HEADER = '[fragment shader]' +TEST_HEADER = '[test]' + +# Draw command for piglit to draw the shader's output. Required in the test header. +DRAW_COMMAND = 'draw rect -1 -1 2 2' + +# Strings used to specify the GL version to use in a piglit test's +# [require] header. +GLSL_VERSION_STRING = 'GLSL >= ' +GLSLES_VERSION_STRING = 'GLSL ES >= ' + +# GLSL preprocessor version flag. +SHADER_VERSION_FLAG = '#version' +# GLES specifier in version string. +ES_SPECIFIER = ' es' +# GLSL version headers specify a version without a decimal point, when piglit takes a version +# string with a decimal point. The easiest way of getting this is by dividing by 100. +SHADER_VERSION_FACTOR = 100 +# Uniform types to be found in the JSON. +UNIFORM_TYPES = { + 'glUniform1i': 'int', + 'glUniform2i': 'ivec2', + 'glUniform3i': 'ivec3', + 'glUniform4i': 'ivec4', + 'glUniform1f': 'float', + 'glUniform2f': 'vec2', + 'glUniform3f': 'vec3', + 'glUniform4f': 'vec4', +} +UNIFORM_DEC = 'uniform' + + +def make_shader_test_string(shader_job: str) -> str: + """ + Makes a piglit shader_test from a shader job and shader. + :param shader_job: The path to the shader job file. + :return: the shader_test + """ + shader_job_json_parsed = get_json_properties(shader_job) + + with open(get_shader_from_job(shader_job), 'r', + encoding='utf-8', errors='ignore') as shader: + shader_file_string = shader.read() + + shader_lines = shader_file_string.split('\n') + shader_test_string = str('') + # The version header always has to be on the first line of the shader. + shader_version_header = shader_lines[0] + + shader_test_string += make_require_header(shader_version_header) + '\n' + shader_test_string += make_vertex_shader_header() + '\n' + shader_test_string += make_fragment_shader_header(shader_file_string) + '\n' + shader_test_string += make_test_header(shader_job_json_parsed) + + return shader_test_string + + +def make_require_header(shader_version_header: str) -> str: + """ + Creates the piglit [require] header as well as the required GL and GLSL version strings. + Note: GLSL version strings are formatted as '#version ### (es)', where ### is a + specific GLSL version multiplied by 100, and (es) is an optional specifier that + determines whether to use GLES or not. + :param shader_version_header: the version string in the GLSL file. + :return: the shader_test require header string. + """ + require_header = REQUIRE_HEADER + '\n' + # Piglit requires a version number with 1 digit of precision for the GL version, and + # 2 digits of precision for the GLSL version. + require_header += GLSLES_VERSION_STRING if ES_SPECIFIER in shader_version_header \ + else GLSL_VERSION_STRING + try: + shader_version = shader_version_header.split(' ')[1] + except IndexError: + raise IOError('Malformed shader - invalid GLSL version string.') + if not shader_version.isdigit(): + raise IOError('Malformed shader - invalid GLSL version string.') + require_header += format(float(shader_version) / SHADER_VERSION_FACTOR, '.2f') + '\n' + return require_header + + +def make_vertex_shader_header() -> str: + """ + Creates the piglit [vertex shader] header. Currently uses passthrough, but this function can + be modified later if we have a need for an explicit vertex shader. + :return: the shader_test vertex header string. + """ + return VERTEX_HEADER + '\n' + + +def make_fragment_shader_header(fragment_shader: str) -> str: + """ + Creates the piglit [fragment shader] header. + :param fragment_shader: the fragment shader code. + :return: the shader_test fragment header string. + """ + frag_header = FRAGMENT_HEADER + '\n' + frag_header += fragment_shader + return frag_header + + +def make_test_header(shader_job_json_parsed: dict) -> str: + """ + Creates the [test] header. Loads uniforms based on the uniforms found in the JSON file. + :param shader_job_json_parsed: the parsed JSON properties. + :return: the shader_test test header string. + """ + test_header = TEST_HEADER + '\n' + for uniform_name, value in shader_job_json_parsed.items(): + test_header += UNIFORM_DEC + ' {type} {uniform_name} {args}\n'.format( + type=get_uniform_type_from_gl_func(value['func']), + uniform_name=uniform_name, + args=' '.join([str(arg) for arg in value['args']]) + ) + test_header += DRAW_COMMAND + return test_header + + +def get_uniform_type_from_gl_func(func: str) -> str: + """ + Helper function to traverse the dict of JSON funcs to determine a uniform's type. + Throws AssertionError if the uniform type is not known. + :param func: the function to check. + :return: the GLSL type of the uniform. + """ + if func not in UNIFORM_TYPES.keys(): + raise AssertionError('Unknown uniform type: ' + func) + return UNIFORM_TYPES[func] + + +def is_version_header(line: str) -> bool: + """ + Helper function to see if a given string is a GLSL preprocessor version string. + :param line: the line of code to check. + :return: True if the string is a GLSL preprocessor version string, False otherwise. + """ + return SHADER_VERSION_FLAG in line + + +def get_json_properties(shader_job: str) -> List: + """ + Helper function to parse a shader job JSON file into a list of properties. + Throws IOError if the file can't be parsed. + :param shader_job: the path to the shader job file. + :return: a list of JSON properties. + """ + with open(shader_job, 'r', encoding='utf-8', errors='ignore') as job: + json_parsed = json.load(job) + if not json_parsed: + raise IOError('Malformed shader job file.') + return json_parsed + + +def get_shader_from_job(shader_job: str) -> str: + """ + Helper function to get the filename of a shader file from its corresponding shader job. + :param shader_job: the path of the shader job file. + :return: the path of the fragment shader file. + """ + return runspv.remove_end(shader_job, '.json') + '.frag' + + +def get_shader_test_from_job(shader_job: str) -> str: + """ + Helper function to get the filename of the new shader test from the given shader job. + :param shader_job: the path of the shader job file. + :return: the path of the shader_test file. + """ + return runspv.remove_end(shader_job, '.json') + '.shader_test' + + +def main_helper(args: List[str]) -> None: + """ + Main function. Parses arguments, delegates to other functions to write the shader_test string, + and writes the string to file. + :param args: the command line arguments. + """ + description = ( + 'Given a GraphicsFuzz shader job JSON file, produce a Mesa piglit shader_test file. ' + 'The shader test will be the same name as the shader job.') + + argparser = argparse.ArgumentParser(description=description) + + argparser.add_argument( + 'shader_job', + help='Path to the GraphicsFuzz shader job JSON file.') + + args = argparser.parse_args(args) + + test_string = make_shader_test_string(args.shader_job) + + with open(get_shader_test_from_job(args.shader_job), 'w', + encoding='utf-8', errors='ignore') as shader_test: + shader_test.write(test_string) From 2741ae5d1a29795f612c3e5baf64d002ba6c7563 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 26 Jun 2019 14:03:40 -0500 Subject: [PATCH 047/180] Ensure bitwise shift opaque one shifts left/right by same value (#580) --- .../generator/fuzzer/OpaqueExpressionGenerator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index db1b75704..b5addf8e3 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -228,8 +228,6 @@ private Optional opaqueZeroOrOneBitwiseShift(BasicType type, boolean const operator))); } else { // We're going to shift twice in opposite directions by the same value. - final Expr shiftBackValue = generator.nextBoolean() ? shiftValue.clone() - : makeClampedFuzzedExpr(type, constContext, depth, fuzzer, maxValue); return Optional.of( new ParenExpr( new BinaryExpr( @@ -238,7 +236,7 @@ private Optional opaqueZeroOrOneBitwiseShift(BasicType type, boolean const makeOpaqueOne(type, constContext, depth, fuzzer), shiftValue, BinOp.SHL)), - shiftBackValue, + shiftValue.clone(), BinOp.SHR))); } } From 1ffb8d5279000ac8789f79d129e02a575ffa477d Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Thu, 27 Jun 2019 00:44:10 +0200 Subject: [PATCH 048/180] Refactor geometric builtins (#581) --- .../common/typing/TyperHelper.java | 117 ++++++++++-------- 1 file changed, 64 insertions(+), 53 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index 555727a93..5fd02353b 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -654,59 +654,7 @@ private static Map> getBuiltinsForGlslVersion( // 8.5: Geometric Functions - { - final String name = "length"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t); - } - } - - { - final String name = "distance"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); - } - } - - { - final String name = "dot"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); - } - } - - { - final String name = "cross"; - addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.VEC3); - } - - { - final String name = "normalize"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "faceforward"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t, t); - } - } - - { - final String name = "reflect"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - - { - final String name = "refract"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t, BasicType.FLOAT); - } - } + getBuiltinsForGlslVersionGeometric(builtinsForVersion); // 8.6: Matrix Functions @@ -1273,6 +1221,69 @@ private static void getBuiltinsForGlslVersionMatrix( } } + /** + * Helper function to register built-in function prototypes for Geometric Functions, + * as specified in section 8.5 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + */ + private static void getBuiltinsForGlslVersionGeometric( + Map> builtinsForVersion) { + { + final String name = "length"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t); + } + } + + { + final String name = "distance"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); + } + } + + { + final String name = "dot"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); + } + } + + { + final String name = "cross"; + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.VEC3); + } + + { + final String name = "normalize"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "faceforward"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, t, t); + } + } + + { + final String name = "reflect"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + + { + final String name = "refract"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, t, BasicType.FLOAT); + } + } + } + private static void addBuiltin(Map> builtinsForVersion, String name, Type resultType, Type... args) { if (!builtinsForVersion.containsKey(name)) { From 9e58bd6d922a9885474e3c3ee4511097794ca8af Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Thu, 27 Jun 2019 12:07:15 -0500 Subject: [PATCH 049/180] Fix piglit converter script not inserting GL ES version (#584) --- .../python/drivers/graphicsfuzz_piglit_converter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py index a4512c7ce..f0801ed73 100644 --- a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py +++ b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py @@ -36,6 +36,7 @@ # Strings used to specify the GL version to use in a piglit test's # [require] header. +GLES_VERSION_STRING = 'GL ES >= ' GLSL_VERSION_STRING = 'GLSL >= ' GLSLES_VERSION_STRING = 'GLSL ES >= ' @@ -97,14 +98,19 @@ def make_require_header(shader_version_header: str) -> str: require_header = REQUIRE_HEADER + '\n' # Piglit requires a version number with 1 digit of precision for the GL version, and # 2 digits of precision for the GLSL version. - require_header += GLSLES_VERSION_STRING if ES_SPECIFIER in shader_version_header \ - else GLSL_VERSION_STRING try: shader_version = shader_version_header.split(' ')[1] except IndexError: raise IOError('Malformed shader - invalid GLSL version string.') if not shader_version.isdigit(): raise IOError('Malformed shader - invalid GLSL version string.') + # Piglit requires GL version to be specified explicitly if ES is in use. + if ES_SPECIFIER in shader_version_header: + require_header += GLES_VERSION_STRING + \ + format(float(shader_version) / SHADER_VERSION_FACTOR, '.1f') + '\n' + require_header += GLSLES_VERSION_STRING + else: + require_header += GLSL_VERSION_STRING require_header += format(float(shader_version) / SHADER_VERSION_FACTOR, '.2f') + '\n' return require_header From 742d18f25deae7540b152dbd1d7ffee24bf04186 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Fri, 28 Jun 2019 14:34:10 +0200 Subject: [PATCH 050/180] Refactor floating-point pack and unpack built-ins (#585) --- .../common/typing/TyperHelper.java | 112 ++++++++++-------- 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index 5fd02353b..094b290de 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -602,55 +602,7 @@ private static Map> getBuiltinsForGlslVersion( // 8.4: Floating-Point Pack and Unpack Functions - if (shadingLanguageVersion.supportedPackUnorm2x16()) { - final String name = "packUnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); - } - - if (shadingLanguageVersion.supportedPackSnorm2x16()) { - final String name = "packSnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); - } - - if (shadingLanguageVersion.supportedPackUnorm4x8()) { - final String name = "packUnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedPackSnorm4x8()) { - final String name = "packSnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedUnpackUnorm2x16()) { - final String name = "unpackUnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); - } - - if (shadingLanguageVersion.supportedUnpackSnorm2x16()) { - final String name = "unpackSnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); - } - - if (shadingLanguageVersion.supportedUnpackUnorm4x8()) { - final String name = "unpackUnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); - } - - if (shadingLanguageVersion.supportedUnpackSnorm4x8()) { - final String name = "unpackSnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); - } - - if (shadingLanguageVersion.supportedPackHalf2x16()) { - final String name = "packHalf2x16"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); - } - - if (shadingLanguageVersion.supportedUnpackHalf2x16()) { - final String name = "unpackHalf2x16"; - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); - } + getBuiltinsForGlslVersionFloatingPointPackAndUnpack(builtinsForVersion, shadingLanguageVersion); // 8.5: Geometric Functions @@ -1284,6 +1236,68 @@ private static void getBuiltinsForGlslVersionGeometric( } } + /** + * Helper function to register built-in function prototypes for Floating-Point Pack and + * Unpack Functions, as specified in section 8.4 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionFloatingPointPackAndUnpack( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + + if (shadingLanguageVersion.supportedPackUnorm2x16()) { + final String name = "packUnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + } + + if (shadingLanguageVersion.supportedPackSnorm2x16()) { + final String name = "packSnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + } + + if (shadingLanguageVersion.supportedPackUnorm4x8()) { + final String name = "packUnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); + } + + if (shadingLanguageVersion.supportedPackSnorm4x8()) { + final String name = "packSnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); + } + + if (shadingLanguageVersion.supportedUnpackUnorm2x16()) { + final String name = "unpackUnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedUnpackSnorm2x16()) { + final String name = "unpackSnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedUnpackUnorm4x8()) { + final String name = "unpackUnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedUnpackSnorm4x8()) { + final String name = "unpackSnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedPackHalf2x16()) { + final String name = "packHalf2x16"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + } + + if (shadingLanguageVersion.supportedUnpackHalf2x16()) { + final String name = "unpackHalf2x16"; + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + } + } + private static void addBuiltin(Map> builtinsForVersion, String name, Type resultType, Type... args) { if (!builtinsForVersion.containsKey(name)) { From c597c7f9772de889a22426d615d0ef5ff1ed868e Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Mon, 1 Jul 2019 22:30:53 +0100 Subject: [PATCH 051/180] Script to convert a directory of glsl-fuzz shaders into a form suitable for use with spirv-fuzz Added a script that turns a directory of GLSL shaders into SPIR-V, creating spirv-fuzz initial facts for them. Refactored runspv somewhat to enable this. --- .../convert-shaders-to-spirv-fuzz-form.py | 79 +++++++++++++++++++ .../main/python/drivers/graphicsfuzz-tool.py | 33 ++------ .../main/python/drivers/graphicsfuzz_tool.py | 43 ++++++++++ python/src/main/python/drivers/runspv.py | 50 +++++++++--- ...hader_job_uniforms_to_spirv_fuzz_facts.py} | 24 ++++-- 5 files changed, 186 insertions(+), 43 deletions(-) create mode 100644 python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py create mode 100644 python/src/main/python/drivers/graphicsfuzz_tool.py rename python/src/main/python/drivers/{shader-job-uniforms-to-spirv-fuzz-facts.py => shader_job_uniforms_to_spirv_fuzz_facts.py} (87%) diff --git a/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py b/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py new file mode 100644 index 000000000..2efccfbe8 --- /dev/null +++ b/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import glob +import os +import shutil +import sys + +import runspv +import graphicsfuzz_tool +import shader_job_uniforms_to_spirv_fuzz_facts + + +HERE = os.path.abspath(__file__) + + +def main_helper(args): + description = ( + 'Convert a directory of GLSL shader jobs to shader jobs suitable as inputs to spirv-fuzz.') + + parser = argparse.ArgumentParser(description=description) + + # Required arguments + parser.add_argument('input_dir', help="A directory of GLSL shader jobs.") + parser.add_argument('output_dir', help="The name of a directory that will be created and that " + "will contain the resulting spirv-fuzz shader jobs.") + + args = parser.parse_args(args) + + if not os.path.isdir(args.input_dir): + raise ValueError("Input directory " + "'" + args.input_dir + "' not found.") + + # Make the output directory if it does not yet exist. + if not os.path.isdir(args.output_dir): + os.makedirs(args.output_dir) + + for shader_job in glob.glob(os.path.join(args.input_dir, "*.json")): + print(shader_job) + shader_job_prefix = os.path.splitext(os.path.basename(shader_job))[0] + json_in_output_dir = os.path.join(args.output_dir, shader_job_prefix + ".json") + graphicsfuzz_tool.main_helper(["com.graphicsfuzz.generator.tool.PrepareReference", + "--generate-uniform-bindings", "--max-uniforms", "10", + shader_job, json_in_output_dir]) + shader_job_uniforms_to_spirv_fuzz_facts.main_helper([json_in_output_dir, + os.path.join(args.output_dir, + shader_job_prefix + + ".facts")]) + runspv.convert_glsl_to_spv(os.path.join(args.output_dir, shader_job_prefix + ".frag"), + os.path.join(args.output_dir, shader_job_prefix + ".spv")) + + for opt_settings in ["-O", "-Os"]: + shutil.copyfile(os.path.join(args.output_dir, shader_job_prefix + ".json"), + os.path.join(args.output_dir, shader_job_prefix + opt_settings + + ".json")) + shutil.copyfile(os.path.join(args.output_dir, shader_job_prefix + ".facts"), + os.path.join(args.output_dir, shader_job_prefix + opt_settings + + ".facts")) + runspv.run_spirv_opt(os.path.join(args.output_dir, shader_job_prefix + ".spv"), + [opt_settings], + os.path.join(args.output_dir, shader_job_prefix + opt_settings + + ".spv")) + + +if __name__ == '__main__': + main_helper(sys.argv[1:]) diff --git a/python/src/main/python/drivers/graphicsfuzz-tool.py b/python/src/main/python/drivers/graphicsfuzz-tool.py index 2ca507658..0ea2c34ca 100644 --- a/python/src/main/python/drivers/graphicsfuzz-tool.py +++ b/python/src/main/python/drivers/graphicsfuzz-tool.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018 The GraphicsFuzz Project Authors +# Copyright 2019 The GraphicsFuzz Project Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,30 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import subprocess +import graphicsfuzz_tool import sys -HERE = os.path.abspath(__file__) - -sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) - -import cmd_helpers - - -def go(argv): - java_tool_path = cmd_helpers.get_tool_path() - print(java_tool_path) - - # Run the tool - - cmd = ["java", "-ea", "-cp", java_tool_path] + argv - print(cmd) - - generate_proc = subprocess.Popen(cmd) - generate_proc.communicate() - return generate_proc.returncode - - -if __name__ == "__main__": - sys.exit(go(sys.argv[1:])) +try: + sys.exit(graphicsfuzz_tool.main_helper(sys.argv[1:])) +except ValueError as value_error: + sys.stderr.write(str(value_error)) + sys.exit(1) diff --git a/python/src/main/python/drivers/graphicsfuzz_tool.py b/python/src/main/python/drivers/graphicsfuzz_tool.py new file mode 100644 index 000000000..2bb81d6c2 --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz_tool.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +# Copyright 2018 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +HERE = os.path.abspath(__file__) + +sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) + +import cmd_helpers + + +def main_helper(argv): + java_tool_path = cmd_helpers.get_tool_path() + print(java_tool_path) + + # Run the tool + + cmd = ["java", "-ea", "-cp", java_tool_path] + argv + print(cmd) + + generate_proc = subprocess.Popen(cmd) + generate_proc.communicate() + return generate_proc.returncode + + +if __name__ == "__main__": + sys.exit(main_helper(sys.argv[1:])) diff --git a/python/src/main/python/drivers/runspv.py b/python/src/main/python/drivers/runspv.py index 3d92c7268..f6301e368 100755 --- a/python/src/main/python/drivers/runspv.py +++ b/python/src/main/python/drivers/runspv.py @@ -329,19 +329,47 @@ def remove_end(str_in: str, str_end: str): def run_spirv_opt( spv_file: str, - spirv_opt_args: List[str] -) -> str: + spirv_opt_args: List[str], + output: str +) -> None: + """ + Optimizes a SPIR-V file. - log('Running optimizer.') + :param spv_file: name of SPIR-V file to be optimized + :param spirv_opt_args: arguments to be passed to spirv-opt + :param output: name of file into which optimized SPIR-V will be written + :return: None + """ - result = spv_file + '.opt.spv' + log('Running optimizer.') - cmd = [spirvopt_path(), spv_file, '-o', result] + cmd = [spirvopt_path(), spv_file, '-o', output] cmd += spirv_opt_args subprocess_helper(cmd, timeout=TIMEOUT_SPIRV_OPT_SECONDS) - return result + +def convert_glsl_to_spv( + glsl_shader: str, + output: str +) -> None: + + """ + Runs glslangValidator on a GLSL shader to convert it to SPIR-V. + + :param glsl_shader: name of GLSL file on which to run glslangValidator + :param output: name of file into which generated SPIR-V will be written + :return: None + """ + + log('Running glslangValidator.') + + cmd = [ + glslang_path(), + '-V', glsl_shader, + '-o', output + ] + subprocess_helper(cmd, timeout=TIMEOUT_RUN) def filename_extension_suggests_glsl(file: str): @@ -408,7 +436,9 @@ def prepare_shader( assert len(result) > 0 if spirv_opt_args: - result = run_spirv_opt(result, spirv_opt_args) + optimized_spv = result + '.opt.spv' + run_spirv_opt(result, spirv_opt_args, optimized_spv) + result = optimized_spv return result @@ -430,7 +460,7 @@ def prepare_shader( def adb_helper( adb_args: List[str], check: bool, - verbose: bool=False + verbose: bool = False ) -> subprocess.CompletedProcess: adb_cmd = [adb_path()] + adb_args @@ -445,7 +475,7 @@ def adb_helper( def adb_check( adb_args: List[str], - verbose: bool=False + verbose: bool = False ) -> subprocess.CompletedProcess: return adb_helper(adb_args, check=True, verbose=verbose) @@ -453,7 +483,7 @@ def adb_check( def adb_can_fail( adb_args: List[str], - verbose: bool=False + verbose: bool = False ) -> subprocess.CompletedProcess: return adb_helper(adb_args, check=False, verbose=verbose) diff --git a/python/src/main/python/drivers/shader-job-uniforms-to-spirv-fuzz-facts.py b/python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py similarity index 87% rename from python/src/main/python/drivers/shader-job-uniforms-to-spirv-fuzz-facts.py rename to python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py index 4862a3bfc..d2e2eb61a 100644 --- a/python/src/main/python/drivers/shader-job-uniforms-to-spirv-fuzz-facts.py +++ b/python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py @@ -16,16 +16,22 @@ import argparse import json -import os import struct +import sys from typing import Any, Dict, List import runspv -# Turns a GraphicsFuzz .json file into a spirv-fuzz .facts file, with one fact per word of uniform -# data. -def main() -> None: +def main_helper(args) -> None: + """ + Turn a GraphicsFuzz .json file into a spirv-fuzz .facts file, with one fact per word of uniform + data. + + :param args: command-line arguments + :return: None + """ + parser = argparse.ArgumentParser() # Required arguments @@ -33,7 +39,11 @@ def main() -> None: 'shader_job', help='Shader job (.json) file.') - args = parser.parse_args() + parser.add_argument( + 'output_file', + help='Output file for facts.') + + args = parser.parse_args(args) # Generate uniform facts from .json file fact_list = [] # type: List[dict] @@ -79,9 +89,9 @@ def main() -> None: constantWord=[int_representation]) fact_list.append(dict(constantUniformFact=fact_constant_uniform)) - with open(os.path.splitext(args.shader_job)[0] + ".facts", "w") as f: + with runspv.open_helper(args.output_file, 'w') as f: f.write(json.dumps(dict(fact=fact_list), indent=1, sort_keys=True)) if __name__ == '__main__': - main() + main_helper(sys.argv[1:]) From 2da8350d47db1740307613314df944ee89afeee4 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 2 Jul 2019 09:27:09 -0500 Subject: [PATCH 052/180] Refactor common python script functions into one common lib (#587) --- python/src/main/python/drivers/gapidfuzz.py | 14 +- .../src/main/python/drivers/gfuzz_common.py | 135 ++++++++++++++++++ .../main/python/drivers/glsl-to-spv-worker.py | 81 ++++------- .../drivers/graphicsfuzz_piglit_converter.py | 15 +- .../python/drivers/inspect_compute_results.py | 16 +-- 5 files changed, 179 insertions(+), 82 deletions(-) create mode 100644 python/src/main/python/drivers/gfuzz_common.py diff --git a/python/src/main/python/drivers/gapidfuzz.py b/python/src/main/python/drivers/gapidfuzz.py index fed3d2426..7cca0e474 100755 --- a/python/src/main/python/drivers/gapidfuzz.py +++ b/python/src/main/python/drivers/gapidfuzz.py @@ -22,6 +22,8 @@ import typing import io import pprint + +import gfuzz_common glsl_generate = __import__("glsl-generate") # Types: @@ -75,16 +77,6 @@ def nz(s): return s -def remove_end(s: str, end: str): - assert s.endswith(end) - return s[:-len(end)] - - -def remove_start(s: str, start: str): - assert s.startswith(start) - return s[len(start):] - - def call(args: List[str], cwd=None): print(" ".join(args)) res = subprocess.run( @@ -223,7 +215,7 @@ def create_traces(params: Params): handle_to_variant_list = {} # type: typing.Dict[str, typing.List[str]] for family in Path(params.families_dir).iterdir(): # type: Path if family.name.startswith("family_"): - shader_handle = remove_start(family.name, "family_") + shader_handle = gfuzz_common.remove_start(family.name, "family_") if params.specific_handle is not None and params.specific_handle != shader_handle: continue variant_shaders = sorted(list(family.glob("variant_*.frag"))) diff --git a/python/src/main/python/drivers/gfuzz_common.py b/python/src/main/python/drivers/gfuzz_common.py new file mode 100644 index 000000000..a1f8e5a1e --- /dev/null +++ b/python/src/main/python/drivers/gfuzz_common.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +from typing import List + +import runspv + + +def open_helper(file: str, mode: str): + """ + Helper function to open a file with UTF-8 format, ignoring errors. + Wraps around runspv.open_helper(), which wraps open() in turn. + :param file: the file to open - same as the file argument of open(). + :param mode: the mode to open the file in - same as the mode argument of open(). + :return: a File object opened in UTF-8 format and ignoring errors. + """ + return runspv.open_helper(file, mode) + + +def open_bin_helper(file: str, mode: str): + """ + Helper function to open a file in binary mode. Wraps around runspv.open_bin_helper(). + Will cause an assertion failure if 'mode' does not contain the open() binary symbol 'b'. + :param file: the file to open - same as the file argument of open(). + :param mode: the mode to open the file in - same as the mode argument of open(). + :return: a File object opened in binary mode. + """ + return runspv.open_bin_helper(file, mode) + + +def get_platform() -> str: + """ + Helper function to determine the current platform in use. Wraps around runspv.get_platform(). + :return: the platform in use. Throws AssertionError if platform is not Linux/Windows/Mac. + """ + return runspv.get_platform() + + +def get_bin_dir(): + """ + Helper function to get the directory of binaries (e.g. glslangvalidator) for the current + platform. Wraps around runspv.get_bin_dir(). + :return: the path of binaries for the platform in use. + """ + return runspv.get_bin_dir() + + +def tool_on_path(tool: str) -> str: + """ + Helper function to determine if a given tool is on the user's PATH variable. Wraps around + runspv.tool_on_path(). + :param tool: the tool's filename to look for. + :return: the path of the tool, else ToolNotOnPathError if the tool isn't on the PATH. + """ + return runspv.tool_on_path(tool) + + +def remove_end(str_in: str, str_end: str) -> str: + """ + Helper function to remove the end of a string. Useful for removing file extensions if you + know what extension your file should be. Wraps around runspv.remove_end(). + :param str_in: the string to remove the end from. + :param str_end: the suffix that you want to remove from str_in. + :return: str_in with str_end removed. + """ + return runspv.remove_end(str_in, str_end) + + +def filename_extension_suggests_glsl(file: str) -> bool: + """ + Helper function to determine if a file is a vertex, fragment, or compute shader. Wraps around + runspv.filename_extension_suggests_glsl(). + :param file: The filename/path to check. + :return: True if the file is a vertex/fragment/compute shader file, false otherwise. + """ + return runspv.filename_extension_suggests_glsl(file) + + +def write_to_file(content, filename) -> None: + """ + Helper function to write a string to a file opened in UTF-8 format. + :param content: The string to write to the file. + :param filename: The name/path of the file to write to. + """ + with open_helper(filename, 'w') as f: + f.write(content) + + +def remove(file) -> None: + """ + Helper function to delete a file. If the file is a directory, the directory will be recursively + deleted. + :param file: The file to be deleted. + """ + if os.path.isdir(file): + shutil.rmtree(file) + elif os.path.isfile(file): + os.remove(file) + + +def check_input_files_exist(filenames: List[str]) -> None: + """ + Helper function to determine if a list of files all exist. + :param filenames: The list of filenames to check. + :return: Nothing - throws FileNotFoundError if a file is not found. + """ + for filename in filenames: + if not os.path.isfile(filename): + raise FileNotFoundError('Input file "' + filename + '" not found') + + +def remove_start(s: str, start: str): + """ + Helper function to remove the beginning of a string. + :param s: the string to remove the prefix from. + :param start: the prefix of the string to remove. + :return: s with the start of the string removed. + """ + assert s.startswith(start) + return s[len(start):] diff --git a/python/src/main/python/drivers/glsl-to-spv-worker.py b/python/src/main/python/drivers/glsl-to-spv-worker.py index 422729654..871a00886 100755 --- a/python/src/main/python/drivers/glsl-to-spv-worker.py +++ b/python/src/main/python/drivers/glsl-to-spv-worker.py @@ -24,6 +24,7 @@ from typing import Optional, List, Union import runspv +import gfuzz_common HERE = os.path.abspath(__file__) @@ -62,24 +63,6 @@ def print(s): ################################################################################ -def write_to_file(content, filename): - with runspv.open_helper(filename, 'w') as f: - f.write(content) - - -################################################################################ - - -def remove(f): - if os.path.isdir(f): - shutil.rmtree(f) - elif os.path.isfile(f): - os.remove(f) - - -################################################################################ - - def prepare_vert_file(output_dir: str) -> str: vert_file = os.path.join(output_dir, 'test.vert') if not os.path.isfile(vert_file): @@ -89,18 +72,10 @@ def prepare_vert_file(output_dir: str) -> str: gl_Position = a_position; } ''' - write_to_file(vert_file_default_content, vert_file) + gfuzz_common.write_to_file(vert_file_default_content, vert_file) return vert_file -################################################################################ - - -def remove_end(str_in: str, str_end: str): - assert str_in.endswith(str_end), 'Expected {} to end with {}'.format(str_in, str_end) - return str_in[:-len(str_end)] - - ################################################################################ OPT_OPTIONS = ['--ccp', @@ -168,12 +143,12 @@ def do_image_job( output_dir = os.path.join(work_dir, image_job.name) # Delete and create output directory. - remove(output_dir) + gfuzz_common.remove(output_dir) os.makedirs(output_dir, exist_ok=True) name = image_job.name if name.endswith('.frag'): - name = remove_end(name, '.frag') + name = gfuzz_common.remove_end(name, '.frag') # TODO(324): the worker currently assumes that no vertex shader is present in the image job. vert_file = prepare_vert_file(output_dir) if args.legacy_worker else None @@ -193,12 +168,12 @@ def do_image_job( res.passSanityCheck = True res.log = 'Start: ' + name + '\n' - write_to_file(image_job.fragmentSource, frag_file) - write_to_file(image_job.uniformsInfo, json_file) + gfuzz_common.write_to_file(image_job.fragmentSource, frag_file) + gfuzz_common.write_to_file(image_job.uniformsInfo, json_file) # Set runspv logger. Use try-finally to clean up. - with runspv.open_helper(log_file, 'w') as f: + with gfuzz_common.open_helper(log_file, 'w') as f: try: runspv.log_to_file = f @@ -240,21 +215,21 @@ def do_image_job( except Exception as ex: runspv.log('Exception: ' + str(ex)) runspv.log('Removing STATUS file.') - remove(status_file) + gfuzz_common.remove(status_file) runspv.log('Continuing.') finally: runspv.log_to_file = None if os.path.isfile(log_file): - with runspv.open_helper(log_file, 'r') as f: + with gfuzz_common.open_helper(log_file, 'r') as f: res.log += f.read() if os.path.isfile(png_file): - with runspv.open_bin_helper(png_file, 'rb') as f: + with gfuzz_common.open_bin_helper(png_file, 'rb') as f: res.PNG = f.read() if os.path.isfile(status_file): - with runspv.open_helper(status_file, 'r') as f: + with gfuzz_common.open_helper(status_file, 'r') as f: status = f.read().rstrip() if status == 'SUCCESS': res.status = tt.JobStatus.SUCCESS @@ -268,9 +243,9 @@ def do_image_job( res.status = tt.JobStatus.UNEXPECTED_ERROR elif status == 'NONDET': res.status = tt.JobStatus.NONDET - with runspv.open_bin_helper(nondet_0, 'rb') as f: + with gfuzz_common.open_bin_helper(nondet_0, 'rb') as f: res.PNG = f.read() - with runspv.open_bin_helper(nondet_1, 'rb') as f: + with gfuzz_common.open_bin_helper(nondet_1, 'rb') as f: res.PNG2 = f.read() else: res.log += '\nUnknown status value: ' + status + '\n' @@ -297,7 +272,7 @@ def do_compute_job( output_dir = os.path.join(work_dir, comp_job.name) # Delete and create output directory. - remove(output_dir) + gfuzz_common.remove(output_dir) os.makedirs(output_dir, exist_ok=True) tmpcomp = os.path.join(output_dir, 'tmp.comp') @@ -308,9 +283,9 @@ def do_compute_job( # Output files from running the app. status_file = os.path.join(output_dir, 'STATUS') - write_to_file(comp_job.computeSource, tmpcomp) + gfuzz_common.write_to_file(comp_job.computeSource, tmpcomp) - write_to_file(comp_job.computeInfo, tmpjson) + gfuzz_common.write_to_file(comp_job.computeInfo, tmpjson) res = tt.ImageJobResult() res.log = '#### Start compute shader\n\n' @@ -319,7 +294,7 @@ def do_compute_job( # Set runspv logger. Use try-finally to clean up. - with runspv.open_helper(log_file, 'w') as f: + with gfuzz_common.open_helper(log_file, 'w') as f: try: runspv.log_to_file = f @@ -336,22 +311,22 @@ def do_compute_job( except Exception as ex: runspv.log('Exception: ' + str(ex)) runspv.log('Removing STATUS file.') - remove(status_file) + gfuzz_common.remove(status_file) runspv.log('Continuing.') finally: runspv.log_to_file = None if os.path.isfile(log_file): - with runspv.open_helper(log_file, 'r') as f: + with gfuzz_common.open_helper(log_file, 'r') as f: res.log += f.read() if os.path.isfile(status_file): - with runspv.open_helper(status_file, 'r') as f: + with gfuzz_common.open_helper(status_file, 'r') as f: status = f.read().rstrip() if status == 'SUCCESS': res.status = tt.JobStatus.SUCCESS assert (os.path.isfile(ssbo_json_file)) - with runspv.open_helper(ssbo_json_file, 'r') as f: + with gfuzz_common.open_helper(ssbo_json_file, 'r') as f: res.computeOutputs = f.read() elif status == 'CRASH': @@ -515,7 +490,7 @@ def main(): # Get worker info worker_info_file = 'worker_info.json' - remove(worker_info_file) + gfuzz_common.remove(worker_info_file) worker_info_json_string = '{}' @@ -531,7 +506,7 @@ def main(): 'the app permission to write to external storage is enabled.' ) - with runspv.open_helper(worker_info_file, 'r') as f: + with gfuzz_common.open_helper(worker_info_file, 'r') as f: worker_info_json_string = f.read() except Exception as ex: @@ -557,23 +532,23 @@ def main(): assert args.local_shader_job.endswith('.json'), \ 'Expected local shader job "{}" to end with .json' - shader_job_prefix = remove_end(args.local_shader_job, '.json') + shader_job_prefix = gfuzz_common.remove_end(args.local_shader_job, '.json') fake_job = tt.ImageJob() fake_job.name = os.path.basename(shader_job_prefix) assert os.path.isfile(args.local_shader_job), \ 'Shader job {} does not exist'.format(args.local_shader_job) - with runspv.open_helper(args.local_shader_job) as f: + with gfuzz_common.open_helper(args.local_shader_job) as f: fake_job.uniformsInfo = f.read() if os.path.isfile(shader_job_prefix + '.frag'): - with runspv.open_helper(shader_job_prefix + '.frag', 'r') as f: + with gfuzz_common.open_helper(shader_job_prefix + '.frag', 'r') as f: fake_job.fragmentSource = f.read() if os.path.isfile(shader_job_prefix + '.vert'): - with runspv.open_helper(shader_job_prefix + '.vert', 'r') as f: + with gfuzz_common.open_helper(shader_job_prefix + '.vert', 'r') as f: fake_job.vertexSource = f.read() if os.path.isfile(shader_job_prefix + '.comp'): - with runspv.open_helper(shader_job_prefix + '.comp', 'r') as f: + with gfuzz_common.open_helper(shader_job_prefix + '.comp', 'r') as f: fake_job.computeSource = f.read() fake_job.computeInfo = fake_job.uniformsInfo do_image_job(args, fake_job, spirvopt_args, work_dir='out') diff --git a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py index f0801ed73..fb4239b54 100644 --- a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py +++ b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py @@ -14,11 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import runspv import argparse import json from typing import List +import gfuzz_common + # Note: We define a 'shader job' as the JSON file of a GraphicsFuzz shader. # Each shader job has a corresponding shader file that has the same base name # as the shader job - for example, a shader job file named 'variant_005.json' will @@ -69,8 +70,7 @@ def make_shader_test_string(shader_job: str) -> str: """ shader_job_json_parsed = get_json_properties(shader_job) - with open(get_shader_from_job(shader_job), 'r', - encoding='utf-8', errors='ignore') as shader: + with gfuzz_common.open_helper(get_shader_from_job(shader_job), 'r') as shader: shader_file_string = shader.read() shader_lines = shader_file_string.split('\n') @@ -180,7 +180,7 @@ def get_json_properties(shader_job: str) -> List: :param shader_job: the path to the shader job file. :return: a list of JSON properties. """ - with open(shader_job, 'r', encoding='utf-8', errors='ignore') as job: + with gfuzz_common.open_helper(shader_job, 'r') as job: json_parsed = json.load(job) if not json_parsed: raise IOError('Malformed shader job file.') @@ -193,7 +193,7 @@ def get_shader_from_job(shader_job: str) -> str: :param shader_job: the path of the shader job file. :return: the path of the fragment shader file. """ - return runspv.remove_end(shader_job, '.json') + '.frag' + return gfuzz_common.remove_end(shader_job, '.json') + '.frag' def get_shader_test_from_job(shader_job: str) -> str: @@ -202,7 +202,7 @@ def get_shader_test_from_job(shader_job: str) -> str: :param shader_job: the path of the shader job file. :return: the path of the shader_test file. """ - return runspv.remove_end(shader_job, '.json') + '.shader_test' + return gfuzz_common.remove_end(shader_job, '.json') + '.shader_test' def main_helper(args: List[str]) -> None: @@ -225,6 +225,5 @@ def main_helper(args: List[str]) -> None: test_string = make_shader_test_string(args.shader_job) - with open(get_shader_test_from_job(args.shader_job), 'w', - encoding='utf-8', errors='ignore') as shader_test: + with gfuzz_common.open_helper(get_shader_test_from_job(args.shader_job), 'w') as shader_test: shader_test.write(test_string) diff --git a/python/src/main/python/drivers/inspect_compute_results.py b/python/src/main/python/drivers/inspect_compute_results.py index ba4238986..9781ffd73 100644 --- a/python/src/main/python/drivers/inspect_compute_results.py +++ b/python/src/main/python/drivers/inspect_compute_results.py @@ -21,13 +21,15 @@ import sys from typing import Callable, List, Optional, Tuple +import gfuzz_common + DEFAULT_REL_TOL = '1e-9' DEFAULT_ABS_TOL = '1e-20' def get_ssbo(result_json_filename: str) -> List: - with open(result_json_filename, 'r', encoding='utf-8', errors='ignore') as f: + with gfuzz_common.open_helper(result_json_filename, 'r') as f: parsed = json.load(f) if not parsed or 'outputs' not in parsed or 'ssbo' not in parsed['outputs']: raise ValueError('No SSBO data found') @@ -91,12 +93,6 @@ def fuzzydiff_ssbos(result_json_filename_1: str, lambda x, y: math.isclose(x, y, rel_tol=rel_tol, abs_tol=abs_tol)) -def check_input_files_exist(filenames: List[str]) -> None: - for filename in filenames: - if not os.path.isfile(filename): - raise FileNotFoundError('Input file "' + filename + '" not found') - - def main_helper(args: List[str]) -> int: description = ( 'Inspect and compare compute shader outputs.') @@ -154,7 +150,7 @@ def main_helper(args: List[str]) -> int: if len(args.inputs) != 1: raise ValueError( 'Command "show" requires exactly 1 input; ' + str(len(args.inputs)) + ' provided') - check_input_files_exist([args.inputs[0]]) + gfuzz_common.check_input_files_exist([args.inputs[0]]) show_ssbo(args.inputs[0]) return 0 @@ -162,7 +158,7 @@ def main_helper(args: List[str]) -> int: if len(args.inputs) != 2: raise ValueError('Command "exactdiff" requires exactly 2 inputs; ' + str( len(args.inputs)) + ' provided') - check_input_files_exist([args.inputs[0], args.inputs[1]]) + gfuzz_common.check_input_files_exist([args.inputs[0], args.inputs[1]]) result, msg = exactdiff_ssbos(args.inputs[0], args.inputs[1]) if result: return 0 @@ -173,7 +169,7 @@ def main_helper(args: List[str]) -> int: if len(args.inputs) != 2: raise ValueError('Command "fuzzydiff" requires exactly 2 inputs; ' + str( len(args.inputs)) + ' provided') - check_input_files_exist([args.inputs[0], args.inputs[1]]) + gfuzz_common.check_input_files_exist([args.inputs[0], args.inputs[1]]) result, msg = fuzzydiff_ssbos(args.inputs[0], args.inputs[1], abs_tol, rel_tol) if result: return 0 From 0c49444fc5b02b8686b92e41e6614ae74bf2f170 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Tue, 2 Jul 2019 22:55:03 +0200 Subject: [PATCH 053/180] Reducer: move global variable initializers to expressions in main Adds to the reducer the ability to find opportunities to remove global variable initializers and put the initializer expressions at the start of main. Fixes #480 --- ...iableDeclToExprReductionOpportunities.java | 73 +++++++++++++++++++ ...ariableDeclToExprReductionOpportunity.java | 33 ++++++++- ...eDeclToExprReductionOpportunitiesTest.java | 39 +++++----- 3 files changed, 123 insertions(+), 22 deletions(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java index 08d864ca6..9761fbb16 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java @@ -17,18 +17,56 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +/* + * This class finds opportunities to remove initializers from global variable declarations and + * replace them with assignment statements which will be inserted as the first statement in main + * function. If main function is not found, do not consider finding this opportunities. + * For example, in: + * int a = 1; + * int b = foo(); + * void main(){ + * int c = 1; + * } + * + *

Because each of a and b has an initializer, we can transform the code fragment + * into the following: + * int a; + * int b; + * void main(){ + * a = 1; + * b = foo(); + * int c = 1; + * } + */ + public class GlobalVariableDeclToExprReductionOpportunities extends ReductionOpportunitiesBase { + private final List globalVariableDecl; + public GlobalVariableDeclToExprReductionOpportunities(TranslationUnit tu, ReducerContext context) { super(tu, context); + this.globalVariableDecl = new ArrayList<>(); } + /** + * Find all initialized global variable declaration opportunities for the given translation unit. + * + * @param shaderJob The shader job to be searched. + * @param context Includes info such as whether we reduce everywhere or only reduce injections + * @return The opportunities that can be reduced + */ static List findOpportunities( ShaderJob shaderJob, ReducerContext context) { @@ -46,4 +84,39 @@ private static List findOpportunit finder.visit(tu); return finder.getOpportunities(); } + + + @Override + public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { + super.visitVariablesDeclaration(variablesDeclaration); + if (!context.reduceEverywhere()) { + // Replacing initializers with a new assignment statement might change semantics, + // do not consider these reduction opportunities if we are not reducing everywhere. + return; + } + + // As constant must always be initialized, we will not consider global variable declarations + // that have const qualifier (i.e., const int a = 1). + if (atGlobalScope() && !variablesDeclaration.getBaseType().hasQualifier(TypeQualifier.CONST)) { + globalVariableDecl.add(variablesDeclaration); + } + } + + @Override + public void visitFunctionDefinition(FunctionDefinition functionDefinition) { + super.visitFunctionDefinition(functionDefinition); + // We consider only the global variable declarations with the initializer that have been + // found before main function. + if (functionDefinition.getPrototype().getName().equals("main")) { + for (VariablesDeclaration variablesDeclaration : globalVariableDecl) { + for (VariableDeclInfo variableDeclInfo : variablesDeclaration.getDeclInfos()) { + if (variableDeclInfo.hasInitializer() + && variableDeclInfo.getInitializer() instanceof ScalarInitializer) { + addOpportunity(new GlobalVariableDeclToExprReductionOpportunity( + getVistitationDepth(), variableDeclInfo, functionDefinition)); + } + } + } + } + } } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java index 6f56ce6be..f5d19b81b 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java @@ -16,20 +16,47 @@ package com.graphicsfuzz.reducer.reductionopportunities; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; public class GlobalVariableDeclToExprReductionOpportunity extends AbstractReductionOpportunity { - GlobalVariableDeclToExprReductionOpportunity(VisitationDepth depth) { + + private final FunctionDefinition mainFunction; + // The initialized global variable declaration info. + private final VariableDeclInfo variableDeclInfo; + + GlobalVariableDeclToExprReductionOpportunity(VisitationDepth depth, + VariableDeclInfo variableDeclInfo, + FunctionDefinition mainFunction) { super(depth); + this.variableDeclInfo = variableDeclInfo; + this.mainFunction = mainFunction; } @Override void applyReductionImpl() { - throw new RuntimeException("Not implemented yet."); + // Given the variable declaration info of global variable, we unset its initializer and + // derive a new assignment statement which will be inserted as the first statement in + // main function. + assert variableDeclInfo.getInitializer() instanceof ScalarInitializer; + final BinaryExpr binaryExpr = new BinaryExpr( + new VariableIdentifierExpr(variableDeclInfo.getName()), + ((ScalarInitializer) variableDeclInfo.getInitializer()).getExpr(), + BinOp.ASSIGN + ); + mainFunction.getBody().insertStmt(0, new ExprStmt(binaryExpr)); + variableDeclInfo.setInitializer(null); } @Override public boolean preconditionHolds() { - throw new RuntimeException("Not implemented yet."); + return mainFunction.getPrototype().getName().equals("main") + && variableDeclInfo.hasInitializer(); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java index f9b019606..b65541415 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java @@ -22,15 +22,12 @@ import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; -import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.*; public class GlobalVariableDeclToExprReductionOpportunitiesTest { - // TODO(519): Enable this test when the issue is solved. - @Ignore @Test public void testDoNotReplace() throws Exception { final String original = "int a = 1; void main() { }"; @@ -44,7 +41,6 @@ public void testDoNotReplace() throws Exception { assertTrue(ops.isEmpty()); } - @Ignore @Test public void testZeroMethod() throws Exception { final String original = "int a = 1;"; @@ -60,7 +56,6 @@ public void testZeroMethod() throws Exception { assertTrue(ops.isEmpty()); } - @Ignore @Test public void testNoMainMethod() throws Exception { final String original = "int a = 1; void foo() { }"; @@ -77,7 +72,6 @@ public void testNoMainMethod() throws Exception { assertTrue(ops.isEmpty()); } - @Ignore @Test public void testInsertAsFirstStatement() throws Exception { final String program = "" @@ -104,7 +98,6 @@ public void testInsertAsFirstStatement() throws Exception { CompareAsts.assertEqualAsts(expected, tu); } - @Ignore @Test public void testMultipleDeclarations() throws Exception { final String program = "" @@ -117,8 +110,8 @@ public void testMultipleDeclarations() throws Exception { + "int b;" + "int c;" // Uninitialized variable declaration. + "void main() {" - + " a = 1;" + " b = foo();" + + " a = 1;" + "}"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = @@ -133,7 +126,6 @@ public void testMultipleDeclarations() throws Exception { CompareAsts.assertEqualAsts(expected, tu); } - @Ignore @Test public void testDoNotReplaceConst() throws Exception { final String original = "const int a = 1; void main() { }"; @@ -148,7 +140,6 @@ public void testDoNotReplaceConst() throws Exception { assertTrue(ops.isEmpty()); } - @Ignore @Test public void testMultipleLineDeclarationsOneLine() throws Exception { final String program = "" @@ -159,9 +150,9 @@ public void testMultipleLineDeclarationsOneLine() throws Exception { final String expected = "" + "int b, c, d, e, f;" + "void main() {" - + " b = 1;" - + " d = foo();" + " f = bar();" + + " d = foo();" + + " b = 1;" + "}"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = @@ -174,7 +165,6 @@ public void testMultipleLineDeclarationsOneLine() throws Exception { CompareAsts.assertEqualAsts(expected, tu); } - @Ignore @Test public void testAssignVariableIdentifier() throws Exception { final String program = "" @@ -182,18 +172,16 @@ public void testAssignVariableIdentifier() throws Exception { + "int b = a, c = b;" // b depends on a and c depends on b. + "int d = c;" // d depends on c. + "void main() { }"; - // As here we have the variable identifier as the initializer, we need to - // make sure that new expressions generated by the reducer are added - // in the correct order. + // No need to retain the order of the assignment as we are running the semantics-changing mode. final String expected = "" + "int a;" + "int b, c;" + "int d;" + "void main() {" - + " a = 1;" - + " b = a;" - + " c = b;" + " d = c;" + + " c = b;" + + " b = a;" + + " a = 1;" + "}"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = @@ -205,4 +193,17 @@ public void testAssignVariableIdentifier() throws Exception { ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); CompareAsts.assertEqualAsts(expected, tu); } + + @Test + public void testGlobalVariableDeclAfterMain() throws Exception { + // The global variable declarations that have found after main method will not be considered. + final String original = "void main() { } int a = 1;"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), null, true)); + assertTrue(ops.isEmpty()); + } } From 7e5c176b55f48ad7b6474fce54ee098e345fb147 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 5 Jul 2019 15:47:01 +0100 Subject: [PATCH 054/180] Make available builtins aware of shader kind (#589) Now that we more comprehensively support builtins, we have the problem that not all builtins are available for every kind of shader. This change adds a ShaderKind parameter to the function that produces available builtins, so that only relevant builtins will be returned. This has required quite a lot of plumbing through, as sometimes builtins are asked for at relatively deep points in the code, where the relevant ShaderKind was not always available. The tests for validity of builtin functions have been extended to try not just every shading language version and also every kind of shader; this would catch the (previously uncaught) problem where a fragment processing builtin could be generated in a compute shader. Fixes #588. --- .../common/ast/inliner/Inliner.java | 2 +- .../CompositeShadingLanguageVersion.java | 5 + .../common/glslversion/Essl100.java | 5 + .../common/glslversion/Essl310.java | 5 + .../common/glslversion/Glsl110.java | 5 + .../common/glslversion/Glsl430.java | 6 + .../glslversion/ShadingLanguageVersion.java | 6 + .../com/graphicsfuzz/common/typing/Typer.java | 28 ++--- .../common/typing/TyperHelper.java | 35 ++++-- .../graphicsfuzz/common/typing/TyperTest.java | 114 +++++++++++------- .../graphicsfuzz/common/util/Obfuscator.java | 2 +- .../common/util/SideEffectChecker.java | 17 +-- .../common/util/SideEffectCheckerTest.java | 6 +- .../graphicsfuzz/generator/fuzzer/Fuzzer.java | 61 +++++----- .../fuzzer/OpaqueExpressionGenerator.java | 3 +- .../generator/fuzzer/Templates.java | 20 ++- .../InterchangeExprMutationFinder.java | 3 +- .../AddLiveOutputWriteMutation.java | 4 +- .../graphicsfuzz/generator/tool/Generate.java | 2 +- .../DonateCodeTransformation.java | 8 +- .../util/RestrictFragmentShaderColors.java | 6 +- .../MakeArrayAccessesInBoundsTest.java | 22 ++-- ...CompoundToBlockReductionOpportunities.java | 3 +- .../FoldConstantReductionOpportunities.java | 2 +- .../FunctionReductionOpportunities.java | 2 +- ...eStructifiedFieldReductionOpportunity.java | 3 +- .../ReductionOpportunitiesBase.java | 11 +- .../SimplifyExprReductionOpportunities.java | 2 +- .../StmtReductionOpportunities.java | 2 +- 29 files changed, 246 insertions(+), 144 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java index c29a186bf..0e1be8345 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java @@ -71,7 +71,7 @@ private Inliner(FunctionCallExpr call, TranslationUnit tu, this.call = call; this.tu = tu; this.shadingLanguageVersion = shadingLanguageVersion; - this.typer = new Typer(tu, shadingLanguageVersion); + this.typer = new Typer(tu); this.parentMap = IParentMap.createParentMap(tu); } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java index e984245d1..7625b319d 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java @@ -73,6 +73,11 @@ public boolean supportedClampUint() { return prototype.supportedClampUint(); } + @Override + public boolean supportedComputeShaders() { + return prototype.supportedComputeShaders(); + } + @Override public boolean supportedDerivativeFunctions() { return prototype.supportedDerivativeFunctions(); diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java index c4e35a4ad..0646d385f 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java @@ -74,6 +74,11 @@ public boolean supportedClampUint() { return false; } + @Override + public boolean supportedComputeShaders() { + return false; + } + @Override public boolean supportedDerivativeFunctions() { return false; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java index 8fe4fa368..4a46ad448 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java @@ -30,6 +30,11 @@ public String getVersionString() { return "310 es"; } + @Override + public boolean supportedComputeShaders() { + return true; + } + @Override public boolean supportedIntegerFunctions() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java index ddaff22e7..a32bceb99 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java @@ -74,6 +74,11 @@ public boolean supportedClampUint() { return false; } + @Override + public boolean supportedComputeShaders() { + return false; + } + @Override public boolean supportedDerivativeFunctions() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl430.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl430.java index e4859c7b8..e1502c523 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl430.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl430.java @@ -29,4 +29,10 @@ private Glsl430(ShadingLanguageVersion prototype) { public String getVersionString() { return "430"; } + + @Override + public boolean supportedComputeShaders() { + return true; + } + } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java index ce402864d..eac0f6977 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java @@ -163,6 +163,12 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedClampUint(); + /** + * GLSL versions 4.3+ and ESSL versions 3.1+ support compute shaders. + * @return true if the shading language version allows compute shaders - false otherwise. + */ + boolean supportedComputeShaders(); + /** * Derivative Functions are a subset of fragment processing functions that compute * the rate of change between pixels in a given fragment. diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java index 0b1f78597..3b754ff99 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java @@ -43,6 +43,7 @@ import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.OpenGlConstants; +import com.graphicsfuzz.common.util.ShaderKind; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -54,28 +55,24 @@ public class Typer extends ScopeTreeBuilder { - private Map types; + private final TranslationUnit tu; - private Map> userDefinedFunctions; + private final Map types; - private Map structDeclarationMap; + private final Map> userDefinedFunctions; - private ShadingLanguageVersion shadingLanguageVersion; + private final Map structDeclarationMap; public Map> getUserDefinedFunctions() { return userDefinedFunctions; } - public Typer(IAstNode node, ShadingLanguageVersion shadingLanguageVersion) { + public Typer(TranslationUnit tu) { + this.tu = tu; this.types = new HashMap<>(); this.userDefinedFunctions = new HashMap<>(); this.structDeclarationMap = new HashMap<>(); - this.shadingLanguageVersion = shadingLanguageVersion; - visit(node); - } - - public Typer(TranslationUnit tu) { - this(tu, tu.getShadingLanguageVersion()); + visit(tu); } @Override @@ -111,7 +108,8 @@ public void visitFunctionDefinition(FunctionDefinition functionDefinition) { public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { super.visitFunctionCallExpr(functionCallExpr); - List candidateBuiltins = TyperHelper.getBuiltins(shadingLanguageVersion) + List candidateBuiltins = + TyperHelper.getBuiltins(tu.getShadingLanguageVersion(), tu.getShaderKind()) .get(functionCallExpr.getCallee()); if (candidateBuiltins != null) { for (FunctionPrototype prototype : candidateBuiltins) { @@ -471,8 +469,10 @@ public Set getPrototypes(String name) { if (userDefinedFunctions.containsKey(name)) { result.addAll(userDefinedFunctions.get(name)); } - if (TyperHelper.getBuiltins(shadingLanguageVersion).containsKey(name)) { - result.addAll(TyperHelper.getBuiltins(shadingLanguageVersion).get(name)); + final Map> builtins = + TyperHelper.getBuiltins(tu.getShadingLanguageVersion(), tu.getShaderKind()); + if (builtins.containsKey(name)) { + result.addAll(builtins.get(name)); } return result; } diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index 094b290de..8ba70aba7 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -23,6 +23,7 @@ import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.ShaderKind; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -39,8 +40,11 @@ */ public final class TyperHelper { + // Maps a (shading language version, shader kind) pair to a mapping from builtin names to builtin + // function prototypes. private static ConcurrentMap>> builtins = new ConcurrentHashMap<>(); + ConcurrentMap>>> builtins = + new ConcurrentHashMap<>(); private TyperHelper() { // Utility class @@ -284,17 +288,32 @@ public static Type resolveTypeOfMul(Type lhsType, Type rhsType) { return null; } + /** + * Yield the builtins available for the given shading language version and kind of shader. + * + * @param shadingLanguageVersion version of GLSL for which relevant builtins should be returned. + * @param shaderKind kind of shader (e.g. fragment or compute) for which relevant builtins + * should be returned. + * @return a mapping from name of builtin to sequence of function prototypes. + */ public static Map> getBuiltins(ShadingLanguageVersion - shadingLanguageVersion) { + shadingLanguageVersion, ShaderKind shaderKind) { + + assert shadingLanguageVersion != null; + assert shaderKind != null; + if (!builtins.containsKey(shadingLanguageVersion)) { - builtins.putIfAbsent(shadingLanguageVersion, - getBuiltinsForGlslVersion(shadingLanguageVersion)); + builtins.putIfAbsent(shadingLanguageVersion, new ConcurrentHashMap<>()); + } + if (!builtins.get(shadingLanguageVersion).containsKey(shaderKind)) { + builtins.get(shadingLanguageVersion).putIfAbsent(shaderKind, + getBuiltinsForGlslVersion(shadingLanguageVersion, shaderKind)); } - return Collections.unmodifiableMap(builtins.get(shadingLanguageVersion)); + return Collections.unmodifiableMap(builtins.get(shadingLanguageVersion).get(shaderKind)); } private static Map> getBuiltinsForGlslVersion( - ShadingLanguageVersion shadingLanguageVersion) { + ShadingLanguageVersion shadingLanguageVersion, ShaderKind shaderKind) { Map> builtinsForVersion = new HashMap<>(); // 8.1: Angle and Trigonometric Functions @@ -622,7 +641,9 @@ private static Map> getBuiltinsForGlslVersion( // 8.13: Fragment Processing Functions (only available in fragment shaders) - getBuiltinsForGlslVersionFragmentProcessing(builtinsForVersion, shadingLanguageVersion); + if (shaderKind == ShaderKind.FRAGMENT) { + getBuiltinsForGlslVersionFragmentProcessing(builtinsForVersion, shadingLanguageVersion); + } // 8.14: Noise Functions - deprecated, so we do not consider them diff --git a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java index 02791dae3..1f7fc29c2 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java @@ -49,6 +49,7 @@ import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.util.ExecHelper.RedirectType; import com.graphicsfuzz.util.ExecResult; import com.graphicsfuzz.common.util.OpenGlConstants; @@ -72,7 +73,8 @@ public class TyperTest { @Test public void visitMemberLookupExpr() throws Exception { - String prog = "struct S { float a; float b; };\n" + String prog = "#version 100\n" + + "struct S { float a; float b; };\n" + "struct T { S s; float c; };\n" + "void main() {\n" + " T myT = T(S(1.0, 2.0), 3.0);\n" @@ -82,7 +84,7 @@ public void visitMemberLookupExpr() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); int actualCount = - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_100) { + new NullCheckTyper(tu) { private int count; @@ -105,7 +107,8 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void visitMemberLookupExprAnonymous() throws Exception { - String prog = "struct { float a; float b; } myStruct;\n" + String prog = "#version 100\n" + + "struct { float a; float b; } myStruct;\n" + "void main() {\n" + " myStruct.a = 2.0;\n" + " myStruct.b = 3.0;\n" @@ -115,7 +118,7 @@ public void visitMemberLookupExprAnonymous() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); int actualCount = - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_100) { + new NullCheckTyper(tu) { private int count; @@ -137,14 +140,15 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testTypeOfScalarConstructors() throws Exception { - String program = "void main() { float(1); int(1); uint(1); bool(1); }"; + String program = "#version 440\n" + + "void main() { float(1); int(1); uint(1); bool(1); }"; for (BasicType b : Arrays.asList(BasicType.FLOAT, BasicType.INT, BasicType.UINT, BasicType.BOOL)) { try { - new NullCheckTyper(ParseHelper.parse(program), ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(ParseHelper.parse(program)) { @Override public void visitTypeConstructorExpr(TypeConstructorExpr typeConstructorExpr) { @@ -171,14 +175,15 @@ public void visitTypeConstructorExpr(TypeConstructorExpr typeConstructorExpr) { @Test public void testMemberLookupTypeFloat() throws Exception { - final String program = "void main() { vec2 v2 = vec2(1.0);" + final String program = "#version 100\n" + + "void main() { vec2 v2 = vec2(1.0);" + " v2.x; v2.y;" + " vec3 v3 = vec3(1.0);" + " v3.x; v3.y; v3.z;" + " vec4 v4 = vec4(1.0);" + " v4.x; v4.y; v4.z; v4.w; }"; final TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_100) { + new NullCheckTyper(tu) { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); @@ -190,14 +195,15 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testMemberLookupTypeInt() throws Exception { - final String program = "void main() { ivec2 v2 = ivec2(1);" + final String program = "#version 440\n" + + "void main() { ivec2 v2 = ivec2(1);" + " v2.x; v2.y;" + " ivec3 v3 = ivec3(1);" + " v3.x; v3.y; v3.z;" + " ivec4 v4 = ivec4(1);" + " v4.x; v4.y; v4.z; v4.w; }"; final TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(tu) { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); @@ -209,14 +215,15 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testMemberLookupTypeUint() throws Exception { - final String program = "void main() { uvec2 v2 = uvec2(1u);" + final String program = "#version 440\n" + + "void main() { uvec2 v2 = uvec2(1u);" + " v2.x; v2.y;" + " uvec3 v3 = uvec3(1u);" + " v3.x; v3.y; v3.z;" + " uvec4 v4 = uvec4(1u);" + " v4.x; v4.y; v4.z; v4.w; }"; final TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(tu) { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); @@ -229,14 +236,15 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testMemberLookupTypeBool() throws Exception { - final String program = "void main() { bvec2 v2 = bvec2(true);" + final String program = "#version 440\n" + + "void main() { bvec2 v2 = bvec2(true);" + " v2.x; v2.y;" + " bvec3 v3 = bvec3(true);" + " v3.x; v3.y; v3.z;" + " bvec4 v4 = bvec4(true);" + " v4.x; v4.y; v4.z; v4.w; }"; final TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(tu) { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); @@ -248,9 +256,10 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testBooleanVectorType() throws Exception { - final String program = "void main() { vec3(1.0) > vec3(2.0); }"; + final String program = "#version 440\n" + + "void main() { vec3(1.0) > vec3(2.0); }"; TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(tu) { @Override public void visitBinaryExpr(BinaryExpr binaryExpr) { super.visitBinaryExpr(binaryExpr); @@ -261,9 +270,10 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { @Test public void testBooleanVectorType2() throws Exception { - final String program = "void main() { vec3(1.0) > 2.0; }"; + final String program = "#version 440\n" + + "void main() { vec3(1.0) > 2.0; }"; TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(tu) { @Override public void visitBinaryExpr(BinaryExpr binaryExpr) { super.visitBinaryExpr(binaryExpr); @@ -275,7 +285,7 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { @Test public void testSwizzleTyped() throws Exception { TranslationUnit tu = ParseHelper.parse("void main() { vec2 v; v.xy = v.yx; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_100); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @@ -287,7 +297,8 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testAssignTyped() throws Exception { - TranslationUnit tu = ParseHelper.parse("void main() {" + TranslationUnit tu = ParseHelper.parse("#version 440\n" + + "void main() {" + "int x;" + "x = 2;" + "float f;" @@ -298,7 +309,7 @@ public void testAssignTyped() throws Exception { + "v3 /= v3;" + "ivec2 i2;" + "i2 -= i2; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitBinaryExpr(BinaryExpr binaryExpr) { @@ -328,11 +339,12 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { @Test public void testCommaTyped() throws Exception { - TranslationUnit tu = ParseHelper.parse("void main() {" + TranslationUnit tu = ParseHelper.parse("#version 440\n" + + "void main() {" + "int x;" + "int y;" + "x, y; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitBinaryExpr(BinaryExpr binaryExpr) { @@ -345,8 +357,10 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { @Test public void testGlPositionTyped() throws Exception { - TranslationUnit tu = ParseHelper.parse("void main() { gl_Position = vec4(0.0); }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300); + TranslationUnit tu = ParseHelper.parse("#version 300 es\n" + + "void main() { gl_Position = vec4(0.0)" + + "; }"); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { @@ -361,8 +375,9 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie @Test public void testGlPointSizeTyped() throws Exception { - TranslationUnit tu = ParseHelper.parse("void main() { gl_PointSize = 1.0; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300); + TranslationUnit tu = ParseHelper.parse("#version 300 es\n" + + "void main() { gl_PointSize = 1.0; }"); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { @@ -434,7 +449,7 @@ public void testOctalIntLiteralTyped() throws Exception { + "void main() {" + " int x = 031;" + "}"); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300) { + new NullCheckTyper(tu) { @Override public void visitScalarInitializer(ScalarInitializer scalarInitializer) { super.visitScalarInitializer(scalarInitializer); @@ -449,7 +464,7 @@ public void testHexIntLiteralTyped() throws Exception { + "void main() {" + " int x = 0xA03B;" + "}"); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300) { + new NullCheckTyper(tu) { @Override public void visitScalarInitializer(ScalarInitializer scalarInitializer) { super.visitScalarInitializer(scalarInitializer); @@ -464,7 +479,7 @@ public void testOctalUnsignedIntLiteralTyped() throws Exception { + "void main() {" + " int x = 031u;" + "}"); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300) { + new NullCheckTyper(tu) { @Override public void visitScalarInitializer(ScalarInitializer scalarInitializer) { super.visitScalarInitializer(scalarInitializer); @@ -479,7 +494,7 @@ public void testHexUnsignedIntLiteralTyped() throws Exception { + "void main() {" + " int x = 0xA03Bu;" + "}"); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300) { + new NullCheckTyper(tu) { @Override public void visitScalarInitializer(ScalarInitializer scalarInitializer) { super.visitScalarInitializer(scalarInitializer); @@ -491,8 +506,9 @@ public void visitScalarInitializer(ScalarInitializer scalarInitializer) { private void checkComputeShaderBuiltin(String builtin, String builtinConstant, BasicType baseType, TypeQualifier qualifier) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - TranslationUnit tu = ParseHelper.parse("void main() { " + builtin + "; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_310); + TranslationUnit tu = ParseHelper.parse("#version 310 es\n" + + "void main() { " + builtin + "; }"); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { @@ -508,9 +524,8 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie class NullCheckTyper extends Typer { - public NullCheckTyper(IAstNode node, - ShadingLanguageVersion shadingLanguageVersion) { - super(node, shadingLanguageVersion); + NullCheckTyper(TranslationUnit tu) { + super(tu); } @Override @@ -684,24 +699,31 @@ public void testBuiltinsGlsl450() throws Exception { private void testBuiltins(ShadingLanguageVersion shadingLanguageVersion) throws IOException, InterruptedException { - final File tempFile = temporaryFolder.newFile("shader.frag"); - FileUtils.writeStringToFile( - tempFile, - makeBuiltinsProgram(shadingLanguageVersion).toString(), - StandardCharsets.UTF_8); - final ExecResult result = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, tempFile); - assertEquals(0, result.res); + for (ShaderKind shaderKind : ShaderKind.values()) { + if (shaderKind == ShaderKind.COMPUTE && !shadingLanguageVersion.supportedComputeShaders()) { + // Compute shaders are not supported for older GLSL versions. + continue; + } + final File tempFile = temporaryFolder.newFile("shader." + shaderKind.getFileExtension()); + FileUtils.writeStringToFile( + tempFile, + makeBuiltinsProgram(shadingLanguageVersion, shaderKind).toString(), + StandardCharsets.UTF_8); + final ExecResult result = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, tempFile); + assertEquals(0, result.res); + } } - private StringBuilder makeBuiltinsProgram(ShadingLanguageVersion shadingLanguageVersion) { + private StringBuilder makeBuiltinsProgram(ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { StringBuilder result = new StringBuilder(); result.append("#version " + shadingLanguageVersion.getVersionString() + "\n"); result.append("#ifdef GL_ES\n"); result.append("precision mediump float;\n"); result.append("#endif\n"); int counter = 0; - for (String name : TyperHelper.getBuiltins(shadingLanguageVersion).keySet()) { - for (FunctionPrototype fp : TyperHelper.getBuiltins(shadingLanguageVersion).get(name)) { + for (String name : TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind).keySet()) { + for (FunctionPrototype fp : TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind).get(name)) { counter++; result.append(fp.getReturnType() + " test" + counter + "_" + fp.getName() + "("); boolean first = true; diff --git a/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java b/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java index 46fadb7cf..2ac0bb4eb 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java @@ -107,7 +107,7 @@ private TranslationUnitObfuscator() { private void obfuscateTranslationUnit(TranslationUnit tu) { - this.typer = new Typer(tu, tu.getShadingLanguageVersion()); + this.typer = new Typer(tu); visit(tu); for (VariableDeclInfo declInfo : varDeclMapping.keySet()) { assert varDeclMapping.containsKey(declInfo); diff --git a/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java b/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java index 5dc406a43..0450ed97d 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java @@ -38,18 +38,19 @@ public class SideEffectChecker { private static boolean isSideEffectFreeVisitor(IAstNode node, - ShadingLanguageVersion shadingLanguageVersion) { + ShadingLanguageVersion shadingLanguageVersion, ShaderKind shaderKind) { return !new CheckPredicateVisitor() { @Override public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { - if (!TyperHelper.getBuiltins(shadingLanguageVersion) + if (!TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind) .containsKey(functionCallExpr.getCallee())) { // Assume that any call to a non-builtin might have a side-effect. predicateHolds(); } for (FunctionPrototype p : - TyperHelper.getBuiltins(shadingLanguageVersion).get(functionCallExpr.getCallee())) { + TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind) + .get(functionCallExpr.getCallee())) { // We check each argument of the built-in's prototypes to see if they require lvalues - // if so, they can cause side effects. // We could be more precise here by finding the specific overload of the function rather @@ -113,12 +114,14 @@ public void visitDefaultCaseLabel(DefaultCaseLabel defaultCaseLabel) { }.test(node); } - public static boolean isSideEffectFree(Stmt stmt, ShadingLanguageVersion shadingLanguageVersion) { - return isSideEffectFreeVisitor(stmt, shadingLanguageVersion); + public static boolean isSideEffectFree(Stmt stmt, ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { + return isSideEffectFreeVisitor(stmt, shadingLanguageVersion, shaderKind); } - public static boolean isSideEffectFree(Expr expr, ShadingLanguageVersion shadingLanguageVersion) { - return isSideEffectFreeVisitor(expr, shadingLanguageVersion); + public static boolean isSideEffectFree(Expr expr, ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { + return isSideEffectFreeVisitor(expr, shadingLanguageVersion, shaderKind); } } diff --git a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java index c5dc5dee8..2b2ae5024 100644 --- a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java +++ b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java @@ -50,7 +50,7 @@ public void testAssignmentHasSideEffects() throws Exception { new VariableIdentifierExpr("v"), new IntConstantExpr("12"), BinOp.ASSIGN - )), ShadingLanguageVersion.ESSL_310)); + )), ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); } @Test @@ -69,7 +69,7 @@ public void testCountingLoopHasSideEffects() throws Exception { new VariableIdentifierExpr("i"), UnOp.POST_INC ), new BlockStmt(Collections.emptyList(), false)), - ShadingLanguageVersion.ESSL_310)); + ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); } @Test @@ -92,7 +92,7 @@ public void testOutParamHasSideEffects() throws Exception { @Override public void visitFunctionCallExpr(FunctionCallExpr expr) { assertTrue(expr.getCallee().equals("uaddCarry")); - assertFalse(SideEffectChecker.isSideEffectFree(expr, ShadingLanguageVersion.ESSL_310)); + assertFalse(SideEffectChecker.isSideEffectFree(expr, ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); } }.visit(tu); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java index 17c73b361..00e26f03f 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java @@ -220,47 +220,21 @@ public static boolean isTooDeep(int depth, GenerationParams generationParams, IR private Stream availableTemplatesFromContext() { return Stream.concat(availableTemplatesFromScope(shadingLanguageVersion, + generationParams.getShaderKind(), fuzzingContext.getCurrentScope()), fuzzingContext.getFunctionPrototypes().stream().map(FunctionCallExprTemplate::new)); } public static Stream availableTemplatesFromScope( ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind, Scope scope) { - return Stream.concat(Templates.get(shadingLanguageVersion).stream(), + return Stream.concat(Templates.get(shadingLanguageVersion, shaderKind).stream(), scope.namesOfAllVariablesInScope() .stream() .map(item -> new VariableIdentifierExprTemplate(item, scope.lookupType(item)))); } - public static void main(String[] args) { - try { - //testFuzzExpr(args); - showTemplates(args); - - } catch (Throwable throwable) { - throwable.printStackTrace(); - System.exit(1); - } - - } - - private static void showTemplates(String[] args) { - // Call this from main to produce a list of templates - List templates = new ArrayList<>(); - templates.addAll(Templates.get(ShadingLanguageVersion.fromVersionString(args[0]))); - Collections.sort(templates, new Comparator() { - @Override - public int compare(IExprTemplate t1, IExprTemplate t2) { - return t1.toString().compareTo(t2.toString()); - } - }); - - for (IExprTemplate t : templates) { - System.out.println(t); - } - } - public TranslationUnit fuzzTranslationUnit() { List decls = new ArrayList<>(); @@ -557,4 +531,33 @@ private Type stripQualifiers(Type type) { return type; } + public static void main(String[] args) { + try { + //testFuzzExpr(args); + showTemplates(args); + + } catch (Throwable throwable) { + throwable.printStackTrace(); + System.exit(1); + } + + } + + private static void showTemplates(String[] args) { + // Call this from main to produce a list of templates + List templates = new ArrayList<>(); + templates.addAll(Templates.get(ShadingLanguageVersion.fromVersionString(args[0]), + ShaderKind.fromExtension(args[1]))); + Collections.sort(templates, new Comparator() { + @Override + public int compare(IExprTemplate t1, IExprTemplate t2) { + return t1.toString().compareTo(t2.toString()); + } + }); + + for (IExprTemplate t : templates) { + System.out.println(t); + } + } + } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index b5addf8e3..6f075b13d 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -746,7 +746,8 @@ public boolean preconditionHolds(Expr expr, BasicType basicType) { if (!exprMustBeSideEffectFree) { return true; } - return SideEffectChecker.isSideEffectFree(expr, shadingLanguageVersion); + return SideEffectChecker.isSideEffectFree(expr, shadingLanguageVersion, + generationParams.getShaderKind()); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Templates.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Templates.java index a00ab5fc2..5eccce3e5 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Templates.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Templates.java @@ -24,6 +24,7 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.SupportedTypes; import com.graphicsfuzz.common.typing.TyperHelper; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.fuzzer.templates.BinaryExprTemplate; import com.graphicsfuzz.generator.fuzzer.templates.ConstantExprTemplate; import com.graphicsfuzz.generator.fuzzer.templates.FunctionCallExprTemplate; @@ -46,21 +47,28 @@ public class Templates { - private static ConcurrentMap> templates + private static ConcurrentMap>> templates = new ConcurrentHashMap<>(); private Templates() { // Utility class } - public static List get(ShadingLanguageVersion shadingLanguageVersion) { + public static List get(ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { if (!templates.containsKey(shadingLanguageVersion)) { - templates.putIfAbsent(shadingLanguageVersion, makeTemplates(shadingLanguageVersion)); + templates.putIfAbsent(shadingLanguageVersion, new ConcurrentHashMap<>()); } - return Collections.unmodifiableList(templates.get(shadingLanguageVersion)); + if (!templates.get(shadingLanguageVersion).containsKey(shaderKind)) { + templates.get(shadingLanguageVersion).putIfAbsent(shaderKind, + makeTemplates(shadingLanguageVersion, shaderKind)); + } + return Collections.unmodifiableList(templates.get(shadingLanguageVersion).get(shaderKind)); } - public static List makeTemplates(ShadingLanguageVersion shadingLanguageVersion) { + private static List makeTemplates(ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { // TODO: assignment operators, array, vector and matrix lookups @@ -69,7 +77,7 @@ public static List makeTemplates(ShadingLanguageVersion shadingLa // Builtins { Map> builtins = TyperHelper.getBuiltins( - shadingLanguageVersion); + shadingLanguageVersion, shaderKind); List keys = builtins.keySet().stream().collect(Collectors.toList()); keys.sort(String::compareTo); for (String key : keys) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java index 2680d1a6d..177ecbb3c 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java @@ -63,7 +63,8 @@ protected void visitExpr(Expr expr) { private Expr applyInterchange(Expr expr, List argumentTypes) { final Type resultType = typer.lookupType(expr).getWithoutQualifiers(); List templates = Fuzzer.availableTemplatesFromScope( - getTranslationUnit().getShadingLanguageVersion(), currentScope) + getTranslationUnit().getShadingLanguageVersion(), getTranslationUnit().getShaderKind(), + currentScope) // Ignore templates that require l-values, so that invalidity is not too likely .filter(InterchangeExprMutationFinder::doesNotRequireLvalue) // Ignore the possibility of replacing a one-argument expression with parentheses, as it diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddLiveOutputWriteMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddLiveOutputWriteMutation.java index 4533afd39..4697e91db 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddLiveOutputWriteMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddLiveOutputWriteMutation.java @@ -30,6 +30,7 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.fuzzer.Fuzzer; import com.graphicsfuzz.generator.fuzzer.FuzzingContext; import com.graphicsfuzz.generator.fuzzer.OpaqueExpressionGenerator; @@ -63,7 +64,8 @@ outputVariableType, new VariableDeclInfo(backupName, null, null)))); stmts.add(new ExprStmt(new BinaryExpr(new VariableIdentifierExpr(backupName), new VariableIdentifierExpr(outputVariableName), BinOp.ASSIGN))); - final Fuzzer fuzzer = new Fuzzer(new FuzzingContext(), shadingLanguageVersion, random, + final Fuzzer fuzzer = new Fuzzer(new FuzzingContext(), shadingLanguageVersion, + random, generationParams); stmts.add(new ExprStmt(new BinaryExpr( new VariableIdentifierExpr(outputVariableName), diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java index 1b9504bd8..7c5f727ab 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java @@ -453,7 +453,7 @@ private static boolean canTypeCheckWithoutFailure(TranslationUnit reference, ShadingLanguageVersion shadingLanguageVersion) { // Debugging aid: fail early if we end up messing up the translation unit so that type checking // does not work. - new Typer(reference, shadingLanguageVersion); + new Typer(reference); return true; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java index 5a1c1a00d..01986c8b6 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java @@ -50,6 +50,7 @@ import com.graphicsfuzz.common.util.OpenGlConstants; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.StructUtils; import com.graphicsfuzz.generator.fuzzer.FuzzedIntoACornerException; import com.graphicsfuzz.generator.fuzzer.Fuzzer; @@ -250,7 +251,7 @@ public boolean apply(TranslationUnit tu, } donateFunctionsAndGlobals(tu); eliminateUsedDonors(); - makeInjectedArrayAccessesInBounds(tu, injectedStmts, tu.getShadingLanguageVersion()); + makeInjectedArrayAccessesInBounds(tu, injectedStmts); return !injectionPoints.isEmpty(); @@ -266,9 +267,8 @@ private void eliminateUsedDonors() { } private void makeInjectedArrayAccessesInBounds(TranslationUnit tu, - List injectedStmts, - ShadingLanguageVersion shadingLanguageVersion) { - Typer typer = new Typer(tu, shadingLanguageVersion); + List injectedStmts) { + Typer typer = new Typer(tu); for (Stmt stmt : injectedStmts) { MakeArrayAccessesInBounds.makeInBounds(stmt, typer); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java index eed772dac..604f12656 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java @@ -40,6 +40,7 @@ import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.common.typing.TyperHelper; import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.fuzzer.Fuzzer; import com.graphicsfuzz.generator.fuzzer.FuzzingContext; import com.graphicsfuzz.generator.fuzzer.OpaqueExpressionGenerator; @@ -156,7 +157,7 @@ private void addNewWrites() { private boolean adaptExistingWrites() { - final Typer typer = new Typer(shaderJob.getFragmentShader().get(), shadingLanguageVersion); + final Typer typer = new Typer(shaderJob.getFragmentShader().get()); return new ScopeTreeBuilder() { @@ -164,7 +165,8 @@ private boolean adaptExistingWrites() { public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { super.visitFunctionCallExpr(functionCallExpr); Set acceptableFunctionNames = new HashSet<>(); - acceptableFunctionNames.addAll(TyperHelper.getBuiltins(shadingLanguageVersion).keySet()); + acceptableFunctionNames.addAll(TyperHelper.getBuiltins(shadingLanguageVersion, + ShaderKind.FRAGMENT).keySet()); acceptableFunctionNames.add(Constants.GLF_FUZZED); acceptableFunctionNames.add(Constants.GLF_IDENTITY); if (acceptableFunctionNames.contains(functionCallExpr.getCallee()) diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java index 3fe3d3d87..32fcba6bd 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java @@ -29,10 +29,10 @@ public class MakeArrayAccessesInBoundsTest { @Test public void testBasic() throws Exception { - final String shader = "void main() { int A[5]; int x = 17; A[x] = 2; }"; - final String expected = "void main() { int A[5]; int x = 17; A[(x) >= 0 && (x) < 5 ? x : 0] = 2; }"; + final String shader = "#version 300 es\nvoid main() { int A[5]; int x = 17; A[x] = 2; }"; + final String expected = "#version 300 es\nvoid main() { int A[5]; int x = 17; A[(x) >= 0 && (x) < 5 ? x : 0] = 2; }"; final TranslationUnit tu = ParseHelper.parse(shader); - final Typer typer = new Typer(tu, ShadingLanguageVersion.ESSL_300); + final Typer typer = new Typer(tu); MakeArrayAccessesInBounds.makeInBounds(tu, typer); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -40,13 +40,13 @@ public void testBasic() throws Exception { @Test public void testMatrixVector() throws Exception { - final String shader = "void main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100; As[x][y][z] = 2.0; }"; - final String expected = "void main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100;" + final String shader = "#version 300 es\nvoid main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100; As[x][y][z] = 2.0; }"; + final String expected = "#version 300 es\nvoid main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100;" + "As[(x) >= 0 && (x) < 5 ? x : 0]" + " /* column */ [(y) >= 0 && (y) < 4 ? y : 0]" + " /* row */ [(z) >= 0 && (z) < 2 ? z : 0] = 2.0; }"; final TranslationUnit tu = ParseHelper.parse(shader); - final Typer typer = new Typer(tu, ShadingLanguageVersion.ESSL_300); + final Typer typer = new Typer(tu); MakeArrayAccessesInBounds.makeInBounds(tu, typer); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -54,7 +54,8 @@ public void testMatrixVector() throws Exception { @Test public void testMatrixVector2() throws Exception { - final String shader = "void main() { mat3x4 As[5];" + final String shader = "#version 300 es\n" + + "void main() { mat3x4 As[5];" + " int x = 17;" + " int y = -22;" + " int z = 100;" @@ -64,7 +65,8 @@ public void testMatrixVector2() throws Exception { + " float f;" + " f = v[z];" + "}"; - final String expected = "void main() { mat3x4 As[5];" + final String expected = "#version 300 es\n" + + "void main() { mat3x4 As[5];" + " int x = 17;" + " int y = -22;" + " int z = 100;" @@ -75,10 +77,10 @@ public void testMatrixVector2() throws Exception { + " f = v[(z) >= 0 && (z) < 4 ? z : 0];" + "}"; final TranslationUnit tu = ParseHelper.parse(shader); - final Typer typer = new Typer(tu, ShadingLanguageVersion.ESSL_300); + final Typer typer = new Typer(tu); MakeArrayAccessesInBounds.makeInBounds(tu, typer); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } -} \ No newline at end of file +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java index 84f998d6a..89364befc 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java @@ -112,7 +112,8 @@ private boolean allowedToReduce(Stmt compoundStmt) { || (StmtReductionOpportunities.isLiveCodeInjection(compoundStmt) && !isLoopLimiterCheck(compoundStmt)) || enclosingFunctionIsDead() - || SideEffectChecker.isSideEffectFree(compoundStmt, context.getShadingLanguageVersion()); + || SideEffectChecker.isSideEffectFree(compoundStmt, context.getShadingLanguageVersion(), + shaderKind); } private static boolean isLoopLimiterCheck(Stmt compoundStmt) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java index a3fe7d1de..a7a062236 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java @@ -230,7 +230,7 @@ private void findReplaceTypeConstructorWithElementOpportunities( // We could handle cases such as vec2(0.0).x resolving to 0.0; but for now we do not. return; } - if (!SideEffectChecker.isSideEffectFree(tce, context.getShadingLanguageVersion())) { + if (!SideEffectChecker.isSideEffectFree(tce, context.getShadingLanguageVersion(), shaderKind)) { // We mustn't eliminate side-effects from elements of the vector that we are not popping out. return; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunities.java index e3884b32c..266ed6a95 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunities.java @@ -46,7 +46,7 @@ public class FunctionReductionOpportunities extends StandardVisitor { private final List declaredFunctions; // All functions declared in the shader private FunctionReductionOpportunities(TranslationUnit tu, ReducerContext context) { - this.typer = new Typer(tu, context.getShadingLanguageVersion()); + this.typer = new Typer(tu); this.opportunities = new ArrayList<>(); this.calledFunctions = new HashSet<>(); this.declaredFunctions = Collections diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunity.java index 98ef0f319..5b0c6408f 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunity.java @@ -64,8 +64,7 @@ public InlineStructifiedFieldReductionOpportunity(StructDefinitionType outerStru @Override public void applyReductionImpl() { - // The GLSL version is irrelevant; really we want a Typer that doesn't require this. - final Typer typer = new Typer(tu, ShadingLanguageVersion.ESSL_100); + final Typer typer = new Typer(tu); final int indexOfInlinedField = outerStruct.getFieldIndex(fieldToInline); outerStruct.removeField(fieldToInline); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java index 812c077a2..c38a2c620 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java @@ -36,6 +36,7 @@ import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.SideEffectChecker; import com.graphicsfuzz.util.Constants; import java.util.ArrayList; @@ -52,6 +53,8 @@ public abstract class ReductionOpportunitiesBase protected final ReducerContext context; + protected final ShaderKind shaderKind; + protected String enclosingFunctionName; private int numEnclosingLValues; @@ -66,9 +69,10 @@ public ReductionOpportunitiesBase(TranslationUnit tu, ReducerContext context) { this.opportunities = new ArrayList<>(); this.injectionTracker = new InjectionTracker(); this.notReferencedFromLiveContext = new NotReferencedFromLiveContext(tu); + this.parentMap = IParentMap.createParentMap(tu); this.context = context; + this.shaderKind = tu.getShaderKind(); this.enclosingFunctionName = null; - this.parentMap = IParentMap.createParentMap(tu); this.numEnclosingLValues = 0; } @@ -77,7 +81,7 @@ public void visitFunctionDefinition(FunctionDefinition functionDefinition) { assert enclosingFunctionName == null; enclosingFunctionName = functionDefinition.getPrototype().getName(); super.visitFunctionDefinition(functionDefinition); - assert enclosingFunctionName == functionDefinition.getPrototype().getName(); + assert enclosingFunctionName.equals(functionDefinition.getPrototype().getName()); enclosingFunctionName = null; } @@ -212,7 +216,8 @@ boolean initializerIsScalarAndSideEffectFree(VariableDeclInfo variableDeclInfo) } return SideEffectChecker.isSideEffectFree( ((ScalarInitializer) variableDeclInfo.getInitializer()).getExpr(), - context.getShadingLanguageVersion()); + context.getShadingLanguageVersion(), + shaderKind); } boolean typeIsReducibleToConst(Type type) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java index 46e995a35..dc8ae474c 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java @@ -41,7 +41,7 @@ abstract class SimplifyExprReductionOpportunities TranslationUnit tu, ReducerContext context) { super(tu, context); - this.typer = new Typer(tu, context.getShadingLanguageVersion()); + this.typer = new Typer(tu); this.inLiveInjectedStmtOrDeclaration = false; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java index 6b9a3e265..ad62cdfe5 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java @@ -110,7 +110,7 @@ private boolean allowedToReduceStmt(BlockStmt block, int childIndex) { return true; } - if (SideEffectChecker.isSideEffectFree(stmt, context.getShadingLanguageVersion())) { + if (SideEffectChecker.isSideEffectFree(stmt, context.getShadingLanguageVersion(), shaderKind)) { return true; } From 1a554ed788be2a08ebe4b6c819088a81a3569ce3 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 5 Jul 2019 16:12:26 +0100 Subject: [PATCH 055/180] Simplify loop limiters (#590) We had functionality that was not used anywhere to avoid adding limiters to short-running loops, and associated tests for this that took quite a long time to run. This change removes that unused functionality. --- ast/pom.xml | 4 - .../common/util/TruncateLoops.java | 171 +----------------- .../common/util/TruncateLoopsTest.java | 102 ++++------- .../DonateLiveCodeTransformation.java | 2 +- 4 files changed, 34 insertions(+), 245 deletions(-) diff --git a/ast/pom.xml b/ast/pom.xml index a7d5077c2..ee3f2ba1a 100644 --- a/ast/pom.xml +++ b/ast/pom.xml @@ -68,10 +68,6 @@ limitations under the License. net.sourceforge.argparse4j argparse4j - - org.apache.commons - commons-lang3 - diff --git a/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java b/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java index 4751e8cd5..62683ca72 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java +++ b/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java @@ -23,10 +23,7 @@ import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; -import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; -import com.graphicsfuzz.common.ast.expr.Op; -import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.UnOp; import com.graphicsfuzz.common.ast.expr.UnaryExpr; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; @@ -45,23 +42,18 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Optional; -import org.apache.commons.lang3.tuple.ImmutablePair; public class TruncateLoops extends StandardVisitor { private final int limit; private final TranslationUnit tu; private final String prefix; - private final boolean ignoreShortRunningForLoops; private int counter; - public TruncateLoops(int limit, String prefix, TranslationUnit tu, - boolean ignoreShortRunningForLoops) { + public TruncateLoops(int limit, String prefix, TranslationUnit tu) { this.limit = limit; this.tu = tu; this.prefix = prefix; - this.ignoreShortRunningForLoops = ignoreShortRunningForLoops; counter = 0; visit(tu); } @@ -69,9 +61,7 @@ public TruncateLoops(int limit, String prefix, TranslationUnit tu, @Override public void visitForStmt(ForStmt forStmt) { super.visitForStmt(forStmt); - if (!ignoreShortRunningForLoops || maybeLongRunning(forStmt)) { - handleLoop(forStmt); - } + handleLoop(forStmt); } @Override @@ -124,161 +114,4 @@ private void handleLoop(LoopStmt loopStmt) { parentMap.getParent(loopStmt).replaceChild(loopStmt, replacementBlock); } - private boolean maybeLongRunning(ForStmt forStmt) { - final Optional> initValueAndLoopCounterName - = getLoopCounterNameAndInitValue(forStmt.getInit()); - if (!initValueAndLoopCounterName.isPresent()) { - return false; - } - final String loopCounterName = initValueAndLoopCounterName.get().left; - final int initValue = initValueAndLoopCounterName.get().right; - - final Optional> condTestTypeAndLimitValue - = getCondTestTypeAndLimitValue(forStmt.getCondition(), loopCounterName); - if (!condTestTypeAndLimitValue.isPresent()) { - return false; - } - final BinOp condTestType = condTestTypeAndLimitValue.get().left; - final int condLimitValue = condTestTypeAndLimitValue.get().right; - if (condTestType.isSideEffecting()) { - return false; - } - - final Optional incrementValue - = getIncrementValue(forStmt.getIncrement(), loopCounterName); - if (!incrementValue.isPresent()) { - return false; - } - - return (((condTestType == BinOp.LT || condTestType == BinOp.LE) && incrementValue.get() <= 0) - || ((condTestType == BinOp.GT || condTestType == BinOp.GE) && incrementValue.get() >= 0) - || ((condLimitValue - initValue) / incrementValue.get() >= limit)); - } - - private Optional> getLoopCounterNameAndInitValue(Stmt init) { - String name = null; - Expr expr = null; - if (init instanceof ExprStmt - && ((ExprStmt) init).getExpr() instanceof BinaryExpr - && ((BinaryExpr)(((ExprStmt) init).getExpr())).getOp() == BinOp.ASSIGN - && ((BinaryExpr)(((ExprStmt) init).getExpr())).getLhs() - instanceof VariableIdentifierExpr) { - name = ((VariableIdentifierExpr) (((BinaryExpr)(((ExprStmt) init) - .getExpr())).getLhs())).getName(); - expr = ((BinaryExpr)(((ExprStmt) init) - .getExpr())).getRhs(); - } else if (init instanceof DeclarationStmt - && ((DeclarationStmt) init).getVariablesDeclaration().getNumDecls() == 1 - && ((DeclarationStmt) init).getVariablesDeclaration().getDeclInfo(0) - .getInitializer() instanceof ScalarInitializer) { - name = ((DeclarationStmt) init).getVariablesDeclaration() - .getDeclInfo(0).getName(); - expr = ((ScalarInitializer)((DeclarationStmt) init) - .getVariablesDeclaration().getDeclInfo(0) - .getInitializer()).getExpr(); - } - if (name == null || expr == null) { - return Optional.empty(); - } - Optional constant = getAsConstant(expr); - if (!constant.isPresent()) { - return Optional.empty(); - } - return Optional.of(new ImmutablePair<>(name, constant.get())); - } - - private Optional> getCondTestTypeAndLimitValue(Expr cond, - String loopCounterName) { - if (!(cond instanceof BinaryExpr)) { - return Optional.empty(); - } - final BinaryExpr binaryExprCond = (BinaryExpr) cond; - Optional condLimitValue = getAsConstant(binaryExprCond.getRhs()); - if (condLimitValue.isPresent() - && isSameVarIdentifier(binaryExprCond.getLhs(), loopCounterName)) { - return Optional.of(new ImmutablePair<>(binaryExprCond.getOp(), condLimitValue.get())); - } - condLimitValue = getAsConstant(binaryExprCond.getLhs()); - if (condLimitValue.isPresent() - && isSameVarIdentifier(binaryExprCond.getRhs(), loopCounterName)) { - return Optional.of(new ImmutablePair<>(switchCondTestType(binaryExprCond.getOp()), - condLimitValue.get())); - } - - return Optional.empty(); - } - - private Optional getIncrementValue(Expr incr, String loopCounterName) { - if (incr instanceof UnaryExpr - && isSameVarIdentifier(((UnaryExpr) incr).getExpr(), loopCounterName)) { - switch (((UnaryExpr) incr).getOp()) { - case POST_INC: - case PRE_INC: - return Optional.of(1); - case POST_DEC: - case PRE_DEC: - return Optional.of(-1); - default: - return Optional.empty(); - } - } - if (incr instanceof BinaryExpr - && isSameVarIdentifier(((BinaryExpr) incr).getLhs(), loopCounterName)) { - final Optional incrementValue = getAsConstant(((BinaryExpr) incr).getRhs()); - if (!incrementValue.isPresent()) { - return Optional.empty(); - } - switch (((BinaryExpr) incr).getOp()) { - case ADD_ASSIGN: - return incrementValue; - case SUB_ASSIGN: - return incrementValue.map(item -> -item); - default: - return Optional.empty(); - } - } - return Optional.empty(); - } - - private boolean isSameVarIdentifier(Expr expr, String loopCounterName) { - return expr instanceof VariableIdentifierExpr - && ((VariableIdentifierExpr)expr).getName().equals(loopCounterName); - } - - private Optional getAsConstant(Expr expr) { - if (expr instanceof ParenExpr) { - return getAsConstant(((ParenExpr)expr).getExpr()); - } - if (expr instanceof IntConstantExpr) { - return Optional.of(Integer.valueOf(((IntConstantExpr) expr).getValue())); - } - if (expr instanceof UnaryExpr) { - final UnaryExpr unaryExpr = (UnaryExpr) expr; - switch (unaryExpr.getOp()) { - case MINUS: - return getAsConstant(unaryExpr.getExpr()).map(item -> -item); - case PLUS: - return getAsConstant(unaryExpr.getExpr()); - default: - return Optional.empty(); - } - } - return Optional.empty(); - } - - private BinOp switchCondTestType(BinOp binOp) { - switch (binOp) { - case LT: - return BinOp.GT; - case GT: - return BinOp.LT; - case LE: - return BinOp.GE; - case GE: - return BinOp.LE; - default: - return binOp; - } - } - } diff --git a/ast/src/test/java/com/graphicsfuzz/common/util/TruncateLoopsTest.java b/ast/src/test/java/com/graphicsfuzz/common/util/TruncateLoopsTest.java index 17aaa6b17..8b47e072d 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/util/TruncateLoopsTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/util/TruncateLoopsTest.java @@ -26,60 +26,6 @@ public class TruncateLoopsTest { - private static final String[] CONDITIONS = {"x < -(-20)", "20 > x", "x <= 20", "20 >= x", - "x > -2", "-2 < x","x >= -2", "-2 <= x"}; - private static final String[] INCREMETS = {"x++", "++x", "x += 1", "x += -(-1)", "x += 5", - "x--", "--x", "x -= 1", "x -= 5"}; - private static final String[] INIT_CONSTS = {"x = -1", "x = 0", "int x = 0", "x = -(+(-10))", - "int x = 10"}; - - @Test - public void intConstantsTest() throws Exception { - final String programBody = - "void main() {" - + "int u = 10;" - + "int x;" - + " for($INIT; $COND; $INCREMENT) {" - + " u = u * 2;" - + " }" - + "}"; - - for (int cond_index = 0; cond_index < CONDITIONS.length; ++cond_index) { - for (int incr_index = 0; incr_index < INCREMETS.length; ++incr_index) { - for (int init_index = 0; init_index < INIT_CONSTS.length; ++init_index) { - final boolean isSane = (cond_index < 4 && incr_index < 5) - || (4 <= cond_index && 5 <= incr_index); - testProgram(programBody - .replace("$INIT", INIT_CONSTS[init_index]) - .replace("$COND", CONDITIONS[cond_index]) - .replace("$INCREMENT", INCREMETS[incr_index]), - isSane); - - } - } - } - } - - private void testProgram(String program, boolean isSane) throws IOException, - ParseTimeoutException, InterruptedException, GlslParserException { - TranslationUnit tu = ParseHelper.parse(program); - new TruncateLoops(30, "webGL_", tu, true); - if(isSane) { - CompareAstsDuplicate.assertEqualAsts(program, tu); - } else { - assertProgramsNotEqual(program, tu); - } - tu = ParseHelper.parse(program); - new TruncateLoops(0, "webGL_", tu, true); - assertProgramsNotEqual(program, tu); - } - - private void assertProgramsNotEqual(String program, TranslationUnit otherProgram) - throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - assert !PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(program)) - .equals(PrettyPrinterVisitor.prettyPrintAsString(otherProgram)); - } - @Test public void testTruncateLoops() throws Exception { final String program = "void main() {" @@ -132,7 +78,7 @@ public void testTruncateLoops() throws Exception { + " }\n" + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); - new TruncateLoops(3, "pre", tu, true); + new TruncateLoops(3, "pre", tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -189,7 +135,7 @@ public void testTruncateLoops2() throws Exception { + " }\n" + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); - new TruncateLoops(3, "pre", tu, false); + new TruncateLoops(3, "pre", tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -210,29 +156,43 @@ public void testTruncateLoops3() throws Exception { final String expected = "" + "void main() {\n" + " int x = 0;\n" - + " for (int i = 0; i < 2; i++) {\n" - + " int pre_looplimiter1 = 0;\n" - + " while (x < i) {\n" - + " if (pre_looplimiter1 >= 3) {\n" + + " {" + + " int pre_looplimiter3 = 0;\n" + + " for (int i = 0; i < 2; i++) {\n" + + " if (pre_looplimiter3 >= 3) {\n" + " break;\n" + " }\n" - + " pre_looplimiter1++;\n" - + " int pre_looplimiter0 = 0;\n" - + " do {\n" - + " if (pre_looplimiter0 >= 3) {\n" + + " pre_looplimiter3++;\n" + + " int pre_looplimiter2 = 0;\n" + + " while (x < i) {\n" + + " if (pre_looplimiter2 >= 3) {\n" + " break;\n" + " }\n" - + " pre_looplimiter0++;\n" - + " x++;\n" - + " for (int j = 0; j < 2; j++) {\n" - + " ;\n" - + " }\n" - + " } while (x > 0);\n" + + " pre_looplimiter2++;\n" + + " int pre_looplimiter1 = 0;\n" + + " do {\n" + + " if (pre_looplimiter1 >= 3) {\n" + + " break;\n" + + " }\n" + + " pre_looplimiter1++;\n" + + " x++;\n" + + " {\n" + + " int pre_looplimiter0 = 0;\n" + + " for (int j = 0; j < 2; j++) {\n" + + " if (pre_looplimiter0 >= 3) {\n" + + " break;\n" + + " }\n" + + " pre_looplimiter0++;\n" + + " ;\n" + + " }\n" + + " }\n" + + " } while (x > 0);\n" + + " }\n" + " }\n" + " }\n" + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); - new TruncateLoops(3, "pre", tu, true); + new TruncateLoops(3, "pre", tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java index 4618db351..0fcc55937 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java @@ -101,7 +101,7 @@ String getPrefix() { @Override void adaptTranslationUnitForSpecificDonation(TranslationUnit tu, IRandom generator) { if (!allowLongLoops) { - new TruncateLoops(3 + generator.nextInt(5), addPrefix(""), tu, false); + new TruncateLoops(3 + generator.nextInt(5), addPrefix(""), tu); } } From dfb7cfaa9e2a8006b6fbb1974a56846a2beb3f39 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 5 Jul 2019 16:12:49 +0100 Subject: [PATCH 056/180] Avoid pop-up on Windows during testing (#591) A test uses an empty file as a custom interestingness judge. Due to not having the .bat extension, this would lead to a pop-up on Windows asking what kind of program should be used to open the file. As other platforms don't care about the file extension of an executable script, this change uses .bat to avoid the pop-up on Windows. Fixes #557 --- .../java/com/graphicsfuzz/reducer/tool/GlslReduceTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/tool/GlslReduceTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/tool/GlslReduceTest.java index 2d7003321..da9dca7bc 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/tool/GlslReduceTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/tool/GlslReduceTest.java @@ -145,7 +145,10 @@ public void checkCustomJudgeIsExecutable() throws Exception { @Test public void testAlwaysReduceJudgeMaximallyReduces() throws Exception { final File jsonFile = getShaderJobReady(); - final File emptyFile = temporaryFolder.newFile(); + // We make this a .bat file to avoid a "what application would you like to use to open this + // file?" pop-up on Windows. (On other platforms the fact that it has the .bat extension does + // not matter.) + final File emptyFile = temporaryFolder.newFile("judge.bat"); emptyFile.setExecutable(true); GlslReduce.mainHelper(new String[]{ jsonFile.getAbsolutePath(), From a66087c2192a6871d7c54d9b78e75bdb5bc517ca Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Mon, 8 Jul 2019 20:39:08 +0700 Subject: [PATCH 057/180] Complete common builtins (#592) * Support all common built-in functions * Add function java docs --- .../CompositeShadingLanguageVersion.java | 15 + .../common/glslversion/Essl100.java | 15 + .../common/glslversion/Essl300.java | 5 + .../common/glslversion/Essl310.java | 10 + .../common/glslversion/Glsl110.java | 15 + .../common/glslversion/Glsl130.java | 5 + .../common/glslversion/Glsl400.java | 9 + .../glslversion/ShadingLanguageVersion.java | 6 + .../common/typing/TyperHelper.java | 1121 +++++++++-------- .../common/util/SideEffectCheckerTest.java | 50 + 10 files changed, 707 insertions(+), 544 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java index 7625b319d..08e4527bc 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java @@ -292,4 +292,19 @@ public boolean supportedAngleAndTrigonometricFunctions() { public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { return prototype.supportedHyperbolicAngleAndTrigonometricFunctions(); } + + @Override + public boolean supportedModf() { + return prototype.supportedModf(); + } + + @Override + public boolean supportedFrexp() { + return prototype.supportedFrexp(); + } + + @Override + public boolean supportedLdexp() { + return prototype.supportedLdexp(); + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java index 0646d385f..fdbce3f9e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java @@ -293,4 +293,19 @@ public boolean supportedAngleAndTrigonometricFunctions() { public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { return false; } + + @Override + public boolean supportedModf() { + return false; + } + + @Override + public boolean supportedFrexp() { + return false; + } + + @Override + public boolean supportedLdexp() { + return false; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java index 4596f0661..0b9277180 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java @@ -219,4 +219,9 @@ public boolean supportedUnsigned() { public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { return true; } + + @Override + public boolean supportedModf() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java index 4a46ad448..bdbfc039f 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java @@ -65,4 +65,14 @@ public boolean supportedUnpackUnorm4x8() { return true; } + @Override + public boolean supportedFrexp() { + return true; + } + + @Override + public boolean supportedLdexp() { + return true; + } + } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java index a32bceb99..d4beaec68 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java @@ -296,4 +296,19 @@ public boolean supportedAngleAndTrigonometricFunctions() { public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { return false; } + + @Override + public boolean supportedModf() { + return false; + } + + @Override + public boolean supportedFrexp() { + return false; + } + + @Override + public boolean supportedLdexp() { + return false; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java index c390e4d7a..36cb869fc 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java @@ -119,4 +119,9 @@ public boolean supportedUnsigned() { public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { return true; } + + @Override + public boolean supportedModf() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java index 3bb98a75e..58ee30cbb 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java @@ -75,4 +75,13 @@ public boolean supportedUnpackUnorm4x8() { return true; } + @Override + public boolean supportedFrexp() { + return true; + } + + @Override + public boolean supportedLdexp() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java index eac0f6977..6013da153 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java @@ -305,6 +305,12 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedUnsigned(); + boolean supportedModf(); + + boolean supportedFrexp(); + + boolean supportedLdexp(); + /** * Angle and Trigonometric Functions are a set of built-in functions related to the calculation * of an angle. For example, sin(angle) - computes the sine value of the angle provided. diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index 8ba70aba7..24ab430fc 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -327,540 +327,250 @@ private static Map> getBuiltinsForGlslVersion( // 8.3: Common Functions - { - final String name = "abs"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + getBuiltinsForGlslVersionCommon(shadingLanguageVersion, builtinsForVersion); + + // 8.4: Floating-Point Pack and Unpack Functions + + getBuiltinsForGlslVersionFloatingPointPackAndUnpack(builtinsForVersion, shadingLanguageVersion); + + // 8.5: Geometric Functions + + getBuiltinsForGlslVersionGeometric(builtinsForVersion); + + // 8.6: Matrix Functions + + getBuiltinsForGlslVersionMatrix(builtinsForVersion, shadingLanguageVersion); + + // 8.7: Vector Relational Functions + + getBuiltinsForGlslVersionVectorRelational(builtinsForVersion, shadingLanguageVersion); + + // 8.8: Integer Functions + + getBuiltinsForGlslVersionInteger(builtinsForVersion, shadingLanguageVersion); + + // 8.13: Fragment Processing Functions (only available in fragment shaders) + + if (shaderKind == ShaderKind.FRAGMENT) { + getBuiltinsForGlslVersionFragmentProcessing(builtinsForVersion, shadingLanguageVersion); + } + + // 8.14: Noise Functions - deprecated, so we do not consider them + + return builtinsForVersion; + } + + /** + * Helper function to register built-in function prototypes for Angle and Trigonometric + * Functions, as specified in section 8.1 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionAngleAndTrigonometric( + ShadingLanguageVersion shadingLanguageVersion, + Map> builtinsForVersion) { + if (shadingLanguageVersion.supportedAngleAndTrigonometricFunctions()) { + { + final String name = "radians"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } } - if (shadingLanguageVersion.supportedAbsInt()) { - for (Type t : igenType()) { + + { + final String name = "degrees"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "sin"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "cos"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "tan"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "asin"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "acos"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "atan"; + for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, t, t, t); } } } - { - final String name = "sign"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + if (shadingLanguageVersion.supportedHyperbolicAngleAndTrigonometricFunctions()) { + { + final String name = "sinh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } } - if (shadingLanguageVersion.supportedSignInt()) { - for (Type t : igenType()) { + + { + final String name = "cosh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "tanh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "asinh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "acosh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "atanh"; + for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } } + } + /** + * Helper function to register built-in function prototypes for Exponential Functions, as + * specified in section 8.2 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + */ + private static void getBuiltinsForGlslVersionExponential( + Map> builtinsForVersion) { { - final String name = "floor"; + final String name = "pow"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, t, t, t); } } - if (shadingLanguageVersion.supportedTrunc()) { - final String name = "trunc"; + { + final String name = "exp"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } - if (shadingLanguageVersion.supportedRound()) { - final String name = "round"; + { + final String name = "log"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } - if (shadingLanguageVersion.supportedRoundEven()) { - final String name = "roundEven"; + { + final String name = "exp2"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } { - final String name = "ceil"; + final String name = "log2"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } { - final String name = "fract"; + final String name = "sqrt"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } { - final String name = "mod"; + final String name = "inversesqrt"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.FLOAT); - if (t != BasicType.FLOAT) { - addBuiltin(builtinsForVersion, name, t, t, t); - } + addBuiltin(builtinsForVersion, name, t, t); } } + } - // TODO: genType modf(genType, out genType) + private static void getBuiltinsForGlslVersionVectorRelational( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + // We need these for every function, so instead of constantly calling the functions, + // we'll just cache them to reduce cruft. + final List genVectors = genType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final List igenVectors = igenType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final List ugenVectors = ugenType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final List bgenVectors = bgenType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final boolean supportsUnsigned = shadingLanguageVersion.supportedUnsigned(); { - final String name = "min"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.FLOAT); - if (t != BasicType.FLOAT) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - if (shadingLanguageVersion.supportedMinInt()) { - for (Type t : igenType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.INT); - if (t != BasicType.INT) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - } - if (shadingLanguageVersion.supportedMinUint()) { - for (Type t : ugenType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.UINT); - if (t != BasicType.UINT) { - addBuiltin(builtinsForVersion, name, t, t, t); - } + final String name = "lessThan"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); } } } { - final String name = "max"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.FLOAT); - if (t != BasicType.FLOAT) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - if (shadingLanguageVersion.supportedMaxInt()) { - for (Type t : igenType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.INT); - if (t != BasicType.INT) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - } - if (shadingLanguageVersion.supportedMaxUint()) { - for (Type t : ugenType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.UINT); - if (t != BasicType.UINT) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - } - } - - { - final String name = "clamp"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.FLOAT, BasicType.FLOAT); - if (t != BasicType.FLOAT) { - addBuiltin(builtinsForVersion, name, t, t, t, t); - } - } - if (shadingLanguageVersion.supportedClampInt()) { - for (Type t : igenType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); - if (t != BasicType.INT) { - addBuiltin(builtinsForVersion, name, t, t, t, t); - } - } - } - if (shadingLanguageVersion.supportedClampUint()) { - for (Type t : ugenType()) { - addBuiltin(builtinsForVersion, name, t, t, BasicType.UINT, BasicType.UINT); - if (t != BasicType.UINT) { - addBuiltin(builtinsForVersion, name, t, t, t, t); - } - } - } - } - - { - final String name = "mix"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t, BasicType.FLOAT); - if (t != BasicType.FLOAT) { - addBuiltin(builtinsForVersion, name, t, t, t, t); - } - } - if (shadingLanguageVersion.supportedMixFloatBool()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.FLOAT, BasicType.FLOAT, - BasicType.BOOL); - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.VEC2, BasicType.VEC2, - BasicType.BVEC2); - addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.VEC3, - BasicType.BVEC3); - addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.VEC4, BasicType.VEC4, - BasicType.BVEC4); - } - - if (shadingLanguageVersion.supportedMixNonfloatBool()) { - addBuiltin(builtinsForVersion, name, BasicType.INT, BasicType.INT, BasicType.INT, - BasicType.BOOL); - addBuiltin(builtinsForVersion, name, BasicType.IVEC2, BasicType.IVEC2, BasicType.IVEC2, - BasicType.BVEC2); - addBuiltin(builtinsForVersion, name, BasicType.IVEC3, BasicType.IVEC3, BasicType.IVEC3, - BasicType.BVEC3); - addBuiltin(builtinsForVersion, name, BasicType.IVEC4, BasicType.IVEC4, BasicType.IVEC4, - BasicType.BVEC4); - - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.UINT, BasicType.UINT, - BasicType.BOOL); - addBuiltin(builtinsForVersion, name, BasicType.UVEC2, BasicType.UVEC2, BasicType.UVEC2, - BasicType.BVEC2); - addBuiltin(builtinsForVersion, name, BasicType.UVEC3, BasicType.UVEC3, BasicType.UVEC3, - BasicType.BVEC3); - addBuiltin(builtinsForVersion, name, BasicType.UVEC4, BasicType.UVEC4, BasicType.UVEC4, - BasicType.BVEC4); - - addBuiltin(builtinsForVersion, name, BasicType.BOOL, BasicType.BOOL, BasicType.BOOL, - BasicType.BOOL); - addBuiltin(builtinsForVersion, name, BasicType.BVEC2, BasicType.BVEC2, BasicType.BVEC2, - BasicType.BVEC2); - addBuiltin(builtinsForVersion, name, BasicType.BVEC3, BasicType.BVEC3, BasicType.BVEC3, - BasicType.BVEC3); - addBuiltin(builtinsForVersion, name, BasicType.BVEC4, BasicType.BVEC4, BasicType.BVEC4, - BasicType.BVEC4); - } - } - - { - final String name = "step"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, BasicType.FLOAT, t); - if (t != BasicType.FLOAT) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - } - - { - final String name = "smoothstep"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, BasicType.FLOAT, BasicType.FLOAT, t); - if (t != BasicType.FLOAT) { - addBuiltin(builtinsForVersion, name, t, t, t, t); - } - } - } - - if (shadingLanguageVersion.supportedIsnan()) { - final String name = "isnan"; - addBuiltin(builtinsForVersion, name, BasicType.BOOL, BasicType.FLOAT); - addBuiltin(builtinsForVersion, name, BasicType.BVEC2, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.BVEC3, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.BVEC4, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedIsinf()) { - final String name = "isinf"; - addBuiltin(builtinsForVersion, name, BasicType.BOOL, BasicType.FLOAT); - addBuiltin(builtinsForVersion, name, BasicType.BVEC2, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.BVEC3, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.BVEC4, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedFloatBitsToInt()) { - final String name = "floatBitsToInt"; - addBuiltin(builtinsForVersion, name, BasicType.INT, BasicType.FLOAT); - addBuiltin(builtinsForVersion, name, BasicType.IVEC2, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.IVEC3, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.IVEC4, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedFloatBitsToUint()) { - final String name = "floatBitsToUint"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.FLOAT); - addBuiltin(builtinsForVersion, name, BasicType.UVEC2, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.UVEC3, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.UVEC4, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedIntBitsToFloat()) { - final String name = "intBitsToFloat"; - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.INT); - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.IVEC2); - addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.IVEC3); - addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.IVEC4); - } - - if (shadingLanguageVersion.supportedUintBitsToFloat()) { - final String name = "uintBitsToFloat"; - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.UINT); - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UVEC2); - addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.UVEC3); - addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UVEC4); - } - - if (shadingLanguageVersion.supportedFma()) { - final String name = "fma"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t, t); - } - } - - { - @SuppressWarnings("unused") - final String name = "frexp"; - // TODO: genType frexp(genType, out genIType) - } - - { - @SuppressWarnings("unused") - final String name = "ldexp"; - // TODO: genType frexp(genType, in genIType) - } - - // 8.4: Floating-Point Pack and Unpack Functions - - getBuiltinsForGlslVersionFloatingPointPackAndUnpack(builtinsForVersion, shadingLanguageVersion); - - // 8.5: Geometric Functions - - getBuiltinsForGlslVersionGeometric(builtinsForVersion); - - // 8.6: Matrix Functions - - getBuiltinsForGlslVersionMatrix(builtinsForVersion, shadingLanguageVersion); - - // 8.7: Vector Relational Functions - - getBuiltinsForGlslVersionVectorRelational(builtinsForVersion, shadingLanguageVersion); - - // 8.8: Integer Functions - - getBuiltinsForGlslVersionInteger(builtinsForVersion, shadingLanguageVersion); - - // 8.13: Fragment Processing Functions (only available in fragment shaders) - - if (shaderKind == ShaderKind.FRAGMENT) { - getBuiltinsForGlslVersionFragmentProcessing(builtinsForVersion, shadingLanguageVersion); - } - - // 8.14: Noise Functions - deprecated, so we do not consider them - - return builtinsForVersion; - } - - /** - * Helper function to register built-in function prototypes for Angle and Trigonometric - * Functions, as specified in section 8.1 of the GLSL 4.6 and ESSL 3.2 specifications. - * - * @param builtinsForVersion the list of builtins to add prototypes to - * @param shadingLanguageVersion the version of GLSL in use - */ - private static void getBuiltinsForGlslVersionAngleAndTrigonometric( - ShadingLanguageVersion shadingLanguageVersion, - Map> builtinsForVersion) { - if (shadingLanguageVersion.supportedAngleAndTrigonometricFunctions()) { - { - final String name = "radians"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "degrees"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "sin"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "cos"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "tan"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "asin"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "acos"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "atan"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - } - - if (shadingLanguageVersion.supportedHyperbolicAngleAndTrigonometricFunctions()) { - { - final String name = "sinh"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "cosh"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "tanh"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "asinh"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "acosh"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "atanh"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - } - } - - /** - * Helper function to register built-in function prototypes for Exponential Functions, as - * specified in section 8.2 of the GLSL 4.6 and ESSL 3.2 specifications. - * - * @param builtinsForVersion the list of builtins to add prototypes to - */ - private static void getBuiltinsForGlslVersionExponential( - Map> builtinsForVersion) { - { - final String name = "pow"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - - { - final String name = "exp"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "log"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "exp2"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "log2"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "sqrt"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "inversesqrt"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - } - - private static void getBuiltinsForGlslVersionVectorRelational( - Map> builtinsForVersion, - ShadingLanguageVersion shadingLanguageVersion) { - // We need these for every function, so instead of constantly calling the functions, - // we'll just cache them to reduce cruft. - final List genVectors = genType().stream().filter( - item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); - final List igenVectors = igenType().stream().filter( - item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); - final List ugenVectors = ugenType().stream().filter( - item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); - final List bgenVectors = bgenType().stream().filter( - item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); - final boolean supportsUnsigned = shadingLanguageVersion.supportedUnsigned(); - - { - final String name = "lessThan"; - for (int i = 0; i < bgenVectors.size(); i++) { - addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), - genVectors.get(i)); - addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), - igenVectors.get(i)); - if (supportsUnsigned) { - addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), - ugenVectors.get(i)); - } - } - } - - { - final String name = "lessThanEqual"; - for (int i = 0; i < bgenVectors.size(); i++) { - addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), - genVectors.get(i)); - addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), - igenVectors.get(i)); - if (supportsUnsigned) { - addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), - ugenVectors.get(i)); + final String name = "lessThanEqual"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); } } } @@ -1192,130 +902,453 @@ private static void getBuiltinsForGlslVersionMatrix( addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.MAT3X3); addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.MAT4X4); } - } + } + + /** + * Helper function to register built-in function prototypes for Geometric Functions, + * as specified in section 8.5 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + */ + private static void getBuiltinsForGlslVersionGeometric( + Map> builtinsForVersion) { + { + final String name = "length"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t); + } + } + + { + final String name = "distance"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); + } + } + + { + final String name = "dot"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); + } + } + + { + final String name = "cross"; + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.VEC3); + } + + { + final String name = "normalize"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "faceforward"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, t, t); + } + } + + { + final String name = "reflect"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + + { + final String name = "refract"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, t, BasicType.FLOAT); + } + } + } + + /** + * Helper function to register built-in function prototypes for Floating-Point Pack and + * Unpack Functions, as specified in section 8.4 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionFloatingPointPackAndUnpack( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + + if (shadingLanguageVersion.supportedPackUnorm2x16()) { + final String name = "packUnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + } + + if (shadingLanguageVersion.supportedPackSnorm2x16()) { + final String name = "packSnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + } + + if (shadingLanguageVersion.supportedPackUnorm4x8()) { + final String name = "packUnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); + } + + if (shadingLanguageVersion.supportedPackSnorm4x8()) { + final String name = "packSnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); + } + + if (shadingLanguageVersion.supportedUnpackUnorm2x16()) { + final String name = "unpackUnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedUnpackSnorm2x16()) { + final String name = "unpackSnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedUnpackUnorm4x8()) { + final String name = "unpackUnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedUnpackSnorm4x8()) { + final String name = "unpackSnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedPackHalf2x16()) { + final String name = "packHalf2x16"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + } + + if (shadingLanguageVersion.supportedUnpackHalf2x16()) { + final String name = "unpackHalf2x16"; + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + } + } + + /** + * Helper function to register built-in function prototypes for Common Functions, + * as specified in section 8.3 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionCommon( + ShadingLanguageVersion shadingLanguageVersion, + Map> builtinsForVersion) { + { + final String name = "abs"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + if (shadingLanguageVersion.supportedAbsInt()) { + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + + { + final String name = "sign"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + if (shadingLanguageVersion.supportedSignInt()) { + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + + { + final String name = "floor"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + if (shadingLanguageVersion.supportedTrunc()) { + final String name = "trunc"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + if (shadingLanguageVersion.supportedRound()) { + final String name = "round"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } - /** - * Helper function to register built-in function prototypes for Geometric Functions, - * as specified in section 8.5 of the GLSL 4.6 and ESSL 3.2 specifications. - * - * @param builtinsForVersion the list of builtins to add prototypes to - */ - private static void getBuiltinsForGlslVersionGeometric( - Map> builtinsForVersion) { - { - final String name = "length"; + if (shadingLanguageVersion.supportedRoundEven()) { + final String name = "roundEven"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t); + addBuiltin(builtinsForVersion, name, t, t); } } { - final String name = "distance"; + final String name = "ceil"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); + addBuiltin(builtinsForVersion, name, t, t); } } { - final String name = "dot"; + final String name = "fract"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); + addBuiltin(builtinsForVersion, name, t, t); } } { - final String name = "cross"; - addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.VEC3); + final String name = "mod"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.FLOAT); + if (t != BasicType.FLOAT) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + } + + if (shadingLanguageVersion.supportedModf()) { + { + final String name = "modf"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, new QualifiedType(t, + Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } } { - final String name = "normalize"; + final String name = "min"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, t, t, BasicType.FLOAT); + if (t != BasicType.FLOAT) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + if (shadingLanguageVersion.supportedMinInt()) { + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.INT); + if (t != BasicType.INT) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + } + if (shadingLanguageVersion.supportedMinUint()) { + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.UINT); + if (t != BasicType.UINT) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } } } { - final String name = "faceforward"; + final String name = "max"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t, t); + addBuiltin(builtinsForVersion, name, t, t, BasicType.FLOAT); + if (t != BasicType.FLOAT) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + if (shadingLanguageVersion.supportedMaxInt()) { + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.INT); + if (t != BasicType.INT) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + } + if (shadingLanguageVersion.supportedMaxUint()) { + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.UINT); + if (t != BasicType.UINT) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } } } { - final String name = "reflect"; + final String name = "clamp"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t); + addBuiltin(builtinsForVersion, name, t, t, BasicType.FLOAT, BasicType.FLOAT); + if (t != BasicType.FLOAT) { + addBuiltin(builtinsForVersion, name, t, t, t, t); + } + } + if (shadingLanguageVersion.supportedClampInt()) { + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); + if (t != BasicType.INT) { + addBuiltin(builtinsForVersion, name, t, t, t, t); + } + } + } + if (shadingLanguageVersion.supportedClampUint()) { + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.UINT, BasicType.UINT); + if (t != BasicType.UINT) { + addBuiltin(builtinsForVersion, name, t, t, t, t); + } + } } } { - final String name = "refract"; + final String name = "mix"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t, t, BasicType.FLOAT); + if (t != BasicType.FLOAT) { + addBuiltin(builtinsForVersion, name, t, t, t, t); + } + } + if (shadingLanguageVersion.supportedMixFloatBool()) { + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.FLOAT, BasicType.FLOAT, + BasicType.BOOL); + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.VEC2, BasicType.VEC2, + BasicType.BVEC2); + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.VEC3, + BasicType.BVEC3); + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.VEC4, BasicType.VEC4, + BasicType.BVEC4); + } + + if (shadingLanguageVersion.supportedMixNonfloatBool()) { + addBuiltin(builtinsForVersion, name, BasicType.INT, BasicType.INT, BasicType.INT, + BasicType.BOOL); + addBuiltin(builtinsForVersion, name, BasicType.IVEC2, BasicType.IVEC2, BasicType.IVEC2, + BasicType.BVEC2); + addBuiltin(builtinsForVersion, name, BasicType.IVEC3, BasicType.IVEC3, BasicType.IVEC3, + BasicType.BVEC3); + addBuiltin(builtinsForVersion, name, BasicType.IVEC4, BasicType.IVEC4, BasicType.IVEC4, + BasicType.BVEC4); + + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.UINT, BasicType.UINT, + BasicType.BOOL); + addBuiltin(builtinsForVersion, name, BasicType.UVEC2, BasicType.UVEC2, BasicType.UVEC2, + BasicType.BVEC2); + addBuiltin(builtinsForVersion, name, BasicType.UVEC3, BasicType.UVEC3, BasicType.UVEC3, + BasicType.BVEC3); + addBuiltin(builtinsForVersion, name, BasicType.UVEC4, BasicType.UVEC4, BasicType.UVEC4, + BasicType.BVEC4); + + addBuiltin(builtinsForVersion, name, BasicType.BOOL, BasicType.BOOL, BasicType.BOOL, + BasicType.BOOL); + addBuiltin(builtinsForVersion, name, BasicType.BVEC2, BasicType.BVEC2, BasicType.BVEC2, + BasicType.BVEC2); + addBuiltin(builtinsForVersion, name, BasicType.BVEC3, BasicType.BVEC3, BasicType.BVEC3, + BasicType.BVEC3); + addBuiltin(builtinsForVersion, name, BasicType.BVEC4, BasicType.BVEC4, BasicType.BVEC4, + BasicType.BVEC4); } } - } - /** - * Helper function to register built-in function prototypes for Floating-Point Pack and - * Unpack Functions, as specified in section 8.4 of the GLSL 4.6 and ESSL 3.2 specifications. - * - * @param builtinsForVersion the list of builtins to add prototypes to - * @param shadingLanguageVersion the version of GLSL in use - */ - private static void getBuiltinsForGlslVersionFloatingPointPackAndUnpack( - Map> builtinsForVersion, - ShadingLanguageVersion shadingLanguageVersion) { + { + final String name = "step"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, BasicType.FLOAT, t); + if (t != BasicType.FLOAT) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + } - if (shadingLanguageVersion.supportedPackUnorm2x16()) { - final String name = "packUnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + { + final String name = "smoothstep"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, BasicType.FLOAT, BasicType.FLOAT, t); + if (t != BasicType.FLOAT) { + addBuiltin(builtinsForVersion, name, t, t, t, t); + } + } } - if (shadingLanguageVersion.supportedPackSnorm2x16()) { - final String name = "packSnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + if (shadingLanguageVersion.supportedIsnan()) { + final String name = "isnan"; + addBuiltin(builtinsForVersion, name, BasicType.BOOL, BasicType.FLOAT); + addBuiltin(builtinsForVersion, name, BasicType.BVEC2, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.BVEC3, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.BVEC4, BasicType.VEC4); } - if (shadingLanguageVersion.supportedPackUnorm4x8()) { - final String name = "packUnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); + if (shadingLanguageVersion.supportedIsinf()) { + final String name = "isinf"; + addBuiltin(builtinsForVersion, name, BasicType.BOOL, BasicType.FLOAT); + addBuiltin(builtinsForVersion, name, BasicType.BVEC2, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.BVEC3, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.BVEC4, BasicType.VEC4); } - if (shadingLanguageVersion.supportedPackSnorm4x8()) { - final String name = "packSnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); + if (shadingLanguageVersion.supportedFloatBitsToInt()) { + final String name = "floatBitsToInt"; + addBuiltin(builtinsForVersion, name, BasicType.INT, BasicType.FLOAT); + addBuiltin(builtinsForVersion, name, BasicType.IVEC2, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.IVEC3, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.IVEC4, BasicType.VEC4); } - if (shadingLanguageVersion.supportedUnpackUnorm2x16()) { - final String name = "unpackUnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + if (shadingLanguageVersion.supportedFloatBitsToUint()) { + final String name = "floatBitsToUint"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.FLOAT); + addBuiltin(builtinsForVersion, name, BasicType.UVEC2, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.UVEC3, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.UVEC4, BasicType.VEC4); } - if (shadingLanguageVersion.supportedUnpackSnorm2x16()) { - final String name = "unpackSnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + if (shadingLanguageVersion.supportedIntBitsToFloat()) { + final String name = "intBitsToFloat"; + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.INT); + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.IVEC2); + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.IVEC3); + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.IVEC4); } - if (shadingLanguageVersion.supportedUnpackUnorm4x8()) { - final String name = "unpackUnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); + if (shadingLanguageVersion.supportedUintBitsToFloat()) { + final String name = "uintBitsToFloat"; + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.UINT); + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UVEC2); + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.UVEC3); + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UVEC4); } - if (shadingLanguageVersion.supportedUnpackSnorm4x8()) { - final String name = "unpackSnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); + if (shadingLanguageVersion.supportedFma()) { + final String name = "fma"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, t, t); + } } - if (shadingLanguageVersion.supportedPackHalf2x16()) { - final String name = "packHalf2x16"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + if (shadingLanguageVersion.supportedFrexp()) { + { + final String name = "frexp"; + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.FLOAT, + new QualifiedType(BasicType.INT, Arrays.asList(TypeQualifier.OUT_PARAM))); + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.VEC2, + new QualifiedType(BasicType.IVEC2, Arrays.asList(TypeQualifier.OUT_PARAM))); + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, + new QualifiedType(BasicType.IVEC3, Arrays.asList(TypeQualifier.OUT_PARAM))); + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.VEC4, + new QualifiedType(BasicType.IVEC4, Arrays.asList(TypeQualifier.OUT_PARAM))); + } } - if (shadingLanguageVersion.supportedUnpackHalf2x16()) { - final String name = "unpackHalf2x16"; - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + if (shadingLanguageVersion.supportedLdexp()) { + { + final String name = "ldexp"; + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.FLOAT, BasicType.INT); + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.VEC2, BasicType.IVEC2); + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.IVEC3); + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.VEC4, BasicType.IVEC4); + } } } diff --git a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java index 2b2ae5024..17e894abe 100644 --- a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java +++ b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java @@ -96,4 +96,54 @@ public void visitFunctionCallExpr(FunctionCallExpr expr) { } }.visit(tu); } + + @Test + public void testOutParamModfHasSideEffects() throws Exception { + final String shader = "void main() { " + + " float out1;" + + " vec2 out2;" + + " vec3 out3;" + + " vec4 out4;" + + " modf(0.0, out1);" + + " modf(vec2(0.0), out2);" + + " modf(vec3(0.0), out3);" + + " modf(vec4(0.0), out4);" + + "}"; + + final TranslationUnit tu = ParseHelper.parse(shader, ShaderKind.FRAGMENT); + tu.setShadingLanguageVersion(ShadingLanguageVersion.ESSL_310); + + new StandardVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr expr) { + assertTrue(expr.getCallee().equals("modf")); + assertFalse(SideEffectChecker.isSideEffectFree(expr, ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); + } + }.visit(tu); + } + + @Test + public void testOutParamFrexpHasSideEffects() throws Exception { + final String shader = "void main() { " + + " int out1;" + + " ivec2 out2;" + + " ivec3 out3;" + + " ivec4 out4;" + + " frexp(0.0, out1);" + + " frexp(vec2(0.0), out2);" + + " frexp(vec3(0.0), out3);" + + " frexp(vec4(0.0), out4);" + + "}"; + + final TranslationUnit tu = ParseHelper.parse(shader, ShaderKind.FRAGMENT); + tu.setShadingLanguageVersion(ShadingLanguageVersion.ESSL_310); + + new StandardVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr expr) { + assertTrue(expr.getCallee().equals("frexp")); + assertFalse(SideEffectChecker.isSideEffectFree(expr, ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); + } + }.visit(tu); + } } From 8e110596a38052df17550509c2de114388daf12e Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Mon, 8 Jul 2019 20:42:50 +0700 Subject: [PATCH 058/180] Fix glsl-to-spv-worker --local-shader-job (#593) --- python/src/main/python/drivers/glsl-to-spv-worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/main/python/drivers/glsl-to-spv-worker.py b/python/src/main/python/drivers/glsl-to-spv-worker.py index 871a00886..a5fae4011 100755 --- a/python/src/main/python/drivers/glsl-to-spv-worker.py +++ b/python/src/main/python/drivers/glsl-to-spv-worker.py @@ -539,7 +539,7 @@ def main(): assert os.path.isfile(args.local_shader_job), \ 'Shader job {} does not exist'.format(args.local_shader_job) - with gfuzz_common.open_helper(args.local_shader_job) as f: + with gfuzz_common.open_helper(args.local_shader_job, 'r') as f: fake_job.uniformsInfo = f.read() if os.path.isfile(shader_job_prefix + '.frag'): with gfuzz_common.open_helper(shader_job_prefix + '.frag', 'r') as f: From 5620fd6fff84e97908799b26765950761295b27e Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 9 Jul 2019 09:44:55 +0100 Subject: [PATCH 059/180] Use ShaderJobFileOperations more consistently (#594) This change reduces the number of places in the code base where PrettyPrinterVisitor.emitShader() is directly invoked, towards a situation where ShaderJobFileOperations is the sole place where shaders are emitted. --- .../common/util/ShaderJobFileOperations.java | 3 - .../common/util/PruneUniformsTest.java | 44 +++------ .../graphicsfuzz/generator/tool/Mutate.java | 20 ++-- .../fuzzer/OpaqueExpressionGeneratorTest.java | 34 ++----- .../tester/GeneratorUnitTest.java | 65 ++++++------ .../tester/ObfuscatorUnitTest.java | 5 +- .../graphicsfuzz/tester/ReducerUnitTest.java | 80 +++++++-------- .../java/com/graphicsfuzz/tester/Util.java | 99 +++---------------- 8 files changed, 114 insertions(+), 236 deletions(-) diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java index 64d5ec8b6..0261ac0c5 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java @@ -55,11 +55,8 @@ import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import java.util.stream.Stream; import javax.imageio.ImageIO; import javax.imageio.stream.FileImageOutputStream; diff --git a/common/src/test/java/com/graphicsfuzz/common/util/PruneUniformsTest.java b/common/src/test/java/com/graphicsfuzz/common/util/PruneUniformsTest.java index 86800e805..422313110 100755 --- a/common/src/test/java/com/graphicsfuzz/common/util/PruneUniformsTest.java +++ b/common/src/test/java/com/graphicsfuzz/common/util/PruneUniformsTest.java @@ -16,16 +16,10 @@ package com.graphicsfuzz.common.util; -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.transformreduce.GlslShaderJob; -import com.graphicsfuzz.util.ExecHelper.RedirectType; -import com.graphicsfuzz.util.ExecResult; -import com.graphicsfuzz.util.ToolHelper; +import com.graphicsfuzz.common.transformreduce.ShaderJob; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; @@ -183,39 +177,27 @@ private void doPruneTest(String program, String uniforms, String expectedProgram String expectedUniforms, List prefixList, int limit) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - final File uniformsFile = temporaryFolder.newFile("uniforms.json"); - FileUtils.writeStringToFile(uniformsFile, uniforms, StandardCharsets.UTF_8); - final PipelineInfo pipelineInfo = new PipelineInfo(uniformsFile); - final TranslationUnit tu = ParseHelper.parse(program); + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), new PipelineInfo(uniforms), + ParseHelper.parse(program)); assertTrue(PruneUniforms.prune( - new GlslShaderJob(Optional.empty(), pipelineInfo, tu), + shaderJob, limit, prefixList)); - final File shaderFile = temporaryFolder.newFile("shader.frag"); - - try (PrintStream stream = new PrintStream(new FileOutputStream(shaderFile))) { - PrettyPrinterVisitor.emitShader( - tu, - Optional.empty(), - stream, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - true - ); - } - final ExecResult execResult = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, shaderFile); - assertEquals(0, execResult.res); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); final File expectedUniformsFile = temporaryFolder.newFile("expecteduniforms.json"); FileUtils.writeStringToFile(expectedUniformsFile, expectedUniforms, StandardCharsets.UTF_8); - assertEquals(PrettyPrinterVisitor.prettyPrintAsString( - ParseHelper.parse(expectedProgram)), - PrettyPrinterVisitor.prettyPrintAsString(tu)); - assertEquals(new PipelineInfo(expectedUniformsFile).toString(), - pipelineInfo.toString()); + CompareAsts.assertEqualAsts(expectedProgram, shaderJob.getFragmentShader().get()); + assertEquals(new PipelineInfo(expectedUniforms).toString(), + shaderJob.getPipelineInfo().toString()); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java index f839f09c1..996fdffa4 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java @@ -18,12 +18,15 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.generator.mutateapi.Mutation; import com.graphicsfuzz.generator.mutateapi.MutationFinder; import com.graphicsfuzz.generator.semanticschanging.AddArrayMutationFinder; @@ -62,6 +65,7 @@ import net.sourceforge.argparse4j.inf.ArgumentParser; import net.sourceforge.argparse4j.inf.ArgumentParserException; import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -131,16 +135,12 @@ public static void mainHelper(String[] args) throws ArgumentParserException, IOE mutate(tu, new RandomWrapper(seed)); - try (PrintStream stream = new PrintStream(new FileOutputStream(output))) { - PrettyPrinterVisitor.emitShader( - tu, - Optional.empty(), - stream, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - false - ); - } + final File shaderJobFile = new File(FilenameUtils.removeExtension(output.getName()) + ".json"); + + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo("{}"), + tu), shaderJobFile, false); } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java index fda14a1bc..bc9ad5b20 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java @@ -33,23 +33,17 @@ import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.typing.Scope; import com.graphicsfuzz.common.typing.SupportedTypes; import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.generator.fuzzer.Fuzzer; -import com.graphicsfuzz.generator.fuzzer.FuzzingContext; -import com.graphicsfuzz.generator.fuzzer.OpaqueExpressionGenerator; import com.graphicsfuzz.generator.tool.Generate; import com.graphicsfuzz.generator.util.GenerationParams; -import com.graphicsfuzz.util.ExecHelper.RedirectType; -import com.graphicsfuzz.util.ExecResult; -import com.graphicsfuzz.util.ToolHelper; import java.io.File; -import java.io.FileOutputStream; -import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -58,7 +52,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class OpaqueExpressionGeneratorTest { @@ -83,20 +77,12 @@ public void testGeneratesValidExpressions() throws Exception { shadingLanguageVersion, 1000), false)))); Generate.addInjectionSwitchIfNotPresent(tu); - final File file = temporaryFolder.newFile("ex.frag"); - try (PrintStream stream = new PrintStream(new FileOutputStream(file))) { - PrettyPrinterVisitor.emitShader( - tu, - Optional.empty(), - stream, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - true - ); - } - ExecResult execResult = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, file); - assertEquals(0, execResult.res); - file.delete(); + final File shaderJobFile = temporaryFolder.newFile("ex.json"); + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo("{}"), + tu), shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); + fileOps.deleteShaderJobFile(shaderJobFile); } } } diff --git a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java index 618f7811e..425d71115 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java @@ -17,9 +17,12 @@ package com.graphicsfuzz.tester; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; @@ -38,9 +41,7 @@ import com.graphicsfuzz.generator.transformation.VectorizeTransformation; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; @@ -48,17 +49,17 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; import org.bytedeco.javacpp.indexer.UByteIndexer; import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_imgcodecs; -import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class GeneratorUnitTest { @@ -77,18 +78,19 @@ public class GeneratorUnitTest { @Test public void testRenderBlackImage() throws Exception { - File shaderFile = temporaryFolder.newFile("shader.frag"); - + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); final String shader = "#version 100\n" + "precision mediump float;\n" + "void main() {\n" + " gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);\n" + "}\n"; - BufferedWriter bw = new BufferedWriter(new FileWriter(shaderFile)); - bw.write(shader); - bw.close(); + fileOps.writeShaderJobFile( + new GlslShaderJob(Optional.empty(), new PipelineInfo(), ParseHelper.parse(shader)), + shaderJobFile, + false); - final File image = Util.getImageUsingSwiftshader(shaderFile, temporaryFolder); + final File image = Util.getImage(shaderJobFile, temporaryFolder, fileOps); final Mat mat = opencv_imgcodecs.imread(image.getAbsolutePath()); final UByteIndexer sI = mat.createIndexer(); @@ -106,32 +108,32 @@ public void testRenderBlackImage() throws Exception { @Test public void testStructify() throws Exception { testTransformationMultiVersions(() -> new StructificationTransformation(), TransformationProbabilities.DEFAULT_PROBABILITIES, - "structs.frag"); + "structs"); } @Test public void testDeadJumps() throws Exception { testTransformationMultiVersions(() -> new AddJumpTransformation(), TransformationProbabilities.onlyAddJumps(), - "jumps.frag"); + "jumps"); } @Test public void testIdentity() throws Exception { testTransformationMultiVersions(() -> new IdentityTransformation(), TransformationProbabilities - .onlyMutateExpressions(), "mutate.frag"); + .onlyMutateExpressions(), "mutate"); } @Test public void testOutlineStatements() throws Exception { testTransformationMultiVersions(() -> new OutlineStatementTransformation(), TransformationProbabilities.onlyOutlineStatements(), - "outline.frag"); + "outline"); } @Test public void testSplitForLoops() throws Exception { testTransformationMultiVersions(() -> new SplitForLoopTransformation(), TransformationProbabilities - .onlySplitLoops(), "split.frag"); + .onlySplitLoops(), "split"); } @Test @@ -140,7 +142,7 @@ public void testDonateDeadCode() throws Exception { TransformationProbabilities.likelyDonateDeadCode()::donateDeadCodeAtStmt, Util.getDonorsFolder(), GenerationParams.normal(ShaderKind.FRAGMENT, true)), TransformationProbabilities.likelyDonateDeadCode(), - "donatedead.frag", + "donatedead", Arrays.asList("bubblesort_flag.json", "squares.json"), Arrays.asList("bubblesort_flag.json", "squares.json")); // Reason for blacklisting^: slow. @@ -153,7 +155,7 @@ public void testDonateLiveCode() throws Exception { Util.getDonorsFolder(), GenerationParams.normal(ShaderKind.FRAGMENT, true), false), TransformationProbabilities.likelyDonateLiveCode(), - "donatelive.frag", + "donatelive", Arrays.asList("squares.json"), Arrays.asList("squares.json")); // Reason for blacklisting^: slow. @@ -162,27 +164,27 @@ public void testDonateLiveCode() throws Exception { @Test public void testAddDeadFragColorWrites() throws Exception { testTransformationMultiVersions(() -> new AddDeadOutputWriteTransformation(), TransformationProbabilities - .onlyAddDeadFragColorWrites(), "deadfragcolor.frag", Arrays.asList(), Arrays.asList()); + .onlyAddDeadFragColorWrites(), "deadfragcolor", Arrays.asList(), Arrays.asList()); } @Test public void testAddLiveOutputVariableWrites() throws Exception { testTransformationMultiVersions(() -> new AddLiveOutputWriteTransformation(), TransformationProbabilities - .onlyAddLiveFragColorWrites(), "liveoutvar.frag", Arrays.asList(), Arrays.asList()); + .onlyAddLiveFragColorWrites(), "liveoutvar", Arrays.asList(), Arrays.asList()); } @Test public void testVectorize() throws Exception { testTransformationMultiVersions(() -> new VectorizeTransformation(), TransformationProbabilities.onlyVectorize(), - "vectorize.frag", Arrays.asList(), Arrays.asList()); + "vectorize", Arrays.asList(), Arrays.asList()); } @Test public void mutateAndVectorize() throws Exception { testTransformationMultiVersions(Arrays.asList(() -> new IdentityTransformation(), () -> new VectorizeTransformation()), TransformationProbabilities.onlyVectorizeAndMutate(), - "mutate_and_vectorize.frag", + "mutate_and_vectorize", Arrays.asList(), Arrays.asList()); } @@ -190,14 +192,14 @@ public void mutateAndVectorize() throws Exception { public void testStructification() throws Exception { testTransformationMultiVersions(() -> new StructificationTransformation(), TransformationProbabilities.onlyStructify(), - "structify.frag", Arrays.asList(), Arrays.asList()); + "structify", Arrays.asList(), Arrays.asList()); } @Test public void testWrap() throws Exception { testTransformationMultiVersions(() -> new AddWrappingConditionalTransformation(), TransformationProbabilities.onlyWrap(), - "wrap.frag", + "wrap", Arrays.asList("bubblesort_flag.json"), Arrays.asList("bubblesort_flag.json")); // Reason for blacklisting^: slow. @@ -242,8 +244,7 @@ private void testTransformation(List transformations, } File referenceImage = null; if (!skipRender) { - referenceImage = Util.renderShader( - originalShaderJobFile, + referenceImage = Util.validateAndGetImage(originalShaderJobFile, temporaryFolder, fileOps); } @@ -331,29 +332,21 @@ private void generateAndCheckVariant( Generate.setInjectionSwitch(shaderJob.getPipelineInfo()); Generate.randomiseUnsetUniforms(tu, shaderJob.getPipelineInfo(), generator); - // Using fileOps, even though the rest of the code here does not use it yet. // Write shaders to shader job file and validate. - - Assert.assertTrue(suffix.endsWith(".frag")); - // e.g. "_matrix_mult" - final String suffixNoExtension = FilenameUtils.removeExtension(suffix); - // e.g. "temp/orig_matrix_mult.json" final File shaderJobFile = Paths.get( temporaryFolder.getRoot().getAbsolutePath(), - originalShaderJobFile.getName() + suffixNoExtension + ".json" + originalShaderJobFile.getName() + suffix + ".json" ).toFile(); fileOps.writeShaderJobFile( shaderJob, shaderJobFile ); - fileOps.areShadersValid(shaderJobFile, true); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); if (!skipRender) { - File underlyingFragFile = fileOps.getUnderlyingShaderFile(shaderJobFile, ShaderKind.FRAGMENT); - // TODO: Use fileOps. - final File variantImage = Util.getImage(underlyingFragFile, temporaryFolder, fileOps); + final File variantImage = Util.getImage(shaderJobFile, temporaryFolder, fileOps); Util.assertImagesSimilar(referenceImage, variantImage); } diff --git a/tester/src/test/java/com/graphicsfuzz/tester/ObfuscatorUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/ObfuscatorUnitTest.java index 92ea26d4d..de0b6033a 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/ObfuscatorUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/ObfuscatorUnitTest.java @@ -42,14 +42,13 @@ public void testObfuscate() throws Exception { final IRandom generator = new RandomWrapper(0); for (File originalShaderJobFile : Util.getReferenceShaderJobFiles100es(fileOps)) { final File originalImage = - Util.renderShader( - originalShaderJobFile, temporaryFolder, fileOps); + Util.validateAndGetImage(originalShaderJobFile, temporaryFolder, fileOps); final ShaderJob shaderJob = fileOps.readShaderJobFile(originalShaderJobFile); final ShaderJob obfuscated = Obfuscator.obfuscate(shaderJob, generator); final File obfuscatedImage = Util.validateAndGetImage( obfuscated, - originalShaderJobFile.getName() + ".obfuscated.frag", + originalShaderJobFile.getName() + ".obfuscated.json", temporaryFolder, fileOps); assertTrue(FileUtils.contentEquals(originalImage, obfuscatedImage)); diff --git a/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java index e079529ce..c8dd68e50 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java @@ -16,8 +16,6 @@ package com.graphicsfuzz.tester; -import static org.junit.Assert.assertEquals; - import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; @@ -26,32 +24,28 @@ import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import com.graphicsfuzz.common.util.GlslParserException; -import com.graphicsfuzz.common.util.PipelineInfo; -import com.graphicsfuzz.reducer.tool.GlslReduce; -import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.ShaderJobFileOperations; -import com.graphicsfuzz.util.ExecHelper.RedirectType; -import com.graphicsfuzz.util.ExecResult; +import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.util.ToolHelper; +import com.graphicsfuzz.common.util.StatsVisitor; import com.graphicsfuzz.generator.tool.Generate; -import com.graphicsfuzz.generator.transformation.ITransformation; import com.graphicsfuzz.generator.transformation.AddDeadOutputWriteTransformation; import com.graphicsfuzz.generator.transformation.AddJumpTransformation; import com.graphicsfuzz.generator.transformation.AddLiveOutputWriteTransformation; -import com.graphicsfuzz.generator.transformation.SplitForLoopTransformation; import com.graphicsfuzz.generator.transformation.DonateDeadCodeTransformation; import com.graphicsfuzz.generator.transformation.DonateLiveCodeTransformation; +import com.graphicsfuzz.generator.transformation.ITransformation; import com.graphicsfuzz.generator.transformation.IdentityTransformation; import com.graphicsfuzz.generator.transformation.OutlineStatementTransformation; +import com.graphicsfuzz.generator.transformation.SplitForLoopTransformation; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.reducer.CheckAstFeatureVisitor; @@ -59,9 +53,14 @@ import com.graphicsfuzz.reducer.IFileJudge; import com.graphicsfuzz.reducer.ReductionDriver; import com.graphicsfuzz.reducer.reductionopportunities.IReductionOpportunity; -import com.graphicsfuzz.reducer.reductionopportunities.ReductionOpportunities; import com.graphicsfuzz.reducer.reductionopportunities.ReducerContext; +import com.graphicsfuzz.reducer.reductionopportunities.ReductionOpportunities; +import com.graphicsfuzz.reducer.tool.GlslReduce; import com.graphicsfuzz.reducer.tool.RandomFileJudge; +import com.graphicsfuzz.util.Constants; +import com.graphicsfuzz.util.ExecHelper.RedirectType; +import com.graphicsfuzz.util.ExecResult; +import com.graphicsfuzz.util.ToolHelper; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -82,6 +81,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.assertEquals; + public class ReducerUnitTest { private static final Logger LOGGER = LoggerFactory.getLogger(ReducerUnitTest.class); @@ -106,29 +107,29 @@ private void testGenerateAndReduce(File originalShaderJobFile, List transformations, RandomWrapper generator) throws Exception { - // TODO: We reduced this due to slow shader run time. Consider reverting to 100000 when run - // time is improved. - final int maxBytes = 10000; + final int maxAstNodes = 2000; final File referenceImage = - Util.renderShader( - originalShaderJobFile, + Util.validateAndGetImage(originalShaderJobFile, temporaryFolder, fileOps); - final PipelineInfo pipelineInfo = new PipelineInfo(originalShaderJobFile); - final TranslationUnit tu = generateSizeLimitedShader( - fileOps.getUnderlyingShaderFile(originalShaderJobFile, ShaderKind.FRAGMENT), - transformations, generator, maxBytes); - Generate.addInjectionSwitchIfNotPresent(tu); + final ShaderJob shaderJob = generateSizeLimitedShader(originalShaderJobFile, + transformations, generator, maxAstNodes); + final PipelineInfo pipelineInfo = shaderJob.getPipelineInfo(); + final TranslationUnit fragmentShader = shaderJob.getFragmentShader().get(); + Generate.addInjectionSwitchIfNotPresent(fragmentShader); Generate.setInjectionSwitch(pipelineInfo); - Generate.randomiseUnsetUniforms(tu, pipelineInfo, generator); + Generate.randomiseUnsetUniforms(fragmentShader, + pipelineInfo, + generator); final IdGenerator idGenerator = new IdGenerator(); for (int step = 0; step < 10; step++) { List ops = ReductionOpportunities.getReductionOpportunities( - new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu), - new ReducerContext(false, tu.getShadingLanguageVersion(), generator, idGenerator, true), + new GlslShaderJob(Optional.empty(), pipelineInfo, + fragmentShader), + new ReducerContext(false, fragmentShader.getShadingLanguageVersion(), generator, idGenerator, true), fileOps); if (ops.isEmpty()) { break; @@ -136,12 +137,10 @@ private void testGenerateAndReduce(File originalShaderJobFile, LOGGER.info("Step: {}; ops: {}", step, ops.size()); ops.get(generator.nextInt(ops.size())).applyReduction(); - final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), pipelineInfo, tu); - final File variantImage = Util.validateAndGetImage( shaderJob, - originalShaderJobFile.getName() + "_reduced_" + step + ".frag", + originalShaderJobFile.getName() + "_reduced_" + step + ".json", temporaryFolder, fileOps); Util.assertImagesSimilar(referenceImage, variantImage); @@ -149,31 +148,24 @@ private void testGenerateAndReduce(File originalShaderJobFile, } - private TranslationUnit generateSizeLimitedShader( - File fragmentShader, + private ShaderJob generateSizeLimitedShader( + File shaderJobFile, List transformations, IRandom generator, - final int maxBytes + final int maxAstNodes ) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { while (true) { List transformationsCopy = new ArrayList<>(); transformationsCopy.addAll(transformations); - final TranslationUnit tu = ParseHelper.parse(fragmentShader); + final ShaderJob shaderJob = fileOps.readShaderJobFile(shaderJobFile); + final TranslationUnit fragmentShader = shaderJob.getFragmentShader().get(); for (int i = 0; i < 4; i++) { getTransformation(transformationsCopy, generator).apply( - tu, TransformationProbabilities.DEFAULT_PROBABILITIES, + fragmentShader, TransformationProbabilities.DEFAULT_PROBABILITIES, generator, GenerationParams.normal(ShaderKind.FRAGMENT, true)); } - File tempFile = temporaryFolder.newFile(); - PrettyPrinterVisitor.emitShader(tu, Optional.empty(), - new PrintStream( - new FileOutputStream(tempFile)), - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - true - ); - if (tempFile.length() <= maxBytes) { - return tu; + if (new StatsVisitor(fragmentShader).getNumNodes() <= maxAstNodes) { + return shaderJob; } } } diff --git a/tester/src/test/java/com/graphicsfuzz/tester/Util.java b/tester/src/test/java/com/graphicsfuzz/tester/Util.java index b382541f8..2ce2b9566 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/Util.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/Util.java @@ -16,13 +16,8 @@ package com.graphicsfuzz.tester; -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.FileHelper; -import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.ImageUtil; -import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.util.ExecHelper.RedirectType; @@ -33,10 +28,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Paths; -import java.util.Optional; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.junit.Assert; import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; @@ -48,99 +39,37 @@ private Util() { // Utility class } - static File renderShader(File originalShader, - TemporaryFolder temporaryFolder, ShaderJobFileOperations fileOps) - throws IOException, InterruptedException, ParseTimeoutException, GlslParserException { - final ShaderJob shaderJob = fileOps.readShaderJobFile(originalShader); - - return validateAndGetImage( - shaderJob, - originalShader.getName() + ".reference.frag", - temporaryFolder, - fileOps); - } - static File validateAndGetImage( - File shaderFile, + File shaderJobFile, TemporaryFolder temporaryFolder, ShaderJobFileOperations fileOps) throws IOException, InterruptedException { - - validate(shaderFile); - return getImage(shaderFile, temporaryFolder, fileOps); - } - - static File getImage( - File shaderFile, - TemporaryFolder temporaryFolder, - ShaderJobFileOperations fileOps) throws IOException, InterruptedException { - - final Optional shaderTranslatorArg = getShaderTranslatorArg(shaderFile, fileOps); - if (shaderTranslatorArg.isPresent()) { - final ExecResult shaderTranslatorResult = ToolHelper - .runShaderTranslatorOnShader(RedirectType.TO_LOG, shaderFile, - shaderTranslatorArg.get()); - assertEquals(0, shaderTranslatorResult.res); - } - return getImageUsingSwiftshader(shaderFile, temporaryFolder); - } - - static void validate(File shaderFile) throws IOException, InterruptedException { - final ExecResult validatorResult = ToolHelper - .runValidatorOnShader(RedirectType.TO_LOG, shaderFile); - assertEquals(validatorResult.res, 0); - } - - private static Optional getShaderTranslatorArg( - File shaderFile, - ShaderJobFileOperations fileOps) throws IOException { - Optional shaderTranslatorArgs; - - // Using fileOps here, even though the rest of the code does not use it yet. - Assert.assertTrue(shaderFile.getName().endsWith(".frag")); - final File shaderJobFile = FileHelper.replaceExtension(shaderFile, ".json"); - - final ShadingLanguageVersion shadingLanguageVersionFromShader = - ShadingLanguageVersion.getGlslVersionFromFirstTwoLines( - fileOps.getFirstTwoLinesOfShader(shaderJobFile, ShaderKind.FRAGMENT)); - // TODO: Use fileOps. - - if (shadingLanguageVersionFromShader == ShadingLanguageVersion.ESSL_300 - || shadingLanguageVersionFromShader == ShadingLanguageVersion.WEBGL2_SL) { - shaderTranslatorArgs = Optional.of("-s=w2"); - } else if(shadingLanguageVersionFromShader == ShadingLanguageVersion.WEBGL_SL) { - shaderTranslatorArgs = Optional.of("-s=w"); - } else { - shaderTranslatorArgs = Optional.empty(); - } - return shaderTranslatorArgs; + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); + assertTrue(fileOps.areShadersValidShaderTranslator(shaderJobFile, false)); + return getImage(shaderJobFile, temporaryFolder, fileOps); } static File validateAndGetImage( ShaderJob shaderJob, - String fileName, + String shaderJobFilename, TemporaryFolder temporaryFolder, ShaderJobFileOperations fileOps) throws IOException, InterruptedException { - - final File shaderJobFileOutput = new File( + final File shaderJobFile = new File( temporaryFolder.getRoot(), - FilenameUtils.removeExtension(fileName) + ".json"); - - fileOps.writeShaderJobFile(shaderJob, shaderJobFileOutput); - - // TODO: Use fileOps more. - - final File tempFile = FileHelper.replaceExtension(shaderJobFileOutput, ".frag"); - - return validateAndGetImage(tempFile, temporaryFolder, fileOps); + shaderJobFilename); + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); + return validateAndGetImage(shaderJobFile, temporaryFolder, fileOps); } - static File getImageUsingSwiftshader(File shaderFile, TemporaryFolder temporaryFolder) throws IOException, InterruptedException { + static File getImage( + File shaderJobFile, + TemporaryFolder temporaryFolder, + ShaderJobFileOperations fileOps) throws IOException, InterruptedException { File imageFile = temporaryFolder.newFile(); ExecResult res = ToolHelper.runSwiftshaderOnShader(RedirectType.TO_LOG, - shaderFile, + fileOps.getUnderlyingShaderFile(shaderJobFile, ShaderKind.FRAGMENT), imageFile, false, 32, From ddecb81f14f57f86a8f0afeafdf29a01636b431d Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 9 Jul 2019 18:29:35 +0100 Subject: [PATCH 060/180] Emit GraphicsFuzz defines only when necessary (#595) This change introduces functionality to check whether a shader uses one or more of the GraphicsFuzz macros, and only emit the macros if so. This involved refactoring PrettyPrinterVisitor.emitShader() and ShaderJobFileOperations.writeShaderJobFile() so that they no longer take a boolean argument specifying whether the macros should be written. In the course of doing this refactoring, many places where null parameters were being passed to the constructor of ReducerContext (in tests, due to them not being needed) were replaced with non-null, standard parameters. The change also gets rid of some never-used 'main' methods that would otherwise have needed to be fixed up, and fixes a few cases where shaders were being emitted in non-standard ways. Fixes #479. --- .../graphicsfuzz/common/tool/PrettyPrint.java | 14 --- .../common/tool/PrettyPrinterVisitor.java | 17 +++- .../graphicsfuzz/common/util}/MacroNames.java | 21 +++- .../common/tool/PrettyPrinterVisitorTest.java | 99 +++++++++++++++++++ .../graphicsfuzz/common/util/AddBraces.java | 6 -- .../common/util/ShaderJobFileOperations.java | 18 +--- .../common/util/AddBracesTest.java | 11 --- .../generator/tool/CustomMutatorServer.java | 3 +- .../generator/tool/FuzzShader.java | 44 --------- .../graphicsfuzz/generator/tool/Mutate.java | 2 +- .../generator/tool/PrepareReference.java | 2 +- .../graphicsfuzz/reducer/ReductionDriver.java | 3 +- .../EliminateInjectionMacrosVisitor.java | 2 +- ...CompoundToBlockReductionOpportunities.java | 1 + ...dentityMutationReductionOpportunities.java | 1 + .../IdentityMutationReductionOpportunity.java | 1 + ...tputVariableWriteReductionOpportunity.java | 1 + .../NotReferencedFromLiveContext.java | 1 + .../ReducerContext.java | 17 +--- .../ReductionOpportunitiesBase.java | 1 + .../StmtReductionOpportunities.java | 1 + .../UnswitchifyReductionOpportunities.java | 1 + .../UnwrapReductionOpportunities.java | 4 +- .../graphicsfuzz/reducer/tool/GlslReduce.java | 3 +- .../reducer/tool/ReducerBugPoint.java | 2 +- .../reducer/tool/ReducerBugPointBasic.java | 2 +- .../reducer/ReductionDriverTest.java | 15 ++- .../com/graphicsfuzz/reducer/TestUtils.java | 48 --------- ...prToSubExprReductionOpportunitiesTest.java | 5 +- ...oundToBlockReductionOpportunitiesTest.java | 5 +- ...DestructifyReductionOpportunitiesTest.java | 11 ++- ...rToConstantReductionOpportunitiesTest.java | 12 ++- ...oldConstantReductionOpportunitiesTest.java | 4 +- .../FunctionReductionOpportunitiesTest.java | 6 +- ...eDeclToExprReductionOpportunitiesTest.java | 19 ++-- ...DeclarationReductionOpportunitiesTest.java | 18 ++-- ...ityMutationReductionOpportunitiesTest.java | 3 +- ...ineFunctionReductionOpportunitiesTest.java | 6 +- ...InitializerReductionOpportunitiesTest.java | 19 ++-- ...tifiedFieldReductionOpportunitiesTest.java | 43 +++++--- ...lineUniformReductionOpportunitiesTest.java | 2 +- ...riableWriteReductionOpportunitiesTest.java | 17 +++- .../LoopMergeReductionOpportunitiesTest.java | 6 +- ...edStatementReductionOpportunitiesTest.java | 19 +++- .../ReductionOpportunitiesTest.java | 66 ++++++------- ...ormMetadataReductionOpportunitiesTest.java | 14 +-- ...StructFieldReductionOpportunitiesTest.java | 11 ++- ...veStructFieldReductionOpportunityTest.java | 16 +-- ...edParameterReductionOpportunitiesTest.java | 3 +- .../StmtReductionOpportunitiesTest.java | 42 ++++---- ...UnswitchifyReductionOpportunitiesTest.java | 5 +- .../UnwrapReductionOpportunitiesTest.java | 28 +++--- ...ariableDeclReductionOpportunitiesTest.java | 11 +-- ...eDeclToExprReductionOpportunitiesTest.java | 11 ++- ...ctorizationReductionOpportunitiesTest.java | 32 +++--- .../tester/GeneratorUnitTest.java | 3 +- .../MiscellaneousGenerateThenReduceTest.java | 2 +- .../graphicsfuzz/tester/ReducerUnitTest.java | 6 +- 58 files changed, 409 insertions(+), 377 deletions(-) rename {reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities => ast/src/main/java/com/graphicsfuzz/common/util}/MacroNames.java (77%) delete mode 100644 generator/src/main/java/com/graphicsfuzz/generator/tool/FuzzShader.java delete mode 100755 reducer/src/test/java/com/graphicsfuzz/reducer/TestUtils.java diff --git a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrint.java b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrint.java index e4e73ca81..ec62c2c36 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrint.java +++ b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrint.java @@ -44,11 +44,6 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .help("Target file name.") .type(String.class); - // Optional arguments - parser.addArgument("--glsl_version") - .help("Version of GLSL to target.") - .type(String.class); - return parser.parseArgs(args); } @@ -75,18 +70,9 @@ private static void prettyPrintShader(Namespace ns, TranslationUnit tu) throws FileNotFoundException { try (PrintStream stream = new PrintStream(new FileOutputStream(new File(ns.getString("output"))))) { - if (getGlslVersion(ns) != null) { - throw new RuntimeException(); - //Helper.emitDefines(stream, new ShadingLanguageVersion(getGlslVersion(ns), false), - // false); - } PrettyPrinterVisitor ppv = new PrettyPrinterVisitor(stream); ppv.visit(tu); } } - private static String getGlslVersion(Namespace ns) { - return ns.getString("glsl_version"); - } - } diff --git a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java index 4a9032dcb..0e68bb368 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java +++ b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java @@ -70,7 +70,9 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.type.VoidType; +import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.util.Constants; import java.io.ByteArrayOutputStream; @@ -128,10 +130,19 @@ public static void emitShader(TranslationUnit shader, Optional license, PrintStream stream, int indentationWidth, - Supplier newlineSupplier, - boolean emitGraphicsFuzzDefines) { + Supplier newlineSupplier) { + final boolean usesGraphicsFuzzDefines = new CheckPredicateVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { + if (MacroNames.isGraphicsFuzzMacro(functionCallExpr)) { + predicateHolds(); + } + super.visitFunctionCallExpr(functionCallExpr); + } + }.test(shader); + new PrettyPrinterVisitor(stream, indentationWidth, newlineSupplier, - emitGraphicsFuzzDefines, license).visit(shader); + usesGraphicsFuzzDefines, license).visit(shader); } private String newLine() { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/MacroNames.java b/ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java similarity index 77% rename from reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/MacroNames.java rename to ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java index 459408b94..0fc4f9f51 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/MacroNames.java +++ b/ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.graphicsfuzz.reducer.reductionopportunities; +package com.graphicsfuzz.common.util; import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; @@ -22,6 +22,25 @@ public class MacroNames { + /** + * Determines whether the given expression is an invocation of a GraphicsFuzz macro. + * @param expr an expression to be tested. + * @return true if and only if the expression is an invocation of a GraphicsFuzz macro. + */ + public static boolean isGraphicsFuzzMacro(Expr expr) { + return isIdentity(expr) + || isZero(expr) + || isOne(expr) + || isFalse(expr) + || isTrue(expr) + || isFuzzed(expr) + || isDeadByConstruction(expr) + || isLoopWrapper(expr) + || isIfWrapperTrue(expr) + || isIfWrapperFalse(expr) + || isSwitch(expr); + } + public static boolean isIdentity(Expr expr) { return isCallToNamedFunction(expr, Constants.GLF_IDENTITY); } diff --git a/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java b/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java index dc9701743..1cc0274f0 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java @@ -18,17 +18,28 @@ import com.graphicsfuzz.common.ast.CompareAstsDuplicate; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.util.ExecHelper; import com.graphicsfuzz.util.ToolHelper; +import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.IOException; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; +import java.util.Optional; import org.apache.commons.io.FileUtils; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class PrettyPrinterVisitorTest { @@ -353,4 +364,92 @@ public void testSamplers() throws Exception { ))); } + /** + * To allow testing of the 'emitShader' method, this parses a shader from the given string, + * invokes 'emitShader' on the resulting parsed shader, and returns the result as a string. + */ + private String getStringViaEmitShader(String shader) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + PrettyPrinterVisitor.emitShader(ParseHelper.parse(shader), Optional.empty(), + new PrintStream(bytes), PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, + PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER); + return new String(bytes.toByteArray(), StandardCharsets.UTF_8); + } + + @Test + public void testNoMacrosUsedSoNoGraphicsFuzzHeader() throws Exception { + assertFalse(getStringViaEmitShader("void main() { }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testNoMacrosUsedSoNoGraphicsFuzzHeader2() throws Exception { + // Even though this uses a macro name, it doesn't use it as a function invocation. + assertFalse(getStringViaEmitShader("void main() { int " + Constants.GLF_FUZZED + "; }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToIdentityMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_IDENTITY + "(1, 1); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToZeroMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_ZERO + "(0); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToOneMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_ONE + "(1); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToFalseMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_FALSE + "(false); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToTrueMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_TRUE + "(true); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToFuzzedMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_FUZZED + "(1234); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToDeadByConstructionMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { if(" + Constants.GLF_DEAD + "(false)) { } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToLoopWrapperMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { while(" + Constants.GLF_WRAPPED_LOOP + + "(false))" + + " {" + + " } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToIfFalseWrapperMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { if(" + Constants.GLF_WRAPPED_IF_FALSE + "(false)) { } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToIfTrueWrapperMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { if(" + Constants.GLF_WRAPPED_IF_TRUE + + "(true)) { } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToSwitchMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { switch(" + Constants.GLF_SWITCH + + "(0)) { case 0: break; default: break; } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/AddBraces.java b/common/src/main/java/com/graphicsfuzz/common/util/AddBraces.java index 2b235274a..123c42cdc 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/AddBraces.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/AddBraces.java @@ -98,10 +98,4 @@ public TranslationUnit addBraces(TranslationUnit tu) { } - public static void main(String[] args) throws IOException, ParseTimeoutException, - InterruptedException, GlslParserException { - TranslationUnit tu = ParseHelper.parse(new File(args[0])); - new PrettyPrinterVisitor(System.out).visit(transform(tu)); - } - } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java index 0261ac0c5..8e153accb 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java @@ -702,8 +702,7 @@ public void writeByteArrayToFile(File file, byte[] contents) throws IOException public void writeShaderJobFile( final ShaderJob shaderJob, - final File outputShaderJobFile, - final boolean emitGraphicsFuzzDefines) throws FileNotFoundException { + final File outputShaderJobFile) throws FileNotFoundException { assertIsShaderJobFile(outputShaderJobFile); @@ -713,8 +712,7 @@ public void writeShaderJobFile( writeShader( tu, shaderJob.getLicense(), - new File(outputFileNoExtension + "." + tu.getShaderKind().getFileExtension()), - emitGraphicsFuzzDefines + new File(outputFileNoExtension + "." + tu.getShaderKind().getFileExtension()) ); } @@ -725,12 +723,6 @@ public void writeShaderJobFile( shaderJob.getPipelineInfo().toString()); } - public void writeShaderJobFile( - final ShaderJob shaderJob, - final File outputShaderJobFile) throws FileNotFoundException { - writeShaderJobFile(shaderJob, outputShaderJobFile, true); - } - public void writeShaderJobFileFromImageJob( final ImageJob imageJob, final File outputShaderJobFile) throws IOException { @@ -873,8 +865,7 @@ private static PrintStream ps(File file) throws FileNotFoundException { private static void writeShader( TranslationUnit tu, Optional license, - File outputFile, - boolean emitGraphicsFuzzDefines + File outputFile ) throws FileNotFoundException { try (PrintStream stream = ps(outputFile)) { PrettyPrinterVisitor.emitShader( @@ -882,8 +873,7 @@ private static void writeShader( license, stream, PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - emitGraphicsFuzzDefines + PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER ); } } diff --git a/common/src/test/java/com/graphicsfuzz/common/util/AddBracesTest.java b/common/src/test/java/com/graphicsfuzz/common/util/AddBracesTest.java index 602d8b056..12f9b3b1d 100644 --- a/common/src/test/java/com/graphicsfuzz/common/util/AddBracesTest.java +++ b/common/src/test/java/com/graphicsfuzz/common/util/AddBracesTest.java @@ -61,17 +61,6 @@ public void loops() throws Exception { @Rule public TemporaryFolder testFolder = new TemporaryFolder(); - @Test - public void testMain() throws Exception { - File input = testFolder.newFile("input.frag"); - String program = "void main() { for (a; b; c) while (d) do e; while (f); }"; - BufferedWriter bw = new BufferedWriter(new FileWriter(input)); - bw.write(program); - bw.flush(); - bw.close(); - AddBraces.main(new String[] { input.getAbsolutePath() }); - } - @Test public void testIsUtilityClass() throws Exception { CheckUtilityClass.assertUtilityClassWellDefined(AddBraces.class); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/CustomMutatorServer.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/CustomMutatorServer.java index 21b02eee6..a32b71997 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/CustomMutatorServer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/CustomMutatorServer.java @@ -130,8 +130,7 @@ private static String mutate(String inputShader, int seed, boolean isFragment) { Optional.empty(), stream, INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - false); + PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER); } final String outputShader = new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/FuzzShader.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/FuzzShader.java deleted file mode 100644 index 91e21e7b9..000000000 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/FuzzShader.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.generator.tool; - -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import com.graphicsfuzz.common.util.RandomWrapper; -import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.generator.fuzzer.Fuzzer; -import com.graphicsfuzz.generator.fuzzer.FuzzingContext; -import com.graphicsfuzz.generator.util.GenerationParams; - -public class FuzzShader { - - public static void main(String[] args) { - - if (args.length != 1) { - System.err.print("Usage: seed"); - System.exit(1); - } - - new PrettyPrinterVisitor(System.out) - .visit(new Fuzzer(new FuzzingContext(), - ShadingLanguageVersion.GLSL_440, - new RandomWrapper(Integer.parseInt(args[0])), - GenerationParams.normal(ShaderKind.FRAGMENT, true)).fuzzTranslationUnit()); - - } - -} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java index 996fdffa4..22e46e849 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java @@ -140,7 +140,7 @@ public static void mainHelper(String[] args) throws ArgumentParserException, IOE final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo("{}"), - tu), shaderJobFile, false); + tu), shaderJobFile); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java index 9bb7e56cb..edf877f9b 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java @@ -119,7 +119,7 @@ public static void prepareReference( maxUniforms, generateUniformBindings); - fileOps.writeShaderJobFile(shaderJob, outputShaderJobFile, false); + fileOps.writeShaderJobFile(shaderJob, outputShaderJobFile); } public static void prepareReference(ShaderJob shaderJob, diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java index 3b0944952..d1cd2caed 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java @@ -294,8 +294,7 @@ private void writeState(ShaderJob state, File shaderJobFileOutput, } fileOps.writeShaderJobFile( state, - shaderJobFileOutput, - context.getEmitGraphicsFuzzDefines() + shaderJobFileOutput ); if (requiresUniformBindings) { assert state.getPipelineInfo().getNumUniforms() == 0 || state.hasUniformBindings(); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java index 869fdcf7b..a32478d5a 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java @@ -25,7 +25,7 @@ import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import com.graphicsfuzz.reducer.reductionopportunities.MacroNames; +import com.graphicsfuzz.common.util.MacroNames; public class EliminateInjectionMacrosVisitor extends StandardVisitor { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java index 89364befc..02102727b 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java @@ -27,6 +27,7 @@ import com.graphicsfuzz.common.util.ContainsTopLevelBreak; import com.graphicsfuzz.common.util.ContainsTopLevelContinue; import com.graphicsfuzz.common.util.ListConcat; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.SideEffectChecker; import java.util.Arrays; import java.util.List; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunities.java index 28bb8578f..80d66b4c7 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunities.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; +import com.graphicsfuzz.common.util.MacroNames; import java.util.Arrays; import java.util.List; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunity.java index 1057d7899..2e8bb7fad 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunity.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import com.graphicsfuzz.common.util.MacroNames; public final class IdentityMutationReductionOpportunity extends AbstractReductionOpportunity { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java index eede0d738..fb89522e2 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java @@ -29,6 +29,7 @@ import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.typing.ScopeEntry; import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.util.Constants; import java.util.ArrayList; import java.util.List; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/NotReferencedFromLiveContext.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/NotReferencedFromLiveContext.java index 2929493a2..c29d3b4f0 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/NotReferencedFromLiveContext.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/NotReferencedFromLiveContext.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.util.MacroNames; import java.util.HashSet; import java.util.Set; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReducerContext.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReducerContext.java index 8ff3e3bd9..85f07e106 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReducerContext.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReducerContext.java @@ -22,8 +22,8 @@ public class ReducerContext { - static final int DEFAULT_MAX_PERCENTAGE_TO_REDUCE = 50; - static final int DEFAULT_AGGRESSION_DECREASE_STEP = 5; + private static final int DEFAULT_MAX_PERCENTAGE_TO_REDUCE = 50; + private static final int DEFAULT_AGGRESSION_DECREASE_STEP = 5; private final boolean reduceEverywhere; private final ShadingLanguageVersion shadingLanguageVersion; @@ -31,27 +31,24 @@ public class ReducerContext { private final IdGenerator idGenerator; private final int maxPercentageToReduce; private final int aggressionDecreaseStep; - private final boolean emitGraphicsFuzzDefines; public ReducerContext(boolean reduceEverywhere, ShadingLanguageVersion shadingLanguageVersion, IRandom random, IdGenerator idGenerator, int maxPercentageToReduce, - int aggressionDecreaseStep, boolean emitGraphicsFuzzDefines) { + int aggressionDecreaseStep) { this.reduceEverywhere = reduceEverywhere; this.shadingLanguageVersion = shadingLanguageVersion; this.random = random; this.idGenerator = idGenerator; this.maxPercentageToReduce = maxPercentageToReduce; this.aggressionDecreaseStep = aggressionDecreaseStep; - this.emitGraphicsFuzzDefines = emitGraphicsFuzzDefines; } public ReducerContext(boolean reduceEverywhere, ShadingLanguageVersion shadingLanguageVersion, - IRandom random, IdGenerator idGenerator, boolean emitGraphicsFuzzDefines) { + IRandom random, IdGenerator idGenerator) { this(reduceEverywhere, shadingLanguageVersion, random, idGenerator, - DEFAULT_MAX_PERCENTAGE_TO_REDUCE, DEFAULT_AGGRESSION_DECREASE_STEP, - emitGraphicsFuzzDefines); + DEFAULT_MAX_PERCENTAGE_TO_REDUCE, DEFAULT_AGGRESSION_DECREASE_STEP); } public boolean reduceEverywhere() { @@ -81,8 +78,4 @@ public int getAggressionDecreaseStep() { return aggressionDecreaseStep; } - public boolean getEmitGraphicsFuzzDefines() { - return emitGraphicsFuzzDefines; - } - } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java index c38a2c620..c93f97cf4 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java @@ -36,6 +36,7 @@ import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.SideEffectChecker; import com.graphicsfuzz.util.Constants; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java index ad62cdfe5..af38009f0 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java @@ -40,6 +40,7 @@ import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.SideEffectChecker; import com.graphicsfuzz.common.util.StructUtils; import com.graphicsfuzz.util.Constants; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java index a36337507..45dd33042 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; +import com.graphicsfuzz.common.util.MacroNames; import java.util.Arrays; import java.util.List; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java index 72299cbe7..4b99de6b9 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java @@ -17,9 +17,7 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.stmt.BlockStmt; -import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; import com.graphicsfuzz.common.ast.stmt.DoStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.IfStmt; @@ -27,12 +25,12 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; +import com.graphicsfuzz.common.util.MacroNames; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; public class UnwrapReductionOpportunities extends ReductionOpportunitiesBase { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java index 68b929025..58c02616f 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java @@ -523,8 +523,7 @@ public static void doReductionHelper( reduceEverywhere, shadingLanguageVersion, random, - idGenerator, - emitGraphicsFuzzDefines), + idGenerator), verbose, fileOps, fileJudge, diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java index e2a6c5540..4551a40af 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java @@ -142,7 +142,7 @@ public static void main(String[] args) generator, null, 10, - 1, true), + 1), verbose, fileOps, new RandomFileJudge( diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java index 54a0d0889..d3a32a57e 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java @@ -140,7 +140,7 @@ public static void main(String[] args) reduceEverywhere, shadingLanguageVersion, generator, - idGenerator, true), + idGenerator), fileOps); } catch (Exception exception) { diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java index 9b25d422b..101f50b44 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java @@ -111,7 +111,7 @@ public boolean isInteresting( ReductionOpportunities. getReductionOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, version, generator, new IdGenerator(), true), + new ReducerContext(false, version, generator, new IdGenerator()), fileOps); assertEquals(3, ops.size()); @@ -120,8 +120,7 @@ public boolean isInteresting( false, version, generator, - null, - true), + new IdGenerator()), false, fileOps, pessimist, testFolder.getRoot()) .doReduction(state, getPrefix(tempFile), 0, -1); @@ -231,7 +230,7 @@ private String reduce(IFileJudge judge, translationUnits); return new ReductionDriver(new ReducerContext(reduceEverywhere, version, - generator, new IdGenerator(), true), false, fileOps, + generator, new IdGenerator()), false, fileOps, judge, testFolder.getRoot()) .doReduction(state, getPrefix(tempFragmentShaderFile), 0, stepLimit); } @@ -278,8 +277,8 @@ public void testInitializersAreInlined() throws Exception { }; final String reducedFilesPrefix = new ReductionDriver( - new ReducerContext(false, version, generator, null, - true), false, fileOps, + new ReducerContext(false, version, generator, new IdGenerator()), + false, fileOps, referencesSinCosAnd3, testFolder.getRoot()) .doReduction(state, getPrefix(tempFile), 0,-1); @@ -553,7 +552,7 @@ public void testReductionWithUniformBindings() throws Exception { final String resultsPrefix = new ReductionDriver(new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), - new IdGenerator(), true), + new IdGenerator()), false, fileOps, (unused, item) -> true, workDir) @@ -585,7 +584,7 @@ public void testReductionOfUnreferencedUniform() throws Exception { final String resultsPrefix = new ReductionDriver(new ReducerContext(true, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - new IdGenerator(), true), + new IdGenerator()), false, fileOps, (unused, item) -> true, workDir) diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/TestUtils.java b/reducer/src/test/java/com/graphicsfuzz/reducer/TestUtils.java deleted file mode 100755 index f6b1a3fe2..000000000 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/TestUtils.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer; - -import static org.junit.Assert.assertEquals; - -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import com.graphicsfuzz.util.ExecHelper.RedirectType; -import com.graphicsfuzz.util.ExecResult; -import com.graphicsfuzz.util.ToolHelper; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Optional; -import org.junit.rules.TemporaryFolder; - -public class TestUtils { - - public static void checkValid(TemporaryFolder testFolder, TranslationUnit tu) throws IOException, InterruptedException { - File tempFile = testFolder.newFile("temp.frag"); - new PrettyPrinterVisitor(System.out).visit(tu); - PrintStream ps = new PrintStream(new FileOutputStream(tempFile)); - PrettyPrinterVisitor ppv = new PrettyPrinterVisitor(ps, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, true, Optional.empty()); - ppv.visit(tu); - ps.close(); - ExecResult result = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, tempFile); - assertEquals(result.stderr.toString() + result.stdout.toString(), 0, result.res); - } - -} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java index ce5d2dcdf..baf5af88d 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.RandomWrapper; @@ -40,7 +41,7 @@ public void noEffectInRegularCode() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); final List ops = CompoundExprToSubExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertTrue(ops.isEmpty()); } @@ -168,7 +169,7 @@ private List getOps(TranslationUnit tu, boolean reduceEverywhere) { return CompoundExprToSubExprReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(reduceEverywhere, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java index 18f4a1fc1..df45063fd 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java @@ -499,8 +499,7 @@ private List getOps(TranslationUnit tu, reduceEverywhere, ShadingLanguageVersion.GLSL_440, new RandomWrapper(0), - new IdGenerator(), - true), + new IdGenerator()), fileOps ).stream() .filter(item -> item instanceof CompoundToBlockReductionOpportunity) @@ -531,7 +530,7 @@ public void testDoNotTurnForLoopIntoBlock() throws Exception { final List ops = CompoundToBlockReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, - new RandomWrapper(0), new IdGenerator(), true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunitiesTest.java index 75f47604a..47f387dec 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunitiesTest.java @@ -19,6 +19,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -40,7 +41,7 @@ public void checkForVariableInScope() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(), new IdGenerator())); // There should be no opportunities as there is already a variable called 'dist' in scope assertEquals(0, ops.size()); } @@ -70,7 +71,7 @@ public void checkForVariableInScope2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(), new IdGenerator())); // There should be one opportunity as variable dist is in a different scope and not used in this scope. assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -94,7 +95,7 @@ public void checkForVariableInScope3() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(), new IdGenerator())); // There should be no opportunities as there is already a variable called 'dist' in scope, // and it is used. assertEquals(0, ops.size()); @@ -126,7 +127,7 @@ public void misc() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(), new IdGenerator())); // There should be no opportunities as there is already a variable called 'dist' in scope, // and it is used. assertEquals(1, ops.size()); @@ -135,4 +136,4 @@ public void misc() throws Exception { PrettyPrinterVisitor.prettyPrintAsString(tu)); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java index 1305310c6..cc3730258 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -35,7 +36,7 @@ public void testOut() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); for (SimplifyExprReductionOpportunity op : ops) { op.applyReduction(); } @@ -48,7 +49,7 @@ public void testInOut() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); for (SimplifyExprReductionOpportunity op : ops) { op.applyReduction(); } @@ -62,7 +63,7 @@ public void testIn() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); for (SimplifyExprReductionOpportunity op : ops) { op.applyReduction(); } @@ -74,10 +75,11 @@ public void testSingleLiveVariable() throws Exception { final String program = "void main() { int GLF_live3_a; GLF_live3_a; }"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts("void main() { int GLF_live3_a; 1; }", tu); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java index 4a70a8833..7334ff9b1 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java @@ -20,8 +20,10 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.RandomWrapper; import java.io.IOException; import java.util.List; import org.junit.Test; @@ -627,7 +629,7 @@ private void check(String before, int numOps, String after) throws IOException, final TranslationUnit tu = ParseHelper.parse(before); final List ops = FoldConstantReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, null, null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); ops.forEach(item -> item.applyReduction()); CompareAsts.assertEqualAsts(after, tu); assertEquals(numOps, ops.size()); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunitiesTest.java index 9d74c87e0..6da1a7fbf 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunitiesTest.java @@ -18,7 +18,9 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -31,8 +33,8 @@ public void testRemovable() throws Exception { + "void main() { }"; TranslationUnit tu = ParseHelper.parse(program); assertEquals(1, FunctionReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)) + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())) .size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java index b65541415..a55f53013 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java @@ -19,6 +19,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -36,7 +37,7 @@ public void testDoNotReplace() throws Exception { GlobalVariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as the preserve semantics is enabled. assertTrue(ops.isEmpty()); } @@ -49,7 +50,7 @@ public void testZeroMethod() throws Exception { GlobalVariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); // Since the new assignment statement must be only inserted as the first statement of // the main function, thus we have to ensure that main function exists. // In this case, there is no method at all. @@ -64,7 +65,7 @@ public void testNoMainMethod() throws Exception { GlobalVariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); // Since the new assignment statement must be only inserted as the first statement of // the main function, thus we have to ensure that main function exists. // In this case, there is one method but it is not main though so there should be no @@ -92,7 +93,7 @@ public void testInsertAsFirstStatement() throws Exception { GlobalVariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); CompareAsts.assertEqualAsts(expected, tu); @@ -118,7 +119,7 @@ public void testMultipleDeclarations() throws Exception { GlobalVariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); // Only variable declarations a and b have the initializer. // Thus, we expect the reducer to find only 2 opportunities. assertEquals(2, ops.size()); @@ -134,7 +135,7 @@ public void testDoNotReplaceConst() throws Exception { GlobalVariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as it is invalid to declare constant variable // without an initial value. assertTrue(ops.isEmpty()); @@ -159,7 +160,7 @@ public void testMultipleLineDeclarationsOneLine() throws Exception { GlobalVariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(3, ops.size()); ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); CompareAsts.assertEqualAsts(expected, tu); @@ -188,7 +189,7 @@ public void testAssignVariableIdentifier() throws Exception { GlobalVariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(4, ops.size()); ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); CompareAsts.assertEqualAsts(expected, tu); @@ -203,7 +204,7 @@ public void testGlobalVariableDeclAfterMain() throws Exception { GlobalVariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertTrue(ops.isEmpty()); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariablesDeclarationReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariablesDeclarationReductionOpportunitiesTest.java index 7b0bf47d2..afc67c5eb 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariablesDeclarationReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariablesDeclarationReductionOpportunitiesTest.java @@ -47,7 +47,7 @@ public void testDoNotRemoveLocalInitialization() throws Exception { List ops = ReductionOpportunities .getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); assertEquals(0, ops.size()); } @@ -60,7 +60,7 @@ public void testDoNotRemoveUsedLiveCodeDecl() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -73,7 +73,7 @@ public void testUsedStructIsNotRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -84,7 +84,7 @@ public void testUnusedStructIsRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); final List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -99,7 +99,7 @@ public void testUsedAnonymousStructIsNotRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -110,7 +110,7 @@ public void testUnusedAnonymousStructIsRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); final List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -125,15 +125,15 @@ public void testVarIsRemovedButStructLeftBehind() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); final List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); final List moreOps = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(0, moreOps.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java index ed3da6a01..96d6aae3d 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java @@ -163,8 +163,7 @@ private ReducerContext getReducerContext() { return new ReducerContext(false, ShadingLanguageVersion.ESSL_310, new RandomWrapper(), - new IdGenerator(), - true); + new IdGenerator()); } private void checkOneReductionOpportunity(String program, String afterReduction) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineFunctionReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineFunctionReductionOpportunitiesTest.java index 7c9e4c2dc..2e4f04c21 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineFunctionReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineFunctionReductionOpportunitiesTest.java @@ -53,7 +53,7 @@ public void testBasicInline() throws Exception { final List ops = InlineFunctionReductionOpportunities.findOpportunities(shaderJob, new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), false)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -77,10 +77,10 @@ public void testTooLargeToInline() throws Exception { final List ops = InlineFunctionReductionOpportunities.findOpportunities(shaderJob, new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), false)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java index ef56cdd91..3b0e08bd3 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -35,7 +36,7 @@ public void testGlobalScope() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -53,12 +54,12 @@ public void testDoNotInlineLargeInitializer() throws Exception { List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(largeProgramTu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(smallProgramTu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(3, ops.size()); ops.get(0).applyReduction(); ops.get(1).applyReduction(); @@ -94,7 +95,7 @@ public void testCanInlineAllExceptLValueIfReducingEverywhere() throws Exception List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(5, ops.size()); ops.forEach(SimplifyExprReductionOpportunity::applyReduction); CompareAsts.assertEqualAsts(expected, tu); @@ -109,7 +110,7 @@ public void testNoInliningIfLValueUsageExistsWhenPreservingSemantics() throws Ex List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -120,7 +121,7 @@ public void testNoInliningIfPartsOfInitializerAreModifiedWhenPreservingSemantics List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -131,7 +132,7 @@ public void testNoInliningWithNameShadowing1() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -143,8 +144,8 @@ public void testNoInliningWithNameShadowing2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunitiesTest.java index f37c48c59..287708ef3 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunitiesTest.java @@ -17,10 +17,16 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.PipelineInfo; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.util.ParseHelper; -import com.graphicsfuzz.reducer.TestUtils; import com.graphicsfuzz.util.ExecHelper.RedirectType; import com.graphicsfuzz.util.ExecResult; import com.graphicsfuzz.util.ToolHelper; @@ -34,9 +40,12 @@ import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class InlineStructifiedFieldReductionOpportunitiesTest { + private final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + @Rule public TemporaryFolder testFolder = new TemporaryFolder(); @@ -72,9 +81,11 @@ public void checkInliningAppliedCorrectly() throws Exception { + "}\n"; TranslationUnit tu = ParseHelper.parse(program); - assertEquals(1, InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)).size()); + assertEquals(1, + InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())).size()); InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)).get(0).applyReduction(); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())).get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(programAfter)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -108,7 +119,8 @@ public void inlineRegressionTest() throws Exception { TranslationUnit tu = ParseHelper.parse(program).clone(); - List ops = InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = + InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(3, ops.size()); File tempFile = testFolder.newFile("temp.frag"); @@ -118,16 +130,10 @@ public void inlineRegressionTest() throws Exception { } op.applyReduction(); - new PrettyPrinterVisitor(System.out).visit(tu); - - PrintStream ps = new PrintStream(new FileOutputStream(tempFile)); - PrettyPrinterVisitor ppv = new PrettyPrinterVisitor(ps, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, true, Optional.empty()); - ppv.visit(tu); - ps.close(); - ExecResult result = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, tempFile); - assertEquals(result.stderr.toString() + result.stdout.toString(), 0, result.res); + final File shaderJobFile = testFolder.newFile("temp.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu), + shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); break; } @@ -151,11 +157,16 @@ public void inlineRegressionTest2() throws Exception { TranslationUnit tu = ParseHelper.parse(program).clone(); List ops = InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); - TestUtils.checkValid(testFolder, tu); + + final File shaderJobFile = testFolder.newFile("temp.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu), + shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java index 001bac84a..5798949dc 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java @@ -203,7 +203,7 @@ private ShaderJob checkCanReduceToTarget(ShaderJob shaderJob, int expectedSize, List ops = InlineUniformReductionOpportunities.findOpportunities(temp, new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null)); assertEquals(expectedSize, ops.size()); ops.get(i).applyReduction(); if (CompareAsts.isEqualAsts(target, temp.getShaders().get(0))) { diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunitiesTest.java index 7783f6119..e8a99c2f3 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunitiesTest.java @@ -17,7 +17,10 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.OpenGlConstants; @@ -48,7 +51,8 @@ public void testLiveGLFragColorWriteOpportunity() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -69,7 +73,8 @@ public void testLiveGLFragColorWriteOpportunityAtRootOfMain() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext( + false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -94,7 +99,8 @@ public void testOpportunityIsPresent1() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -115,7 +121,8 @@ public void testOpportunityIsPresent2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -137,7 +144,7 @@ public void testOpportunityIsPresent3() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunitiesTest.java index fd188f586..cffe03673 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunitiesTest.java @@ -19,7 +19,10 @@ import static org.junit.Assert.assertEquals; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.util.ParseHelper; import java.util.List; @@ -62,7 +65,8 @@ public void findOpportunities() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List opportunities = - LoopMergeReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + LoopMergeReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, opportunities.size()); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunitiesTest.java index f482dfeb7..bc1648d2a 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunitiesTest.java @@ -17,8 +17,11 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; import org.junit.Test; @@ -36,7 +39,8 @@ public void testOutline() throws Exception { TranslationUnit tu = ParseHelper.parse(program); - List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -56,7 +60,8 @@ public void testOutline2() throws Exception { TranslationUnit tu = ParseHelper.parse(program); - List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -78,7 +83,9 @@ public void testOutline3() throws Exception { TranslationUnit tu = ParseHelper.parse(program); - List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -95,9 +102,11 @@ public void testOutline4() throws Exception { TranslationUnit tu = ParseHelper.parse(program); - List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertEquals(0, ops.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java index e7a8907e9..9a09aa5da 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java @@ -60,7 +60,7 @@ public void testDeadConditionalNotReplaced() throws Exception { List opportunities = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(), new IdGenerator(), true), fileOps); + new RandomWrapper(), new IdGenerator()), fileOps); // There should be no ExprToConstant reduction opportunity, because the expressions do not occur // under dead code, and the fuzzed expression is too simple to be reduced. assertFalse(opportunities.stream().anyMatch(item -> item instanceof SimplifyExprReductionOpportunity)); @@ -74,7 +74,7 @@ public void testPopScope() throws Exception { ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(), new IdGenerator(), true), fileOps); + new RandomWrapper(), new IdGenerator()), fileOps); } private void stressTestStructification(String variantProgram, String reducedProgram) throws Exception{ @@ -84,7 +84,7 @@ private void stressTestStructification(String variantProgram, String reducedProg while (true) { List ops = InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); if (ops.isEmpty()) { break; } @@ -93,7 +93,7 @@ private void stressTestStructification(String variantProgram, String reducedProg while (true) { List ops - = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); if (ops.isEmpty()) { break; } @@ -104,7 +104,7 @@ private void stressTestStructification(String variantProgram, String reducedProg List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(), new IdGenerator(), true), fileOps); + new RandomWrapper(), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } @@ -333,8 +333,8 @@ public void testStructInlineWithParam() throws Exception { TranslationUnit tu = ParseHelper.parse(program); List ops = - InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); - InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); @@ -380,7 +380,7 @@ public void testDestructifyWithParam() throws Exception { List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); @@ -425,7 +425,7 @@ public void exprSimplificationAndLoopSplitInteraction() throws Exception { { List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); for (int i = 0; i < ops.size(); i++) { if (ops.get(i) instanceof SimplifyExprReductionOpportunity) { indicesOfExprToConstOps.add(i); @@ -441,13 +441,13 @@ public void exprSimplificationAndLoopSplitInteraction() throws Exception { List ops = ReductionOpportunities.getReductionOpportunities( MakeShaderJobFromFragmentShader.make(aClone), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), true), fileOps); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator()), fileOps); ((SimplifyExprReductionOpportunity) ops.get(index)).applyReduction(); } for (IReductionOpportunity op : ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(aClone), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), true), fileOps)) { + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator()), fileOps)) { if (op instanceof LoopMergeReductionOpportunity) { op.applyReduction(); } @@ -508,7 +508,7 @@ private void tryAllCompatibleOpportunities(String program) TranslationUnit tu = ParseHelper.parse(program); int numOps = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), true), fileOps).size(); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator()), fileOps).size(); for (int i = 0; i < numOps; i++) { for (int j = 0; j < numOps; j++) { @@ -519,7 +519,7 @@ private void tryAllCompatibleOpportunities(String program) List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(aClone), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), true), fileOps); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator()), fileOps); if (Compatibility.compatible(ops.get(i).getClass(), ops.get(j).getClass())) { ops.get(i).applyReduction(); @@ -537,7 +537,7 @@ public void reduceRedundantStuff() throws Exception { while (true) { List ops = ReductionOpportunities.getReductionOpportunities( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } @@ -554,7 +554,7 @@ public void testReduceToConstantInLiveCode() throws Exception { + " float GLF_live3x = sin(4.0);" + " float GLF_live3y = GLF_live3x + GLF_live3x;" + "}"); - List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(4, ops.size()); } @@ -579,7 +579,7 @@ public void testLeaveLoopLimiter() throws Exception { while (true) { List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new RandomWrapper(), new IdGenerator(), true), fileOps); + ShadingLanguageVersion.GLSL_440, new RandomWrapper(), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } @@ -618,7 +618,7 @@ public void testTernary() throws Exception { List ops = ExprToConstantReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); if (i >= ops.size()) { break; } @@ -652,7 +652,7 @@ public void testRemoveLoopLimiter() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), @@ -670,13 +670,13 @@ public void testInterplayBetweenVectorizationAndIdentity() throws Exception { + "}"); { List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } { List ops = IdentityMutationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts("void main() {" @@ -689,7 +689,7 @@ public void testInterplayBetweenVectorizationAndIdentity() throws Exception { } { List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(2, ops.size()); ops.get(0).applyReduction(); ops.get(1).applyReduction(); @@ -704,7 +704,7 @@ public void testInterplayBetweenVectorizationAndIdentity() throws Exception { { List ops = VariableDeclReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts("void main() {" @@ -718,7 +718,7 @@ public void testInterplayBetweenVectorizationAndIdentity() throws Exception { { List ops = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts("void main() {" @@ -757,7 +757,7 @@ public void reduceNestedVectors() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(4, ops.size()); // (1) remove b from GLF_merged2_0_1_1_1_1_1bc @@ -879,7 +879,7 @@ public void reduceNestedVectors() throws Exception { List stmtOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(7, stmtOps.size()); for (StmtReductionOpportunity op : stmtOps) { op.applyReduction(); @@ -887,7 +887,7 @@ public void reduceNestedVectors() throws Exception { CompareAsts.assertEqualAsts(expected5, tu); ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(3, ops.size()); // (6) remove a from GLF_merged3_0_1_1_1_1_1_2_1_1abc @@ -959,7 +959,7 @@ public void reduceNestedVectors() throws Exception { stmtOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(3, stmtOps.size()); for (StmtReductionOpportunity op : stmtOps) { op.applyReduction(); @@ -982,7 +982,7 @@ public void reduceNestedVectors() throws Exception { VariableDeclReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(5, varDeclOps.size()); for (VariableDeclReductionOpportunity op : varDeclOps) { op.applyReduction(); @@ -1000,7 +1000,7 @@ public void reduceNestedVectors() throws Exception { StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(5, finalStmtOps.size()); for (StmtReductionOpportunity op : finalStmtOps) { op.applyReduction(); @@ -1022,14 +1022,14 @@ public void testRemoveDeclarationsInUnreachableFunction() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(4, ops.size()); for (VariableDeclReductionOpportunity op : ops) { op.applyReduction(); } final List moreOps = StmtReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(4, moreOps.size()); } @@ -1076,7 +1076,7 @@ public void testUniformsGetRemoved() throws Exception { List ops = ReductionOpportunities.getReductionOpportunities( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof VariableDeclReductionOpportunity); ops.get(0).applyReduction(); @@ -1113,7 +1113,7 @@ public void testReductionOpportunitiesOnEmptyForLoops() throws Exception { List ops = ReductionOpportunities.getReductionOpportunities( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); ops.forEach(IReductionOpportunity::applyReduction); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java index 4bbe08355..b6dc44313 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; @@ -50,8 +51,8 @@ public void testRemoveUnused() throws Exception { List ops = RemoveRedundantUniformMetadataReductionOpportunities .findOpportunities(shaderJob, - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, - true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -81,8 +82,7 @@ public void testRemoveUnusedNameShadowing() throws Exception { RemoveRedundantUniformMetadataReductionOpportunities .findOpportunities(shaderJob, new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - null, - true)); + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -109,8 +109,8 @@ public void testDoNotRemoveUsed() throws Exception { List ops = RemoveRedundantUniformMetadataReductionOpportunities .findOpportunities(shaderJob, - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, - true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(0, ops.size()); } @@ -140,7 +140,7 @@ public void testDoNotRemoveUsedMultipleShaders() throws Exception { RemoveRedundantUniformMetadataReductionOpportunities .findOpportunities(shaderJob, new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - null, true)); + new IdGenerator())); assertEquals(0, ops.size()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunitiesTest.java index 35493427e..52b70788c 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunitiesTest.java @@ -19,6 +19,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -47,7 +48,7 @@ public void nestedStruct() throws Exception { TranslationUnit tu = ParseHelper.parse(program); assertEquals(2, RemoveStructFieldReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)).size()); + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())).size()); } @@ -67,7 +68,7 @@ public void singleField() throws Exception { TranslationUnit tu = ParseHelper.parse(program); assertEquals(0, RemoveStructFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)).size()); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())).size()); } @@ -99,7 +100,7 @@ public void testNonLocalInitialization() throws Exception { TranslationUnit tu = ParseHelper.parse(shader); List ops = RemoveStructFieldReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(true, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(true, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(tu), PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected))); @@ -132,7 +133,7 @@ public void twoFields() throws Exception { TranslationUnit tu = ParseHelper.parse(program); final List ops = RemoveStructFieldReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(2, ops .size()); ops.stream().filter(item -> item.getFieldToRemove().equals("_f0")) @@ -149,4 +150,4 @@ public void twoFields() throws Exception { PrettyPrinterVisitor.prettyPrintAsString(tu)); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java index 022ed11e4..2a0c635fb 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java @@ -73,7 +73,7 @@ public void applyReduction() throws Exception { declarationStmt), false); assertEquals("foo v1 = foo(1.0, bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)), bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0))), v2 = foo(1.0, bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)), bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)));\n", - getString(declarationStmt)); + declarationStmt.getText()); IReductionOpportunity ro1 = new RemoveStructFieldReductionOpportunity(foo, "a", b, new VisitationDepth(0)); @@ -87,31 +87,25 @@ public void applyReduction() throws Exception { ro1.applyReduction(); assertEquals("foo v1 = foo(bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)), bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0))), v2 = foo(bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)), bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)));\n", - getString(declarationStmt)); + declarationStmt.getText()); assertEquals(2, bar.getNumFields()); assertEquals(2, foo.getNumFields()); ro2.applyReduction(); assertEquals("foo v1 = foo(bar(vec3(0.0, 0.0, 0.0)), bar(vec3(0.0, 0.0, 0.0))), v2 = foo(bar(vec3(0.0, 0.0, 0.0)), bar(vec3(0.0, 0.0, 0.0)));\n", - getString(declarationStmt)); + declarationStmt.getText()); assertEquals(1, bar.getNumFields()); assertEquals(2, foo.getNumFields()); ro3.applyReduction(); assertEquals("foo v1 = foo(bar(vec3(0.0, 0.0, 0.0))), v2 = foo(bar(vec3(0.0, 0.0, 0.0)));\n", - getString(declarationStmt)); + declarationStmt.getText()); assertEquals(1, bar.getNumFields()); assertEquals(1, foo.getNumFields()); } - private String getString(DeclarationStmt declarationStmt) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - new PrettyPrinterVisitor(new PrintStream(output)).visit(declarationStmt); - return new String(output.toByteArray(), StandardCharsets.UTF_8); - } - -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunitiesTest.java index 6d5172e98..b1aed8b22 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunitiesTest.java @@ -19,6 +19,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -139,7 +140,7 @@ private List findOpportunities(Transl boolean reduceEverywhere) { return RemoveUnusedParameterReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(reduceEverywhere, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java index 7d7a25ae3..74bd7c9a3 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java @@ -26,6 +26,7 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.util.Constants; @@ -76,7 +77,7 @@ public void testSwitch1() throws Exception { final TranslationUnit tu = ParseHelper.parse(prog); List ops = StmtReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_130, - new RandomWrapper(), null, true)); + new RandomWrapper(), new IdGenerator())); for (StmtReductionOpportunity op : ops) { op.applyReduction(); @@ -102,7 +103,7 @@ public void testDoNotLeaveDefaultEmpty() throws Exception { List ops = StmtReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_310, - new RandomWrapper(), null, false)); + new RandomWrapper(), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -122,7 +123,7 @@ public void testEmptyBlock() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof StmtReductionOpportunity); ops.get(0).applyReduction(); @@ -136,7 +137,7 @@ public void testNullStmtRemoved() throws Exception { final String reducedProgram = "void main() { }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof StmtReductionOpportunity); ops.get(0).applyReduction(); @@ -149,7 +150,9 @@ public void testNullStmtsInForNotTouched() throws Exception { final String program = "void main() { for(int i = 0; i < 100; i++) ; }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(0, ops.size()); } @@ -158,7 +161,9 @@ public void testNullStmtsInForNotTouched2() throws Exception { final String program = "int x; void foo() { x = 42; } void main() { if (foo()) ; else ; }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(0, ops.size()); } @@ -173,7 +178,7 @@ public void testSimpleLiveCodeRemoval() throws Exception { + "}"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof StmtReductionOpportunity); ops.get(0).applyReduction(); @@ -195,7 +200,7 @@ public void testUnaryIncDecLiveCodeRemoval() throws Exception { + "}"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(4, ops.size()); for (int i = 0; i < ops.size(); i++) { assertTrue(ops.get(i) instanceof StmtReductionOpportunity); @@ -211,7 +216,7 @@ public void testSideEffectFreeTypeInitializerRemoved() throws Exception { final String reducedProgram = "void main() { }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof StmtReductionOpportunity); ops.get(0).applyReduction(); @@ -224,7 +229,7 @@ public void testSideEffectingTypeInitializerNotRemoved() throws Exception { final String program = "void main() { float x; vec4(0.0, x++, 0.0, 0.0); }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -250,8 +255,8 @@ public void testDeadStatementsPickedUp() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, - true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), + new IdGenerator())); assertEquals(4, ops.size()); } @@ -271,7 +276,7 @@ public void testDeadBasicReturnsPickedUp() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), new IdGenerator())); assertEquals(3, ops.size()); } @@ -282,8 +287,8 @@ public void testRemoveDuplicateReturn1() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, - true)); + new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), + new IdGenerator())); assertEquals(3, ops.size()); ops.get(0).applyReduction(); ops.get(1).applyReduction(); @@ -299,8 +304,8 @@ public void testRemoveDuplicateReturn2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, - true)); + new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), + new IdGenerator())); assertEquals(5, ops.size()); ops.get(0).applyReduction(); ops.get(1).applyReduction(); @@ -317,8 +322,7 @@ public void testRemoveArbitraryVoidReturn() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, - true)); + new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), new IdGenerator())); assertEquals(5, ops.size()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunitiesTest.java index 4112d4778..6fa98e910 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunitiesTest.java @@ -18,6 +18,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -36,8 +37,8 @@ public void testNotInjected() throws Exception { new ReducerContext(false, ShadingLanguageVersion.GLSL_130, new RandomWrapper(0), - null, true)); + new IdGenerator())); assertEquals(0, ops.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunitiesTest.java index a1236d0f9..94ed4406d 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunitiesTest.java @@ -45,7 +45,7 @@ public void testBlock1() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(1, ops.size()); } @@ -55,7 +55,7 @@ public void testEmptyBlock() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); // No opportunities because the inner block is empty. Another reduction pass may be able to // delete it, but it cannot be unwrapped. assertEquals(0, ops.size()); @@ -67,7 +67,7 @@ public void testNestedEmptyBlocks() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(1, ops.size()); } @@ -78,7 +78,7 @@ public void testUnwrapWithDeclNoClash() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -91,7 +91,7 @@ public void testNoUnwrapWithDeclClash() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); // The inner block cannot be unwrapped as it would change which variable 'x = 2' refers to. assertEquals(1, ops.size()); } @@ -102,7 +102,7 @@ public void testNoUnwrapWithDeclClash2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -112,7 +112,7 @@ public void testOneUnwrapDisablesAnother() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(2, ops.size()); assertTrue(ops.get(0).preconditionHolds()); assertTrue(ops.get(1).preconditionHolds()); @@ -144,7 +144,7 @@ public void misc() throws Exception { assertTrue(PrettyPrinterVisitor.prettyPrintAsString(tu).contains(expectedStmt)); IRandom generator = new RandomWrapper(1); - List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, null, true)); + List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertTrue(PrettyPrinterVisitor.prettyPrintAsString(tu).contains(expectedStmt)); @@ -156,8 +156,8 @@ public void testDoubleRemoval() throws Exception { final String expected = "void main() { }"; final TranslationUnit tu = ParseHelper.parse(shader); final IRandom generator = new RandomWrapper(0); - List stmtOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, null, true)); - List unwrapOps = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, null, true)); + List stmtOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, new IdGenerator())); + List unwrapOps = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, new IdGenerator())); assertTrue(!stmtOps.isEmpty()); assertTrue(!unwrapOps.isEmpty()); stmtOps.forEach(StmtReductionOpportunity::applyReduction); @@ -190,12 +190,12 @@ public void unwrapFor() throws Exception { final RandomWrapper generator = new RandomWrapper(0); final IdGenerator idGenerator = new IdGenerator(); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, version, - generator, idGenerator, true)); + generator, idGenerator)); assertEquals(1, ops.size()); ops.get(0).applyReduction(); List remainingOps = VariableDeclReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, version, generator, idGenerator, true)); + new ReducerContext(false, version, generator, idGenerator)); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(tu), PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected))); assertEquals(2, remainingOps.size()); remainingOps.get(0).applyReduction(); @@ -209,7 +209,7 @@ public void unwrapFor() throws Exception { assertEquals(PrettyPrinterVisitor.prettyPrintAsString(tu), PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2))); remainingOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, version, generator, idGenerator, true)); + new ReducerContext(false, version, generator, idGenerator)); assertEquals(3, remainingOps.size()); remainingOps.get(0).applyReduction(); remainingOps.get(1).applyReduction(); @@ -219,7 +219,7 @@ public void unwrapFor() throws Exception { assertEquals(PrettyPrinterVisitor.prettyPrintAsString(tu), PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected3))); remainingOps = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, version, generator, idGenerator, true), fileOps); + new ReducerContext(false, version, generator, idGenerator), fileOps); assertEquals(0, remainingOps.size()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunitiesTest.java index ac25d5b25..735bca51d 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunitiesTest.java @@ -40,7 +40,7 @@ public void testRemoveUnusedLiveCodeDecl() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -53,7 +53,7 @@ public void testAnonymousStruct() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -66,7 +66,7 @@ public void testNamedStruct() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -86,10 +86,9 @@ public void testNothingToRemove() throws Exception { false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - new IdGenerator(), - true) + new IdGenerator()) ); assertEquals(0, ops.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java index 9b0c36c7a..5a7a5e270 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java @@ -19,6 +19,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -37,7 +38,7 @@ public void testDoNotReplace() throws Exception { VariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as the preserve semantics is enabled. assertTrue(ops.isEmpty()); } @@ -50,7 +51,7 @@ public void testDoNotReplaceConst() throws Exception { VariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as it is invalid to declare constant variable // without an initial value. assertTrue(ops.isEmpty()); @@ -74,7 +75,7 @@ public void testMultipleDeclarations() throws Exception { List ops = VariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); // Only variable declarations a and b have the initializer. // Thus, we expect the reducer to find only 2 opportunities. assertEquals(2, ops.size()); @@ -103,7 +104,7 @@ public void testMultipleLineDeclarationsOneLine() throws Exception { List ops = VariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(3, ops.size()); ops.forEach(VariableDeclToExprReductionOpportunity::applyReductionImpl); CompareAsts.assertEqualAsts(expected, tu); @@ -132,7 +133,7 @@ public void testAssignVariableIdentifier() throws Exception { List ops = VariableDeclToExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(4, ops.size()); ops.forEach(VariableDeclToExprReductionOpportunity::applyReductionImpl); CompareAsts.assertEqualAsts(expected, tu); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunitiesTest.java index d9528928b..25d686064 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunitiesTest.java @@ -22,6 +22,7 @@ import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.util.CannedRandom; import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ZeroCannedRandom; @@ -57,7 +58,8 @@ public void testInDead() throws Exception { + " }\n" + "}\n"; final TranslationUnit tu = ParseHelper.parse(shader); - final List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, new RandomWrapper(0), null, true)); + final List ops = + VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -75,7 +77,7 @@ public void testIncompatibleDeclaration() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(0, ops.size()); } @@ -94,7 +96,7 @@ public void testProblematicDeclarationPreviousScope() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(0, ops.size()); } @@ -127,7 +129,7 @@ public void testOneProblematicDeclarationPreviousScope() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -176,7 +178,7 @@ public void misc1() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(2, ops.size()); ops = ops.stream().filter(item -> item.getComponentType() == BasicType.FLOAT).collect(Collectors.toList()); assertEquals(2, ops.size()); @@ -218,7 +220,7 @@ public void misc2() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(2, ops.size()); ops.stream().filter(item -> item.getVectorName().equals("GLF_merged2_0_1_1_1_1_1bc") && item.getComponentName().equals("b")).findAny().get().applyReduction(); @@ -228,7 +230,7 @@ public void misc2() throws Exception { assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(0, ops.size()); } @@ -270,7 +272,7 @@ public void misc3() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(shader); final IRandom cannedRandom = new ZeroCannedRandom(); - List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, null, true)); + List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, new IdGenerator())); assertEquals(3, ops.size()); ops.stream().filter(item -> item.getComponentName().equals("P")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -278,7 +280,7 @@ public void misc3() throws Exception { assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops.stream().filter(item -> item.getComponentName().equals("GLF_merged2_0_1_1_1_1_1PQ")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected3)), PrettyPrinterVisitor.prettyPrintAsString(tu)); - ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, null, true)); + ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, new IdGenerator())); assertEquals(0, ops.size()); } @@ -320,7 +322,7 @@ public void misc4() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(shader); final IRandom cannedRandom = new CannedRandom(2); - final List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, null, true)); + final List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, new IdGenerator())); ops.stream().filter(item -> item.getComponentName().equals("a")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops.stream().filter(item -> item.getComponentName().equals("c")).findAny().get().applyReduction(); @@ -367,7 +369,7 @@ public void misc5() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(shader); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(3, ops.size()); ops.stream().filter(item -> item.getComponentName().equals("GLF_merged2_0_1_1_1_1_1PQ")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -406,12 +408,12 @@ public void testCompatibleDeclaration() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(2, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -452,12 +454,12 @@ public void testBasicBehaviour() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(2, ops.size()); ops.stream().filter(item -> item.getComponentName().equals("b")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(1, ops.size()); ops.stream().filter(item -> item.getComponentName().equals("c")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2)), PrettyPrinterVisitor.prettyPrintAsString(tu)); diff --git a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java index 425d71115..ec861c10a 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java @@ -87,8 +87,7 @@ public void testRenderBlackImage() throws Exception { + "}\n"; fileOps.writeShaderJobFile( new GlslShaderJob(Optional.empty(), new PipelineInfo(), ParseHelper.parse(shader)), - shaderJobFile, - false); + shaderJobFile); final File image = Util.getImage(shaderJobFile, temporaryFolder, fileOps); final Mat mat = opencv_imgcodecs.imread(image.getAbsolutePath()); diff --git a/tester/src/test/java/com/graphicsfuzz/tester/MiscellaneousGenerateThenReduceTest.java b/tester/src/test/java/com/graphicsfuzz/tester/MiscellaneousGenerateThenReduceTest.java index af3d1cead..bc5dba37c 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/MiscellaneousGenerateThenReduceTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/MiscellaneousGenerateThenReduceTest.java @@ -78,7 +78,7 @@ private void checkControlFlowWrapElimination(String program) .getReductionOpportunities(new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu), new ReducerContext(false, shadingLanguageVersion, - new SameValueRandom(false, 0), new IdGenerator(), true), fileOps); + new SameValueRandom(false, 0), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } diff --git a/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java index c8dd68e50..0cb603c17 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java @@ -129,7 +129,7 @@ private void testGenerateAndReduce(File originalShaderJobFile, List ops = ReductionOpportunities.getReductionOpportunities( new GlslShaderJob(Optional.empty(), pipelineInfo, fragmentShader), - new ReducerContext(false, fragmentShader.getShadingLanguageVersion(), generator, idGenerator, true), + new ReducerContext(false, fragmentShader.getShadingLanguageVersion(), generator, idGenerator), fileOps); if (ops.isEmpty()) { break; @@ -220,7 +220,7 @@ public void reduceRepeatedly(String shaderJobFileName, int numIterations, new ReductionDriver(new ReducerContext(false, shadingLanguageVersion, generator, - new IdGenerator(), true), false, fileOps, + new IdGenerator()), false, fileOps, new RandomFileJudge(generator, threshold, throwExceptionOnInvalid, fileOps), workDir) .doReduction(initialState, shaderJobShortName, 0, @@ -384,7 +384,7 @@ private String runReductionOnShader(File shaderJobFile, IFileJudge fileJudge) fileOps.copyShaderJobFileTo(shaderJobFile, new File(temporaryFolder.getRoot(), shaderJobFile.getName()), false); return new ReductionDriver(new ReducerContext(false, version, generator, - new IdGenerator(), true), false, fileOps, + new IdGenerator()), false, fileOps, fileJudge, temporaryFolder.getRoot()) .doReduction(state, shaderJobShortName, 0, -1); } From 5e012e2675e3a7d3b11c2ad43beee4af041192d0 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 9 Jul 2019 19:37:16 +0100 Subject: [PATCH 061/180] Fix issue where the reducer was replacing a dead code injection with its body (#599) The reducer used to be very conservative about replacing dead code injections with their bodies and was made less conservative. However this introduced a bug whereby the reducer would always replace 'if(_GLF_DEAD(...)) body' by 'body'. The problem was that the guard of the dead code injection was being regarded as itself being dead code, which is wrong; it should only be the 'then' and 'else' branches of the if statement that are regarded as dead. This change fixes this problem. It was necessary to change a number of tests in the process: tests where the reducer was only managing to do a particularly good job of reducing a shader due to this bug; for example it could turn: if (_GLF_DEAD(...)) { int a; a = 2; } into: { int a; a = 2; } which is OK, because the body of the 'if' statement does not have side-effects to data declared outside the 'if' statement. But this was only working due to the body of the 'if' being unconditionally used to replace the 'if'. Now that that problem is fixed, the side effect analysis is not yet sophisticated enough to determine that the body of the 'if' is side-effect free, thus the transformation does not take place. Fixes #598. --- .../common/util/SideEffectChecker.java | 32 +++++----- .../InjectionTracker.java | 16 ----- .../ReductionOpportunitiesBase.java | 36 +++++++---- .../SimplifyExprReductionOpportunities.java | 3 +- ...oundToBlockReductionOpportunitiesTest.java | 61 ++++++++----------- 5 files changed, 68 insertions(+), 80 deletions(-) diff --git a/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java b/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java index 0450ed97d..f8220bd9c 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java @@ -43,24 +43,26 @@ private static boolean isSideEffectFreeVisitor(IAstNode node, @Override public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { - if (!TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind) + if (TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind) .containsKey(functionCallExpr.getCallee())) { - // Assume that any call to a non-builtin might have a side-effect. - predicateHolds(); - } - for (FunctionPrototype p : - TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind) - .get(functionCallExpr.getCallee())) { - // We check each argument of the built-in's prototypes to see if they require lvalues - - // if so, they can cause side effects. - // We could be more precise here by finding the specific overload of the function rather - // than checking every possible prototype for lvalue parameters. - for (ParameterDecl param : p.getParameters()) { - if (param.getType().hasQualifier(TypeQualifier.OUT_PARAM) - || param.getType().hasQualifier(TypeQualifier.INOUT_PARAM)) { - predicateHolds(); + for (FunctionPrototype p : + TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind) + .get(functionCallExpr.getCallee())) { + // We check each argument of the built-in's prototypes to see if they require lvalues - + // if so, they can cause side effects. + // We could be more precise here by finding the specific overload of the function rather + // than checking every possible prototype for lvalue parameters. + for (ParameterDecl param : p.getParameters()) { + if (param.getType().hasQualifier(TypeQualifier.OUT_PARAM) + || param.getType().hasQualifier(TypeQualifier.INOUT_PARAM)) { + predicateHolds(); + } } } + } else if (!MacroNames.isGraphicsFuzzMacro(functionCallExpr)) { + // Assume that any call to a function that is not a GraphicsFuzz macro or builtin + // might have a side-effect. + predicateHolds(); } super.visitFunctionCallExpr(functionCallExpr); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InjectionTracker.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InjectionTracker.java index 34a7b60f3..5a6277674 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InjectionTracker.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InjectionTracker.java @@ -26,14 +26,12 @@ class InjectionTracker { private int numEnclosingFuzzedMacros; private int numEnclosingDeadCodeInjections; - private boolean underDeadMacro; private Deque> switchStmts; InjectionTracker() { this.numEnclosingFuzzedMacros = 0; this.numEnclosingDeadCodeInjections = 0; - this.underDeadMacro = false; this.switchStmts = new LinkedList<>(); } @@ -41,10 +39,6 @@ boolean underFuzzedMacro() { return numEnclosingFuzzedMacros > 0; } - boolean underDeadMacro() { - return underDeadMacro; - } - void enterFuzzedMacro() { numEnclosingFuzzedMacros++; } @@ -54,16 +48,6 @@ void exitFuzzedMacro() { numEnclosingFuzzedMacros--; } - void enterDeadMacro() { - assert !underDeadMacro; - underDeadMacro = true; - } - - void exitDeadMacro() { - assert underDeadMacro; - underDeadMacro = false; - } - void enterDeadCodeInjection() { numEnclosingDeadCodeInjections++; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java index c93f97cf4..e103dd647 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java @@ -117,13 +117,7 @@ public void visitBlockStmt(BlockStmt block) { injectionTracker.notifySwitchBreak(); } - if (isDeadCodeInjection(child)) { - injectionTracker.enterDeadCodeInjection(); - } visit(child); - if (isDeadCodeInjection(child)) { - injectionTracker.exitDeadCodeInjection(); - } } leaveBlockStmt(block); } @@ -171,18 +165,36 @@ public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { if (MacroNames.isFuzzed(functionCallExpr)) { injectionTracker.enterFuzzedMacro(); } - if (MacroNames.isDeadByConstruction(functionCallExpr)) { - injectionTracker.enterDeadMacro(); - } super.visitFunctionCallExpr(functionCallExpr); - if (MacroNames.isDeadByConstruction(functionCallExpr)) { - injectionTracker.exitDeadMacro(); - } if (MacroNames.isFuzzed(functionCallExpr)) { injectionTracker.exitFuzzedMacro(); } } + @Override + public void visitIfStmt(IfStmt ifStmt) { + // This method is overridden in order to allow tracking of when we are inside a dead code + // injection. + + // Even for an "if(_GLF_DEAD(...))", the condition itself is not inside the dead code injection, + // so we visit it as usual first. + visitChildFromParent(ifStmt.getCondition(), ifStmt); + + if (isDeadCodeInjection(ifStmt)) { + // This is a dead code injection, so update the injection tracker appropriately before + // visiting the 'then' and (possibly) 'else' branches. + injectionTracker.enterDeadCodeInjection(); + } + visit(ifStmt.getThenStmt()); + if (ifStmt.hasElseStmt()) { + visit(ifStmt.getElseStmt()); + } + if (isDeadCodeInjection(ifStmt)) { + // Update the injection tracker to indicate that we have left this dead code injection. + injectionTracker.exitDeadCodeInjection(); + } + } + @Override protected void visitChildFromParent(IAstNode child, IAstNode parent) { super.visitChildFromParent(child, parent); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java index dc8ae474c..63bb31cbb 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java @@ -110,8 +110,7 @@ boolean allowedToReduceExpr(IAstNode parent, Expr child) { return true; } - if (injectionTracker.enclosedByDeadCodeInjection() && !injectionTracker.underDeadMacro()) { - // We want to reduce expressions in dead code blocks, but not inside the dead macro itself + if (injectionTracker.enclosedByDeadCodeInjection()) { return true; } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java index df45063fd..d3b7b8b43 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java @@ -125,22 +125,20 @@ public void testUnderUnreachableSwitch() throws Exception { } @Test - public void testDoRemoveDeadIf() throws Exception { + public void testDoNotRemoveDeadIf() throws Exception { final String original = "" + "void main() {" + + " int a = 2;" + " if (" + Constants.GLF_DEAD + "(false)) {" - + " int a = 2;" + " a = a + 1;" + " }" + "}"; - final String expected = "" - + "void main() {" - + " {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + "}"; - check(false, original, expected); + final List opportunities = + CompoundToBlockReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(ParseHelper.parse(original)), + new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); + assertEquals(0, opportunities.size()); } @Test @@ -154,7 +152,7 @@ public void testInDeadCode() throws Exception { + " }" + " }" + "}"; - final String expected1 = "" + final String expected = "" + "void main() {" + " if (" + Constants.GLF_DEAD + "(false)) {" + " {" @@ -163,16 +161,7 @@ public void testInDeadCode() throws Exception { + " }" + " }" + "}"; - final String expected2 = "" - + "void main() {" - + " {" - + " if (false) {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - check(false, original, expected1, expected2); + check(false, original, expected); } @Test @@ -256,20 +245,7 @@ public void testInDeadCode2() throws Exception { + " }" + " }" + "}"; - final String expected4 = "" - + "void main() {" - + " int a = 4;" - + " {" - + " if (a) {" - + " if (a > 0) {" - + " int a = 2;" - + " a = a + 1;" - + " } else" - + " a++;" - + " }" - + " }" - + "}"; - check(false, original, expected1, expected2, expected3, expected4); + check(false, original, expected1, expected2, expected3); } @Test @@ -534,4 +510,19 @@ public void testDoNotTurnForLoopIntoBlock() throws Exception { assertEquals(0, ops.size()); } + @Test + public void doNotReplaceDeadCodeInjectionWithBody() throws Exception { + final String original = "void main() {" + + " if (" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false))) {" + + " discard;" + + " }" + + "}"; + final List opportunities = + CompoundToBlockReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(ParseHelper.parse(original)), + new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); + assertEquals(0, opportunities.size()); + } + } From 08989bccbc6d5bd69386457209bb75f085b56ec7 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 11 Jul 2019 12:13:57 +0100 Subject: [PATCH 062/180] Fix opaque expressions based on vector length (#600) We recently added the ability to generate zero as length(zero vector), and one as length(normalize(one vector)). However, the code was incorrectly assuming that the length of a vector of dimension n would itself be a vector of dimension n, whereas in reality the length of a vector is always a scalar. As well as fixing this so that the expected return type for these ways to make zero and one is 'float', the change allows the dimension of the vector whose length is taken to be chosen at random. Finer-grained tests for opaque zero and one generation are also added, and the shading language versions for which opaque expression generation is tested has been expanded a bit. Fixes #559 --- .../fuzzer/OpaqueExpressionGenerator.java | 37 ++++-- .../fuzzer/OpaqueExpressionGeneratorTest.java | 105 ++++++++++++++++-- 2 files changed, 127 insertions(+), 15 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 6f075b13d..dc7b0ff91 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -105,7 +105,10 @@ private List waysToMakeZeroOrOne() { ); } - private List waysToMakeZero() { + /** + * This method has non-private visibility for purposes of testing only. + */ + List waysToMakeZero() { List opaqueZeroFactories = new ArrayList<>(); opaqueZeroFactories.addAll(waysToMakeZeroOrOne()); opaqueZeroFactories.add(this::opaqueZeroSin); @@ -115,7 +118,10 @@ private List waysToMakeZero() { return opaqueZeroFactories; } - private List waysToMakeOne() { + /** + * This method has non-private visibility for purposes of testing only. + */ + List waysToMakeOne() { List opaqueOneFactories = new ArrayList<>(); opaqueOneFactories.addAll(waysToMakeZeroOrOne()); opaqueOneFactories.add(this::opaqueOneExponential); @@ -402,10 +408,17 @@ private Optional opaqueZeroVectorLength(BasicType type, boolean constConte Fuzzer fuzzer, boolean isZero) { // represent 0 as the length of zero vector, e.g. length(opaqueZero). assert isZero; - if (!BasicType.allGenTypes().contains(type)) { + if (type != BasicType.FLOAT) { + // 'length' has return type 'float', so we can only create a scalar floating-point zero. return Optional.empty(); } - return Optional.of(new FunctionCallExpr("length", makeOpaqueZero(type, constContext, depth, + + // We can choose any vector size we wish for the vector whose length will be computed. + final BasicType vectorType = + BasicType.allGenTypes().get(generator.nextInt(BasicType.allGenTypes().size())); + + return Optional.of(new FunctionCallExpr("length", makeOpaqueZero(vectorType, constContext, + depth, fuzzer))); } @@ -434,13 +447,21 @@ private Optional opaqueOneCosine(BasicType type, boolean constContext, fin private Optional opaqueOneNormalizedVectorLength(BasicType type, boolean constContext, final int depth, Fuzzer fuzzer, boolean isZero) { - // represent 1 as the length of normalized vector + // represent 1 as the length of a normalized vector assert !isZero; - if (!BasicType.allGenTypes().contains(type)) { + if (type != BasicType.FLOAT) { + // 'length' has return type 'float', so we can only create a scalar floating-point zero. return Optional.empty(); } - Expr normalizedExpr = new FunctionCallExpr("normalize", makeOpaqueZeroOrOne(isZero, - type, constContext, depth, fuzzer)); + + // We can choose any vector size we wish for the vector whose length will be computed. + final BasicType vectorType = + BasicType.allGenTypes().get(generator.nextInt(BasicType.allGenTypes().size())); + + // We create a vector of ones and normalize it. Note that we could be more general and create + // any non-zero vector and normalize it. + Expr normalizedExpr = new FunctionCallExpr("normalize", makeOpaqueZeroOrOne(false, + vectorType, constContext, depth, fuzzer)); return Optional.of(new FunctionCallExpr("length", normalizedExpr)); } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java index bc9ad5b20..491b2d98a 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; +import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -31,12 +32,16 @@ import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.QualifiedType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.typing.Scope; import com.graphicsfuzz.common.typing.SupportedTypes; import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderJobFileOperations; @@ -46,6 +51,7 @@ import java.io.File; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import org.junit.Rule; @@ -61,25 +67,34 @@ public class OpaqueExpressionGeneratorTest { @Test public void testGeneratesValidExpressions() throws Exception { + // We make a random number generator here and pass it into functions below in order to maximise + // diversity - even though there are differences between shading languages, there is still a + // good chance that until such a difference is hit, generating expressions for different + // shading languages but from the same seed will lead to identical results. + final IRandom generator = new RandomWrapper(0); + + // These are the shading language versions we support best, so test them thoroughly. for (ShadingLanguageVersion shadingLanguageVersion : Arrays.asList( + ShadingLanguageVersion.ESSL_100, ShadingLanguageVersion.ESSL_300, - ShadingLanguageVersion.ESSL_100)) { + ShadingLanguageVersion.ESSL_310, + ShadingLanguageVersion.ESSL_320)) { for (BasicType t : BasicType.allBasicTypes()) { if (!SupportedTypes.supported(t, shadingLanguageVersion)) { continue; } - final TranslationUnit tu = new TranslationUnit(Optional.of(ShadingLanguageVersion.ESSL_310), + final TranslationUnit tu = new TranslationUnit(Optional.of(shadingLanguageVersion), Arrays.asList( new PrecisionDeclaration("precision mediump float;"), new FunctionDefinition( - new FunctionPrototype("main", VoidType.VOID, new ArrayList<>()), + new FunctionPrototype("main", VoidType.VOID, Collections.emptyList()), new BlockStmt(makeMutatedExpressionAssignments(t, - shadingLanguageVersion, 1000), + shadingLanguageVersion, 1000, generator), false)))); Generate.addInjectionSwitchIfNotPresent(tu); final File shaderJobFile = temporaryFolder.newFile("ex.json"); final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo("{}"), + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu), shaderJobFile); assertTrue(fileOps.areShadersValid(shaderJobFile, false)); fileOps.deleteShaderJobFile(shaderJobFile); @@ -89,7 +104,8 @@ public void testGeneratesValidExpressions() throws Exception { private List makeMutatedExpressionAssignments(BasicType basicType, ShadingLanguageVersion shadingLanguageVersion, - int numberOfAssignments) { + int numberOfAssignments, + IRandom generator) { // We declare only one variable and generate a number of assignments to this variable, // instead of making multiple declarations and assignments. // numberOfAssignments should be large enough to get high coverage, e.g. 1000. @@ -99,7 +115,6 @@ private List makeMutatedExpressionAssignments(BasicType basicType, new VariableDeclInfo("x", null, null)))); for (int i = 0; i < numberOfAssignments; i++) { - final IRandom generator = new RandomWrapper(i); final OpaqueExpressionGenerator opaqueExpressionGenerator = new OpaqueExpressionGenerator(generator, generationParams, shadingLanguageVersion); @@ -114,4 +129,80 @@ private List makeMutatedExpressionAssignments(BasicType basicType, } return newStmts; } + + @Test + public void testWaysToMakeZeroAndOne() throws Exception { + + final IRandom generator = new RandomWrapper(0); + final GenerationParams generationParams = GenerationParams.large(ShaderKind.FRAGMENT, true); + + // These are the shading language versions we support best, so test them thoroughly. + for (ShadingLanguageVersion shadingLanguageVersion : Arrays.asList( + ShadingLanguageVersion.ESSL_100, + ShadingLanguageVersion.ESSL_300, + ShadingLanguageVersion.ESSL_310, + ShadingLanguageVersion.ESSL_320)) { + final IdGenerator idGenerator = new IdGenerator(); + final List stmts = new ArrayList<>(); + final OpaqueExpressionGenerator opaqueExpressionGenerator = + new OpaqueExpressionGenerator(generator, + generationParams, shadingLanguageVersion); + for (BasicType basicType : BasicType.allNumericTypes()) { + if (!SupportedTypes.supported(basicType, shadingLanguageVersion)) { + continue; + } + for (boolean constContext : Arrays.asList(true, false)) { + stmts.addAll(makeStatementsFromFactories(generator, generationParams, + shadingLanguageVersion, + idGenerator, basicType, constContext, opaqueExpressionGenerator.waysToMakeZero(), true)); + // We do not allow making one for non-square matrices. + if (!BasicType.allNonSquareMatrixTypes().contains(basicType)) { + stmts.addAll(makeStatementsFromFactories(generator, generationParams, + shadingLanguageVersion, + idGenerator, basicType, constContext, opaqueExpressionGenerator.waysToMakeOne(), + false)); + } + } + } + final TranslationUnit tu = new TranslationUnit(Optional.of(shadingLanguageVersion), + Arrays.asList( + new PrecisionDeclaration("precision mediump float;"), + new FunctionDefinition( + new FunctionPrototype("main", VoidType.VOID, Collections.emptyList()), + new BlockStmt(stmts, + false)))); + Generate.addInjectionSwitchIfNotPresent(tu); + final File shaderJobFile = temporaryFolder.newFile("ex.json"); + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + tu), shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); + fileOps.deleteShaderJobFile(shaderJobFile); + } + } + + private List makeStatementsFromFactories(IRandom generator, GenerationParams generationParams, ShadingLanguageVersion shadingLanguageVersion, IdGenerator idGenerator, BasicType typeToGenerate, boolean constContext, List factories, boolean makingZero) { + List result = new ArrayList<>(); + for (OpaqueZeroOneFactory factory : factories) { + final Optional expr = factory.tryMakeOpaque(typeToGenerate, constContext, 0, + new Fuzzer(new FuzzingContext(new Scope(null)), shadingLanguageVersion, + generator, generationParams), makingZero); + if (expr.isPresent()) { + final Type baseType = constContext ? new QualifiedType(typeToGenerate, + Collections.singletonList(TypeQualifier.CONST)) : typeToGenerate; + result.add(new DeclarationStmt( + new VariablesDeclaration( + baseType, + new VariableDeclInfo( + "v" + idGenerator.freshId(), + null, + new ScalarInitializer(expr.get()) + ) + ) + )); + } + } + return result; + } + } From 69536daacb175050562e87b0565053a71906d824 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Fri, 12 Jul 2019 15:02:23 +0100 Subject: [PATCH 063/180] Update SPIRV-Tools (#606) --- assembly-binaries/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assembly-binaries/pom.xml b/assembly-binaries/pom.xml index 5491452e9..94ee8d19d 100644 --- a/assembly-binaries/pom.xml +++ b/assembly-binaries/pom.xml @@ -58,7 +58,7 @@ limitations under the License. - 422f2fe0f0f32494fa687a12ba343d24863b330a + f2170cc791d0eaa5789ec7528862ae00b984b3b8 9d3202492d78aae5ced08ff14679b81c98d71c15 803082da4f73920729938905cb1727e11ff79530 1586e566f4949b1957e7c32454cbf27e501ed632 From 27fd7f2ca59d5b8cc3cd7ad5fb0f1e1482043eb6 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Mon, 15 Jul 2019 10:50:06 +0100 Subject: [PATCH 064/180] Fix WebUi paths so variant image shows in wrong image result view (#609) Fixes #607 --- .../com/graphicsfuzz/server/webui/WebUi.java | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java index 31c5931ba..2a8d4c04c 100755 --- a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java +++ b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java @@ -905,6 +905,19 @@ private void viewShader(HttpServletRequest request, HttpServletResponse response response.getWriter().println(html); } + private static String posixPath(String path, String... otherParts) { + StringBuilder result = new StringBuilder(path); + for (String part : otherParts) { + result.append("/"); + result.append(part); + } + return result.toString(); + } + + private static File posixPathToFile(String path, String... otherParts) { + return new File(FilenameUtils.separatorsToSystem(posixPath(path, otherParts))); + } + // Page to view the result from a single shader by a worker - /webui/result/ private void viewResult(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException, TException { @@ -919,18 +932,17 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response final String worker = path[3]; final String shaderFamily = path[4]; final String variant = path[5]; - final Path variantDir = - Paths.get(WebUiConstants.WORKER_DIR, worker, shaderFamily); - final Path variantFullPathNoExtension = Paths.get(variantDir.toString(), variant); - - File infoFile = new File(variantDir.toString(), variant + ".info.json"); + final String variantDir = + posixPath(WebUiConstants.WORKER_DIR, worker, shaderFamily); + final String variantFullPathNoExtension = posixPath(variantDir, variant); + File infoFile = posixPathToFile(variantDir, variant + ".info.json"); if (!infoFile.isFile()) { err404(request, response, "Invalid result path: cannot find corresponding info file"); return; } final boolean isCompute = - new File(Paths.get("shaderfamilies", shaderFamily, variant + ".comp").toString()) + posixPathToFile("shaderfamilies", shaderFamily, variant + ".comp") .isFile(); JsonObject info = accessFileInfo.getResultInfo(infoFile); @@ -952,7 +964,7 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response "\n", "

", "Delete this result
\n", "", @@ -1013,14 +1025,13 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response } } else { - final String referencePngPath = - Paths.get(variantDir.toString(), "reference.png").toString(); + final String referencePngPath = posixPath(variantDir, "reference.png"); htmlAppendLn("

Reference image:

", ""); - String pngPath = variantDir + variant + ".png"; - File pngFile = new File(pngPath); + String pngPath = posixPath(variantDir, variant + ".png"); + File pngFile = posixPathToFile(pngPath); if (!variant.equals("reference")) { if (pngFile.exists()) { @@ -1029,8 +1040,8 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response } } - String gifPath = variantDir + variant + ".gif"; - File gifFile = new File(gifPath); + String gifPath = posixPath(variantDir, variant + ".gif"); + File gifFile = posixPathToFile(gifPath); if (gifFile.exists()) { htmlAppendLn("

Results non-deterministic animation:

", "", @@ -1043,7 +1054,7 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response "'> "); } - if (!(pngFile.exists()) && !(gifFile.exists())) { + if (!pngFile.exists() && !gifFile.exists()) { htmlAppendLn("

No image to display for this result status

"); } @@ -1098,13 +1109,12 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response "
\n", "

Run log

\n", "\n", "
"); // Get result file - File result = variantFullPathNoExtension.toFile(); + File result = posixPathToFile(variantFullPathNoExtension); // Information/links for result File referenceRes = new File(result.getParentFile(), "reference.info.json"); From 913ca3fb420a9ed7a3f33624ffa4862501f9f6b5 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Mon, 15 Jul 2019 11:54:35 +0100 Subject: [PATCH 065/180] Replace ArrayInitializer and ScalarInitializer with Initializer (#601) The Initializer class had a never-used subclass, ArrayInitializer (which was in fact intended to handle the case of initializer lists), and the ScalarInitializer subclass was being used for all initializers (including for array initialization). As initializer lists are not supported in OpenGL ES, this change simplifies things so that there is just an Initializer class, which does the job ScalarInitializer was doing before. Support for initializer lists can be added in the future if they are deemed important. Fixes #520 --- .../common/ast/decl/ArrayInitializer.java | 33 ---------- .../common/ast/decl/Initializer.java | 40 +++++++++++- .../common/ast/decl/ScalarInitializer.java | 62 ------------------- .../common/ast/inliner/Inliner.java | 4 +- .../common/ast/inliner/ReturnRemover.java | 4 +- .../common/ast/visitors/AstBuilder.java | 3 +- .../common/ast/visitors/IAstVisitor.java | 7 +-- .../common/ast/visitors/StandardVisitor.java | 12 +--- .../common/util/TruncateLoops.java | 4 +- .../graphicsfuzz/common/typing/TyperTest.java | 29 ++++----- .../common/util/ParseHelperTest.java | 38 ++++++------ .../common/util/PruneUniforms.java | 5 +- .../AddWrappingConditionalMutation.java | 5 +- .../IdentityMutationFinder.java | 15 +---- .../SplitForLoopMutation.java | 8 +-- .../StructificationMutation.java | 6 +- .../DonateCodeTransformation.java | 17 +++-- .../DonateDeadCodeTransformation.java | 4 +- .../DonateLiveCodeTransformation.java | 8 +-- .../generator/util/ConstCleaner.java | 7 +-- .../fuzzer/OpaqueExpressionGeneratorTest.java | 4 +- .../StructificationMutationTest.java | 5 +- .../DonateLiveCodeTransformationTest.java | 6 +- .../SplitForLoopTransformationTest.java | 6 +- .../DestructifyReductionOpportunity.java | 10 +-- ...iableDeclToExprReductionOpportunities.java | 4 +- ...ariableDeclToExprReductionOpportunity.java | 5 +- ...lineInitializerReductionOpportunities.java | 3 +- .../LoopMergeReductionOpportunities.java | 7 +-- .../ReductionOpportunitiesBase.java | 6 +- .../StructifiedVariableInfo.java | 8 +-- ...iableDeclToExprReductionOpportunities.java | 4 +- ...ariableDeclToExprReductionOpportunity.java | 5 +- ...uctifiedFieldReductionOpportunityTest.java | 6 +- ...veStructFieldReductionOpportunityTest.java | 10 +-- 35 files changed, 145 insertions(+), 255 deletions(-) delete mode 100644 ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInitializer.java delete mode 100644 ast/src/main/java/com/graphicsfuzz/common/ast/decl/ScalarInitializer.java diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInitializer.java b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInitializer.java deleted file mode 100644 index df369467b..000000000 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInitializer.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.common.ast.decl; - -import com.graphicsfuzz.common.ast.visitors.IAstVisitor; - -public class ArrayInitializer extends Initializer { - - @Override - public void accept(IAstVisitor visitor) { - visitor.visitArrayInitializer(this); - } - - @Override - public ArrayInitializer clone() { - return new ArrayInitializer(); - } - -} diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/Initializer.java b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/Initializer.java index 4c5645672..90e2c14bd 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/Initializer.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/Initializer.java @@ -17,10 +17,46 @@ package com.graphicsfuzz.common.ast.decl; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.visitors.IAstVisitor; -public abstract class Initializer implements IAstNode { +public class Initializer implements IAstNode { + + private Expr expr; + + public Initializer(Expr expr) { + this.expr = expr; + } + + public Expr getExpr() { + return expr; + } + + public void setExpr(Expr expr) { + this.expr = expr; + } + + @Override + public void accept(IAstVisitor visitor) { + visitor.visitInitializer(this); + } + + @Override + public Initializer clone() { + return new Initializer(expr.clone()); + } + + @Override + public void replaceChild(IAstNode child, IAstNode newChild) { + if (!(child == expr && newChild instanceof Expr)) { + throw new IllegalArgumentException(); + } + setExpr((Expr) newChild); + } @Override - public abstract Initializer clone(); + public boolean hasChild(IAstNode candidateChild) { + return candidateChild == expr; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ScalarInitializer.java b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ScalarInitializer.java deleted file mode 100644 index 94456a1c2..000000000 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ScalarInitializer.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.common.ast.decl; - -import com.graphicsfuzz.common.ast.IAstNode; -import com.graphicsfuzz.common.ast.expr.Expr; -import com.graphicsfuzz.common.ast.visitors.IAstVisitor; - -public class ScalarInitializer extends Initializer { - - private Expr expr; - - public ScalarInitializer(Expr expr) { - this.expr = expr; - } - - public Expr getExpr() { - return expr; - } - - public void setExpr(Expr expr) { - this.expr = expr; - } - - @Override - public void accept(IAstVisitor visitor) { - visitor.visitScalarInitializer(this); - } - - @Override - public ScalarInitializer clone() { - return new ScalarInitializer(expr.clone()); - } - - @Override - public void replaceChild(IAstNode child, IAstNode newChild) { - if (!(child == expr && newChild instanceof Expr)) { - throw new IllegalArgumentException(); - } - setExpr((Expr) newChild); - } - - @Override - public boolean hasChild(IAstNode candidateChild) { - return candidateChild == expr; - } - -} diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java index 0e1be8345..662f4ac07 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java @@ -20,8 +20,8 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.ParameterDecl; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -186,7 +186,7 @@ private List getInlinedStmts(FunctionDefinition functionDefinition, new VariablesDeclaration( pd.getType().getWithoutQualifiers(), new VariableDeclInfo(pd.getName(), null, - new ScalarInitializer(call.getArg(i).clone()))))); + new Initializer(call.getArg(i).clone()))))); } for (Stmt stmt : functionDefinition.getBody().getStmts()) { if (stmt instanceof ReturnStmt) { diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/ReturnRemover.java b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/ReturnRemover.java index 1e4ddb8d1..b4b4d3142 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/ReturnRemover.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/ReturnRemover.java @@ -19,7 +19,7 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -170,7 +170,7 @@ private void addSpecialDeclarations() { fd.getBody().insertStmt(0, new DeclarationStmt( new VariablesDeclaration(BasicType.BOOL, new VariableDeclInfo(makeHasReturnedName(), null, - new ScalarInitializer(new BoolConstantExpr(false)))))); + new Initializer(new BoolConstantExpr(false)))))); } private void addReturnInstrumentation() { diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java index fa29dc4a9..846449b83 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java @@ -26,7 +26,6 @@ import com.graphicsfuzz.common.ast.decl.InterfaceBlock; import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; @@ -835,7 +834,7 @@ public Initializer visitInitializer(InitializerContext ctx) { return null; } if (ctx.assignment_expression() != null) { - return new ScalarInitializer(visitAssignment_expression(ctx.assignment_expression())); + return new Initializer(visitAssignment_expression(ctx.assignment_expression())); } throw new RuntimeException(); } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/IAstVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/IAstVisitor.java index 98784af46..1df2911a2 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/IAstVisitor.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/IAstVisitor.java @@ -19,14 +19,13 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.ArrayInfo; -import com.graphicsfuzz.common.ast.decl.ArrayInitializer; import com.graphicsfuzz.common.ast.decl.DefaultLayout; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.InterfaceBlock; import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; @@ -90,9 +89,7 @@ public interface IAstVisitor { void visitPrecisionDeclaration(PrecisionDeclaration precisionDeclaration); - void visitArrayInitializer(ArrayInitializer arrayInitializer); - - void visitScalarInitializer(ScalarInitializer scalarInitializer); + void visitInitializer(Initializer initializer); void visitBinaryExpr(BinaryExpr binaryExpr); diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/StandardVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/StandardVisitor.java index 52b6574d4..f06dadde6 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/StandardVisitor.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/StandardVisitor.java @@ -19,15 +19,14 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.ArrayInfo; -import com.graphicsfuzz.common.ast.decl.ArrayInitializer; import com.graphicsfuzz.common.ast.decl.Declaration; import com.graphicsfuzz.common.ast.decl.DefaultLayout; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.InterfaceBlock; import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; @@ -163,13 +162,8 @@ public void visitPrecisionDeclaration(PrecisionDeclaration precisionDeclaration) } @Override - public void visitArrayInitializer(ArrayInitializer arrayInitializer) { - throw new RuntimeException(); - } - - @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - visitChildFromParent(scalarInitializer.getExpr(), scalarInitializer); + public void visitInitializer(Initializer initializer) { + visitChildFromParent(initializer.getExpr(), initializer); } @Override diff --git a/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java b/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java index 62683ca72..6bb96053e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java +++ b/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java @@ -18,7 +18,7 @@ import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -84,7 +84,7 @@ private void handleLoop(LoopStmt loopStmt) { final DeclarationStmt limiterDeclaration = new DeclarationStmt( new VariablesDeclaration(BasicType.INT, new VariableDeclInfo(limiterName, null, - new ScalarInitializer(new IntConstantExpr("0"))))); + new Initializer(new IntConstantExpr("0"))))); final List limitCheckAndIncrement = Arrays.asList( new IfStmt( diff --git a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java index 1f7fc29c2..c0b6984dd 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java @@ -22,11 +22,10 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.ParameterDecl; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; @@ -38,14 +37,12 @@ import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.TernaryExpr; import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; -import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; import com.graphicsfuzz.common.ast.expr.UnaryExpr; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.QualifiedType; import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.type.VoidType; -import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.GlslParserException; @@ -451,9 +448,9 @@ public void testOctalIntLiteralTyped() throws Exception { + "}"); new NullCheckTyper(tu) { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - assertSame(lookupType(scalarInitializer.getExpr()), BasicType.INT); + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + assertSame(lookupType(initializer.getExpr()), BasicType.INT); } }.visit(tu); } @@ -466,9 +463,9 @@ public void testHexIntLiteralTyped() throws Exception { + "}"); new NullCheckTyper(tu) { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - assertSame(lookupType(scalarInitializer.getExpr()), BasicType.INT); + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + assertSame(lookupType(initializer.getExpr()), BasicType.INT); } }.visit(tu); } @@ -481,9 +478,9 @@ public void testOctalUnsignedIntLiteralTyped() throws Exception { + "}"); new NullCheckTyper(tu) { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - assertSame(lookupType(scalarInitializer.getExpr()), BasicType.UINT); + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + assertSame(lookupType(initializer.getExpr()), BasicType.UINT); } }.visit(tu); } @@ -496,9 +493,9 @@ public void testHexUnsignedIntLiteralTyped() throws Exception { + "}"); new NullCheckTyper(tu) { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - assertSame(lookupType(scalarInitializer.getExpr()), BasicType.UINT); + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + assertSame(lookupType(initializer.getExpr()), BasicType.UINT); } }.visit(tu); } diff --git a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java index ad65e1827..18c64d3a6 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java @@ -21,7 +21,7 @@ import com.graphicsfuzz.common.ast.decl.Declaration; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.FloatConstantExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; @@ -381,10 +381,10 @@ public void testParseOctalIntLiteral() throws Exception { assertTrue( new CheckPredicateVisitor() { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (scalarInitializer.getExpr() instanceof IntConstantExpr - && ((IntConstantExpr) scalarInitializer.getExpr()).getValue().equals("031")) { + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + if (initializer.getExpr() instanceof IntConstantExpr + && ((IntConstantExpr) initializer.getExpr()).getValue().equals("031")) { predicateHolds(); } } @@ -401,10 +401,10 @@ public void testParseHexIntLiteral() throws Exception { assertTrue( new CheckPredicateVisitor() { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (scalarInitializer.getExpr() instanceof IntConstantExpr - && ((IntConstantExpr) scalarInitializer.getExpr()).getValue().equals("0xa03b")) { + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + if (initializer.getExpr() instanceof IntConstantExpr + && ((IntConstantExpr) initializer.getExpr()).getValue().equals("0xa03b")) { predicateHolds(); } } @@ -421,10 +421,10 @@ public void testParseOctalUnsignedIntLiteral() throws Exception { assertTrue( new CheckPredicateVisitor() { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (scalarInitializer.getExpr() instanceof UIntConstantExpr - && ((UIntConstantExpr) scalarInitializer.getExpr()).getValue().equals("031u")) { + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + if (initializer.getExpr() instanceof UIntConstantExpr + && ((UIntConstantExpr) initializer.getExpr()).getValue().equals("031u")) { predicateHolds(); } } @@ -441,10 +441,10 @@ public void testParseHexUnsignedIntLiteral() throws Exception { assertTrue( new CheckPredicateVisitor() { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (scalarInitializer.getExpr() instanceof UIntConstantExpr - && ((UIntConstantExpr) scalarInitializer.getExpr()).getValue().equals("0xA03Bu")) { + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + if (initializer.getExpr() instanceof UIntConstantExpr + && ((UIntConstantExpr) initializer.getExpr()).getValue().equals("0xA03Bu")) { predicateHolds(); } } @@ -517,8 +517,8 @@ public void testParseInPlacePrecisionAndFloatSuffix() throws Exception { variablesDeclaration.getBaseType(); assertTrue(baseType.hasQualifier(TypeQualifier.MEDIUMP)); assertEquals("1.00f", - ((FloatConstantExpr) ((ScalarInitializer) variablesDeclaration.getDeclInfo(0) - .getInitializer()).getExpr()).getValue()); + ((FloatConstantExpr) variablesDeclaration.getDeclInfo(0) + .getInitializer().getExpr()).getValue()); } @Test diff --git a/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java b/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java index a78cd74ac..49e75cdd5 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java @@ -20,7 +20,6 @@ import com.graphicsfuzz.common.ast.decl.ArrayInfo; import com.graphicsfuzz.common.ast.decl.Declaration; import com.graphicsfuzz.common.ast.decl.Initializer; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; @@ -131,11 +130,11 @@ private static Initializer makeInitializer(BasicType baseType, args.subList(index * baseType.getNumElements(), (index + 1) * baseType.getNumElements()))); } - return new ScalarInitializer(new ArrayConstructorExpr( + return new Initializer(new ArrayConstructorExpr( new ArrayType(baseType.getWithoutQualifiers(), arrayInfo.clone()), argExprs)); } - return new ScalarInitializer(getBasicTypeLiteralExpr(baseType, args)); + return new Initializer(getBasicTypeLiteralExpr(baseType, args)); } public static Expr getBasicTypeLiteralExpr(BasicType baseType, List args) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddWrappingConditionalMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddWrappingConditionalMutation.java index 6c5766b14..aae749311 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddWrappingConditionalMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddWrappingConditionalMutation.java @@ -16,7 +16,7 @@ package com.graphicsfuzz.generator.semanticspreserving; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -38,7 +38,6 @@ import com.graphicsfuzz.common.util.ContainsTopLevelBreak; import com.graphicsfuzz.common.util.ContainsTopLevelContinue; import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.generator.fuzzer.Fuzzer; import com.graphicsfuzz.generator.fuzzer.FuzzingContext; import com.graphicsfuzz.generator.fuzzer.OpaqueExpressionGenerator; @@ -167,7 +166,7 @@ private Stmt makeSingleIterationForStmt(Stmt stmt, : opaqueExpressionGenerator .makeOpaqueOne(BasicType.INT, loopBoundsMustBeConst, 0, fuzzer); DeclarationStmt init = new DeclarationStmt(new VariablesDeclaration(BasicType.INT, - new VariableDeclInfo(loopVariableName, null, new ScalarInitializer(start)))); + new VariableDeclInfo(loopVariableName, null, new Initializer(start)))); Expr end = up ? opaqueExpressionGenerator .makeOpaqueOne(BasicType.INT, loopBoundsMustBeConst, 0, fuzzer) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java index dd2a44b08..8f0b44a5c 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java @@ -17,9 +17,8 @@ package com.graphicsfuzz.generator.semanticspreserving; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ArrayInitializer; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; import com.graphicsfuzz.common.ast.expr.BinaryExpr; @@ -228,18 +227,10 @@ public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) } @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { + public void visitInitializer(Initializer initializer) { assert !inInitializer; inInitializer = true; - super.visitScalarInitializer(scalarInitializer); - inInitializer = false; - } - - @Override - public void visitArrayInitializer(ArrayInitializer arrayInitializer) { - assert !inInitializer; - inInitializer = true; - super.visitArrayInitializer(arrayInitializer); + super.visitInitializer(initializer); inInitializer = false; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/SplitForLoopMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/SplitForLoopMutation.java index ae356524f..e1a9da067 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/SplitForLoopMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/SplitForLoopMutation.java @@ -16,7 +16,7 @@ package com.graphicsfuzz.generator.semanticspreserving; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -102,7 +102,7 @@ private void adjustInitializer(ForStmt loop, int numIterationsToSplitAfter, + (loopSplitInfo.getIncreasing() ? 1 : -1) * numIterationsToSplitAfter; VariablesDeclaration varDecl = ((DeclarationStmt) loop.getInit()).getVariablesDeclaration(); - varDecl.getDeclInfo(0).setInitializer(new ScalarInitializer(new IntConstantExpr( + varDecl.getDeclInfo(0).setInitializer(new Initializer(new IntConstantExpr( String.valueOf(newStart)))); } @@ -179,12 +179,12 @@ private static Optional maybeGetLoopSplitInfo(ForStmt forStmt) { } // Now we grab the initial value, which needs to be an integer. - if (!(declInfo.getInitializer() instanceof ScalarInitializer)) { + if (!(declInfo.getInitializer() instanceof Initializer)) { return Optional.empty(); } // Now we get its integer value, if it has one - final Optional maybeStartValue = maybeGetIntegerValue(((ScalarInitializer) declInfo + final Optional maybeStartValue = maybeGetIntegerValue((declInfo .getInitializer()).getExpr()); if (!maybeStartValue.isPresent()) { return Optional.empty(); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java index 9f1d74fd0..4b5151504 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java @@ -18,7 +18,7 @@ import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.Expr; @@ -177,9 +177,9 @@ private void structifyDeclaration(String enclosingStructVariableName, declInfo.setName(enclosingStructVariableName); if (declInfo.hasInitializer()) { declInfo.setInitializer( - new ScalarInitializer( + new Initializer( makeInitializationExpr(enclosingStructType, - ((ScalarInitializer) declInfo.getInitializer()).getExpr()) + (declInfo.getInitializer()).getExpr()) ) ); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java index 01986c8b6..49d32f21b 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java @@ -22,9 +22,9 @@ import com.graphicsfuzz.common.ast.decl.Declaration; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; @@ -50,7 +50,6 @@ import com.graphicsfuzz.common.util.OpenGlConstants; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; -import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.StructUtils; import com.graphicsfuzz.generator.fuzzer.FuzzedIntoACornerException; import com.graphicsfuzz.generator.fuzzer.Fuzzer; @@ -304,12 +303,12 @@ Map getGlobalsFromShader(TranslationUnit shader) { }.getGlobalsFromShader(shader); } - final ScalarInitializer getScalarInitializer(IInjectionPoint injectionPoint, - DonationContext donationContext, - Type type, - boolean restrictToConst, - IRandom generator, - ShadingLanguageVersion shadingLanguageVersion) { + final Initializer getInitializer(IInjectionPoint injectionPoint, + DonationContext donationContext, + Type type, + boolean restrictToConst, + IRandom generator, + ShadingLanguageVersion shadingLanguageVersion) { final boolean isConst = type.hasQualifier(TypeQualifier.CONST); try { @@ -322,7 +321,7 @@ final ScalarInitializer getScalarInitializer(IInjectionPoint injectionPoint, scopeForFuzzing.addStructDefinition(sdt); } - return new ScalarInitializer( + return new Initializer( new OpaqueExpressionGenerator(generator, generationParams, shadingLanguageVersion) .fuzzedConstructor( new Fuzzer(new FuzzingContext(scopeForFuzzing), diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java index c16d9a689..2750aba22 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java @@ -18,7 +18,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.stmt.BlockStmt; @@ -97,7 +97,7 @@ Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, DonationContext do String newName = "donor_replacement" + name; substitution.put(name, newName); - final ScalarInitializer initializer = getScalarInitializer( + final Initializer initializer = getInitializer( injectionPoint, donationContext, type, diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java index 0fcc55937..1dbbda5ea 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java @@ -17,7 +17,7 @@ package com.graphicsfuzz.generator.transformation; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; @@ -72,13 +72,13 @@ public Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, } type = dropQualifiersThatCannotBeUsedForLocalVariable(type); - ScalarInitializer initializer; + Initializer initializer; // We fuzz a const expression because we need to ensure we don't generate side-effects to // non-injected code if (isLoopLimiter(vars.getKey(), type.getWithoutQualifiers())) { - initializer = new ScalarInitializer(new IntConstantExpr("0")); + initializer = new Initializer(new IntConstantExpr("0")); } else { - initializer = getScalarInitializer(injectionPoint, donationContext, type, true, + initializer = getInitializer(injectionPoint, donationContext, type, true, generator, shadingLanguageVersion); } donatedStmts.add(new DeclarationStmt( diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java b/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java index 126bf1e69..a832f5d89 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java @@ -18,7 +18,6 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -33,7 +32,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; /** * Attempts to minimally remove const qualifiers and move global declaration initializers into @@ -111,12 +109,9 @@ private void addGlobalInitializers(FunctionDefinition mainFunction) { for (int i = globalsToBeReInitialized.size() - 1; i >= 0; i--) { for (int j = globalsToBeReInitialized.get(i).getNumDecls() - 1; j >= 0; j--) { final VariableDeclInfo vdi = globalsToBeReInitialized.get(i).getDeclInfo(j); - if (!(vdi.getInitializer() instanceof ScalarInitializer)) { - throw new RuntimeException("Only know how to deal with scalar initializers at present."); - } mainFunction.getBody().insertStmt(0, new ExprStmt(new BinaryExpr(new VariableIdentifierExpr(vdi.getName()), - ((ScalarInitializer) vdi.getInitializer()).getExpr(), BinOp.ASSIGN))); + (vdi.getInitializer()).getExpr(), BinOp.ASSIGN))); vdi.setInitializer(null); } } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java index 491b2d98a..b1e58d2f6 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java @@ -19,8 +19,8 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -196,7 +196,7 @@ private List makeStatementsFromFactories(IRandom generator, GenerationPara new VariableDeclInfo( "v" + idGenerator.freshId(), null, - new ScalarInitializer(expr.get()) + new Initializer(expr.get()) ) ) )); diff --git a/generator/src/test/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutationTest.java index 20fd576ac..76c131f3e 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutationTest.java @@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; @@ -37,7 +37,6 @@ import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.generator.semanticspreserving.StructificationMutation; import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.util.CannedRandom; import com.graphicsfuzz.common.util.IdGenerator; @@ -72,7 +71,7 @@ public void applyStructification() { new VariableDeclInfo( "v", null, - new ScalarInitializer( + new Initializer( new FloatConstantExpr("3.0"))))); // float v = 3.0; diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java index 00ac78081..410040495 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java @@ -20,7 +20,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; @@ -138,7 +138,7 @@ BlockInjectionPoint getBlockInjectionPoint(TranslationUnit tu) { new VariablesDeclaration( BasicType.INT, new VariableDeclInfo("a", null, - new ScalarInitializer( + new Initializer( new BinaryExpr( new BinaryExpr( new BinaryExpr( @@ -429,7 +429,7 @@ public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { ((VariableIdentifierExpr) arrayIndexExpr.getIndex()).getName()); assertTrue(scopeEntry.hasVariableDeclInfo()); assertNotNull(scopeEntry.getVariableDeclInfo().getInitializer()); - assertFalse(((ScalarInitializer) scopeEntry.getVariableDeclInfo().getInitializer()) + assertFalse((scopeEntry.getVariableDeclInfo().getInitializer()) .getExpr() instanceof FunctionCallExpr); } } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java index 86e91d3f3..8ca5cd516 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java @@ -19,8 +19,8 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.ParameterDecl; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -38,13 +38,11 @@ import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.Scope; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.semanticspreserving.SplitForLoopMutation; -import com.graphicsfuzz.generator.transformation.SplitForLoopTransformation; import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; import com.graphicsfuzz.generator.transformation.injection.InjectionPoints; import com.graphicsfuzz.generator.util.GenerationParams; @@ -135,7 +133,7 @@ private static TranslationUnit makeExampleTranslationUnit() { new DeclarationStmt( new VariablesDeclaration( new QualifiedType(BasicType.INT, - new ArrayList()), new VariableDeclInfo("i", null, new ScalarInitializer(new IntConstantExpr("0"))))), + new ArrayList()), new VariableDeclInfo("i", null, new Initializer(new IntConstantExpr("0"))))), new BinaryExpr( new VariableIdentifierExpr("i"), new IntConstantExpr("10"), diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java index 773e7bee5..75076afee 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java @@ -19,7 +19,7 @@ import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.expr.MemberLookupExpr; @@ -84,22 +84,22 @@ private StructifiedVariableInfo findOriginalVariableInfo() { return findOriginalVariableInfo( (StructNameType) declaration.getVariablesDeclaration().getBaseType() .getWithoutQualifiers(), - Optional.ofNullable((ScalarInitializer) declaration.getVariablesDeclaration() + Optional.ofNullable(declaration.getVariablesDeclaration() .getDeclInfo(0).getInitializer())) .get(); } private Optional findOriginalVariableInfo( StructNameType type, - Optional initializer) { + Optional initializer) { final StructDefinitionType structDefinitionType = tu.getStructDefinition(type); for (int i = 0; i < structDefinitionType.getNumFields(); i++) { final int currentIndex = i; - final Optional componentInitializer = initializer.map(item -> - new ScalarInitializer(((TypeConstructorExpr) item.getExpr()) + final Optional componentInitializer = initializer.map(item -> + new Initializer(((TypeConstructorExpr) item.getExpr()) .getArg(currentIndex))); if (structDefinitionType.getFieldName(i).startsWith(Constants.STRUCTIFICATION_FIELD_PREFIX)) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java index 9761fbb16..7293b28dc 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java @@ -18,7 +18,6 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.type.TypeQualifier; @@ -110,8 +109,7 @@ public void visitFunctionDefinition(FunctionDefinition functionDefinition) { if (functionDefinition.getPrototype().getName().equals("main")) { for (VariablesDeclaration variablesDeclaration : globalVariableDecl) { for (VariableDeclInfo variableDeclInfo : variablesDeclaration.getDeclInfos()) { - if (variableDeclInfo.hasInitializer() - && variableDeclInfo.getInitializer() instanceof ScalarInitializer) { + if (variableDeclInfo.hasInitializer()) { addOpportunity(new GlobalVariableDeclToExprReductionOpportunity( getVistitationDepth(), variableDeclInfo, functionDefinition)); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java index f5d19b81b..b32c2ff91 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java @@ -17,7 +17,6 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; @@ -44,10 +43,10 @@ void applyReductionImpl() { // Given the variable declaration info of global variable, we unset its initializer and // derive a new assignment statement which will be inserted as the first statement in // main function. - assert variableDeclInfo.getInitializer() instanceof ScalarInitializer; + assert variableDeclInfo.hasInitializer(); final BinaryExpr binaryExpr = new BinaryExpr( new VariableIdentifierExpr(variableDeclInfo.getName()), - ((ScalarInitializer) variableDeclInfo.getInitializer()).getExpr(), + (variableDeclInfo.getInitializer()).getExpr(), BinOp.ASSIGN ); mainFunction.getBody().insertStmt(0, new ExprStmt(binaryExpr)); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java index 8be577160..4fb36ae2e 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java @@ -18,7 +18,6 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.Initializer; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; @@ -79,7 +78,7 @@ private void addOpportunities() { if (!blackList.contains(pair.getLeft())) { addOpportunity(new SimplifyExprReductionOpportunity( parentMap.getParent(pair.getRight()), - new ParenExpr(((ScalarInitializer) pair.getLeft().getInitializer()).getExpr().clone()), + new ParenExpr((pair.getLeft().getInitializer()).getExpr().clone()), pair.getRight(), getVistitationDepth())); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java index 11a02b377..b3becbdfd 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java @@ -17,7 +17,6 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; @@ -106,9 +105,9 @@ private boolean canMergeLoops(Stmt first, Stmt second) { final BinOp firstLoopOp = ((BinaryExpr) firstLoop.getCondition()).getOp(); - final Integer secondLoopStart = new Integer(((IntConstantExpr) ((ScalarInitializer) - ((DeclarationStmt) secondLoop.getInit()).getVariablesDeclaration().getDeclInfo(0) - .getInitializer()).getExpr()).getValue()); + final Integer secondLoopStart = new Integer(((IntConstantExpr) ( + (DeclarationStmt) secondLoop.getInit()).getVariablesDeclaration().getDeclInfo(0) + .getInitializer().getExpr()).getValue()); assert firstLoopOp == BinOp.LT || firstLoopOp == BinOp.GT : "Unexpected operator in split loops."; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java index e103dd647..c0586ccaa 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java @@ -20,7 +20,6 @@ import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.ConstantExpr; @@ -224,11 +223,8 @@ boolean initializerIsScalarAndSideEffectFree(VariableDeclInfo variableDeclInfo) if (!variableDeclInfo.hasInitializer()) { return false; } - if (!(variableDeclInfo.getInitializer() instanceof ScalarInitializer)) { - return false; - } return SideEffectChecker.isSideEffectFree( - ((ScalarInitializer) variableDeclInfo.getInitializer()).getExpr(), + (variableDeclInfo.getInitializer()).getExpr(), context.getShadingLanguageVersion(), shaderKind); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StructifiedVariableInfo.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StructifiedVariableInfo.java index 57bfca543..2d13ff75f 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StructifiedVariableInfo.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StructifiedVariableInfo.java @@ -16,7 +16,7 @@ package com.graphicsfuzz.reducer.reductionopportunities; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.type.Type; import java.util.Optional; @@ -24,9 +24,9 @@ class StructifiedVariableInfo { private final String name; private final Type type; - private final Optional initializer; + private final Optional initializer; - StructifiedVariableInfo(String name, Type type, Optional initializer) { + StructifiedVariableInfo(String name, Type type, Optional initializer) { this.name = name; this.type = type; this.initializer = initializer; @@ -40,7 +40,7 @@ public Type getType() { return type; } - public Optional getInitializer() { + public Optional getInitializer() { return initializer; } } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java index 6db2ba13b..fe770b992 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java @@ -17,7 +17,6 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; import com.graphicsfuzz.common.ast.type.TypeQualifier; @@ -94,8 +93,7 @@ public void visitDeclarationStmt(DeclarationStmt declarationStmt) { // correct order with respect to its original order in the variable declaration info list. for (int i = declInfos.size() - 1; i >= 0; i--) { final VariableDeclInfo variableDeclInfo = declInfos.get(i); - if (variableDeclInfo.hasInitializer() - && variableDeclInfo.getInitializer() instanceof ScalarInitializer) { + if (variableDeclInfo.hasInitializer()) { addOpportunity(new VariableDeclToExprReductionOpportunity( variableDeclInfo, currentBlock(), diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java index 9a2f17098..2e832196a 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java @@ -16,7 +16,6 @@ package com.graphicsfuzz.reducer.reductionopportunities; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; @@ -48,10 +47,10 @@ public class VariableDeclToExprReductionOpportunity extends AbstractReductionOpp void applyReductionImpl() { // Given the variable declaration info, we unset its initializer and derive a new assignment // statement which will be inserted right after the declaration in the block statement. - assert variableDeclInfo.getInitializer() instanceof ScalarInitializer; + assert variableDeclInfo.hasInitializer(); final BinaryExpr binaryExpr = new BinaryExpr( new VariableIdentifierExpr(variableDeclInfo.getName()), - ((ScalarInitializer) variableDeclInfo.getInitializer()).getExpr(), + (variableDeclInfo.getInitializer()).getExpr(), BinOp.ASSIGN ); enclosingBlock.insertAfter(declarationStmt, new ExprStmt(binaryExpr)); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunityTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunityTest.java index ac372c73e..e554b304c 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunityTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunityTest.java @@ -21,7 +21,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; @@ -41,8 +41,6 @@ import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.util.Constants; -import com.graphicsfuzz.common.util.ParseTimeoutException; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Optional; @@ -86,7 +84,7 @@ public void applyReduction() throws Exception { Arrays.asList( new DeclarationStmt(new VariablesDeclaration(outerStructDefinitionType.getStructNameType(), new VariableDeclInfo("myOuter", null, - new ScalarInitializer( + new Initializer( new TypeConstructorExpr(outerStructTypeName, Arrays.asList( new TypeConstructorExpr(innerStructTypeName, diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java index 2a0c635fb..5da758de8 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertEquals; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; @@ -30,10 +30,6 @@ import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import org.junit.Test; @@ -61,9 +57,9 @@ public void applyReduction() throws Exception { barConstructor.clone(), barConstructor.clone())); VariableDeclInfo v1 = new VariableDeclInfo("v1", null, - new ScalarInitializer(fooConstructor.clone())); + new Initializer(fooConstructor.clone())); VariableDeclInfo v2 = new VariableDeclInfo("v2", null, - new ScalarInitializer(fooConstructor.clone())); + new Initializer(fooConstructor.clone())); DeclarationStmt declarationStmt = new DeclarationStmt(new VariablesDeclaration(foo.getStructNameType(), Arrays.asList(v1, v2))); From 2ed29eb2b16277a040833f6ce13ba486f41b9ff1 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Mon, 15 Jul 2019 14:33:04 +0100 Subject: [PATCH 066/180] Fix image tests by extending blacklist (#604) Also, update SwiftShader. The bugs we encounter are known SwiftShader bugs that are unlikely to be fixed soon. --- assembly-binaries/pom.xml | 2 +- .../java/com/graphicsfuzz/tester/GeneratorUnitTest.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/assembly-binaries/pom.xml b/assembly-binaries/pom.xml index 94ee8d19d..dcade1a0f 100644 --- a/assembly-binaries/pom.xml +++ b/assembly-binaries/pom.xml @@ -60,7 +60,7 @@ limitations under the License. f2170cc791d0eaa5789ec7528862ae00b984b3b8 9d3202492d78aae5ced08ff14679b81c98d71c15 - 803082da4f73920729938905cb1727e11ff79530 + 0565d5aee279a4782689297ada0aa2489e24ad3e 1586e566f4949b1957e7c32454cbf27e501ed632 d987dc0e5f8aeae0dc2f5aac2013e9a3edd54465 diff --git a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java index ec861c10a..fb573ab5e 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java @@ -142,8 +142,8 @@ public void testDonateDeadCode() throws Exception { Util.getDonorsFolder(), GenerationParams.normal(ShaderKind.FRAGMENT, true)), TransformationProbabilities.likelyDonateDeadCode(), "donatedead", - Arrays.asList("bubblesort_flag.json", "squares.json"), - Arrays.asList("bubblesort_flag.json", "squares.json")); + Arrays.asList("bubblesort_flag.json", "squares.json", "mandelbrot_blurry.json"), + Arrays.asList("bubblesort_flag.json", "squares.json", "mandelbrot_blurry.json")); // Reason for blacklisting^: slow. } @@ -199,8 +199,8 @@ public void testWrap() throws Exception { testTransformationMultiVersions(() -> new AddWrappingConditionalTransformation(), TransformationProbabilities.onlyWrap(), "wrap", - Arrays.asList("bubblesort_flag.json"), - Arrays.asList("bubblesort_flag.json")); + Arrays.asList("bubblesort_flag.json", "colorgrid_modulo.json"), + Arrays.asList("bubblesort_flag.json", "colorgrid_modulo.json")); // Reason for blacklisting^: slow. } From b35747c352a190ab19808064c953e186232abf8c Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 16 Jul 2019 04:34:11 -0500 Subject: [PATCH 067/180] Fix piglit converter not accepting blank JSON (#613) We don't need to error check the variable - json.load() just returns nothing if the JSON is empty, and errors out by itself if it detects malformed JSON. --- python/src/main/python/drivers/graphicsfuzz_piglit_converter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py index fb4239b54..ac0c029f4 100644 --- a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py +++ b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py @@ -182,8 +182,6 @@ def get_json_properties(shader_job: str) -> List: """ with gfuzz_common.open_helper(shader_job, 'r') as job: json_parsed = json.load(job) - if not json_parsed: - raise IOError('Malformed shader job file.') return json_parsed From 969af2a92866726692453f33f665c42a0988aece Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 16 Jul 2019 10:39:26 +0100 Subject: [PATCH 068/180] Improve RNG; use commons-rng (#614) * 64 bit seed. * Use commons-rng ISAAC random number generator. * Use rng.spawnChild() wherever possible instead of generating a seed. * Pass a rng down instead of a seed where possible. * Let the seed argument be a String and parse it as an unsigned 64 bit long integer. * Output the seed as an unsigned long. * Update some tests that were dependent on having the same RNG. * Update some Apache Java packages. * Add missing 0 seeds in tests. * Fix license information. * Update WebUi hard-coded seed to be unsigned. --- build/travis/licenses.py | 37 ++++ client-tests/pom.xml | 2 +- common/pom.xml | 8 + .../common/util/CannedRandom.java | 5 + .../com/graphicsfuzz/common/util/IRandom.java | 6 + .../common/util/RandomWrapper.java | 32 +-- .../common/util/SameValueRandom.java | 5 + .../common/util/ZeroCannedRandom.java | 5 + .../generator/ShaderProducer.java | 8 +- .../generator/tool/Fragment2Compute.java | 8 +- .../graphicsfuzz/generator/tool/Generate.java | 19 +- .../generator/tool/GenerateShaderFamily.java | 25 +-- .../generator/tool/GlslGenerate.java | 3 +- .../graphicsfuzz/generator/tool/Mutate.java | 11 +- .../Expr2BinaryMutationFinderTest.java | 5 + ...lockStmtsWithSwitchMutationFinderTest.java | 200 +++++++++++------- .../DonateLiveCodeTransformationTest.java | 16 +- .../SplitForLoopTransformationTest.java | 2 +- parent-all/pom.xml | 21 +- .../graphicsfuzz/reducer/tool/GlslReduce.java | 13 +- .../reducer/tool/ReducerBugPoint.java | 8 +- .../reducer/tool/ReducerBugPointBasic.java | 11 +- ...DestructifyReductionOpportunitiesTest.java | 8 +- ...ityMutationReductionOpportunitiesTest.java | 2 +- ...lineUniformReductionOpportunitiesTest.java | 2 +- .../ReductionOpportunitiesTest.java | 10 +- .../StmtReductionOpportunitiesTest.java | 4 +- .../com/graphicsfuzz/server/webui/WebUi.java | 4 +- shadersets-util/pom.xml | 2 +- util/pom.xml | 4 + .../java/com/graphicsfuzz/util/ArgsUtil.java | 10 +- 31 files changed, 317 insertions(+), 179 deletions(-) diff --git a/build/travis/licenses.py b/build/travis/licenses.py index c7f6afbf6..808071719 100644 --- a/build/travis/licenses.py +++ b/build/travis/licenses.py @@ -622,6 +622,43 @@ def get_maven_dependencies_populated(): 'license_file': '', 'skipped': '', }, + 'org.apache.commons:commons-rng-core': { + 'comment': '', + 'name': 'Apache Commons RNG', + 'url': 'https://github.com/apache/commons-rng', + 'license_url': [ + 'https://raw.githubusercontent.com/apache/commons-rng' + '/838f60e09ce458ced96ea05bf09e576c4283136f/NOTICE.txt', + 'https://raw.githubusercontent.com/apache/commons-rng' + '/838f60e09ce458ced96ea05bf09e576c4283136f/LICENSE.txt' + ], + 'license_file': '', + 'skipped': '', + }, + 'org.apache.commons:commons-rng-simple': { + 'comment': '', + 'name': '', + 'url': '', + 'license_url': '', + 'license_file': '', + 'skipped': 'Same project as commons-rng-core.', + }, + 'org.apache.commons:commons-rng-sampling': { + 'comment': '', + 'name': '', + 'url': '', + 'license_url': '', + 'license_file': '', + 'skipped': 'Same project as commons-rng-core.', + }, + 'org.apache.commons:commons-rng-client-api': { + 'comment': '', + 'name': '', + 'url': '', + 'license_url': '', + 'license_file': '', + 'skipped': 'Same project as commons-rng-core.', + }, 'commons-logging:commons-logging': { 'comment': '', 'name': 'Apache Commons Logging', diff --git a/client-tests/pom.xml b/client-tests/pom.xml index df0c2baa1..806bbfe04 100755 --- a/client-tests/pom.xml +++ b/client-tests/pom.xml @@ -28,7 +28,7 @@ limitations under the License. 1.0 ../parent-checkstyle/pom.xml - + diff --git a/common/pom.xml b/common/pom.xml index 5f08ed857..dcaf50db6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -55,6 +55,14 @@ limitations under the License. org.apache.commons commons-lang3 + + org.apache.commons + commons-rng-simple + + + org.apache.commons + commons-rng-client-api + com.google.code.gson gson diff --git a/common/src/main/java/com/graphicsfuzz/common/util/CannedRandom.java b/common/src/main/java/com/graphicsfuzz/common/util/CannedRandom.java index 9d2c8b360..404038925 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/CannedRandom.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/CannedRandom.java @@ -70,6 +70,11 @@ public IRandom spawnChild() { return this; } + @Override + public String getDescription() { + return "CannedRandom"; + } + public boolean isExhausted() { return !items.hasNext(); } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/IRandom.java b/common/src/main/java/com/graphicsfuzz/common/util/IRandom.java index 0ad65814b..e3f196f71 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/IRandom.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/IRandom.java @@ -40,4 +40,10 @@ default int nextPositiveInt(int bound) { */ IRandom spawnChild(); + + /** + * @return A description of the random number generator, ideally including the seed. + */ + String getDescription(); + } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/RandomWrapper.java b/common/src/main/java/com/graphicsfuzz/common/util/RandomWrapper.java index 4d539c15d..95731a4bd 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/RandomWrapper.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/RandomWrapper.java @@ -16,46 +16,46 @@ package com.graphicsfuzz.common.util; -import java.util.Random; +import org.apache.commons.rng.UniformRandomProvider; +import org.apache.commons.rng.simple.RandomSource; /** - * Random generator that uses java.util.Random, to be used when genuine pseudo-random generation - * is required (as opposed to mocking for testing). + * Random generator to be used when genuine pseudo-random generation is required (as opposed to + * mocking for testing). */ public class RandomWrapper implements IRandom { - private final Random generator; + private final long seed; + private final UniformRandomProvider provider; - public RandomWrapper(int seed) { - this.generator = new Random(seed); - } - - public RandomWrapper() { - this.generator = new Random(); + public RandomWrapper(long seed) { + this.seed = seed; + this.provider = RandomSource.create(RandomSource.ISAAC, seed); } @Override public int nextInt(int bound) { - return generator.nextInt(bound); + return provider.nextInt(bound); } @Override public Float nextFloat() { - return generator.nextFloat(); + return provider.nextFloat(); } @Override public boolean nextBoolean() { - return generator.nextBoolean(); + return provider.nextBoolean(); } @Override public IRandom spawnChild() { - return new RandomWrapper(generator.nextInt()); + return new RandomWrapper(provider.nextLong()); } - public void setSeed(int seed) { - generator.setSeed(seed); + @Override + public String getDescription() { + return "RandomWrapper with seed: " + Long.toUnsignedString(seed); } } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/SameValueRandom.java b/common/src/main/java/com/graphicsfuzz/common/util/SameValueRandom.java index 608e6b98f..e6407914e 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/SameValueRandom.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/SameValueRandom.java @@ -46,4 +46,9 @@ public boolean nextBoolean() { public IRandom spawnChild() { throw new UnsupportedOperationException("Child spawning not available"); } + + @Override + public String getDescription() { + return "SameValueRandom: " + intValue + " " + boolValue; + } } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ZeroCannedRandom.java b/common/src/main/java/com/graphicsfuzz/common/util/ZeroCannedRandom.java index af7adbe41..112a1af1a 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ZeroCannedRandom.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ZeroCannedRandom.java @@ -38,4 +38,9 @@ public IRandom spawnChild() { throw new UnsupportedOperationException(); } + @Override + public String getDescription() { + return "ZeroCannedRandom"; + } + } diff --git a/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/ShaderProducer.java b/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/ShaderProducer.java index f96466bc2..3068fc0b6 100755 --- a/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/ShaderProducer.java +++ b/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/ShaderProducer.java @@ -89,8 +89,7 @@ public void run() { !ns.getBoolean("no_injection_switch") ); - final int outerSeed = ArgsUtil.getSeedArgument(ns); - final IRandom generator = new RandomWrapper(outerSeed); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); int sent = 0; for (int counter = 0; sent < limit; counter++) { @@ -107,8 +106,9 @@ public void run() { new RemoveDiscardStatements(shaderJob.getFragmentShader().get()); // Create a variant. - Generate.generateVariant(shaderJob, generatorArguments, - generator.nextInt(Integer.MAX_VALUE)); + IRandom childRandom = generator.spawnChild(); + LOGGER.info("Calling generateVariant with RNG: {}", childRandom.getDescription()); + Generate.generateVariant(shaderJob, generatorArguments, childRandom); // Restrict the colors that the variant can emit. final float probabilityOfAddingNewColorWrite = 0.01f; diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java index a3ff3e972..692983232 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java @@ -56,6 +56,7 @@ import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.SsboFieldData; import com.graphicsfuzz.generator.util.RemoveDiscardStatements; +import com.graphicsfuzz.util.ArgsUtil; import java.io.File; import java.io.IOException; import java.util.Arrays; @@ -99,8 +100,8 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .type(File.class); parser.addArgument("--seed") - .help("Seed for random number generator.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) for random number generator.") + .type(String.class); parser.addArgument("--generate-uniform-bindings") .help("Put all uniforms in uniform blocks and generate bindings; required for Vulkan " @@ -307,8 +308,7 @@ public static void mainHelper(String... args) throws ArgumentParserException, InterruptedException, GlslParserException, ParseTimeoutException, IOException { final Namespace ns = parse(args); final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - final IRandom generator = new RandomWrapper(ns.getInt("seed") == null - ? new Random().nextInt() : ns.getInt("seed")); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final ShaderJob transformedShaderJob = transform( fileOps.readShaderJobFile(ns.get("fragment_json")), generator); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java index 7c5f727ab..d5b34e050 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java @@ -54,6 +54,7 @@ import com.graphicsfuzz.generator.util.FloatLiteralReplacer; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; +import com.graphicsfuzz.util.ArgsUtil; import com.graphicsfuzz.util.Constants; import java.io.File; import java.io.IOException; @@ -106,8 +107,8 @@ private static Namespace parse(String[] args) throws ArgumentParserException { public static void addGeneratorCommonArguments(ArgumentParser parser) { parser.addArgument("--seed") - .help("Seed to initialize random number generator with.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) for the random number generator.") + .type(String.class); parser.addArgument("--small") .help("Try to generate small shaders.") @@ -169,9 +170,8 @@ public static void addGeneratorCommonArguments(ArgumentParser parser) { */ public static StringBuilder generateVariant(ShaderJob shaderJob, GeneratorArguments args, - int seed) { + IRandom random) { final StringBuilder result = new StringBuilder(); - final IRandom random = new RandomWrapper(seed); if (args.getAddInjectionSwitch()) { for (TranslationUnit shader : shaderJob.getShaders()) { @@ -224,7 +224,7 @@ public static void generateVariant(ShaderJobFileOperations fileOps, File referenceShaderJobFile, File outputShaderJobFile, GeneratorArguments generatorArguments, - int seed, + IRandom random, boolean writeProbabilities) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { // This is mutated into the variant. @@ -233,7 +233,7 @@ public static void generateVariant(ShaderJobFileOperations fileOps, final StringBuilder generationInfo = generateVariant( variantShaderJob, generatorArguments, - seed); + random); fileOps.writeShaderJobFile( variantShaderJob, @@ -325,17 +325,14 @@ public static void mainHelper(String[] args) GlslParserException { final Namespace ns = parse(args); - Integer seed = ns.get("seed"); - if (seed == null) { - seed = new Random().nextInt(); - } + final IRandom random = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); generateVariant( new ShaderJobFileOperations(), ns.get("reference_json"), ns.get("output"), getGeneratorArguments(ns), - seed, + random, ns.getBoolean("write_probabilities")); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java index 68f70f514..386f1f283 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java @@ -128,7 +128,7 @@ public static void mainHelper(String[] args) throws ArgumentParserException, final boolean writeProbabilities = ns.getBoolean("write_probabilities"); final boolean keepBadVariants = ns.getBoolean("keep_bad_variants"); final boolean stopOnFail = ns.getBoolean("stop_on_fail"); - final int seed = ArgsUtil.getSeedArgument(ns); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final int numVariants = ns.getInt("num_variants"); Optional maxBytes = ns.get("max_bytes") == null ? Optional.empty() : Optional.of(ns.getInt("max_bytes")); @@ -137,7 +137,7 @@ public static void mainHelper(String[] args) throws ArgumentParserException, final GeneratorArguments generatorArguments = Generate.getGeneratorArguments(ns); if (verbose) { - LOGGER.info("Using seed " + seed); + LOGGER.info("Using random: " + generator.getDescription()); } if (!referenceShaderJob.isFile()) { @@ -207,8 +207,6 @@ public static void mainHelper(String[] args) throws ArgumentParserException, int generatedVariants = 0; int triedVariants = 0; - final IRandom generator = new RandomWrapper(seed); - // Main variant generation loop while (generatedVariants < numVariants) { if (verbose) { @@ -216,29 +214,28 @@ public static void mainHelper(String[] args) throws ArgumentParserException, + " of " + numVariants + ")"); } // Generate a variant - final int innerSeed = generator.nextInt(Integer.MAX_VALUE); - if (verbose) { - LOGGER.info("Generating variant with inner seed " + innerSeed); - } final File variantShaderJobFile = new File(outputDir, "variant_" + String.format("%03d", generatedVariants) + ".json"); + final IRandom childRandom = generator.spawnChild(); + if (verbose) { + LOGGER.info("Generating variant with inner random: " + childRandom.getDescription()); + } triedVariants++; try { Generate.generateVariant(fileOps, referenceShaderJob, variantShaderJobFile, - generatorArguments, innerSeed, writeProbabilities); + generatorArguments, childRandom, writeProbabilities); } catch (Exception exception) { if (verbose) { - LOGGER.error("Failed generating variant: " + exception.getMessage() - + exception.getStackTrace() + LOGGER.error("Failed generating variant: " + "\nGenerator arguments: " + generatorArguments + "\nReference shader job: " + referenceShaderJob - + "\nSeed: " + innerSeed); + + "\nRandom: " + childRandom.getDescription(), exception); } if (stopOnFail) { final String message = "Failed generating a variant, stopping."; LOGGER.info(message); - throw new RuntimeException(message); + throw new RuntimeException(message, exception); } continue; } @@ -283,7 +280,7 @@ public static void mainHelper(String[] args) throws ArgumentParserException, StandardCharsets.UTF_8) : "none"; infoLog.addProperty("git_hash", hashContents); infoLog.addProperty("args", String.join(" ", args)); - infoLog.addProperty("seed", seed); + infoLog.addProperty("seed", generator.getDescription()); // Pretty-print the info log. FileUtils.writeStringToFile(new File(outputDir,"infolog.json"), diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/GlslGenerate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/GlslGenerate.java index d7028ac21..46d602414 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/GlslGenerate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/GlslGenerate.java @@ -83,7 +83,7 @@ public static void mainHelper(String[] args, boolean failOnReferencePreparationE final String prefix = ns.get("prefix"); final int numVariants = ns.getInt("num_variants"); final boolean verbose = ns.getBoolean("verbose"); - final int seed = ArgsUtil.getSeedArgument(ns); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); @@ -100,7 +100,6 @@ public static void mainHelper(String[] args, boolean failOnReferencePreparationE + " variant" + (numVariants == 1 ? "" : "s") + "."); } - final IRandom generator = new RandomWrapper(seed); int referenceCount = 0; for (File shaderJobFile : referenceShaderJobFiles) { LOGGER.info("Generating family " + referenceCount + " from reference " diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java index 22e46e849..31ff13355 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java @@ -88,8 +88,8 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .type(File.class); parser.addArgument("--seed") - .help("Seed to initialize random number generator with.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) for the random number generator.") + .type(String.class); return parser.parseArgs(args); } @@ -127,13 +127,14 @@ public static void mainHelper(String[] args) throws ArgumentParserException, IOE final File input = ns.get("input"); final File output = ns.get("output"); - final int seed = ArgsUtil.getSeedArgument(ns); + IRandom random = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final TranslationUnit tu = ParseHelper.parse(input); - LOGGER.info("Mutating from " + input + " to " + output + " with seed " + seed); + LOGGER.info("Mutating from " + input + " to " + output + " with RNG " + + random.getDescription()); - mutate(tu, new RandomWrapper(seed)); + mutate(tu, random); final File shaderJobFile = new File(FilenameUtils.removeExtension(output.getName()) + ".json"); diff --git a/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/Expr2BinaryMutationFinderTest.java b/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/Expr2BinaryMutationFinderTest.java index 79dcb7d05..ade40d893 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/Expr2BinaryMutationFinderTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/Expr2BinaryMutationFinderTest.java @@ -96,6 +96,11 @@ public boolean nextBoolean() { public IRandom spawnChild() { throw new UnsupportedOperationException(); } + + @Override + public String getDescription() { + return "Just 2"; + } }; } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinderTest.java b/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinderTest.java index 0cb770a32..f1a285301 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinderTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinderTest.java @@ -61,55 +61,78 @@ public void testReplaceBlockStmtsWithSwitchMiner() throws Exception { + "}"; final String expected = "#version 300 es\n" - + "int foo(int x)" - + "{" - + " switch(x)" - + " {" - + " case 84:" - + " case 43:" - + " case 44:" - + " int a = 5;" - + " case 62:" - + " case 77:" - + " int b = 7;" - + " case 75:" - + " case 20:" - + " case 41:" - + " case 73:" - + " case 95:" - + " default:" - + " {" - + " switch(x)" - + " {" - + " case 84:" - + " case 43:" - + " default:" - + " {" - + " switch(0)" - + " {" - + " case 0:" - + " return x;" - + " }" - + " }" - + " }" - + " }" - + " }" - + "}" - + "void main()" - + "{" - + " int a = 0;" - + " int b = 0;" - + " int c = 0;" - + " for(" - + " int i = 0;" - + " i < 10;" - + " i ++" - + " )" - + " {" - + " a ++;" - + " b += 2;" - + " c += 3;" - + " }" + + "int foo(int x)\n" + + "{\n" + + " switch(x)\n" + + " {\n" + + " case 57:\n" + + " case 63:\n" + + " case 60:\n" + + " case 86:\n" + + " case 73:\n" + + " case 78:\n" + + " case 20:\n" + + " case 9:\n" + + " case 24:\n" + + " case 99:\n" + + " case 95:\n" + + " case 75:\n" + + " int a = 5;\n" + + " case 44:\n" + + " case 66:\n" + + " case 84:\n" + + " case 13:\n" + + " case 2:\n" + + " case 18:\n" + + " case 27:\n" + + " case 5:\n" + + " case 93:\n" + + " case 49:\n" + + " case 40:\n" + + " int b = 7;\n" + + " case 11:\n" + + " case 39:\n" + + " case 43:\n" + + " case 34:\n" + + " case 33:\n" + + " case 80:\n" + + " case 1:\n" + + " case 22:\n" + + " case 32:\n" + + " case 31:\n" + + " default:\n" + + " {\n" + + " switch(x)\n" + + " {\n" + + " case 84:\n" + + " case 43:\n" + + " default:\n" + + " {\n" + + " switch(0)\n" + + " {\n" + + " case 0:\n" + + " return x;\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + + "void main()\n" + + "{\n" + + " int a = 0;\n" + + " int b = 0;\n" + + " int c = 0;\n" + + " for(\n" + + " int i = 0;\n" + + " i < 10;\n" + + " i ++\n" + + " )\n" + + " {\n" + + " a ++;\n" + + " b += 2;\n" + + " c += 3;\n" + + " }\n" + "}"; @@ -158,33 +181,58 @@ public void testReplaceBlockStmtsWithSwitchMiner3() throws Exception { + "}"; final String expected = "#version 300 es\n" - + "int foo(int x) {" - + "switch(x)" - + " {" - + " case 84:" - + " case 43:" - + " case 44:" - + " case 62:" - + " case 77:" - + " case 75:" - + " case 20:" - + " case 41:" - + " case 73:" - + " case 95:" - + " default:" - + " {" - + " switch(x)" - + " {" - + " case 1:" - + " case 2:" - + " default:" - + " return 5;" - + " }" - + " }" - + " }" - + "}" - + "void main() {" - + "}"; + + "int foo(int x)\n" + + "{\n" + + " switch(x)\n" + + " {\n" + + " case 57:\n" + + " case 63:\n" + + " case 60:\n" + + " case 86:\n" + + " case 73:\n" + + " case 78:\n" + + " case 20:\n" + + " case 9:\n" + + " case 24:\n" + + " case 99:\n" + + " case 95:\n" + + " case 75:\n" + + " case 44:\n" + + " case 66:\n" + + " case 84:\n" + + " case 13:\n" + + " case 2:\n" + + " case 18:\n" + + " case 27:\n" + + " case 5:\n" + + " case 93:\n" + + " case 49:\n" + + " case 40:\n" + + " case 11:\n" + + " case 39:\n" + + " case 43:\n" + + " case 34:\n" + + " case 33:\n" + + " case 80:\n" + + " case 1:\n" + + " case 22:\n" + + " case 32:\n" + + " case 31:\n" + + " default:\n" + + " {\n" + + " switch(x)\n" + + " {\n" + + " case 1:\n" + + " case 2:\n" + + " default:\n" + + " return 5;\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + + "void main()\n" + + "{\n" + + "}\n"; TranslationUnit tu = ParseHelper.parse(program); diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java index 410040495..a7bfe31c1 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java @@ -387,9 +387,11 @@ public void testArrayAccessesAreInBounds() throws Exception { ); } + int noCodeDonatedCount = 0; + // Try the following a few times, so that there is a good chance of triggering the issue // this test was used to catch, should it return: - for (int seed = 0; seed < 5; seed++) { + for (int seed = 0; seed < 15; seed++) { final ShaderJob referenceShaderJob = fileOps.readShaderJobFile(referenceFile); @@ -407,7 +409,10 @@ public void testArrayAccessesAreInBounds() throws Exception { GenerationParams.normal(ShaderKind.FRAGMENT, true) ); - Assert.assertTrue(result); + if (!result) { + ++noCodeDonatedCount; + continue; + } // An array access injected into the shader must either be (1) already in bounds, or // (2) made in bounds. Only in the former case can the array index be a variable identifier @@ -437,6 +442,13 @@ public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { }.visit(referenceShaderJob.getFragmentShader().get()); } + // The above code tests donation of live code, but there is still a chance that no code will + // be donated. We assert that this happens < 10 times to ensure that we get some test + // coverage, but this could fail due to bad luck. + Assert.assertTrue( + "Donation failure count should be < 10, " + noCodeDonatedCount, + noCodeDonatedCount < 10 + ); } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java index 8ca5cd516..beaaa8eed 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java @@ -115,7 +115,7 @@ public void checkSimpleLoopSplit() { TranslationUnit tu = makeExampleTranslationUnit(); - List ips = new InjectionPoints(tu, new RandomWrapper(), + List ips = new InjectionPoints(tu, new RandomWrapper(0), SplitForLoopMutation::suitableForSplitting).getInjectionPoints( TransformationProbabilities.onlySplitLoops()::splitLoops); diff --git a/parent-all/pom.xml b/parent-all/pom.xml index 02f0fd0b6..24c8d771a 100644 --- a/parent-all/pom.xml +++ b/parent-all/pom.xml @@ -123,12 +123,12 @@ limitations under the License. commons-io commons-io - 2.5 + 2.6 org.apache.commons commons-lang3 - 3.5 + 3.9 org.apache.commons @@ -224,7 +224,22 @@ limitations under the License. commons-codec commons-codec - 1.9 + 1.12 + + + org.apache.commons + commons-rng-simple + 1.2 + + + org.apache.commons + commons-rng-sampling + 1.2 + + + org.apache.commons + commons-rng-client-api + 1.2 diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java index 58c02616f..f11bfafcb 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java @@ -146,9 +146,9 @@ private static ArgumentParser getParser() { .action(Arguments.storeTrue()); parser.addArgument("--seed") - .help("Seed with which to initialize the random number generator that is used to control " - + "reduction decisions.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) with which to initialize the random number " + + "generator that is used to control reduction decisions.") + .type(String.class); parser.addArgument("--timeout") .help( @@ -287,7 +287,7 @@ public static void mainHelper( final Integer retryLimit = ns.get("retry_limit"); final Boolean verbose = ns.get("verbose"); final boolean skipRender = ns.get("skip_render"); - final int seed = ArgsUtil.getSeedArgument(ns); + final IRandom random = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final String errorString = ns.get("error_string"); final boolean reduceEverywhere = !ns.getBoolean("preserve_semantics"); final boolean stopOnError = ns.get("stop_on_error"); @@ -453,7 +453,7 @@ public static void mainHelper( doReductionHelper( inputShaderJobFile, shaderJobShortName, - seed, + random, fileJudge, workDir, maxSteps, @@ -479,7 +479,7 @@ public static void mainHelper( public static void doReductionHelper( File initialShaderJobFile, String outputShortName, - int seed, + IRandom random, IFileJudge fileJudge, File workDir, int stepLimit, @@ -490,7 +490,6 @@ public static void doReductionHelper( throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { final ShadingLanguageVersion shadingLanguageVersion = getGlslVersionForShaderJob(initialShaderJobFile, fileOps); - final IRandom random = new RandomWrapper(seed); final IdGenerator idGenerator = new IdGenerator(); final int fileCountOffset = getFileCountOffset( diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java index 4551a40af..c99a3a5a6 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java @@ -66,8 +66,8 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .type(Integer.class); parser.addArgument("--seed") - .help("Seed to initialize random number generator with.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) to initialize random number generator with.") + .type(String.class); parser.addArgument("--preserve-semantics") .help("Only perform semantics-preserving reductions.") @@ -95,7 +95,7 @@ public static void main(String[] args) final Namespace ns = parse(args); - final int seed = ArgsUtil.getSeedArgument(ns); + IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final int maxIterations = ns.get("max_iterations"); @@ -107,8 +107,6 @@ public static void main(String[] args) ? "" : ns.getString("expected_string"); - final IRandom generator = new RandomWrapper(seed); - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); final File interestingShaderJobFile = new File("interesting.frag"); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java index d3a32a57e..b79331a89 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java @@ -60,8 +60,8 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .type(Integer.class); parser.addArgument("--seed") - .help("Seed to initialize random number generator with.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) to initialize random number generator with.") + .type(String.class); parser.addArgument("--preserve-semantics") .help("Only perform semantics-preserving reductions.") @@ -95,9 +95,7 @@ public static void main(String[] args) .getGlslVersionFromFirstTwoLines( fileOps.getFirstTwoLinesOfShader(shaderJobFile, ShaderKind.FRAGMENT)); - final int seed = ArgsUtil.getSeedArgument(ns); - - final IRandom generator = new RandomWrapper(seed); + IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final ShaderJob originalShaderJob = fileOps.readShaderJobFile(shaderJobFile); @@ -180,7 +178,7 @@ public static void main(String[] args) } continue; } - if (invalid(current, shadingLanguageVersion, fileOps)) { + if (invalid(current, fileOps)) { System.err.println("Invalid shader after reduction step."); if (ignoreInvalid) { System.err.println("Ignoring it and backtracking."); @@ -227,7 +225,6 @@ private static List getOpsToApply(List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), new IdGenerator())); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as there is already a variable called 'dist' in scope assertEquals(0, ops.size()); } @@ -71,7 +71,7 @@ public void checkForVariableInScope2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), new IdGenerator())); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); // There should be one opportunity as variable dist is in a different scope and not used in this scope. assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -95,7 +95,7 @@ public void checkForVariableInScope3() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), new IdGenerator())); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as there is already a variable called 'dist' in scope, // and it is used. assertEquals(0, ops.size()); @@ -127,7 +127,7 @@ public void misc() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), new IdGenerator())); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as there is already a variable called 'dist' in scope, // and it is used. assertEquals(1, ops.size()); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java index 96d6aae3d..127201b2c 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java @@ -162,7 +162,7 @@ public void testReductionOpportunityForIdentityAsExprStmt() throws Exception { private ReducerContext getReducerContext() { return new ReducerContext(false, ShadingLanguageVersion.ESSL_310, - new RandomWrapper(), + new RandomWrapper(0), new IdGenerator()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java index 5798949dc..da4945e08 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java @@ -203,7 +203,7 @@ private ShaderJob checkCanReduceToTarget(ShaderJob shaderJob, int expectedSize, List ops = InlineUniformReductionOpportunities.findOpportunities(temp, new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null)); assertEquals(expectedSize, ops.size()); ops.get(i).applyReduction(); if (CompareAsts.isEqualAsts(target, temp.getShaders().get(0))) { diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java index 9a09aa5da..af4e8e3f6 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java @@ -60,7 +60,7 @@ public void testDeadConditionalNotReplaced() throws Exception { List opportunities = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(), new IdGenerator()), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); // There should be no ExprToConstant reduction opportunity, because the expressions do not occur // under dead code, and the fuzzed expression is too simple to be reduced. assertFalse(opportunities.stream().anyMatch(item -> item instanceof SimplifyExprReductionOpportunity)); @@ -74,13 +74,13 @@ public void testPopScope() throws Exception { ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(), new IdGenerator()), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); } private void stressTestStructification(String variantProgram, String reducedProgram) throws Exception{ TranslationUnit tu = ParseHelper.parse(variantProgram); - IRandom generator = new RandomWrapper(); + IRandom generator = new RandomWrapper(0); while (true) { List ops = InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), @@ -104,7 +104,7 @@ private void stressTestStructification(String variantProgram, String reducedProg List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(), new IdGenerator()), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } @@ -579,7 +579,7 @@ public void testLeaveLoopLimiter() throws Exception { while (true) { List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new RandomWrapper(), new IdGenerator()), fileOps); + ShadingLanguageVersion.GLSL_440, new RandomWrapper(0), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java index 74bd7c9a3..c48575c66 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java @@ -77,7 +77,7 @@ public void testSwitch1() throws Exception { final TranslationUnit tu = ParseHelper.parse(prog); List ops = StmtReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_130, - new RandomWrapper(), new IdGenerator())); + new RandomWrapper(0), new IdGenerator())); for (StmtReductionOpportunity op : ops) { op.applyReduction(); @@ -103,7 +103,7 @@ public void testDoNotLeaveDefaultEmpty() throws Exception { List ops = StmtReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_310, - new RandomWrapper(), new IdGenerator())); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); diff --git a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java index 2a8d4c04c..d00d8f081 100755 --- a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java +++ b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java @@ -1619,7 +1619,7 @@ private void reduce(HttpServletRequest request, HttpServletResponse response) args.add(retryLimit); } final String seed = request.getParameter("seed"); - if (seed != null) { + if (seed != null && seed.length() > 0) { args.add("--seed"); args.add(seed); } @@ -2292,7 +2292,7 @@ private void htmlReductionForm( "

Random Seed:

", "", "", "", "", diff --git a/shadersets-util/pom.xml b/shadersets-util/pom.xml index e8a79bcb5..4dac88efa 100644 --- a/shadersets-util/pom.xml +++ b/shadersets-util/pom.xml @@ -47,7 +47,7 @@ limitations under the License. com.graphicsfuzzserver-thrift-gen - + com.google.code.gson gson diff --git a/util/pom.xml b/util/pom.xml index 30d88e8de..f1d84c2c8 100644 --- a/util/pom.xml +++ b/util/pom.xml @@ -43,6 +43,10 @@ limitations under the License. commons-io commons-io + + org.apache.commons + commons-rng-simple + diff --git a/util/src/main/java/com/graphicsfuzz/util/ArgsUtil.java b/util/src/main/java/com/graphicsfuzz/util/ArgsUtil.java index 13a6ea726..a3616b84e 100644 --- a/util/src/main/java/com/graphicsfuzz/util/ArgsUtil.java +++ b/util/src/main/java/com/graphicsfuzz/util/ArgsUtil.java @@ -16,15 +16,15 @@ package com.graphicsfuzz.util; -import java.util.Random; import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.commons.rng.simple.internal.SeedFactory; public final class ArgsUtil { - public static int getSeedArgument(Namespace ns) { - Integer seed = ns.get("seed"); + public static long getSeedArgument(Namespace ns) { + String seed = ns.getString("seed"); if (seed == null) { - seed = new Random().nextInt(); + return SeedFactory.createLong(); } - return seed; + return Long.parseUnsignedLong(seed); } } From b86e7c7ae682d60aee32cd5fbf5592ff19e0e8c9 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 16 Jul 2019 11:36:33 +0100 Subject: [PATCH 069/180] Remove redundant code to check for GraphicsFuzz defines (#616) A recent change removed the need to explicitly track whether GraphicsFuzz defines are present in a shader, instead opting to emit them if and only if the shader uses at least one GraphicsFuzz macro. Some old code for tracking when the defines were used had been left behind. This change removes that old code. --- .../common/util/ShaderJobFileOperations.java | 23 ------------------- .../graphicsfuzz/reducer/tool/GlslReduce.java | 3 --- 2 files changed, 26 deletions(-) diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java index 8e153accb..d20df620e 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java @@ -309,29 +309,6 @@ public boolean doesShaderJobResultFileExist(File shaderJobResultFile) { return shaderJobResultFile.isFile(); } - /** - * Determines whether the underlying shader files for the shader jobs use GraphicsFuzz defines. - * Assumes that if one shader does, they all do. - */ - public boolean doesShaderJobUseGraphicsFuzzDefines(File shaderJobFile) throws IOException { - for (ShaderKind shaderKind : ShaderKind.values()) { - //noinspection deprecation: fine inside this class. - final File shaderFile = getUnderlyingShaderFile(shaderJobFile, shaderKind); - if (!shaderFile.isFile()) { - continue; - } - try (BufferedReader br = new BufferedReader(new FileReader(shaderFile))) { - String line; - while ((line = br.readLine()) != null) { - if (line.trim().startsWith(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)) { - return true; - } - } - } - } - return false; - } - /** * Does this shaderJobResultFile have an associated image result? * diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java index f11bfafcb..88f71cc46 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java @@ -514,9 +514,6 @@ public static void doReductionHelper( shaderJobFile ); - final boolean emitGraphicsFuzzDefines = - fileOps.doesShaderJobUseGraphicsFuzzDefines(shaderJobFile); - new ReductionDriver( new ReducerContext( reduceEverywhere, From be34975e782159c2bf609b093580d6ab7a946352 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 16 Jul 2019 11:38:19 +0100 Subject: [PATCH 070/180] Rework AstBuilder to better track unsupported language features (#615) The AstBuilder class, which turns an Antlr-parsed shader into a GraphicsFuzz abstract syntax tree, was built on demand in the early phases of the project and initially had many cases where runtime exceptions would be thrown due to features being unsupported. This change introduces a new runtime exception class, UnsupportedLanguageFeatureException, to represent features that are known to be unsupported at present (e.g. initializer lists), and AstBuilder has been re-worked to throw UnsupportedLanguageFeatureExceptions when feature support is lacking, and plain RuntimeExceptions only for cases corresponding to an invalid input shader. In the process, the GLSL grammar file has been cleaned up a git - the '?' operator is used to eliminate optional production rules in a couple of places, some irrelevant comments have been removed, and many commented out 'TODO' token lines have been eliminated. --- .../antlr4/com/graphicsfuzz/parser/GLSL.g4 | 151 ++---------------- .../common/ast/visitors/AstBuilder.java | 47 +++--- .../UnsupportedLanguageFeatureException.java | 29 ++++ .../common/tool/PrettyPrinterVisitorTest.java | 2 +- .../common/util/ParseHelperTest.java | 150 +++++++++++++++++ 5 files changed, 217 insertions(+), 162 deletions(-) create mode 100644 ast/src/main/java/com/graphicsfuzz/common/ast/visitors/UnsupportedLanguageFeatureException.java diff --git a/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 b/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 index 07efaa322..517b450df 100644 --- a/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 +++ b/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 @@ -59,9 +59,6 @@ variable_identifier: primary_expression: variable_identifier - { - // NEW VARIABLE DECLARATION - } | INTCONSTANT | UINTCONSTANT | FLOATCONSTANT @@ -103,9 +100,8 @@ function_call_header: ; function_identifier: - builtin_type_specifier_nonarray // constructor + builtin_type_specifier_nonarray // type constructor | variable_identifier - | FIELD_SELECTION ; method_call_generic: @@ -123,14 +119,10 @@ method_call_header_with_parameters: | method_call_header_with_parameters COMMA assignment_expression ; - // Grammar Note: Constructors look like methods, but lexical - // analysis recognized most of them as keywords. They are now - // recognized through "type_specifier". method_call_header: variable_identifier LPAREN ; - // Grammar Note: No traditional style type casts. unary_expression: postfix_expression | INC_OP unary_expression @@ -138,7 +130,6 @@ unary_expression: | unary_operator unary_expression ; - // Grammar Note: No '*' or '&' unary ops. Pointers are not supported. unary_operator: PLUS_OP | MINUS_OP @@ -555,8 +546,6 @@ declaration_statement: declaration ; - // Grammar Note: labeled statements for SWITCH only; 'goto' is not - // supported. statement: compound_statement | simple_statement @@ -603,7 +592,7 @@ selection_statement: selection_rest_statement: statement ELSE statement - | statement // REVISIT: binding issue with conditionals + | statement ; condition: @@ -611,10 +600,6 @@ condition: | fully_specified_type IDENTIFIER ASSIGN_OP initializer ; -/// - // switch_statement grammar is based on the syntax described in the body - // of the GLSL spec, not in it's appendix!!! - /// switch_statement: SWITCH LPAREN expression RPAREN switch_body ; @@ -655,23 +640,17 @@ for_init_statement: | declaration_statement ; -conditionopt: - condition - | /// empty /// - ; - for_rest_statement: - conditionopt SEMICOLON - | conditionopt SEMICOLON expression + condition? SEMICOLON + | condition? SEMICOLON expression ; - // Grammar Note: No 'goto'. Gotos are not supported. jump_statement: CONTINUE SEMICOLON | BREAK SEMICOLON | RETURN SEMICOLON | RETURN expression SEMICOLON - | DISCARD SEMICOLON // Fragment shader only. + | DISCARD SEMICOLON ; external_declaration: @@ -685,14 +664,13 @@ function_definition: function_prototype compound_statement_no_new_scope ; -/// layout_qualifieropt is packed into this rule /// interface_block: basic_interface_block | layout_qualifier basic_interface_block ; basic_interface_block: - interface_qualifier IDENTIFIER LBRACE member_list RBRACE instance_name_opt SEMICOLON + interface_qualifier IDENTIFIER LBRACE member_list RBRACE instance_name? SEMICOLON ; interface_qualifier: @@ -702,10 +680,8 @@ interface_qualifier: | BUFFER ; -instance_name_opt: - /// empty /// - | IDENTIFIER - | IDENTIFIER array_specifier +instance_name: + IDENTIFIER array_specifier? ; layout_defaults: @@ -749,7 +725,7 @@ VARYING: 'varying' ; READONLY: 'readonly' ; WRITEONLY: 'writeonly' ; SHARED: 'shared' ; -LAYOUT_TOK: 'layout' ; // REVISIT +LAYOUT_TOK: 'layout' ; UINTCONSTANT: (DECIMAL_DIGITS | OCTAL_DIGITS | HEX_DIGITS) 'u'; ROW_MAJOR: 'row_major' ; PACKED_TOK: 'packed' ; @@ -758,7 +734,6 @@ BOOLCONSTANT: 'true' | 'false' ; INC_OP: '++' ; DEC_OP: '--' ; VOID_TOK: 'void' ; -FIELD_SELECTION: 'C_TODO' ; // REVISIT LEFT_OP: '<<' ; RIGHT_OP: '>>' ; LE_OP: '<=' ; @@ -782,65 +757,26 @@ FLOAT_TOK: 'float' ; INT_TOK: 'int' ; UINT_TOK: 'uint' ; BOOL_TOK: 'bool' ; -// VEC: 'TODO' ; -// BVEC: 'TODO' ; -// IVEC: 'TODO' ; -// UVEC: 'TODO' ; -// MATX: 'TODO' ; -// SAMPLERD: 'TODO' ; -// SAMPLERDRECT: 'TODO' ; SAMPLERCUBE: 'samplerCube' ; -SAMPLEREXTERNALOES: 'samplerExternalOES' ; // REVISIT -// SAMPLERDSHADOW: 'TODO' ; -// SAMPLERDRECTSHADOW: 'TODO' ; SAMPLERCUBESHADOW: 'samplerCubeShadow' ; -// SAMPLERDARRAY: 'TODO' ; -// SAMPLERDARRAYSHADOW: 'TODO' ; SAMPLERBUFFER: 'samplerBuffer' ; SAMPLERCUBEARRAY: 'samplerCubeArray' ; SAMPLERCUBEARRAYSHADOW: 'samplerCubeArrayShadow' ; -// ISAMPLERD: 'TODO' ; -// ISAMPLERDRECT: 'TODO' ; ISAMPLERCUBE: 'isamplerCube' ; -// ISAMPLERDARRAY: 'TODO' ; ISAMPLERBUFFER: 'isamplerBuffer' ; ISAMPLERCUBEARRAY: 'isamplerCubeArray' ; -// USAMPLERD: 'TODO' ; -// USAMPLERDRECT: 'TODO' ; USAMPLERCUBE: 'usamplerCube' ; -// USAMPLERDARRAY: 'TODO' ; USAMPLERBUFFER: 'usamplerBuffer' ; USAMPLERCUBEARRAY: 'usamplerCubeArray' ; -// SAMPLERDMS: 'TODO' ; -// ISAMPLERDMS: 'TODO' ; -// USAMPLERDMS: 'TODO' ; -// SAMPLERDMSARRAY: 'TODO' ; -// ISAMPLERDMSARRAY: 'TODO' ; -// USAMPLERDMSARRAY: 'TODO' ; -// IMAGED: 'TODO' ; -// IMAGEDRECT: 'TODO' ; IMAGECUBE: 'imageCube' ; IMAGEBUFFER: 'imageBuffer' ; -// IMAGEDARRAY: 'TODO' ; IMAGECUBEARRAY: 'imageCubeArray' ; -// IMAGEDMS: 'TODO' ; -// IMAGEDMSARRAY: 'TODO' ; -// IIMAGED: 'TODO' ; -// IIMAGEDRECT: 'TODO' ; IIMAGECUBE: 'iimageCube' ; IIMAGEBUFFER: 'iimageBuffer' ; -// IIMAGEDARRAY: 'TODO' ; IIMAGECUBEARRAY: 'iimageCubeArray' ; -// IIMAGEDMS: 'TODO' ; -// IIMAGEDMSARRAY: 'TODO' ; -// UIMAGED: 'TODO' ; -// UIMAGEDRECT: 'TODO' ; UIMAGECUBE: 'uimageCube' ; UIMAGEBUFFER: 'uimageBuffer' ; -// UIMAGEDARRAY: 'TODO' ; UIMAGECUBEARRAY: 'uimageCubeArray' ; -// UIMAGEDMS: 'TODO' ; -// UIMAGEDMSARRAY: 'TODO' ; ATOMIC_UINT: 'atomic_uint' ; STRUCT: 'struct' ; IF: 'if' ; @@ -968,72 +904,3 @@ COMMENT: ('//' ~('\n'|'\r')* '\r'? '\n' | '/*' (.)*? '*/') -> skip ; WS: [\t\r\u000C ]+ { skip(); } ; EOL: '\n' { if(ignoreNewLine) { skip(); } ignoreNewLine = true; } ; - -/* - -%token ATTRIBUTE CONST_TOK BOOL_TOK FLOAT_TOK INT_TOK UINT_TOK -%token BREAK CONTINUE DO ELSE FOR IF DISCARD RETURN SWITCH CASE DEFAULT -%token BVEC2 BVEC3 BVEC4 IVEC2 IVEC3 IVEC4 UVEC2 UVEC3 UVEC4 VEC2 VEC3 VEC4 -%token CENTROID IN_TOK OUT_TOK INOUT_TOK UNIFORM VARYING SAMPLE -%token NOPERSPECTIVE FLAT SMOOTH -%token MAT2X2 MAT2X3 MAT2X4 -%token MAT3X2 MAT3X3 MAT3X4 -%token MAT4X2 MAT4X3 MAT4X4 -%token SAMPLER1D SAMPLER2D SAMPLER3D SAMPLERCUBE SAMPLER1DSHADOW SAMPLER2DSHADOW -%token SAMPLERCUBESHADOW SAMPLER1DARRAY SAMPLER2DARRAY SAMPLER1DARRAYSHADOW -%token SAMPLER2DARRAYSHADOW SAMPLERCUBEARRAY SAMPLERCUBEARRAYSHADOW -%token ISAMPLER1D ISAMPLER2D ISAMPLER3D ISAMPLERCUBE -%token ISAMPLER1DARRAY ISAMPLER2DARRAY ISAMPLERCUBEARRAY -%token USAMPLER1D USAMPLER2D USAMPLER3D USAMPLERCUBE USAMPLER1DARRAY -%token USAMPLER2DARRAY USAMPLERCUBEARRAY -%token SAMPLER2DRECT ISAMPLER2DRECT USAMPLER2DRECT SAMPLER2DRECTSHADOW -%token SAMPLERBUFFER ISAMPLERBUFFER USAMPLERBUFFER -%token SAMPLER2DMS ISAMPLER2DMS USAMPLER2DMS -%token SAMPLER2DMSARRAY ISAMPLER2DMSARRAY USAMPLER2DMSARRAY -%token SAMPLEREXTERNALOES -%token IMAGE1D IMAGE2D IMAGE3D IMAGE2DRECT IMAGECUBE IMAGEBUFFER -%token IMAGE1DARRAY IMAGE2DARRAY IMAGECUBEARRAY IMAGE2DMS IMAGE2DMSARRAY -%token IIMAGE1D IIMAGE2D IIMAGE3D IIMAGE2DRECT IIMAGECUBE IIMAGEBUFFER -%token IIMAGE1DARRAY IIMAGE2DARRAY IIMAGECUBEARRAY IIMAGE2DMS IIMAGE2DMSARRAY -%token UIMAGE1D UIMAGE2D UIMAGE3D UIMAGE2DRECT UIMAGECUBE UIMAGEBUFFER -%token UIMAGE1DARRAY UIMAGE2DARRAY UIMAGECUBEARRAY UIMAGE2DMS UIMAGE2DMSARRAY -%token IMAGE1DSHADOW IMAGE2DSHADOW IMAGE1DARRAYSHADOW IMAGE2DARRAYSHADOW -%token COHERENT VOLATILE RESTRICT READONLY WRITEONLY -%token ATOMIC_UINT -%token STRUCT VOID_TOK WHILE -%token IDENTIFIER TYPE_IDENTIFIER IDENTIFIER -%token FLOATCONSTANT -%token INTCONSTANT UINTCONSTANT BOOLCONSTANT -%token FIELD_SELECTION -%token LEFT_OP RIGHT_OP -%token INC_OP DEC_OP LE_OP GE_OP EQ_OP NE_OP -%token AND_OP OR_OP XOR_OP MUL_ASSIGN DIV_ASSIGN ADD_ASSIGN -%token MOD_ASSIGN LEFT_ASSIGN RIGHT_ASSIGN AND_ASSIGN XOR_ASSIGN OR_ASSIGN -%token SUB_ASSIGN -%token INVARIANT PRECISE -%token LOWP MEDIUMP HIGHP SUPERP PRECISION - -%token VERSION EXTENSION LINE COLON EOL INTERFACE OUTPUT -%token PRAGMA_DEBUG_ON PRAGMA_DEBUG_OFF -%token PRAGMA_OPTIMIZE_ON PRAGMA_OPTIMIZE_OFF -%token PRAGMA_INVARIANT_ALL -%token LAYOUT_TOK - -// Reserved words that are not actually used in the grammar. -%token ASM CLASS UNION ENUM TYPEDEF TEMPLATE THIS PACKED_TOK GOTO -%token INLINE_TOK NOINLINE PUBLIC_TOK STATIC EXTERN EXTERNAL -%token LONG_TOK SHORT_TOK DOUBLE_TOK HALF FIXED_TOK UNSIGNED INPUT_TOK -%token HVEC2 HVEC3 HVEC4 DVEC2 DVEC3 DVEC4 FVEC2 FVEC3 FVEC4 -%token SAMPLER3DRECT -%token SIZEOF CAST NAMESPACE USING -%token RESOURCE PATCH -%token SUBROUTINE - -%token ERROR_TOK - -%token COMMON PARTITION ACTIVE FILTER ROW_MAJOR - -%right THEN ELSE -%% - -*/ diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java index 846449b83..178ac1bdc 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java @@ -277,6 +277,8 @@ public Declaration visitDeclaration(DeclarationContext ctx) { if (ctx.interface_block() != null) { return visitInterface_block(ctx.interface_block()); } + // The above captures all the declaration kinds, so this indicates a bad input + // rather than lack of support. throw new RuntimeException("Unknown declaration at line " + ctx.start.getLine() + ": " + getOriginalSourceText(ctx)); } @@ -289,9 +291,9 @@ public Declaration visitInterface_block(Interface_blockContext ctx) { final Basic_interface_blockContext basicCtx = ctx.basic_interface_block(); final TypeQualifier interfaceQualifier = visitInterface_qualifier(basicCtx.interface_qualifier()); - final Optional maybeInstanceName = basicCtx.instance_name_opt() == null + final Optional maybeInstanceName = basicCtx.instance_name() == null ? Optional.empty() - : Optional.of(basicCtx.instance_name_opt().getText()); + : Optional.of(basicCtx.instance_name().getText()); final Pair, List> members = getMembers(basicCtx.member_list()); return new InterfaceBlock( maybeLayoutQualifier, @@ -314,8 +316,9 @@ public TypeQualifier visitInterface_qualifier(Interface_qualifierContext ctx) { case "buffer": return TypeQualifier.BUFFER; default: - throw new RuntimeException("Interface qualifier: " + ctx.getText() - + " unknown or not yet supported."); + // The above is supposed to capture all the interface qualifiers, so this + // indicates that the input is bad (rather than lack of support). + throw new RuntimeException("Unknown interface qualifier: " + ctx.getText()); } } @@ -357,7 +360,8 @@ private Type getType(Type_qualifierContext qualifiersCtx, private Type getType(Type_specifierContext typeSpecifier, List qualifiers) { if (typeSpecifier.array_specifier() != null) { - throw new RuntimeException(); + throw new UnsupportedLanguageFeatureException("Array information specified at the base type" + + ", e.g. 'int[3] v', is not currently supported; use e.g. 'int A[3]' instead"); } if (typeSpecifier.type_specifier_nonarray().builtin_type_specifier_nonarray() != null) { return new QualifiedType(getBuiltinType(typeSpecifier.type_specifier_nonarray() @@ -415,7 +419,7 @@ private StructDefinitionType makeStructDefinition(Optional struc private ArrayInfo getArrayInfo(Array_specifierContext arraySpecifierContext) { if (arraySpecifierContext.array_specifier() != null) { - throw new RuntimeException("Not yet supporting multi-dimmensional arrays"); + throw new UnsupportedLanguageFeatureException("Not yet supporting multi-dimensional arrays"); } if (arraySpecifierContext.constant_expression() == null) { // An array with unspecified length. @@ -425,8 +429,8 @@ private ArrayInfo getArrayInfo(Array_specifierContext arraySpecifierContext) { if (expr instanceof IntConstantExpr) { return new ArrayInfo(Integer.parseInt(((IntConstantExpr) expr).getValue())); } - throw new RuntimeException("Unable to construct array info for array with size " - + expr.getText()); + throw new UnsupportedLanguageFeatureException("Unable to construct array info for array with " + + "size " + expr.getText()); } private BuiltinType getBuiltinType(Builtin_type_specifier_nonarrayContext ctx) { @@ -791,10 +795,6 @@ public List visitStatement_list(Statement_listContext ctx) { @Override public DeclarationStmt visitDeclaration_statement(Declaration_statementContext ctx) { - if (ctx.declaration().init_declarator_list() == null) { - throw new RuntimeException("Error at line " + ctx.start.getLine() - + ": Only variable declarations are supported in declaration statements"); - } return new DeclarationStmt( visitInit_declarator_list(ctx.declaration().init_declarator_list())); } @@ -836,7 +836,7 @@ public Initializer visitInitializer(InitializerContext ctx) { if (ctx.assignment_expression() != null) { return new Initializer(visitAssignment_expression(ctx.assignment_expression())); } - throw new RuntimeException(); + throw new UnsupportedLanguageFeatureException("Initializer lists are not currently supported."); } @Override @@ -956,8 +956,8 @@ public Stmt visitIteration_statement(Iteration_statementContext ctx) { } assert ctx.FOR() != null; return new ForStmt(visitFor_init_statement(ctx.for_init_statement()), - ctx.for_rest_statement().conditionopt().condition() == null ? null : - visitCondition(ctx.for_rest_statement().conditionopt().condition()), + ctx.for_rest_statement().condition() == null ? null : + visitCondition(ctx.for_rest_statement().condition()), ctx.for_rest_statement().expression() == null ? null : visitExpression(ctx.for_rest_statement().expression()), visitStatement_no_new_scope(ctx.statement_no_new_scope())); @@ -969,14 +969,14 @@ public Expr visitCondition(ConditionContext ctx) { return visitExpression(ctx.expression()); } assert ctx.ASSIGN_OP() != null; - throw new RuntimeException( + throw new UnsupportedLanguageFeatureException( "We do not yet support the case where the condition of a 'for' or 'while' introduces a " + "new variable: " + getOriginalSourceText(ctx)); } @Override public Stmt visitFor_rest_statement(For_rest_statementContext ctx) { - throw new RuntimeException(); + throw new RuntimeException("By construction, this visitor method should never get executed."); } @Override @@ -1036,6 +1036,8 @@ public PragmaStatement visitPragma_statement(Pragma_statementContext ctx) { if (ctx.PRAGMA_INVARIANT_ALL() != null) { return PragmaStatement.INVARIANT_ALL; } + // The above captures all the possibilities for a pragma statement, so reaching the following + // line indicates that the shader is invalid, rather than that support is missing. throw new RuntimeException("Unknown pragma statement " + ctx.getText()); } @@ -1097,8 +1099,8 @@ public Expr visitPostfix_expression(Postfix_expressionContext ctx) { visitExpression(ctx.integer_expression().expression())); } if (ctx.method_call_generic() != null) { - // TODO: check grammar - throw new RuntimeException("Not yet supported: " + getOriginalSourceText(ctx)); + throw new UnsupportedLanguageFeatureException("Method calls are not currently supported: " + + getOriginalSourceText(ctx)); } if (ctx.IDENTIFIER() != null) { return new MemberLookupExpr(visitPostfix_expression(ctx.postfix_expression()), @@ -1130,6 +1132,7 @@ public Expr visitFunction_call_header_no_parameters( Function_call_header_no_parametersContext ctx) { if (isBuiltinTypeConstructor(ctx.function_call_header().function_identifier()) || isStructTypeConstructor(ctx.function_call_header().function_identifier())) { + // This is illegal, so indicates an invalid shader rather than lack of support. throw new RuntimeException( "Found type constructor with no arguments at line " + ctx.start.getLine() + ": " + getOriginalSourceText(ctx)); @@ -1138,6 +1141,8 @@ public Expr visitFunction_call_header_no_parameters( return new FunctionCallExpr(getCallee(ctx.function_call_header().function_identifier()), new ArrayList<>()); } + // The above logic is intended to capture all cases, so the following indicates an invalid + // shader, rather than lack of support. throw new RuntimeException("Unsupported function call at line " + ctx.start.getLine() + ": " + getOriginalSourceText(ctx)); } @@ -1175,6 +1180,8 @@ public Expr visitFunction_call_header_with_parameters( if (isRegularFunction(header.function_identifier())) { return new FunctionCallExpr(getCallee(header.function_identifier()), params); } + // The above logic is intended to capture all cases, so the following indicates an invalid + // shader, rather than lack of support. throw new RuntimeException("Unsupported function call: " + getOriginalSourceText(ctx)); } @@ -1411,6 +1418,8 @@ private BinOp getBinOp(Token token) { return op; } } + // The BinOp class includes all binary operators, so the following indicates an invalid shader, + // rather than lack of support. throw new RuntimeException("Unknown binary operator: " + token.getText()); } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/UnsupportedLanguageFeatureException.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/UnsupportedLanguageFeatureException.java new file mode 100644 index 000000000..b63b345ab --- /dev/null +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/UnsupportedLanguageFeatureException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.common.ast.visitors; + +/** + * A RuntimeException to identify cases where AST building fails due to known unsupported + * language features. + */ +public class UnsupportedLanguageFeatureException extends RuntimeException { + + public UnsupportedLanguageFeatureException(String message) { + super(message); + } + +} diff --git a/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java b/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java index 1cc0274f0..7850251a0 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java @@ -135,7 +135,7 @@ public void testParseAndPrintComputeShader() throws Exception { final String program = "" + "layout(std430, binding = 2) buffer abuf {\n" + " int data[];\n" - + "} ;\n" + + "};\n" + "layout(local_size_x = 128, local_size_y = 1) in;\n" + "void main()\n" + "{\n" diff --git a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java index 18c64d3a6..cc7338b51 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java @@ -31,6 +31,7 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; +import com.graphicsfuzz.common.ast.visitors.UnsupportedLanguageFeatureException; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import java.io.BufferedWriter; @@ -564,4 +565,153 @@ private String getStringFromInputStream(InputStream strippedIs) throws IOExcepti return writer.toString(); } + @Test + public void testUnsupportedArrayLength() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("void main() {\n" + + " int A[3 + 4];\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Unable to construct array info")); + } + } + + @Test + public void testUnsupportedMultiDimensionalArrays() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("void main() {\n" + + " int A[3][4];\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Not yet supporting multi-dimensional arrays")); + } + } + + @Test + public void testUnsupportedArrayInBaseType() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("void main() {\n" + + " int[2] A, B[3];\n" + + " B[2][1] = 3;\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Array information specified at the base type")); + } + } + + @Test + public void testUnsupportedDeclarationInCondition() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("void main() {\n" + + " while(bool b = true) {\n" + + " if(b) {\n" + + " break;\n" + + " }\n" + + " }\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("We do not yet support the case where the " + + "condition of a 'for' or 'while' introduces a new variable")); + } + } + + @Test + public void testUnsupportedInitializerList() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("#version 440\n" + + "\n" + + "void main() {\n" + + "\n" + + " int A[4] = { 1, 2, 3, 4 };\n" + + "\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Initializer lists are not currently supported")); + } + } + + @Test + public void testUnsupportedMethodCall1() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("#version 310 es\n" + + "\n" + + "void main() {\n" + + "\n" + + " int A[4];\n" + + " A.length();\n" + + "\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Method calls are not currently supported")); + } + } + + @Test + public void testUnsupportedMethodCall2() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("#version 310 es\n" + + "\n" + + "void main() {\n" + + "\n" + + " int A[4];\n" + + " A.length(void);\n" + + "\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Method calls are not currently supported")); + } + } + + @Test + public void testUnsupportedMethodCall3() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("#version 320 es\n" + + "\n" + + "precision highp float;\n" + + "\n" + + "struct S { int A[4]; };\n" + + "\n" + + "void main() {\n" + + "\n" + + " S s = S(int[4](1,2,3,4));\n" + + " s.A.length();\n" + + "\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Method calls are not currently supported")); + } + } + } From bdd8f21547d2ef690a18d340dd0adcd15e85680b Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Wed, 17 Jul 2019 11:33:11 +0100 Subject: [PATCH 071/180] Reduce switch statements by replacing switch with do..while(false) (#618) Adds a reduction pass that will replace a switch statement with a loop of the form: do { // Body of switch statement, minus 'case' and 'default' labels } while (false); The loop construct is convenient as it allows for 'break' statements. Existing reduction passes will then try to remove the loop. Does not apply when --preserve-semantics is used, unless the switch statement is part of a dead code injection. Fixes #610 --- .../graphicsfuzz/reducer/ReductionDriver.java | 1 + ...CompoundToBlockReductionOpportunities.java | 2 +- ...iableDeclToExprReductionOpportunities.java | 2 +- .../IReductionOpportunityFinder.java | 19 ++ ...tlinedStatementReductionOpportunities.java | 5 +- .../ReductionOpportunities.java | 3 +- .../ReductionOpportunitiesBase.java | 8 +- ...UniformMetadataReductionOpportunities.java | 5 + .../SwitchToLoopReductionOpportunities.java | 68 ++++++ .../SwitchToLoopReductionOpportunity.java | 60 +++++ .../UnswitchifyReductionOpportunities.java | 2 +- ...iableDeclToExprReductionOpportunities.java | 2 +- .../reducer/ReductionDriverTest.java | 53 +++++ ...witchToLoopReductionOpportunitiesTest.java | 205 ++++++++++++++++++ 14 files changed, 424 insertions(+), 11 deletions(-) create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunities.java create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java create mode 100644 reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java index d1cd2caed..a1c92dd59 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java @@ -112,6 +112,7 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.mutationFinder(), IReductionOpportunityFinder.loopMergeFinder(), IReductionOpportunityFinder.compoundToBlockFinder(), + IReductionOpportunityFinder.switchToLoopFinder(), IReductionOpportunityFinder.outlinedStatementFinder(), IReductionOpportunityFinder.unwrapFinder(), IReductionOpportunityFinder.removeStructFieldFinder(), diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java index 02102727b..c8d81ceba 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java @@ -35,7 +35,7 @@ public class CompoundToBlockReductionOpportunities extends ReductionOpportunitiesBase { - public CompoundToBlockReductionOpportunities( + private CompoundToBlockReductionOpportunities( TranslationUnit tu, ReducerContext context) { super(tu, context); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java index 7293b28dc..2f2d785de 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java @@ -53,7 +53,7 @@ public class GlobalVariableDeclToExprReductionOpportunities extends ReductionOpportunitiesBase { private final List globalVariableDecl; - public GlobalVariableDeclToExprReductionOpportunities(TranslationUnit tu, + private GlobalVariableDeclToExprReductionOpportunities(TranslationUnit tu, ReducerContext context) { super(tu, context); this.globalVariableDecl = new ArrayList<>(); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java index 24443c760..32efb2c68 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java @@ -500,4 +500,23 @@ public String getName() { }; } + static IReductionOpportunityFinder + switchToLoopFinder() { + return new IReductionOpportunityFinder() { + @Override + public List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return SwitchToLoopReductionOpportunities.findOpportunities( + shaderJob, + context); + } + + @Override + public String getName() { + return "switchToLoop"; + } + }; + } + } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunities.java index 727340ba1..c4a3cd759 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunities.java @@ -29,6 +29,7 @@ import com.graphicsfuzz.util.Constants; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -48,8 +49,8 @@ static List findOpportunities( ReducerContext context) { return shaderJob.getShaders() .stream() - .map(item -> findOpportunitiesForShader(item)) - .reduce(Arrays.asList(), ListConcat::concatenate); + .map(OutlinedStatementReductionOpportunities::findOpportunitiesForShader) + .reduce(Collections.emptyList(), ListConcat::concatenate); } private static List findOpportunitiesForShader( diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java index 6597aba5a..948f859eb 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java @@ -67,7 +67,8 @@ public static List getReductionOpportunities( IReductionOpportunityFinder.foldConstantFinder(), IReductionOpportunityFinder.inlineUniformFinder(), IReductionOpportunityFinder.redundantUniformMetadataFinder(), - IReductionOpportunityFinder.variableDeclToExprFinder())) { + IReductionOpportunityFinder.variableDeclToExprFinder(), + IReductionOpportunityFinder.switchToLoopFinder())) { final List currentOpportunities = ros .findOpportunities(shaderJob, context); if (ReductionDriver.DEBUG_REDUCER) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java index c0586ccaa..598904ce6 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java @@ -47,15 +47,15 @@ public abstract class ReductionOpportunitiesBase extends ScopeTreeBuilder { private final List opportunities; - protected final InjectionTracker injectionTracker; - protected final NotReferencedFromLiveContext notReferencedFromLiveContext; + final InjectionTracker injectionTracker; + final NotReferencedFromLiveContext notReferencedFromLiveContext; protected final IParentMap parentMap; protected final ReducerContext context; - protected final ShaderKind shaderKind; + final ShaderKind shaderKind; - protected String enclosingFunctionName; + private String enclosingFunctionName; private int numEnclosingLValues; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java index f025d6166..8d9fddbc4 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java @@ -29,6 +29,11 @@ public class RemoveRedundantUniformMetadataReductionOpportunities { + private RemoveRedundantUniformMetadataReductionOpportunities() { + // This class just provides a static method; there is no cause to create an instance of the + // class. + } + static List findOpportunities( ShaderJob shaderJob, ReducerContext context) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunities.java new file mode 100644 index 000000000..06eca2a2a --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunities.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.ListConcat; +import java.util.Collections; +import java.util.List; + +public class SwitchToLoopReductionOpportunities extends + ReductionOpportunitiesBase { + + /** + * Find all switch-to-loop opportunities for the given translation unit. + * + * @param shaderJob The shader job to be searched. + * @param context Includes info such as whether we reduce everywhere + * @return The opportunities that can be reduced + */ + static List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return shaderJob.getShaders() + .stream() + .map(item -> findOpportunitiesForShader(item, context)) + .reduce(Collections.emptyList(), ListConcat::concatenate); + } + + private static List findOpportunitiesForShader( + TranslationUnit tu, + ReducerContext context) { + final SwitchToLoopReductionOpportunities finder = + new SwitchToLoopReductionOpportunities(tu, context); + finder.visit(tu); + return finder.getOpportunities(); + } + + private SwitchToLoopReductionOpportunities(TranslationUnit tu, ReducerContext context) { + super(tu, context); + } + + @Override + public void visitSwitchStmt(SwitchStmt switchStmt) { + super.visitSwitchStmt(switchStmt); + if (context.reduceEverywhere() || injectionTracker.enclosedByDeadCodeInjection()) { + addOpportunity(new SwitchToLoopReductionOpportunity(getVistitationDepth(), + parentMap.getParent(switchStmt), + switchStmt)); + } + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java new file mode 100644 index 000000000..7a0ddc616 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.expr.BoolConstantExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.CaseLabel; +import com.graphicsfuzz.common.ast.stmt.DoStmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.stream.Collectors; + +/** + * Turns a switch statement into a do...while(false) loop. The body of the loop comprises all + * the non-case/default statements from the switch. The reason a loop is used is to allow for + * break statements. The reduction opportunity is not semantics-preserving. + */ +public class SwitchToLoopReductionOpportunity extends AbstractReductionOpportunity { + + // The parent of the switch statemtnt + private final IAstNode parent; + + // The switch statement to be replaced + private final SwitchStmt switchStmt; + + SwitchToLoopReductionOpportunity(VisitationDepth depth, IAstNode parent, SwitchStmt switchStmt) { + super(depth); + this.parent = parent; + this.switchStmt = switchStmt; + } + + @Override + void applyReductionImpl() { + final BlockStmt loopBody = new BlockStmt( + switchStmt.getBody().getStmts().stream().filter(item -> !(item instanceof CaseLabel)) + .collect(Collectors.toList()), true); + parent.replaceChild(switchStmt, new DoStmt(loopBody, new BoolConstantExpr(false))); + } + + @Override + public boolean preconditionHolds() { + return parent.hasChild(switchStmt); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java index 45dd33042..6a7c75363 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java @@ -27,7 +27,7 @@ public class UnswitchifyReductionOpportunities extends ReductionOpportunitiesBase { - public UnswitchifyReductionOpportunities( + private UnswitchifyReductionOpportunities( TranslationUnit tu, ReducerContext context) { super(tu, context); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java index fe770b992..055cad2df 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java @@ -44,7 +44,7 @@ public class VariableDeclToExprReductionOpportunities extends ReductionOpportunitiesBase { - public VariableDeclToExprReductionOpportunities(TranslationUnit tu, ReducerContext context) { + private VariableDeclToExprReductionOpportunities(TranslationUnit tu, ReducerContext context) { super(tu, context); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java index 101f50b44..4566f8462 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java @@ -599,6 +599,59 @@ public void testReductionOfUnreferencedUniform() throws Exception { } + @Test + public void testSimplificationOfSwitch() throws Exception { + final String shader = "#version 310 es\n" + + "void main() {\n" + + " switch(0) {\n" + + " case 0:\n" + + " mix(0.0, 1.0, 0.0);\n" + + " break;\n" + + " default:\n" + + " 1;\n" + + " }\n" + + "}\n"; + + final String expected = "#version 310 es\n" + + "void main() {\n" + + " mix(0.0, 1.0, 0.0);\n" + + "}\n"; + + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), + ParseHelper.parse(shader)); + + final File workDir = testFolder.getRoot(); + final File tempShaderJobFile = new File(workDir, "temp.json"); + fileOps.writeShaderJobFile(shaderJob, tempShaderJobFile); + + final IFileJudge usesMixFileJudge = + new CheckAstFeaturesFileJudge(Collections.singletonList(() -> new CheckAstFeatureVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { + super.visitFunctionCallExpr(functionCallExpr); + if (functionCallExpr.getCallee().equals("mix")) { + trigger(); + } + } + }), + ShaderKind.FRAGMENT, + fileOps); + + final String resultsPrefix = new ReductionDriver(new ReducerContext(true, + ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), + new IdGenerator()), + false, + fileOps, + usesMixFileJudge, + workDir) + .doReduction(shaderJob, "temp", 0, 100); + + CompareAsts.assertEqualAsts(expected, ParseHelper.parse(new File(testFolder.getRoot(), resultsPrefix + ".frag"))); + + } + private String getPrefix(File tempFile) { return FilenameUtils.removeExtension(tempFile.getName()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java new file mode 100644 index 000000000..90b2b0f37 --- /dev/null +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java @@ -0,0 +1,205 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.util.Constants; +import java.util.List; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class SwitchToLoopReductionOpportunitiesTest { + + @Test + public void testDoNotReduceIfPreservingSemantics() throws Exception { + final String original = "void main() {\n" + + " if (" + Constants.GLF_DEAD + "(false)) {\n" + + " switch (0) {\n" + + " case 0:\n" + + " return;\n" + + " }\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + // There should be no opportunities as the preserve semantics is enabled. + assertTrue(ops.isEmpty()); + } + + @Test + public void testDoReduceWhenPreservingSemanticsIfInDeadCode() throws Exception { + final String original = "void main() {\n" + + " if (" + Constants.GLF_DEAD + "(false)) {\n" + + " switch (0) {\n" + + " case 0:\n" + + " return;\n" + + " }\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " if (" + Constants.GLF_DEAD + "(false)) {\n" + + " do {\n" + + " return;\n" + + " } while (false);\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + SwitchToLoopReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + // There should be an opportunity, as the switch statement is in a dead code block. + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testNoBreaks() throws Exception { + final String original = "void main() {\n" + + " int a;\n" + + " int x = 3;\n" + + " switch (x) {\n" + + " case 0:\n" + + " {\n" + + " x = 1;\n" + + " }\n" + + " case 1:\n" + + " a = 4;\n" + + " x = 2;\n" + + " case 3:\n" + + " default:\n" + + " x = 1;\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " int a;\n" + + " int x = 3;\n" + + " do {\n" + + " {\n" + + " x = 1;\n" + + " }\n" + + " a = 4;\n" + + " x = 2;\n" + + " x = 1;\n" + + " } while (false);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + SwitchToLoopReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testWithBreaks() throws Exception { + final String original = "void main() {\n" + + " int a;\n" + + " int x = 3;\n" + + " switch (x) {\n" + + " case 0:\n" + + " {\n" + + " x = 1;\n" + + " }\n" + + " break;\n" + + " case 1:\n" + + " a = 4;\n" + + " x = 2;\n" + + " break;\n" + + " case 3:\n" + + " default:\n" + + " x = 1;\n" + + " break;\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " int a;\n" + + " int x = 3;\n" + + " do {\n" + + " {\n" + + " x = 1;\n" + + " }\n" + + " break;\n" + + " a = 4;\n" + + " x = 2;\n" + + " break;\n" + + " x = 1;\n" + + " break;\n" + + " } while (false);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + SwitchToLoopReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testTwoSwitches() throws Exception { + final String original = "void main() {\n" + + " int x;\n" + + " switch (0) {\n" + + " case 0:\n" + + " x = 1;\n" + + " }\n" + + " switch (1) {\n" + + " case 1:\n" + + " x = 2;\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " int x;\n" + + " do {\n" + + " x = 1;\n" + + " } while (false);\n" + + " do {\n" + + " x = 2;\n" + + " } while (false);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + SwitchToLoopReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + assertEquals(2, ops.size()); + ops.get(0).applyReduction(); + ops.get(1).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + +} From 916bcbd54a9d820ba5c2f51b2006e3821f8b4189 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Wed, 17 Jul 2019 16:27:34 +0100 Subject: [PATCH 072/180] Eliminate redundant GLSL version arguments (#620) GlslGenerate and GenerateAndRunShaders both required a 'glsl-version' argument that was subsequently never used, due to the shading language version now being inferred from the shader itself. This change removes those arguments, and in passing cleans up various style issues (such as using method references in favour of lambdas), and uses ShaderJobFileOperations more consistently in some tests that were affected by the change. Fixes #619 --- .../generator/GenerateAndRunShaders.java | 8 - .../generator/GenerateAndRunShadersTest.java | 9 +- .../graphicsfuzz/generator/tool/Generate.java | 27 ++- .../generator/tool/GenerateTest.java | 155 ++++++++---------- 4 files changed, 81 insertions(+), 118 deletions(-) diff --git a/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/GenerateAndRunShaders.java b/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/GenerateAndRunShaders.java index 6def15710..60252eed9 100755 --- a/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/GenerateAndRunShaders.java +++ b/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/GenerateAndRunShaders.java @@ -16,12 +16,9 @@ package com.graphicsfuzz.generator; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.generator.tool.Generate; -import com.graphicsfuzz.util.ArgsUtil; import java.io.File; import java.io.IOException; import java.util.HashSet; @@ -29,7 +26,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import net.sourceforge.argparse4j.ArgumentParsers; -import net.sourceforge.argparse4j.impl.Arguments; import net.sourceforge.argparse4j.inf.ArgumentParser; import net.sourceforge.argparse4j.inf.ArgumentParserException; import net.sourceforge.argparse4j.inf.Namespace; @@ -65,10 +61,6 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .help("Worker name.") .type(String.class); - parser.addArgument("glsl-version") - .help("Version of GLSL to target.") - .type(String.class); - // Optional arguments Generate.addGeneratorCommonArguments(parser); diff --git a/generate-and-run-shaders/src/test/java/com/graphicsfuzz/generator/GenerateAndRunShadersTest.java b/generate-and-run-shaders/src/test/java/com/graphicsfuzz/generator/GenerateAndRunShadersTest.java index dbf1a7980..7fc689cbe 100755 --- a/generate-and-run-shaders/src/test/java/com/graphicsfuzz/generator/GenerateAndRunShadersTest.java +++ b/generate-and-run-shaders/src/test/java/com/graphicsfuzz/generator/GenerateAndRunShadersTest.java @@ -44,8 +44,7 @@ public void testNoReferences() throws Exception { donors.getAbsolutePath(), outputDir, "dummy_server", - "dummy_worker", - "100" + "dummy_worker" } ); throw new RuntimeException("Exception expected."); @@ -70,8 +69,7 @@ public void testNoDonors() throws Exception { donors.getAbsolutePath(), outputDir, "dummy_server", - "dummy_worker", - "100" + "dummy_worker" } ); throw new RuntimeException("Exception expected."); @@ -95,8 +93,7 @@ public void testMissingJson() throws Exception { donors.getAbsolutePath(), outputDir, "dummy_server", - "dummy_worker", - "100" + "dummy_worker" } ); throw new RuntimeException("Exception expected."); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java index d5b34e050..c59710edd 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java @@ -27,7 +27,6 @@ import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.PruneUniforms; @@ -62,7 +61,6 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.Random; import java.util.function.Supplier; import java.util.stream.Collectors; import net.sourceforge.argparse4j.ArgumentParsers; @@ -91,10 +89,6 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .help("Path of folder of donor shaders.") .type(File.class); - parser.addArgument("glsl-version") - .help("Version of GLSL to target.") - .type(String.class); - parser.addArgument("output") .help("Output shader job file file (.json.") .type(File.class); @@ -211,7 +205,7 @@ public static StringBuilder generateVariant(ShaderJob shaderJob, * @param referenceShaderJobFile The shader job to be transformed. * @param outputShaderJobFile Output file for the variant. * @param generatorArguments Arguments to control generation. - * @param seed Seed for random number generation. + * @param random Random number generator. * @param writeProbabilities Records whether details about probabilities should be written. * @throws IOException if file reading or writing goes wrong. * @throws ParseTimeoutException if parsing takes too long. @@ -254,7 +248,7 @@ private static StringBuilder transformShader(TranslationUnit shaderToTransform, GeneratorArguments args) { final ShaderKind shaderKind = shaderToTransform.getShaderKind(); StringBuilder result = new StringBuilder(); - result.append("======\n" + shaderKind + ":\n"); + result.append("======\n").append(shaderKind).append(":\n"); if (args.getReplaceFloatLiterals()) { FloatLiteralReplacer.replace( @@ -419,20 +413,20 @@ private static String applyTransformationsMultiPass(GeneratorArguments args, } List nextRoundTransformations = new ArrayList<>(); - String result = ""; + final StringBuilder result = new StringBuilder(); // Keep applying transformations until all transformations cease to be effective, or // we get a large enough shader. while (!transformations.isEmpty() && !shaderLargeEnough(reference, generator)) { ITransformation transformation = transformations.remove(generator.nextInt( transformations.size())); - result += transformation.getName() + "\n"; + result.append(transformation.getName()).append("\n"); if (transformation.apply(reference, probabilities, generator.spawnChild(), generationParams)) { // Keep the size down by stripping unused stuff. StripUnusedFunctions.strip(reference); StripUnusedGlobals.strip(reference); - assert canTypeCheckWithoutFailure(reference, reference.getShadingLanguageVersion()); + assert canTypeCheckWithoutFailure(reference); // Only if the transformation applied successfully (i.e., made a change), do we add it // to the list of transformations to be applied next round. @@ -443,11 +437,10 @@ private static String applyTransformationsMultiPass(GeneratorArguments args, nextRoundTransformations = new ArrayList<>(); } } - return result; + return result.toString(); } - private static boolean canTypeCheckWithoutFailure(TranslationUnit reference, - ShadingLanguageVersion shadingLanguageVersion) { + private static boolean canTypeCheckWithoutFailure(TranslationUnit reference) { // Debugging aid: fail early if we end up messing up the translation unit so that type checking // does not work. new Typer(reference); @@ -592,7 +585,7 @@ private static String applyControlFlowComplication(GeneratorArguments args, public static void randomiseUnsetUniforms(TranslationUnit tu, PipelineInfo pipelineInfo, IRandom generator) { - final Supplier floatSupplier = () -> generator.nextFloat(); + final Supplier floatSupplier = generator::nextFloat; final Supplier intSupplier = () -> generator.nextInt(1 << 15); final Supplier uintSupplier = () -> generator.nextInt(1 << 15); final Supplier boolSupplier = () -> generator.nextInt(2); @@ -611,14 +604,14 @@ public static void addInjectionSwitchIfNotPresent(TranslationUnit tu) { private static boolean alreadyDeclaresInjectionSwitch(TranslationUnit tu) { return tu.getGlobalVarDeclInfos() .stream() - .map(item -> item.getName()) + .map(VariableDeclInfo::getName) .collect(Collectors.toList()) .contains(Constants.INJECTION_SWITCH); } public static void setInjectionSwitch(PipelineInfo pipelineInfo) { pipelineInfo.addUniform(Constants.INJECTION_SWITCH, BasicType.VEC2, Optional.empty(), - Arrays.asList(new Float(0.0), new Float(1.0))); + Arrays.asList(0.0f, 1.0f)); } } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateTest.java index 3eb3a079c..581b5fd78 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateTest.java @@ -17,26 +17,28 @@ package com.graphicsfuzz.generator.tool; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.transformation.DonateLiveCodeTransformation; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import net.sourceforge.argparse4j.inf.ArgumentParserException; import org.junit.Rule; import org.junit.Test; @@ -48,7 +50,7 @@ public class GenerateTest { - public static final String A_VERTEX_SHADER = "#version 300 es\n" + + private static final String A_VERTEX_SHADER = "#version 300 es\n" + "layout(location=0) in highp vec4 a_position;" + "float foo(float b) {" + " for (int i = 0; i < 10; i++) {" + @@ -76,8 +78,7 @@ public class GenerateTest { @Test public void testSynthetic() throws Exception { - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - // TODO: Use fileOps more. + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); final String program = "#version 300 es\n" + "precision highp float;\n" @@ -137,21 +138,18 @@ public void testSynthetic() throws Exception { + " }\n" + "}"; - File shaderFile = temporaryFolder.newFile("shader.frag"); - File jsonFile = temporaryFolder.newFile("shader.json"); - fileOps.writeStringToFile(shaderFile, program); - fileOps.writeStringToFile(jsonFile, json); - - File outputDir = temporaryFolder.getRoot(); - - File outputShaderJobFile = new File(outputDir, "output.json"); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); - File donors = temporaryFolder.newFolder("donors"); + final File outputDir = temporaryFolder.getRoot(); + final File outputShaderJobFile = new File(outputDir, "output.json"); + final File donors = temporaryFolder.newFolder("donors"); Generate.mainHelper(new String[]{"--seed", "0", - jsonFile.toString(), + shaderJobFile.toString(), donors.toString(), - "300 es", outputShaderJobFile.toString(), }); @@ -161,17 +159,16 @@ public void testSynthetic() throws Exception { @Test public void testStructDonation() throws Exception { + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); final String usesStruct = "struct A { int x; }; void main() { { A a = A(1); a.x = 2; } }"; - File donorsFolder = temporaryFolder.newFolder(); + final File donorsFolder = temporaryFolder.newFolder(); for (int i = 0; i < 10; i++) { - File donor = new File( + final File donor = new File( Paths.get(donorsFolder.getAbsolutePath(), "donor" + i + ".frag").toString()); - BufferedWriter bw = new BufferedWriter(new FileWriter(donor)); - bw.write(usesStruct); - bw.close(); + fileOps.writeStringToFile(donor, usesStruct); } - String reference = "void main() { ; { ; ; ; }; ; { ; ; ; }; ; ; ; ; ; ; }"; - TranslationUnit tu = ParseHelper.parse(reference); + final String reference = "void main() { ; { ; ; ; }; ; { ; ; ; }; ; ; ; ; ; ; }"; + final TranslationUnit tu = ParseHelper.parse(reference); new DonateLiveCodeTransformation(TransformationProbabilities.likelyDonateLiveCode()::donateLiveCodeAtStmt, donorsFolder, GenerationParams.normal(ShaderKind.FRAGMENT, true), false) .apply(tu, @@ -183,35 +180,25 @@ public void testStructDonation() throws Exception { @Test public void testFragVertAndUniformsPassedThrough() throws Exception { - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - // TODO: Use fileOps more. + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - final String dummyFragment = "precision mediump float;\n" + final TranslationUnit fragmentShader = ParseHelper.parse("precision mediump float;\n" + "void main()\n" + "{\n" + " float iAmAFragmentShader;" - + "}\n"; + + "}\n", ShaderKind.FRAGMENT); - final String dummyVertex = "void main()\n" + final TranslationUnit vertexShader = ParseHelper.parse("void main()\n" + "{\n" + " float iAmAVertexShader;" - + "}\n"; + + "}\n", ShaderKind.VERTEX); - final String json = "{\n" - + "}"; + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), new PipelineInfo(), + vertexShader, fragmentShader); + + final File shaderJobFile = temporaryFolder.newFile("shader.json"); - File fragmentShaderFile = temporaryFolder.newFile("shader.frag"); - File vertexShaderFile = temporaryFolder.newFile("shader.vert"); - File jsonFile = temporaryFolder.newFile("shader.json"); - BufferedWriter bw = new BufferedWriter(new FileWriter(fragmentShaderFile)); - bw.write(dummyFragment); - bw.close(); - bw = new BufferedWriter(new FileWriter(vertexShaderFile)); - bw.write(dummyVertex); - bw.close(); - bw = new BufferedWriter(new FileWriter(jsonFile)); - bw.write(json); - bw.close(); + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); File outputDir = temporaryFolder.getRoot(); @@ -220,9 +207,8 @@ public void testFragVertAndUniformsPassedThrough() throws Exception { File donors = temporaryFolder.newFolder("donors"); Generate.mainHelper(new String[]{"--seed", "0", - jsonFile.toString(), + shaderJobFile.toString(), donors.toString(), - "100", outputShaderJobFile.toString() }); @@ -254,13 +240,13 @@ public void testValidityOfVertexShaderJumpTransformations() throws Exception { private void testValidityOfVertexShaderTransformations(List extraArgs, int repeatCount) throws IOException, InterruptedException, ParseTimeoutException, ArgumentParserException, GlslParserException { - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - // TODO: Use fileOps more. + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), ParseHelper.parse(A_VERTEX_SHADER, ShaderKind.VERTEX)); - File vertexShaderFile = temporaryFolder.newFile("shader.vert"); - File jsonFile = temporaryFolder.newFile("shader.json"); - fileOps.writeStringToFile(vertexShaderFile, A_VERTEX_SHADER); - fileOps.writeStringToFile(jsonFile, EMPTY_JSON); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); final File outputDir = temporaryFolder.getRoot(); final File outputShaderJobFile = new File(outputDir, "output.json"); @@ -271,9 +257,8 @@ private void testValidityOfVertexShaderTransformations(List extraArgs, i args.addAll( Arrays.asList( "--seed", Integer.toString(seed), - jsonFile.toString(), + shaderJobFile.toString(), donors.toString(), - "300 es", outputShaderJobFile.toString() ) ); @@ -303,31 +288,30 @@ public void testInjectionSwitchAddedByDefault() throws Exception { " x = x + i;" + " }" + "}"; - final String uniforms = "{}"; - final File json = temporaryFolder.newFile("shader.json"); - final File frag = temporaryFolder.newFile("shader.frag"); - fileOps.writeStringToFile(frag, program); - fileOps.writeStringToFile(json, uniforms); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), + new PipelineInfo(), ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); final File donors = temporaryFolder.newFolder(); final File output = temporaryFolder.newFile("output.json"); - Generate.mainHelper(new String[] { json.getAbsolutePath(), donors.getAbsolutePath(), "100", + Generate.mainHelper(new String[] { shaderJobFile.getAbsolutePath(), donors.getAbsolutePath(), output.getAbsolutePath(), "--seed", "0" }); - final ShaderJob shaderJob = fileOps.readShaderJobFile(output); + final ShaderJob outputShaderJob = fileOps.readShaderJobFile(output); - assertTrue(shaderJob.getPipelineInfo().hasUniform("injectionSwitch")); + assertTrue(outputShaderJob.getPipelineInfo().hasUniform("injectionSwitch")); assertTrue(fileOps.areShadersValid(output, false)); - assertTrue(shaderJob.getShaders().get(0).getTopLevelDeclarations() + assertTrue(outputShaderJob.getShaders().get(0).getTopLevelDeclarations() .stream() .filter(item -> item instanceof VariablesDeclaration) .map(item -> ((VariablesDeclaration) item).getDeclInfos()) .reduce(new ArrayList<>(), ListConcat::concatenate) .stream() - .map(item -> item.getName()) + .map(VariableDeclInfo::getName) .anyMatch(item -> item.equals("injectionSwitch"))); } @@ -343,31 +327,30 @@ public void testNoInjectionSwitchIfDisabled() throws Exception { " x = x + i;" + " }" + "}"; - final String uniforms = "{}"; - final File json = temporaryFolder.newFile("shader.json"); - final File frag = temporaryFolder.newFile("shader.frag"); - fileOps.writeStringToFile(frag, program); - fileOps.writeStringToFile(json, uniforms); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); final File donors = temporaryFolder.newFolder(); final File output = temporaryFolder.newFile("output.json"); - Generate.mainHelper(new String[] { json.getAbsolutePath(), donors.getAbsolutePath(), "100", + Generate.mainHelper(new String[] { shaderJobFile.getAbsolutePath(), donors.getAbsolutePath(), output.getAbsolutePath(), "--no-injection-switch", "--seed", "0" }); - final ShaderJob shaderJob = fileOps.readShaderJobFile(output); + final ShaderJob outputShaderJob = fileOps.readShaderJobFile(output); - assertFalse(shaderJob.getPipelineInfo().hasUniform("injectionSwitch")); + assertFalse(outputShaderJob.getPipelineInfo().hasUniform("injectionSwitch")); assertTrue(fileOps.areShadersValid(output, false)); - assertFalse(shaderJob.getShaders().get(0).getTopLevelDeclarations() + assertFalse(outputShaderJob.getShaders().get(0).getTopLevelDeclarations() .stream() .filter(item -> item instanceof VariablesDeclaration) .map(item -> ((VariablesDeclaration) item).getDeclInfos()) .reduce(new ArrayList<>(), ListConcat::concatenate) .stream() - .map(item -> item.getName()) + .map(VariableDeclInfo::getName) .anyMatch(item -> item.equals("injectionSwitch"))); } @@ -377,18 +360,16 @@ public void testBeRobustWhenNoDonors() throws Exception { final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); final String program = "#version 100\n" + "void main() { }"; - final String uniforms = "{}"; - - final File json = temporaryFolder.newFile("shader.json"); - final File frag = temporaryFolder.newFile("shader.frag"); - fileOps.writeStringToFile(frag, program); - fileOps.writeStringToFile(json, uniforms); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); final File donors = new File(temporaryFolder.getRoot(), "does_not_exist"); final File output = temporaryFolder.newFile("output.json"); try { - Generate.mainHelper(new String[]{json.getAbsolutePath(), donors.getAbsolutePath(), "100", + Generate.mainHelper(new String[]{shaderJobFile.getAbsolutePath(), donors.getAbsolutePath(), output.getAbsolutePath(), "--seed", "0"}); fail("An exception should have been thrown."); } catch (RuntimeException runtimeException) { @@ -397,6 +378,7 @@ public void testBeRobustWhenNoDonors() throws Exception { } } + @Test public void testStructUniform() throws Exception { // Checks that the generator does not fall over when presented with struct uniforms. final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); @@ -405,16 +387,15 @@ public void testStructUniform() throws Exception { + "uniform S myS;" + "uniform struct T { int y; } myT;" + "void main() {}"; - final String uniforms = "{}"; - final File json = temporaryFolder.newFile("shader.json"); - final File frag = temporaryFolder.newFile("shader.frag"); - fileOps.writeStringToFile(frag, program); - fileOps.writeStringToFile(json, uniforms); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); final File donors = temporaryFolder.newFolder(); final File output = temporaryFolder.newFile("output.json"); - Generate.mainHelper(new String[] { json.getAbsolutePath(), donors.getAbsolutePath(), - "310 es", output.getAbsolutePath(), "--seed", "0" }); + Generate.mainHelper(new String[] { shaderJobFile.getAbsolutePath(), donors.getAbsolutePath(), + output.getAbsolutePath(), "--seed", "0" }); } } From 0b170189b93268937f8091ffb98b8f4e4f750d88 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Fri, 19 Jul 2019 21:33:52 +0700 Subject: [PATCH 073/180] Add reducer opportunities (#622) Related issue: #519 The already implemented reducer that replaces the global variable declarations with the expression is not yet integrated into GraphicsFuzz. This PR adds the implementation for setting up such reducer with the reduction driver and reduction opportunities. --- .../graphicsfuzz/reducer/ReductionDriver.java | 3 ++- .../IReductionOpportunityFinder.java | 19 +++++++++++++++++++ .../ReductionOpportunities.java | 3 ++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java index a1c92dd59..a8275c5ca 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java @@ -93,7 +93,8 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.unusedParamFinder(), IReductionOpportunityFinder.foldConstantFinder(), IReductionOpportunityFinder.redundantUniformMetadataFinder(), - IReductionOpportunityFinder.variableDeclToExprFinder())) { + IReductionOpportunityFinder.variableDeclToExprFinder(), + IReductionOpportunityFinder.globalVariableDeclToExprFinder())) { cleanupPasses.add(new SystematicReductionPass(context, verbose, finder)); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java index 32efb2c68..5f5f715f0 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java @@ -519,4 +519,23 @@ public String getName() { }; } + static IReductionOpportunityFinder + globalVariableDeclToExprFinder() { + return new IReductionOpportunityFinder() { + @Override + public List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return GlobalVariableDeclToExprReductionOpportunities.findOpportunities( + shaderJob, + context); + } + + @Override + public String getName() { + return "globalVariableDeclToExpr"; + } + }; + } + } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java index 948f859eb..fcc82af7b 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java @@ -68,7 +68,8 @@ public static List getReductionOpportunities( IReductionOpportunityFinder.inlineUniformFinder(), IReductionOpportunityFinder.redundantUniformMetadataFinder(), IReductionOpportunityFinder.variableDeclToExprFinder(), - IReductionOpportunityFinder.switchToLoopFinder())) { + IReductionOpportunityFinder.switchToLoopFinder(), + IReductionOpportunityFinder.globalVariableDeclToExprFinder())) { final List currentOpportunities = ros .findOpportunities(shaderJob, context); if (ReductionDriver.DEBUG_REDUCER) { From 94c2c36bb9bee62e114fedf5494210aa724b94ef Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Fri, 19 Jul 2019 09:48:05 -0500 Subject: [PATCH 074/180] Add piglit shader_runner_gles3 worker script (#617) --- .../drivers/graphicsfuzz_piglit_converter.py | 30 +- python/src/main/python/drivers/piglit-worker | 25 ++ .../src/main/python/drivers/piglit-worker.bat | 29 ++ .../src/main/python/drivers/piglit-worker.py | 368 ++++++++++++++++++ 4 files changed, 443 insertions(+), 9 deletions(-) create mode 100755 python/src/main/python/drivers/piglit-worker create mode 100644 python/src/main/python/drivers/piglit-worker.bat create mode 100644 python/src/main/python/drivers/piglit-worker.py diff --git a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py index ac0c029f4..6deae302d 100644 --- a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py +++ b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py @@ -34,6 +34,8 @@ # Draw command for piglit to draw the shader's output. Required in the test header. DRAW_COMMAND = 'draw rect -1 -1 2 2' +# Command to clear the screen to black before drawing, for the test header. +CLEAR_COMMAND = 'clear color 0.0 0.0 0.0 1.0' # Strings used to specify the GL version to use in a piglit test's # [require] header. @@ -62,10 +64,11 @@ UNIFORM_DEC = 'uniform' -def make_shader_test_string(shader_job: str) -> str: +def make_shader_test_string(shader_job: str, nodraw: bool) -> str: """ Makes a piglit shader_test from a shader job and shader. :param shader_job: The path to the shader job file. + :param nodraw: determines if the draw command is added to draw the shader. :return: the shader_test """ shader_job_json_parsed = get_json_properties(shader_job) @@ -74,14 +77,14 @@ def make_shader_test_string(shader_job: str) -> str: shader_file_string = shader.read() shader_lines = shader_file_string.split('\n') - shader_test_string = str('') + shader_test_string = '' # The version header always has to be on the first line of the shader. shader_version_header = shader_lines[0] shader_test_string += make_require_header(shader_version_header) + '\n' shader_test_string += make_vertex_shader_header() + '\n' shader_test_string += make_fragment_shader_header(shader_file_string) + '\n' - shader_test_string += make_test_header(shader_job_json_parsed) + shader_test_string += make_test_header(shader_job_json_parsed, nodraw) return shader_test_string @@ -135,10 +138,11 @@ def make_fragment_shader_header(fragment_shader: str) -> str: return frag_header -def make_test_header(shader_job_json_parsed: dict) -> str: +def make_test_header(shader_job_json_parsed: dict, nodraw: bool) -> str: """ Creates the [test] header. Loads uniforms based on the uniforms found in the JSON file. :param shader_job_json_parsed: the parsed JSON properties. + :param nodraw: determines if the draw command is added to draw the shader. :return: the shader_test test header string. """ test_header = TEST_HEADER + '\n' @@ -148,7 +152,9 @@ def make_test_header(shader_job_json_parsed: dict) -> str: uniform_name=uniform_name, args=' '.join([str(arg) for arg in value['args']]) ) - test_header += DRAW_COMMAND + if not nodraw: + test_header += CLEAR_COMMAND + '\n' + test_header += DRAW_COMMAND return test_header @@ -173,12 +179,12 @@ def is_version_header(line: str) -> bool: return SHADER_VERSION_FLAG in line -def get_json_properties(shader_job: str) -> List: +def get_json_properties(shader_job: str) -> dict: """ - Helper function to parse a shader job JSON file into a list of properties. + Helper function to parse a shader job JSON file into a dict of properties. Throws IOError if the file can't be parsed. :param shader_job: the path to the shader job file. - :return: a list of JSON properties. + :return: a dict of JSON properties. """ with gfuzz_common.open_helper(shader_job, 'r') as job: json_parsed = json.load(job) @@ -218,10 +224,16 @@ def main_helper(args: List[str]) -> None: argparser.add_argument( 'shader_job', help='Path to the GraphicsFuzz shader job JSON file.') + + argparser.add_argument( + '--nodraw', + action='store_true', + help='Do not draw the shader output when running the test. Useful for crash testing.' + ) args = argparser.parse_args(args) - test_string = make_shader_test_string(args.shader_job) + test_string = make_shader_test_string(args.shader_job, args.nodraw) with gfuzz_common.open_helper(get_shader_test_from_job(args.shader_job), 'w') as shader_test: shader_test.write(test_string) diff --git a/python/src/main/python/drivers/piglit-worker b/python/src/main/python/drivers/piglit-worker new file mode 100755 index 000000000..4ec96a5bb --- /dev/null +++ b/python/src/main/python/drivers/piglit-worker @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if test -n "${PYTHON_GF}"; then + "${PYTHON_GF}" ${BASH_SOURCE}.py "$@" +elif type -P python3 >/dev/null; then + python3 ${BASH_SOURCE}.py "$@" +elif type -P py >/dev/null; then + py -3 ${BASH_SOURCE}.py "$@" +else + python ${BASH_SOURCE}.py "$@" +fi diff --git a/python/src/main/python/drivers/piglit-worker.bat b/python/src/main/python/drivers/piglit-worker.bat new file mode 100644 index 000000000..aded83e44 --- /dev/null +++ b/python/src/main/python/drivers/piglit-worker.bat @@ -0,0 +1,29 @@ +@echo off + +@REM +@REM Copyright 2019 The GraphicsFuzz Project Authors +@REM +@REM Licensed under the Apache License, Version 2.0 (the "License"); +@REM you may not use this file except in compliance with the License. +@REM You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. +@REM + +where /q py +IF ERRORLEVEL 0 ( + py -3 "%~dpn0.py" %* +) ELSE ( + where /q python3 + IF ERRORLEVEL 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) +) diff --git a/python/src/main/python/drivers/piglit-worker.py b/python/src/main/python/drivers/piglit-worker.py new file mode 100644 index 000000000..b5fa16f98 --- /dev/null +++ b/python/src/main/python/drivers/piglit-worker.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +import sys +import subprocess +import shutil +from subprocess import CalledProcessError +import time + +import gfuzz_common +import runspv +import graphicsfuzz_piglit_converter + +HERE = os.path.abspath(__file__) + +# Add directory above to Python path for access to dependencies. +# Prepend it so we override any globally installed dependencies. +sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) + +# noinspection PyPep8 +from fuzzer_service import FuzzerService +# noinspection PyPep8 +import fuzzer_service.ttypes as tt +# noinspection PyPep8 +from thrift.transport import THttpClient, TTransport +# noinspection PyPep8 +from thrift.Thrift import TApplicationException +# noinspection PyPep8 +from thrift.protocol import TBinaryProtocol + +FRAG_SUFFIX = '.frag' +JSON_SUFFIX = '.json' +PNG_SUFFIX = '.png' +SHADER_TEST_SUFFIX = '.shader_test' +PNG_FILENAME = 'shader_runner_gles3000.png' +LOGFILE_NAME = 'piglit_log.txt' +STATUS_FILENAME = 'STATUS' + +SHADER_RUNNER_ARG_PNG = '-png' +SHADER_RUNNER_ARG_AUTO = '-auto' +SHADER_RUNNER_ARG_UNIFORMS = '-ignore-missing-uniforms' +SHADER_RUNNER_ARG_SUBTESTS = '-report-subtests' +WORKER_INFO_FILE = 'worker_info.json' + +NO_DRAW_ARG = '--nodraw' + +RETURNCODE_STR = 'Returncode: ' +STDOUT_STR = 'STDOUT: ' +STDERR_STR = 'STDERR: ' + +STATUS_SUCCESS = 'SUCCESS' +STATUS_CRASH = 'CRASH' +STATUS_TIMEOUT = 'TIMEOUT' +STATUS_SANITYERROR = 'SANITY_ERROR' +STATUS_UNEXPECTED = 'UNEXPECTED_ERROR' +STATUS_NONDET = 'NONDET' + +TIMEOUT = 30 + + +def glxinfo_cmd(): + return [gfuzz_common.tool_on_path('glxinfo'), '-B'] + + +def shader_runner_cmd(): + return [gfuzz_common.tool_on_path('shader_runner_gles3')] + + +def thrift_connect(server: str, worker_name: str, worker_info: str) -> (FuzzerService, str): + """ + Helper function to initiate a connection from this worker to a Thrift server. + Handles sending the worker name and info to the server. If there's a fatal problem with a worker + (such as a worker info mismatch between client/server), this function will terminate the + program. + :param server: The server request URL to connect to. + :param worker_name: The name of the worker to connect with. + :param worker_info: The worker info string. + :return: a FuzzerService object and the confirmed worker name, or None for both if the + connection failed without a fatal error. + """ + try: + http_client = THttpClient.THttpClient(server) + transport = TTransport.TBufferedTransport(http_client) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + service = FuzzerService.Client(protocol) + transport.open() + + # Get worker name + runspv.log("Call getWorkerName()") + worker_res = service.getWorkerName(worker_info, worker_name) + assert type(worker_res) is not None + + if worker_res.workerName is None: + # noinspection PyProtectedMember + runspv.log('Worker error: ' + tt.WorkerNameError._VALUES_TO_NAMES[worker_res.error]) + exit(1) + + worker = worker_res.workerName + + runspv.log("Got worker: " + worker) + assert (worker == worker_name) + + return service, worker + + except (TApplicationException, ConnectionRefusedError, ConnectionResetError): + return None, None + + +def dump_glxinfo(filename: str) -> None: + """ + Helper function that dumps the stable results of 'glxinfo -B' to a JSON file. Removes any + file with the same name as filename before writing. Will throw an exception if 'glxinfo' + fails or the JSON file can't be written. + :param filename: the filename to write to. + """ + # There are some useless or unstable lines in glxinfo we need to remove before trying to parse + # into JSON. + glxinfo_lines = filter( + lambda glx_line: 'OpenGL' in glx_line, + runspv.subprocess_helper(glxinfo_cmd()).stdout.split('\n')) + # We form keys out of the OpenGL info descriptors and values out of the hardware dependent + # strings. For example, "OpenGL version string: 4.6.0 NVIDIA 430.14" would become + # { "OpenGL version string": "4.6.0 NVIDIA 430.14" }. + glx_dict = dict() + for line in glxinfo_lines: + prop = line.split(': ') + assert len(prop) is 2 + glx_dict.update({prop[0]: prop[1]}) + with gfuzz_common.open_helper(filename, 'w') as info_file: + info_file.write(json.JSONEncoder().encode(glx_dict)) + + +def do_image_job(image_job: tt.ImageJob, work_dir: str) -> tt.ImageJobResult: + """ + Does an image job. Sets up directories and some files, then delegates to run_image_job to + convert the job to a shader_test and run it. Sets a global logfile to log to for the lifetime + of the function. Gets the status of the shader job from a file that is written to by + run_image_job. + :param image_job: the image job containing the shader/uniforms. + :param work_dir: the directory to work in. + :return: the result of the image job, including the log, PNG and status. + """ + # Output directory is based on the name of job. + output_dir = os.path.join(work_dir, image_job.name) + + # Delete and create output directory. + gfuzz_common.remove(output_dir) + os.makedirs(output_dir, exist_ok=True) + + name = image_job.name + if name.endswith('.frag'): + name = gfuzz_common.remove_end(name, '.frag') + + frag_file = os.path.join(output_dir, name + FRAG_SUFFIX) + json_file = os.path.join(output_dir, name + JSON_SUFFIX) + log_file = os.path.join(output_dir, LOGFILE_NAME) + status_file = os.path.join(output_dir, STATUS_FILENAME) + png_file = os.path.join(output_dir, name + PNG_SUFFIX) + + gfuzz_common.write_to_file(image_job.fragmentSource, frag_file) + gfuzz_common.write_to_file(image_job.uniformsInfo, json_file) + + res = tt.ImageJobResult() + + # Set nice defaults to fields we will not update anyway + res.passSanityCheck = True + res.log = 'Start: ' + name + '\n' + + with gfuzz_common.open_helper(log_file, 'w') as f: + try: + runspv.log_to_file = f + run_image_job(frag_file, json_file, status_file, png_file, + output_dir, image_job.skipRender) + except Exception as ex: + runspv.log(str(ex)) + runspv.log('Removing status file and continuing...') + gfuzz_common.remove(status_file) + finally: + runspv.log_to_file = None + + if os.path.isfile(log_file): + with gfuzz_common.open_helper(log_file, 'r') as f: + res.log += f.read() + + if os.path.isfile(png_file): + with gfuzz_common.open_bin_helper(png_file, 'rb') as f: + res.PNG = f.read() + + if os.path.isfile(status_file): + with gfuzz_common.open_helper(status_file, 'r') as f: + status = f.read().rstrip() + if status == 'SUCCESS': + res.status = tt.JobStatus.SUCCESS + elif status == 'CRASH': + res.status = tt.JobStatus.CRASH + elif status == 'TIMEOUT': + res.status = tt.JobStatus.TIMEOUT + elif status == 'UNEXPECTED_ERROR': + res.status = tt.JobStatus.UNEXPECTED_ERROR + else: + res.log += '\nUnknown status value: ' + status + '\n' + res.status = tt.JobStatus.UNEXPECTED_ERROR + else: + # Not even a status file? + res.log += '\nNo STATUS file\n' + res.status = tt.JobStatus.UNEXPECTED_ERROR + + return res + + +def run_image_job(frag_file: str, json_file: str, status_file: str, + png_file: str, output_dir: str, skip_render: bool): + """ + Runs an image job. Converts the shader job to a piglit shader_test file, then delegates to + run_shader_test to render with shader_runner. Writes the status of the job to file. + :param frag_file: The fragment shader to convert and run. + :param json_file: The JSON uniforms to use with the shader. + :param status_file: The status file to write to. + :param png_file: The PNG file to write to. + :param output_dir: The directory to use for the job. + :param skip_render: whether to skip rendering or not. + """ + + assert os.path.isdir(output_dir) + assert os.path.isfile(frag_file) + assert os.path.isfile(json_file) + + arglist = [json_file] + if skip_render: + arglist.append(NO_DRAW_ARG) + + shader_test_file = graphicsfuzz_piglit_converter.get_shader_test_from_job(json_file) + + try: + runspv.log('Creating shader_test file...') + graphicsfuzz_piglit_converter.main_helper(arglist) + except Exception as ex: + runspv.log('Could not create shader_test from the given job.') + raise ex + + status = STATUS_SUCCESS + + shader_runner_cmd_list = shader_runner_cmd() + [shader_test_file, SHADER_RUNNER_ARG_AUTO, + SHADER_RUNNER_ARG_UNIFORMS, + SHADER_RUNNER_ARG_SUBTESTS] + if not skip_render: + shader_runner_cmd_list.append(SHADER_RUNNER_ARG_PNG) + try: + runspv.subprocess_helper(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) + except subprocess.TimeoutExpired: + status = STATUS_TIMEOUT + except subprocess.CalledProcessError: + status = STATUS_CRASH + + runspv.log('STATUS: ' + status) + + # Piglit throws the output PNG render into whatever the current working directory is + # (and there's no way to specify a location to write to) - we need to move it to wherever our + # output is. + + if os.path.isfile(PNG_FILENAME): + shutil.move(PNG_FILENAME, png_file) + + with gfuzz_common.open_helper(status_file, 'w') as f: + f.write(status) + + +def main(): + description = ( + 'Uses the piglit GLES3 shader runner to render shader jobs.' + ) + + parser = argparse.ArgumentParser(description=description) + + # Required + parser.add_argument( + 'worker_name', + help='The name that will refer to this worker.' + ) + + # Optional + parser.add_argument( + '--server', + default='http://localhost:8080', + help='Server URL to connect to (default: http://localhost:8080 )' + ) + + args = parser.parse_args() + + gfuzz_common.tool_on_path('shader_runner_gles3') + gfuzz_common.tool_on_path('glxinfo') + + runspv.log('Worker: ' + args.worker_name) + server = args.server + '/request' + runspv.log('server: ' + server) + + # Get worker info + worker_info_json_string = '{}' + + runspv.log('Dumping glxinfo to file for worker info string...') + try: + dump_glxinfo(WORKER_INFO_FILE) + with gfuzz_common.open_helper(WORKER_INFO_FILE, 'r') as info_file: + worker_info_json_string = info_file.read() + except Exception as ex: + runspv.log(str(ex)) + runspv.log('Could not get worker info, continuing without it.') + + service = None + worker = None + + while True: + if not service: + runspv.log('Connecting to server...') + service, worker = thrift_connect(server, args.worker_name, worker_info_json_string) + if not service: + runspv.log('Failed to connect, retrying...') + time.sleep(1) + continue + + assert worker + + os.makedirs(worker, exist_ok=True) + + try: + job = service.getJob(worker) + if job.noJob is not None: + runspv.log("No job") + elif job.skipJob is not None: + runspv.log("Skip job") + service.jobDone(worker, job) + else: + assert job.imageJob + if job.imageJob.computeSource: + runspv.log("Got a compute job, but this worker " + "doesn't support compute shaders.") + job.imageJob.result = tt.ImageJobResult() + job.imageJob.result.status = tt.JobStatus.UNEXPECTED_ERROR + else: + runspv.log("#### Image job: " + job.imageJob.name) + job.imageJob.result = do_image_job(job.imageJob, work_dir=worker) + runspv.log("Sending back, results status: {}".format(job.imageJob.result.status)) + service.jobDone(worker, job) + continue + except (TApplicationException, ConnectionError): + runspv.log("Connection to server lost. Re-initialising client.") + service = None + time.sleep(1) + + +if __name__ == '__main__': + main() From db2c4469889e7f7ba0350584d782b4ab967c1b1e Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 24 Jul 2019 04:36:10 -0500 Subject: [PATCH 075/180] Add documentation for piglit worker (#631) Fixes #630. --- docs/piglit-worker.md | 84 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 docs/piglit-worker.md diff --git a/docs/piglit-worker.md b/docs/piglit-worker.md new file mode 100644 index 000000000..8e3748051 --- /dev/null +++ b/docs/piglit-worker.md @@ -0,0 +1,84 @@ +# GraphicsFuzz Piglit/shader_runner worker + +[Piglit](https://piglit.freedesktop.org/) is a test suite for OpenGL implementations that +is developed as part of the [Mesa](https://mesa.freedesktop.org/) open source graphics +driver project. Along with its huge test suite, Piglit provides a testing framework, +`piglit-gl-framework`, and an associated renderer implementation, `shader_runner`. +Our `piglit-worker` is a Python script that sends shader jobs to `shader_runner`, retrieves +the rendered image/crash logs, and sends the results back to the GraphicsFuzz webserver. + +## Using the worker + +To run the worker, you will first need to build the GraphicsFuzz project - see +[GraphicsFuzz: developer documentation](https://github.com/google/graphicsfuzz/blob/master/docs/glsl-fuzz-develop.md) +for more details. Ensure that `graphicsfuzz/target/graphicsfuzz/python/drivers` +is on your PATH environment variable - this is the final extraction directory of GraphicsFuzz's +Python scripts after building. + +Additionally, you will need to clone and build Piglit - see [Piglit's README.md](https://gitlab.freedesktop.org/mesa/piglit/blob/master/README.md) +for build instructions. Once Piglit is built, you **must also** add the build directory, +`/path/to/piglit/bin`, to your PATH environment variable. To check this, you can run + +```sh +$ shader_runner_gles3 +``` + +in your terminal - if you get +```sh +PIGLIT: {"result": "skip" } +``` +in the output, your PATH is set up properly. + +To start the worker, simply use + ```sh +$ piglit-worker (worker name) + ``` + +A server IP/URL can be specified with an optional --server argument: + ```sh +$ piglit-worker --server (IP:PORT) (worker name) + ``` + +## piglit-worker shader_runner arguments + +These are the arguments that piglit-worker runs `shader_runner_gles3` with: + +`-png`: Dumps the rendered image of the shader to the current working directory, + named `shader_runner_gles3000.png`. + +`-ignore-missing-uniforms`: Piglit prefers to fail quickly in the event of a potentially malformed +`.shader_test` file, causing a test to fail if it tries to load uniform data into a uniform variable +that has been optimized away by the compiler (e.g. if the uniform was an unused variable). This +argument makes `shader_runner_gles3` ignore loading a uniform if it can't be found in the shader, instead of failing. + +`-report_subtests`: Forces `shader_runner_gles3` to render out of screen VRAM, preventing issues +where garbage would be rendered whenever _GLF_color is undefined or a fragment is discarded. + +`-auto`: Prevents `shader_runner_gles3` from rendering to a window, to speed up rendering + and mass processing. + +## graphicsfuzz-piglit-converter + +A GraphicsFuzz test is composed two files - a 'shader job' JSON file that encodes uniform variables +and other data that needs to be supplied to the shader, and the shader source code itself. In +contrast, a `shader_runner` test is a `.shader_test` file that contains the required OpenGL/GLSL +versions for the shader, the shader source code, and a special `[test]` header that can be used to +load data, probe pixels for colors, clear the screen, and much more. + +To convert a GraphicsFuzz shader job to a `.shader_test` file for use with `shader_runner`, +`piglit-worker` uses a script, `graphicsfuzz-piglit-converter`, which formats the GraphicsFuzz test +into a working `.shader_test` file. + +`graphicsfuzz-piglit-converter` can be used standalone as well - once you have GraphicsFuzz built +and the Python scripts on your PATH, you can just use +```sh +$ graphicsfuzz-piglit-converter (GraphicsFuzz shader job JSON file) +``` +and the `.shader_test` file will be output in the same directory as the JSON file. + +To run the `.shader_test` file through shader_runner yourself: +```sh +$ shader_runner_gles3 (.shader_test file) -png +``` + The output will be dumped to a PNG file named +`shader_runner_gles3000.png` in the current working directory. From 279a4cb18650b5adab1b79e32281fc54c2257bc8 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Wed, 24 Jul 2019 16:38:27 +0700 Subject: [PATCH 076/180] Add new sample reference shaders (#602) Adds three new sample shaders. --- .../samples/310es/prefix_sum_checkers.frag | 95 +++++++++++++++++++ .../samples/310es/prefix_sum_checkers.json | 16 ++++ .../samples/310es/selection_sort_struct.frag | 75 +++++++++++++++ .../samples/310es/selection_sort_struct.json | 16 ++++ .../samples/310es/trigonometric_strip.frag | 74 +++++++++++++++ .../samples/310es/trigonometric_strip.json | 16 ++++ 6 files changed, 292 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/prefix_sum_checkers.frag create mode 100644 shaders/src/main/glsl/samples/310es/prefix_sum_checkers.json create mode 100644 shaders/src/main/glsl/samples/310es/selection_sort_struct.frag create mode 100644 shaders/src/main/glsl/samples/310es/selection_sort_struct.json create mode 100644 shaders/src/main/glsl/samples/310es/trigonometric_strip.frag create mode 100644 shaders/src/main/glsl/samples/310es/trigonometric_strip.json diff --git a/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.frag b/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.frag new file mode 100644 index 000000000..05858c552 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.frag @@ -0,0 +1,95 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +vec2 pattern(in vec2 x) { + vec2 n = floor(x); + vec3 m = vec3(1.0); + + for (int j = -1; j <= int(injectionSwitch.y); j++) { + for (int i = -1; i <= int(injectionSwitch.y); i++) { + vec2 g = vec2(float(j), float(i)); + vec2 o = mix(n, g, 0.2); + if (injectionSwitch.x < (m.x)) { + int k = 1; + while (k >= 0) { + o = o + o; + k--; + } + m = vec3(injectionSwitch.x, cos(o)); + } + } + } + return vec2(m.x, m.y - m.z); +} + +void main() { + vec2 uv = gl_FragCoord.xy / resolution.y; + float A[50]; + for (int i = 0; i < 200; i++) { + if (i >= int(resolution.x)) { + break; + } + if ((4 * (i/4)) == i) { + A[i/4] = float(i); + } + } + for (int i = 0; i < 50; i++) { + if (i < int(gl_FragCoord.x)) { + break; + } + if (i > 0) { + A[i] += A[i - 1]; + } + } + + vec2 c = pattern((15.0 + tan(0.2)) * uv); + vec3 col; + if (int(gl_FragCoord.y) < 20) { + col = .5 + cos(c.y + vec3(resolution.x, A[4]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 40) { + col = .5 + cos(c.y + vec3(resolution.x, A[9]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 60) { + col = .5 + cos(c.y + vec3(resolution.x, A[14]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 80) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 100) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 120) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 140) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 160) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 180) { + col = .5 + cos(c.y + vec3(resolution.x, A[44]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 200) { + col = .5 + cos(c.y + vec3(resolution.x, A[49]/resolution.x + 50.0, 22.0)); + } else { + discard; + } + + _GLF_color = vec4(col, 1.0); +} diff --git a/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.json b/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag b/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag new file mode 100644 index 000000000..44f9d0adf --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag @@ -0,0 +1,75 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct Obj { + float odd_numbers[10]; + float even_numbers[10]; +}; + +void main() { + Obj obj; + + // Initialize first 10 odd numbers to the array. + int odd_index = 0; + float odd_number = 1.0; + while (odd_index <= 9) { + obj.odd_numbers[odd_index] = odd_number; + odd_number += 2.0; + odd_index++; + } + // Similarly, initialize even numbers and iterate backward. + int even_index = 9; + float even_number = 0.0; + while (even_index >= 0) { + obj.even_numbers[even_index] = even_number; + even_number += 2.; + even_index--; + } + + // Perform the selection sort. + for (int i = 0; i < 9; i++) { + int index = i; + for (int j = i + 1; j < 10; j++) { + if (obj.even_numbers[j] < obj.even_numbers[index]) { + index = j; + } + } + float smaller_number = obj.even_numbers[index]; + obj.even_numbers[index] = obj.even_numbers[i]; + obj.even_numbers[i] = smaller_number; + } + + vec2 uv = gl_FragCoord.xy/resolution.xy; + vec3 col = tan(pow(uv.xxx, uv.yyy) + + vec3( + obj.odd_numbers[int(floor(gl_FragCoord.x/1000.0))], + obj.even_numbers[int(floor(gl_FragCoord.y/1000.0))], + sinh(uv.x) + )); + + _GLF_color.rgb = col; + _GLF_color.a = 1.0; +} diff --git a/shaders/src/main/glsl/samples/310es/selection_sort_struct.json b/shaders/src/main/glsl/samples/310es/selection_sort_struct.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/selection_sort_struct.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/trigonometric_strip.frag b/shaders/src/main/glsl/samples/310es/trigonometric_strip.frag new file mode 100644 index 000000000..4c05e0d3c --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/trigonometric_strip.frag @@ -0,0 +1,74 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +float compute_derivative_x(float v){ + return dFdx(v) * injectionSwitch.y; +} + +float compute_derivative_y(float v){ + return dFdy(v) * injectionSwitch.y; +} + +float compute_stripe(float v) { + return smoothstep(-.9, 1., (v)/ ((injectionSwitch.y > injectionSwitch.x) ? compute_derivative_x(v): compute_derivative_y(v))); +} + +void main() { + vec2 uv = gl_FragCoord.xy / resolution.x; + vec3 col = vec3(0, 0, 0); + + bool c1 = uv.y < 0.25; + if (c1) { + float stripe = compute_stripe(cos((uv.x + uv.y ) * 20.0 )); + col = mix(vec3(uv.x, 0, 0.75), vec3(.8, .7, uv.x), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c2 = uv.y < 0.5; + if (!c1 && c2) { + float stripe = compute_stripe(tan((uv.x + uv.y) * 20.0 )); + col = mix(vec3(0.5, uv.x, 0.1), vec3(.4, 0, .5), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c3 = uv.y < 0.75; + if (!c1 && !c2 && c3) { + float stripe = compute_stripe(cos((uv.x + uv.y) * 20.0 )); + col = mix(vec3(.7, sinh(uv.x), uv.x ), vec3(.3, .5, uv.x), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c4 = uv.y >= 0.75; + if (!c1 && !c2 && !c3 && c4) { + float stripe = compute_stripe(tan((uv.x + uv.y) * 20.0 )); + col = mix(vec3(uv.x, .8, 0), vec3(1, uv.y, 0), stripe); + _GLF_color = vec4(col, 1.0); + return; + } +} diff --git a/shaders/src/main/glsl/samples/310es/trigonometric_strip.json b/shaders/src/main/glsl/samples/310es/trigonometric_strip.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/trigonometric_strip.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From 5814df43e3b479ff7830b74b18b55bb1e3995e24 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Wed, 24 Jul 2019 13:09:01 +0100 Subject: [PATCH 077/180] Rework ScopeTreeBuilder (#627) The ScopeTreeBuilder class is one of the most critical classes in the GraphicsFuzz framework, but had a poor name and was not well-documented (despite having some subtle behaviours). This change renames the class to ScopeTrackingVisitor, which is a more accurate name, and adds documentary comments to its fields and methods. The fields have all made private, with accessor and mutator methods added where needed. In the process, the ReducerBugPoint and ReducerBugPointBasic classes have been removed; they used a subclass of ScopeTreeBuilder called 'InformativeScopeTreeBuilder', and rather than maintaining this class it seemed better to remove those tools (which have not been used for a long time). --- .../common/typing/ScopeTrackingVisitor.java | 306 ++++++++++++++++++ .../common/typing/ScopeTreeBuilder.java | 206 ------------ .../com/graphicsfuzz/common/typing/Typer.java | 8 +- ...est.java => ScopeTrackingVisitorTest.java} | 6 +- .../common/util/ApplySubstitution.java | 6 +- .../graphicsfuzz/common/util/Obfuscator.java | 6 +- .../common/util/StripUnusedGlobals.java | 11 +- .../mutateapi/MutationFinderBase.java | 6 +- .../Expr2ArrayAccessMutationFinder.java | 2 +- .../InterchangeExprMutationFinder.java | 2 +- ...aceBlockStmtsWithSwitchMutationFinder.java | 5 +- ...SwapVariableIdentifiersMutationFinder.java | 13 +- .../IdentityMutationFinder.java | 2 +- .../OutlineStatementMutationFinder.java | 14 +- .../StructificationMutation.java | 6 +- .../VectorizeMutation.java | 15 +- .../VectorizeMutationFinder.java | 7 +- .../generator/tool/Fragment2Compute.java | 7 +- .../DonateCodeTransformation.java | 8 +- .../donation/DonationContext.java | 6 +- .../injection/InjectionPoints.java | 21 +- .../util/AvailableStructsCollector.java | 8 +- .../generator/util/ConstCleaner.java | 6 +- .../util/FreeVariablesCollector.java | 19 +- .../util/RestrictFragmentShaderColors.java | 26 +- .../util/TransformationProbabilities.java | 34 +- .../generator/fuzzer/FuzzerTest.java | 8 +- .../DonateLiveCodeTransformationTest.java | 12 +- .../reducer/CheckAstFeatureVisitor.java | 10 +- .../DestructifyReductionOpportunities.java | 4 +- .../DestructifyReductionOpportunity.java | 6 +- ...lineInitializerReductionOpportunities.java | 4 +- ...tructifiedFieldReductionOpportunities.java | 9 +- .../InlineUniformReductionOpportunities.java | 2 +- ...tputVariableWriteReductionOpportunity.java | 6 +- .../LoopMergeReductionOpportunities.java | 4 +- .../ReductionOpportunitiesBase.java | 4 +- ...moveStructFieldReductionOpportunities.java | 8 +- ...UnusedParameterReductionOpportunities.java | 3 +- .../reductionopportunities/ShadowChecker.java | 8 +- .../UnwrapReductionOpportunities.java | 3 +- .../VariableDeclReductionOpportunities.java | 16 +- .../VectorizationReductionOpportunities.java | 2 +- .../VectorizationReductionOpportunity.java | 17 +- .../reducer/tool/ReducerBugPoint.java | 189 ----------- .../reducer/tool/ReducerBugPointBasic.java | 286 ---------------- .../reducer/tool/ReportAstDifferences.java | 124 ------- 47 files changed, 485 insertions(+), 996 deletions(-) create mode 100644 ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitor.java delete mode 100644 ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTreeBuilder.java rename ast/src/test/java/com/graphicsfuzz/common/typing/{ScopeTreeBuilderTest.java => ScopeTrackingVisitorTest.java} (94%) delete mode 100755 reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java delete mode 100755 reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java delete mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReportAstDifferences.java diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitor.java new file mode 100644 index 000000000..b5022d677 --- /dev/null +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitor.java @@ -0,0 +1,306 @@ +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.common.typing; + +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.ParameterDecl; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.WhileStmt; +import com.graphicsfuzz.common.ast.type.ArrayType; +import com.graphicsfuzz.common.ast.type.StructDefinitionType; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; + +/** + * This class extends StandardVisitor to track details of what is in scope at each point of + * visitation. + */ +public abstract class ScopeTrackingVisitor extends StandardVisitor { + + // Tracks the scope at the current point of visitation. + private Scope currentScope; + + // Tracks the function, if any, enclosing the current point of visitation. If this field is null, + // this indicates that visitation is at global scope. + private FunctionDefinition enclosingFunction; + + // A stack of the blocks that enclose the current point of visitation. + private final Deque enclosingBlocks; + + // All of the function prototypes that have been encountered during visitation so far. + private final List encounteredFunctionPrototypes; + + // When we visit a function prototype, we want to add its parameters to the current scope if and + // only if it is the prototype component of a function definition. If it is a stand-alone + // function prototype then we do not want to add its parameters to the current scope. This field + // is used to determine whether or not we should record encountered parameters to the current + // scope. + private boolean addEncounteredParametersToScope; + + protected ScopeTrackingVisitor() { + this.currentScope = new Scope(null); + this.enclosingFunction = null; + this.enclosingBlocks = new LinkedList<>(); + this.encounteredFunctionPrototypes = new ArrayList<>(); + this.addEncounteredParametersToScope = false; + } + + // Over-ridden 'visit' methods; keep in alphabetical order: + + @Override + public void visitBlockStmt(BlockStmt stmt) { + enterBlockStmt(stmt); + super.visitBlockStmt(stmt); + leaveBlockStmt(stmt); + } + + @Override + public void visitForStmt(ForStmt forStmt) { + // A 'for' statement is special in that it, rather than its open-curly, starts a new scope. + // For example, this is illegal: + // + // for (int i = 0; i < 10; i++) { + // int i; + // } + // + // because it declares 'i' twice in the same scope. + // + // We thus have to push a new scope before traversing a 'for' statement and pop it afterwards. + pushScope(); + super.visitForStmt(forStmt); + popScope(); + } + + @Override + public void visitFunctionDefinition(FunctionDefinition functionDefinition) { + // Record the fact that we are in a function, and push a scope for that function. + assert enclosingFunction == null; + enclosingFunction = functionDefinition; + pushScope(); + + // As this is a function definition, we *do* want to add the function's parameters to the + // current scope when we visit the function prototpye. + addEncounteredParametersToScope = true; + visitFunctionPrototype(functionDefinition.getPrototype()); + addEncounteredParametersToScope = false; + + visit(functionDefinition.getBody()); + + // Get rid of the function's scope, and record that we are no longer in a function. + popScope(); + enclosingFunction = null; + } + + @Override + public void visitFunctionPrototype(FunctionPrototype functionPrototype) { + encounteredFunctionPrototypes.add(functionPrototype); + for (ParameterDecl p : functionPrototype.getParameters()) { + visitParameterDecl(p); + + // Only add the parameter to the current scope if we have decided to do so, and then only + // if it actually has a name. + if (!addEncounteredParametersToScope || p.getName() == null) { + continue; + } + currentScope.add(p.getName(), + p.getArrayInfo() == null ? p.getType() : new ArrayType(p.getType(), p.getArrayInfo()), + Optional.of(p)); + } + } + + @Override + public void visitStructDefinitionType(StructDefinitionType structDefinitionType) { + super.visitStructDefinitionType(structDefinitionType); + if (structDefinitionType.hasStructNameType()) { + // We add named structs to the current scope. + currentScope.addStructDefinition(structDefinitionType); + } + } + + @Override + public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { + visit(variablesDeclaration.getBaseType()); + for (VariableDeclInfo declInfo : variablesDeclaration.getDeclInfos()) { + // Visit the declInfo both before and after adding it to the current scope, to give + // subclasses the flexibility to examine both cases. + visitVariableDeclInfo(declInfo); + currentScope.add(declInfo.getName(), + declInfo.getArrayInfo() == null + ? variablesDeclaration.getBaseType() + : new ArrayType(variablesDeclaration.getBaseType(), declInfo.getArrayInfo()), + Optional.empty(), + declInfo, variablesDeclaration); + visitVariableDeclInfoAfterAddedToScope(declInfo); + } + } + + @Override + public void visitWhileStmt(WhileStmt whileStmt) { + // A 'while' statement is special in that it, rather than its open-curly, starts a new scope. + // For example, this is illegal: + // + // while (bool b = true) { + // bool b; + // } + // + // because it declares 'b' twice in the same scope. + // + // We thus have to push a new scope before traversing a 'while' statement and pop it + // afterwards. + pushScope(); + super.visitWhileStmt(whileStmt); + popScope(); + } + + // Additional methods for use by subclasses: + + /** + * Tracks entry to a block, updating the current scope and the enclosing blocks. + * Subclasses can override this method to perform additional actions on block entry, but *must* + * invoke this superclass method, otherwise scopes will not be properly tracked. + * @param blockStmt A block statement that is about to be traversed + */ + protected void enterBlockStmt(BlockStmt blockStmt) { + enclosingBlocks.addFirst(blockStmt); + if (blockStmt.introducesNewScope()) { + pushScope(); + } + } + + /** + * Tracks exit from a block, updating the current scope and the enclosing blocks. + * Subclasses can override this method to perform additional actions on block exit, but *must* + * invoke this superclass method, otherwise scopes will not be properly tracked. + * @param blockStmt A block statement that has just been traversed + */ + protected void leaveBlockStmt(BlockStmt blockStmt) { + if (blockStmt.introducesNewScope()) { + popScope(); + } + enclosingBlocks.removeFirst(); + } + + /** + * Returns the closest block enclosing the current point of visitation. + * + * @return The closest block + */ + protected BlockStmt currentBlock() { + return enclosingBlocks.peekFirst(); + } + + /** + * Returns true if and only if visitation is in some block. + * + * @return Whether visitation is in a block + */ + protected boolean inSomeBlock() { + return !enclosingBlocks.isEmpty(); + } + + /** + * Because WebGL places limits on for loops it can be convenient to only visit the body of a + * for loop, skipping the header. Subclasses can call this method to invoke this behaviour + * during visitation, instead of calling super.visitForStmt(...). + * + * @param forStmt For statement whose body should be visited + */ + protected void visitForStmtBodyOnly(ForStmt forStmt) { + pushScope(); + visit(forStmt.getBody()); + popScope(); + } + + /** + * This is a hook for subclasses that need to perform an action after a declaration has been added + * to the current scope. + * + * @param declInfo The declaration info that was just added to the current scope + */ + protected void visitVariableDeclInfoAfterAddedToScope(VariableDeclInfo declInfo) { + // Deliberately empty - to be optionally overridden by subclasses. + } + + /** + * Replace the current scope with its parent. + */ + protected void popScope() { + currentScope = currentScope.getParent(); + } + + /** + * Replace the current scope with a new scope whose parent is the old current scope. + */ + protected void pushScope() { + Scope newScope = new Scope(currentScope); + currentScope = newScope; + } + + /** + * Yield the function prototypes that have been encountered during visitation so far. + * @return The function prototypes that have been encountered during visitation so far + */ + protected List getEncounteredFunctionPrototypes() { + return Collections.unmodifiableList(encounteredFunctionPrototypes); + } + + /** + * Determines whether visitation is at global scope, i.e. not in any function. + * @return true if and only if visitation is at global scope + */ + protected boolean atGlobalScope() { + return !currentScope.hasParent(); + } + + /** + * Yields the current scope. + * @return the current scope + */ + protected Scope getCurrentScope() { + return currentScope; + } + + /** + * Replaces the current scope with the given scope, returning the old current scope. This can + * be useful if one needs to temporarily change the visitor's view of what is in scope, + * @param newScope A scope to replace the current scope + * @return the previous current scope + */ + protected Scope swapCurrentScope(Scope newScope) { + final Scope result = currentScope; + currentScope = newScope; + return result; + } + + /** + * Yields the function enclosing the current point of visitation. + * @return the function enclosing the current point of visitation + */ + protected FunctionDefinition getEnclosingFunction() { + return enclosingFunction; + } + +} diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTreeBuilder.java b/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTreeBuilder.java deleted file mode 100644 index df60c39af..000000000 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTreeBuilder.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.common.typing; - -import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ParameterDecl; -import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; -import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; -import com.graphicsfuzz.common.ast.stmt.BlockStmt; -import com.graphicsfuzz.common.ast.stmt.ForStmt; -import com.graphicsfuzz.common.ast.stmt.WhileStmt; -import com.graphicsfuzz.common.ast.type.ArrayType; -import com.graphicsfuzz.common.ast.type.StructDefinitionType; -import com.graphicsfuzz.common.ast.type.StructNameType; -import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -public abstract class ScopeTreeBuilder extends StandardVisitor { - - protected Scope currentScope; - private Deque enclosingBlocks; - private boolean addEncounteredParametersToScope; - protected FunctionDefinition enclosingFunction; - private List encounteredFunctionPrototypes; - - protected ScopeTreeBuilder() { - this.currentScope = new Scope(null); - this.enclosingBlocks = new LinkedList<>(); - this.addEncounteredParametersToScope = false; - this.enclosingFunction = null; - this.encounteredFunctionPrototypes = new ArrayList<>(); - } - - @Override - public void visitStructDefinitionType(StructDefinitionType structDefinitionType) { - super.visitStructDefinitionType(structDefinitionType); - if (structDefinitionType.hasStructNameType()) { - currentScope.addStructDefinition(structDefinitionType); - } - } - - @Override - public void visitBlockStmt(BlockStmt stmt) { - enterBlockStmt(stmt); - super.visitBlockStmt(stmt); - leaveBlockStmt(stmt); - } - - protected void enterBlockStmt(BlockStmt stmt) { - enclosingBlocks.addFirst(stmt); - if (stmt.introducesNewScope()) { - pushScope(); - } - } - - protected void leaveBlockStmt(BlockStmt stmt) { - if (stmt.introducesNewScope()) { - popScope(); - } - enclosingBlocks.removeFirst(); - } - - /** - * Returns the closest block enclosing the current point of visitation. - * - * @return The closest block - */ - protected BlockStmt currentBlock() { - return enclosingBlocks.peekFirst(); - } - - /** - * Returns true if and only if visitatin is in some block. - * - * @return Whether visitation is in a block - */ - protected boolean inBlock() { - return !enclosingBlocks.isEmpty(); - } - - @Override - public void visitForStmt(ForStmt forStmt) { - pushScope(); - super.visitForStmt(forStmt); - popScope(); - } - - /** - * Because WebGL places limits on for loops it can be convenient to only visit the body of a - * for loop, skipping the header. Subclasses can call this method to invoke this behaviour - * during visitation, instead of calling super.visitForStmt(...). - * - * @param forStmt For statement whose body should be visited - */ - protected void visitForStmtBodyOnly(ForStmt forStmt) { - pushScope(); - visit(forStmt.getBody()); - popScope(); - } - - - @Override - public void visitWhileStmt(WhileStmt whileStmt) { - pushScope(); - super.visitWhileStmt(whileStmt); - popScope(); - } - - @Override - public void visitFunctionDefinition(FunctionDefinition functionDefinition) { - assert enclosingFunction == null; - enclosingFunction = functionDefinition; - pushScope(); - - addEncounteredParametersToScope = true; - visitFunctionPrototype(functionDefinition.getPrototype()); - addEncounteredParametersToScope = false; - visit(functionDefinition.getBody()); - popScope(); - enclosingFunction = null; - } - - @Override - public void visitFunctionPrototype(FunctionPrototype functionPrototype) { - encounteredFunctionPrototypes.add(functionPrototype); - for (ParameterDecl p : functionPrototype.getParameters()) { - visitParameterDecl(p); - if (addEncounteredParametersToScope) { - if (p.getName() == null) { - continue; - } - currentScope.add(p.getName(), - p.getArrayInfo() == null ? p.getType() : new ArrayType(p.getType(), p.getArrayInfo()), - Optional.of(p)); - } - } - } - - @Override - public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { - visit(variablesDeclaration.getBaseType()); - List children = new ArrayList<>(); - children.addAll(variablesDeclaration.getDeclInfos()); - for (VariableDeclInfo declInfo : children) { - visitVariableDeclInfo(declInfo); - currentScope.add(declInfo.getName(), - declInfo.getArrayInfo() == null - ? variablesDeclaration.getBaseType() - : new ArrayType(variablesDeclaration.getBaseType(), declInfo.getArrayInfo()), - Optional.empty(), - declInfo, variablesDeclaration); - visitVariableDeclInfoAfterAddedToScope(declInfo); - } - } - - /** - * This is a hook for subclasses that need to perform an action after a declaration has been added - * to the current scope. - * - * @param declInfo The declaration info that was just added to the current scope - */ - protected void visitVariableDeclInfoAfterAddedToScope(VariableDeclInfo declInfo) { - // Deliberately empty - to be optionally overridden by subclasses. - } - - protected void popScope() { - currentScope = currentScope.getParent(); - } - - protected void pushScope() { - Scope newScope = new Scope(currentScope); - currentScope = newScope; - } - - protected List getEncounteredFunctionPrototypes() { - return Collections.unmodifiableList(encounteredFunctionPrototypes); - } - - protected boolean atGlobalScope() { - return !currentScope.hasParent(); - } - - -} diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java index 3b754ff99..c1c6a8ac3 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java @@ -16,7 +16,6 @@ package com.graphicsfuzz.common.typing; -import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; @@ -41,10 +40,7 @@ import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.TypeQualifier; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.OpenGlConstants; -import com.graphicsfuzz.common.util.ShaderKind; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -53,7 +49,7 @@ import java.util.Optional; import java.util.Set; -public class Typer extends ScopeTreeBuilder { +public class Typer extends ScopeTrackingVisitor { private final TranslationUnit tu; @@ -166,7 +162,7 @@ public boolean prototypeMatches(FunctionPrototype prototype, FunctionCallExpr fu @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - Type type = currentScope.lookupType(variableIdentifierExpr.getName()); + Type type = getCurrentScope().lookupType(variableIdentifierExpr.getName()); if (type != null) { types.put(variableIdentifierExpr, type); return; diff --git a/ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTreeBuilderTest.java b/ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitorTest.java similarity index 94% rename from ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTreeBuilderTest.java rename to ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitorTest.java index 8e2d57032..6fd5a478f 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTreeBuilderTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitorTest.java @@ -20,7 +20,7 @@ import com.graphicsfuzz.common.util.ParseHelper; import org.junit.Test; -public class ScopeTreeBuilderTest { +public class ScopeTrackingVisitorTest { @Test public void testSwitch() throws Exception { @@ -35,9 +35,9 @@ public void testSwitch() throws Exception { + " }" + "}"; final TranslationUnit tu = ParseHelper.parse(program); - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { }.visit(tu); } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ApplySubstitution.java b/common/src/main/java/com/graphicsfuzz/common/util/ApplySubstitution.java index 2066d923f..b8e690af7 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ApplySubstitution.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ApplySubstitution.java @@ -18,10 +18,10 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.Map; -public class ApplySubstitution extends ScopeTreeBuilder { +public class ApplySubstitution extends ScopeTrackingVisitor { private Map substitution; @@ -33,7 +33,7 @@ public ApplySubstitution(Map substitution, IAstNode node) { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { String name = variableIdentifierExpr.getName(); - if (currentScope.lookupType(name) == null) { + if (getCurrentScope().lookupType(name) == null) { if (substitution.containsKey(name)) { variableIdentifierExpr.setName(substitution.get(name)); } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java b/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java index 2ac0bb4eb..2b7c0c16e 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java @@ -33,7 +33,7 @@ import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.util.ExecHelper; import com.graphicsfuzz.util.ExecHelper.RedirectType; @@ -84,7 +84,7 @@ private ShaderJob obfuscate( clonedTus); } - private class TranslationUnitObfuscator extends ScopeTreeBuilder { + private class TranslationUnitObfuscator extends ScopeTrackingVisitor { private final Map functionRenaming; private final Map namedStructRenaming; private final List structDefinitionTypes; @@ -241,7 +241,7 @@ private String applyFunctionNameMapping(String name) { } private String applyVariableNameMapping(String name) { - ScopeEntry scopeEntry = currentScope.lookupScopeEntry(name); + ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry(name); if (scopeEntry == null) { return name; } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/StripUnusedGlobals.java b/common/src/main/java/com/graphicsfuzz/common/util/StripUnusedGlobals.java index bbfce25f3..e1f0618a6 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/StripUnusedGlobals.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/StripUnusedGlobals.java @@ -26,14 +26,14 @@ import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.util.Constants; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -public class StripUnusedGlobals extends ScopeTreeBuilder { +public class StripUnusedGlobals extends ScopeTrackingVisitor { public static void strip(TranslationUnit tu) { new StripUnusedGlobals(tu); @@ -65,7 +65,7 @@ public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) @Override public void visitStructNameType(StructNameType structNameType) { super.visitStructNameType(structNameType); - unusedStructs.remove(currentScope.lookupStructName(structNameType.getName())); + unusedStructs.remove(getCurrentScope().lookupStructName(structNameType.getName())); } @Override @@ -74,7 +74,7 @@ public void visitTypeConstructorExpr(TypeConstructorExpr typeConstructorExpr) { // If a struct name is used in a type constructor, the struct counts as being used. // The type constructor here might not be a struct type, e.g. it could be "float" or "vec2". // That's OK: lookupStructName will just return null so nothing will be removed. - unusedStructs.remove(currentScope.lookupStructName(typeConstructorExpr.getTypename())); + unusedStructs.remove(getCurrentScope().lookupStructName(typeConstructorExpr.getTypename())); } @Override @@ -89,7 +89,8 @@ public void visitVariableDeclInfo(VariableDeclInfo variableDeclInfo) { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - final ScopeEntry scopeEntry = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry(variableIdentifierExpr + .getName()); if (scopeEntry != null && scopeEntry.hasVariableDeclInfo()) { // If this is a global, mark it as used. unusedGlobals.remove(scopeEntry.getVariableDeclInfo()); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/mutateapi/MutationFinderBase.java b/generator/src/main/java/com/graphicsfuzz/generator/mutateapi/MutationFinderBase.java index 5d8abfad9..a577bc996 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/mutateapi/MutationFinderBase.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/mutateapi/MutationFinderBase.java @@ -34,15 +34,13 @@ import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; -import com.graphicsfuzz.generator.mutateapi.Mutation; -import com.graphicsfuzz.generator.mutateapi.MutationFinder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.ArrayList; import java.util.Collections; import java.util.List; public abstract class MutationFinderBase - extends ScopeTreeBuilder implements MutationFinder { + extends ScopeTrackingVisitor implements MutationFinder { private final TranslationUnit tu; private final List mutations; diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/Expr2ArrayAccessMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/Expr2ArrayAccessMutationFinder.java index 66b610f64..744274264 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/Expr2ArrayAccessMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/Expr2ArrayAccessMutationFinder.java @@ -52,7 +52,7 @@ protected void visitVariableDeclInfoAfterAddedToScope(VariableDeclInfo declInfo) if (!atGlobalScope()) { return; } - ScopeEntry se = currentScope.lookupScopeEntry(declInfo.getName()); + ScopeEntry se = getCurrentScope().lookupScopeEntry(declInfo.getName()); if (se.getType() instanceof ArrayType) { globalArrays.put(declInfo.getName(), ((ArrayType) se.getType()).getBaseType() .getWithoutQualifiers()); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java index 177ecbb3c..3ce173047 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java @@ -64,7 +64,7 @@ private Expr applyInterchange(Expr expr, List argumentTypes) { final Type resultType = typer.lookupType(expr).getWithoutQualifiers(); List templates = Fuzzer.availableTemplatesFromScope( getTranslationUnit().getShadingLanguageVersion(), getTranslationUnit().getShaderKind(), - currentScope) + getCurrentScope()) // Ignore templates that require l-values, so that invalidity is not too likely .filter(InterchangeExprMutationFinder::doesNotRequireLvalue) // Ignore the possibility of replacing a one-argument expression with parentheses, as it diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinder.java index 04543c9f3..6cf21f153 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinder.java @@ -128,8 +128,9 @@ public void visitBlockStmt(BlockStmt block) { private Expr getSwitchCondition() { final List candidateVariables = new ArrayList<>(); - currentScope.namesOfAllVariablesInScope().stream() - .filter(item -> currentScope.lookupType(item).getWithoutQualifiers().equals(BasicType.INT)) + getCurrentScope().namesOfAllVariablesInScope().stream() + .filter(item -> getCurrentScope().lookupType(item).getWithoutQualifiers() + .equals(BasicType.INT)) .forEach(candidateVariables::add); if (candidateVariables.isEmpty()) { return new LiteralFuzzer(generator).fuzz(BasicType.INT).get(); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/SwapVariableIdentifiersMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/SwapVariableIdentifiersMutationFinder.java index 8ab4d39c2..d86574934 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/SwapVariableIdentifiersMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/SwapVariableIdentifiersMutationFinder.java @@ -44,11 +44,12 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie return; } - if (currentScope.lookupType(variableIdentifierExpr.getName()) == null) { + if (getCurrentScope().lookupType(variableIdentifierExpr.getName()) == null) { return; } - assert currentScope.lookupType(variableIdentifierExpr.getName()).getWithoutQualifiers() != null; + assert getCurrentScope().lookupType(variableIdentifierExpr.getName()).getWithoutQualifiers() + != null; final List candidateVariables = getCandidateVariables(variableIdentifierExpr.getName()); @@ -65,11 +66,11 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie } private List getCandidateVariables(String varIdentifier) { - return currentScope.namesOfAllVariablesInScope().stream() + return getCurrentScope().namesOfAllVariablesInScope().stream() .filter(item -> !item.equals(varIdentifier) - && currentScope.lookupType(item) != null - && currentScope.lookupType(varIdentifier).getWithoutQualifiers().equals( - currentScope.lookupType(item).getWithoutQualifiers())) + && getCurrentScope().lookupType(item) != null + && getCurrentScope().lookupType(varIdentifier).getWithoutQualifiers().equals( + getCurrentScope().lookupType(item).getWithoutQualifiers())) .collect(Collectors.toList()); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java index 8f0b44a5c..2a9f84724 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java @@ -76,7 +76,7 @@ protected void visitExpr(Expr expr) { } final BasicType basicType = (BasicType) type; - final Scope clonedScope = currentScope.shallowClone(); + final Scope clonedScope = getCurrentScope().shallowClone(); if (getTranslationUnit().getShadingLanguageVersion().restrictedForLoops()) { for (Set iterators : forLoopIterators) { iterators.forEach(clonedScope::remove); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/OutlineStatementMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/OutlineStatementMutationFinder.java index d62bf5306..c2634dd32 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/OutlineStatementMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/OutlineStatementMutationFinder.java @@ -27,19 +27,11 @@ import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.typing.Scope; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; -import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; -import com.graphicsfuzz.generator.mutateapi.MutationFinder; import com.graphicsfuzz.generator.mutateapi.MutationFinderBase; -import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.util.Constants; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Set; -import java.util.stream.Collectors; public class OutlineStatementMutationFinder extends MutationFinderBase { @@ -74,11 +66,11 @@ public void visitExprStmt(ExprStmt exprStmt) { if (!OutlineStatementMutation.assignsDirectlyToVariable(be)) { return; } - if (referencesArray(be.getRhs(), currentScope)) { + if (referencesArray(be.getRhs(), getCurrentScope())) { return; } - addMutation(new OutlineStatementMutation(exprStmt, currentScope, getTranslationUnit(), - enclosingFunction, idGenerator)); + addMutation(new OutlineStatementMutation(exprStmt, getCurrentScope(), getTranslationUnit(), + getEnclosingFunction(), idGenerator)); } private boolean referencesArray(Expr expr, Scope enclosingScope) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java index 4b5151504..055903e12 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java @@ -33,7 +33,7 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.SupportedTypes; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; @@ -207,11 +207,11 @@ private void structifyBlock(Expr structifiedExpr) { final IParentMap parentMap = IParentMap.createParentMap(block); - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); if (se == null) { // We are traversing a block in isolation, so we won't have a scope entry for any variable // declared outside the block. diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutation.java index 43ef01c6d..fb3db2c74 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutation.java @@ -28,7 +28,7 @@ import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.transformreduce.MergeSet; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.generator.mutateapi.Mutation; import java.util.ArrayList; @@ -61,7 +61,7 @@ public void apply() { mergeSet.getMergedType(), new VariableDeclInfo(mergeSet.getMergedName(), null, null)))); } - private class VectorizerVisitor extends ScopeTreeBuilder { + private class VectorizerVisitor extends ScopeTrackingVisitor { private boolean inDeclarationOfTargetVariable = false; private final ScopeEntry currentComponent; @@ -96,19 +96,20 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie } private boolean isCurrentComponentVariable(String name) { - ScopeEntry entry = currentScope.lookupScopeEntry(name); + ScopeEntry entry = getCurrentScope().lookupScopeEntry(name); return entry != null && entry.hasVariableDeclInfo() && currentComponent.getVariableDeclInfo() == entry.getVariableDeclInfo(); } @Override public void visitDeclarationStmt(DeclarationStmt declarationStmt) { - List existingKeys = new ArrayList<>(); - existingKeys.addAll(currentScope.keys()); + List existingKeys = new ArrayList<>(getCurrentScope().keys()); super.visitDeclarationStmt(declarationStmt); - List newKeys = currentScope.keys().stream().filter(key -> !existingKeys.contains(key)) + List newKeys = getCurrentScope().keys() + .stream() + .filter(key -> !existingKeys.contains(key)) + .sorted(String::compareTo) .collect(Collectors.toList()); - newKeys.sort(String::compareTo); for (String newKey : newKeys) { if (isCurrentComponentVariable(newKey)) { final ExprStmt insertedStmt = new ExprStmt( diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutationFinder.java index 513b1c3e0..b056590c4 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutationFinder.java @@ -26,12 +26,9 @@ import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.transformreduce.MergeSet; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.generator.mutateapi.MutationFinder; import com.graphicsfuzz.generator.mutateapi.MutationFinderBase; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -60,8 +57,8 @@ protected void popScope() { assert lastExitedBlock != null; if (!(parentMap.getParent(lastExitedBlock) instanceof SwitchStmt)) { List mergeSetsForThisScope = new ArrayList<>(); - for (String v : currentScope.keys()) { - ScopeEntry entry = currentScope.lookupScopeEntry(v); + for (String v : getCurrentScope().keys()) { + ScopeEntry entry = getCurrentScope().lookupScopeEntry(v); if (!isCandidateForMerging(entry)) { continue; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java index 692983232..a2b4da8a2 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java @@ -45,7 +45,7 @@ import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.OpenGlConstants; @@ -64,7 +64,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Optional; -import java.util.Random; import java.util.stream.Collectors; import net.sourceforge.argparse4j.ArgumentParsers; import net.sourceforge.argparse4j.impl.Arguments; @@ -180,7 +179,7 @@ private static String demoteOutputVariable(TranslationUnit tu) { * Replace all references to gl_FragCoord with gl_GlobalInvocationID. */ private static void replaceFragCoordWithIdLookup(TranslationUnit computeTu) { - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { private IParentMap parentMap = IParentMap.createParentMap(computeTu); @@ -189,7 +188,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie super.visitVariableIdentifierExpr(variableIdentifierExpr); if (variableIdentifierExpr.getName().equals(OpenGlConstants.GL_FRAG_COORD)) { // It has the right name. - if (currentScope.lookupScopeEntry(OpenGlConstants.GL_FRAG_COORD) == null) { + if (getCurrentScope().lookupScopeEntry(OpenGlConstants.GL_FRAG_COORD) == null) { // It has no scope; i.e., it is built-in - so it is the real gl_FragCoord. // Replace it with something made from gl_GlobalInvocationID. parentMap.getParent(variableIdentifierExpr).replaceChild(variableIdentifierExpr, diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java index 49d32f21b..5991c2fae 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java @@ -42,7 +42,7 @@ import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.Scope; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; @@ -291,12 +291,12 @@ private boolean incompatible(IInjectionPoint injectionPoint, DonationContext don } private Map getGlobalVariablesFromShader(TranslationUnit shader) { - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { Map getGlobalsFromShader(TranslationUnit shader) { visit(shader); Map result = new HashMap<>(); - for (String globalName : currentScope.keys()) { - result.put(globalName, currentScope.lookupType(globalName)); + for (String globalName : getCurrentScope().keys()) { + result.put(globalName, getCurrentScope().lookupType(globalName)); } return result; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/DonationContext.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/DonationContext.java index fa04733c5..eb05ff76a 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/DonationContext.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/DonationContext.java @@ -25,7 +25,7 @@ import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -88,7 +88,7 @@ public boolean indexesArrayUsingFreeVariable() { // Note: we don't use the freeVariables member here, because we want to account for // name shadowing. - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { private boolean found = false; @@ -105,7 +105,7 @@ public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); if (arrayIndexDepth > 0 - && currentScope.lookupScopeEntry(variableIdentifierExpr.getName()) == null) { + && getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()) == null) { // A free variable that appears under an array index found = true; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java index de49199a5..ded76e845 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java @@ -25,7 +25,7 @@ import com.graphicsfuzz.common.ast.stmt.IfStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.stmt.WhileStmt; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.IRandom; import java.util.ArrayList; import java.util.Collections; @@ -34,7 +34,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -public class InjectionPoints extends ScopeTreeBuilder { +public class InjectionPoints extends ScopeTrackingVisitor { private final List injectionPoints; private FunctionDefinition currentFunction; @@ -64,7 +64,8 @@ public void visitBlockStmt(BlockStmt stmt) { enterBlockStmt(stmt); // This setup ensures that variables declared inside the block are in scope by the time we // consider each statement as an injection point. - // It is a bit ugly because it has to mimic the "enter ... leave" structure of ScopeTreeBuilder + // It is a bit ugly because it has to mimic the "enter ... leave" structure of + // ScopeTrackingVisitor for (int i = 0; i < stmt.getNumStmts(); i++) { Stmt innerStmt = stmt.getStmt(i); if (i == 0 && innerStmt instanceof CaseLabel) { @@ -72,11 +73,11 @@ public void visitBlockStmt(BlockStmt stmt) { continue; } maybeAddInjectionPoint(new BlockInjectionPoint(stmt, innerStmt, currentFunction, inLoop(), - currentScope)); + getCurrentScope())); visit(innerStmt); } maybeAddInjectionPoint(new BlockInjectionPoint(stmt, null, currentFunction, inLoop(), - currentScope)); + getCurrentScope())); leaveBlockStmt(stmt); } @@ -88,7 +89,7 @@ private boolean inLoop() { public void visitDoStmt(DoStmt doStmt) { if (!(doStmt.getBody() instanceof BlockStmt)) { maybeAddInjectionPoint(new LoopInjectionPoint(doStmt, currentFunction, - currentScope)); + getCurrentScope())); } loopNestingDepth++; super.visitDoStmt(doStmt); @@ -99,7 +100,7 @@ public void visitDoStmt(DoStmt doStmt) { public void visitForStmt(ForStmt forStmt) { if (!(forStmt.getBody() instanceof BlockStmt)) { maybeAddInjectionPoint(new LoopInjectionPoint(forStmt, currentFunction, - currentScope)); + getCurrentScope())); } loopNestingDepth++; super.visitForStmt(forStmt); @@ -110,12 +111,12 @@ public void visitForStmt(ForStmt forStmt) { public void visitIfStmt(IfStmt ifStmt) { if (!(ifStmt.getThenStmt() instanceof BlockStmt)) { maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, true, currentFunction, inLoop(), - currentScope)); + getCurrentScope())); } if (ifStmt.hasElseStmt()) { if (!(ifStmt.getElseStmt() instanceof BlockStmt)) { maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, false, currentFunction, inLoop(), - currentScope)); + getCurrentScope())); } } super.visitIfStmt(ifStmt); @@ -125,7 +126,7 @@ public void visitIfStmt(IfStmt ifStmt) { public void visitWhileStmt(WhileStmt whileStmt) { if (!(whileStmt.getBody() instanceof BlockStmt)) { maybeAddInjectionPoint(new LoopInjectionPoint(whileStmt, currentFunction, - currentScope)); + getCurrentScope())); } loopNestingDepth++; super.visitWhileStmt(whileStmt); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/AvailableStructsCollector.java b/generator/src/main/java/com/graphicsfuzz/generator/util/AvailableStructsCollector.java index 7066c39c1..4673a404e 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/AvailableStructsCollector.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/AvailableStructsCollector.java @@ -19,12 +19,12 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.type.StructDefinitionType; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.ArrayList; import java.util.Collections; import java.util.List; -public class AvailableStructsCollector extends ScopeTreeBuilder { +public class AvailableStructsCollector extends ScopeTrackingVisitor { private final List structDefinitionTypes; private final IAstNode donorFragment; @@ -39,8 +39,8 @@ public AvailableStructsCollector(TranslationUnit donor, IAstNode donorFragment) public void visit(IAstNode node) { if (node == donorFragment) { assert structDefinitionTypes.isEmpty(); - for (String structName : currentScope.namesOfAllStructDefinitionsInScope()) { - structDefinitionTypes.add(currentScope.lookupStructName(structName)); + for (String structName : getCurrentScope().namesOfAllStructDefinitionsInScope()) { + structDefinitionTypes.add(getCurrentScope().lookupStructName(structName)); } } super.visit(node); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java b/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java index a832f5d89..f7a8299af 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java @@ -28,7 +28,7 @@ import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -37,7 +37,7 @@ * Attempts to minimally remove const qualifiers and move global declaration initializers into * main, to make a shader valid. */ -class ConstCleaner extends ScopeTreeBuilder { +class ConstCleaner extends ScopeTrackingVisitor { private boolean atGlobalScope; private Optional currentVariablesDeclaration; @@ -91,7 +91,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie } private boolean nonConst(VariableIdentifierExpr variableIdentifierExpr) { - final ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); return se != null && se.hasVariablesDeclaration() && !se.getVariablesDeclaration().getBaseType() .hasQualifier(TypeQualifier.CONST); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/FreeVariablesCollector.java b/generator/src/main/java/com/graphicsfuzz/generator/util/FreeVariablesCollector.java index fb0d6abf1..89634afae 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/FreeVariablesCollector.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/FreeVariablesCollector.java @@ -20,15 +20,14 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.stmt.Stmt; -import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.typing.Scope; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.OpenGlConstants; import java.util.HashMap; import java.util.Map; -public class FreeVariablesCollector extends ScopeTreeBuilder { +public class FreeVariablesCollector extends ScopeTrackingVisitor { private final Stmt donorFragment; private Scope enclosingScope; @@ -48,14 +47,12 @@ public Map getFreeVariables() { @Override public void visit(IAstNode node) { if (node == donorFragment) { - enclosingScope = currentScope; - currentScope = new Scope(null); - super.visit(node); - // early exit now, if we implement that - currentScope = enclosingScope; + enclosingScope = swapCurrentScope(new Scope(null)); + } + super.visit(node); + if (node == donorFragment) { + swapCurrentScope(enclosingScope); enclosingScope = null; - } else { - super.visit(node); } } @@ -65,7 +62,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie if (isBuiltinVariable(name)) { return; } - if (enclosingScope != null && currentScope.lookupType(name) == null) { + if (enclosingScope != null && getCurrentScope().lookupType(name) == null) { Type type = enclosingScope.lookupType(name); if (type == null) { throw new RuntimeException( diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java index 604f12656..fd6036766 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java @@ -36,7 +36,7 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.typing.Scope; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.common.typing.TyperHelper; import com.graphicsfuzz.common.util.IRandom; @@ -50,7 +50,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Set; public class RestrictFragmentShaderColors { @@ -159,7 +158,7 @@ private boolean adaptExistingWrites() { final Typer typer = new Typer(shaderJob.getFragmentShader().get()); - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { @Override public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { @@ -174,7 +173,7 @@ public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { return; } for (Expr expr : functionCallExpr.getArgs()) { - if (isPartOfOutputVariable(expr, currentScope)) { + if (isPartOfOutputVariable(expr, getCurrentScope())) { throw new AbortVisitationException("We do not yet handle components of the output " + "variable being passed to functions, in case they are out parameters."); } @@ -188,7 +187,7 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { // Do nothing: the binary operator is not side-effecting. return; } - if (!isPartOfOutputVariable(binaryExpr.getLhs(), currentScope)) { + if (!isPartOfOutputVariable(binaryExpr.getLhs(), getCurrentScope())) { return; } final BasicType basicType = getBasicType(binaryExpr.getLhs()); @@ -198,9 +197,9 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { // Replace RHS by // (1.0 + 0.0 * (is_nan_or_inf(original_rhs) ? 0.0 : original_rhs)) binaryExpr.setRhs( - new ParenExpr(new BinaryExpr(opaqueOne(currentScope, basicType), - new BinaryExpr(opaqueZero(currentScope, basicType), - ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), currentScope), + new ParenExpr(new BinaryExpr(opaqueOne(getCurrentScope(), basicType), + new BinaryExpr(opaqueZero(getCurrentScope(), basicType), + ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), getCurrentScope()), BinOp.MUL), BinOp.ADD)) ); @@ -211,8 +210,8 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { binaryExpr.setRhs( new ParenExpr( new BinaryExpr( - opaqueZero(currentScope, basicType), - ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), currentScope), + opaqueZero(getCurrentScope(), basicType), + ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), getCurrentScope()), BinOp.MUL) ) ); @@ -222,9 +221,10 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { binaryExpr.setRhs( new ParenExpr( new BinaryExpr( - makeColorVector(basicType, currentScope), - new BinaryExpr(opaqueZero(currentScope, basicType), - ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), currentScope), + makeColorVector(basicType, getCurrentScope()), + new BinaryExpr(opaqueZero(getCurrentScope(), basicType), + ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), + getCurrentScope()), BinOp.MUL), BinOp.ADD ) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/TransformationProbabilities.java b/generator/src/main/java/com/graphicsfuzz/generator/util/TransformationProbabilities.java index 006b626ff..154796b91 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/TransformationProbabilities.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/TransformationProbabilities.java @@ -61,7 +61,7 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona this.probInjectDeadBarrierAtStmt = probInjectDeadBarrierAtStmt; } - public static final TransformationProbabilities DEFAULT_PROBABILITIES = + public static TransformationProbabilities DEFAULT_PROBABILITIES = new TransformationProbabilities( 80, 20, @@ -79,7 +79,7 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona 20 ); - public static final TransformationProbabilities SMALL_PROBABILITIES = + public static TransformationProbabilities SMALL_PROBABILITIES = new TransformationProbabilities( 80, 5, @@ -96,7 +96,7 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona 5, 5); - public static final TransformationProbabilities AGGRESSIVE_CONTROL_FLOW = + public static TransformationProbabilities AGGRESSIVE_CONTROL_FLOW = new TransformationProbabilities(DEFAULT_PROBABILITIES.probSubstituteFreeVariable, DEFAULT_PROBABILITIES.probDonateDeadCodeAtStmt, DEFAULT_PROBABILITIES.probDonateLiveCodeAtStmt, @@ -113,7 +113,7 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona 70); // Useful for testing; add similar for others when needed - public static final TransformationProbabilities ZERO = + public static TransformationProbabilities ZERO = new TransformationProbabilities( 0, 0, @@ -130,68 +130,68 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona 0, 0); - public static final TransformationProbabilities onlySplitLoops() { + public static TransformationProbabilities onlySplitLoops() { TransformationProbabilities result = ZERO; result.probSplitLoops = 100; return result; } - public static final TransformationProbabilities onlyStructify() { + public static TransformationProbabilities onlyStructify() { TransformationProbabilities result = ZERO; result.probStructify = 100; return result; } - public static final TransformationProbabilities onlyVectorize() { + public static TransformationProbabilities onlyVectorize() { TransformationProbabilities result = ZERO; result.probVectorizeStmts = 100; return result; } - public static final TransformationProbabilities onlyVectorizeAndMutate() { + public static TransformationProbabilities onlyVectorizeAndMutate() { TransformationProbabilities result = ZERO; result.probVectorizeStmts = 100; result.probMutatePoint = 100; return result; } - public static final TransformationProbabilities onlyOutlineStatements() { + public static TransformationProbabilities onlyOutlineStatements() { TransformationProbabilities result = ZERO; result.probOutline = 100; return result; } - public static final TransformationProbabilities onlyMutateExpressions() { + public static TransformationProbabilities onlyMutateExpressions() { TransformationProbabilities result = ZERO; result.probMutatePoint = 100; return result; } - public static final TransformationProbabilities onlyWrap() { + public static TransformationProbabilities onlyWrap() { TransformationProbabilities result = ZERO; result.probWrapStmtInConditional = 100; return result; } - public static final TransformationProbabilities onlyAddJumps() { + public static TransformationProbabilities onlyAddJumps() { TransformationProbabilities result = ZERO; result.probInjectJumpAtStmt = 100; return result; } - public static final TransformationProbabilities onlyAddDeadFragColorWrites() { + public static TransformationProbabilities onlyAddDeadFragColorWrites() { TransformationProbabilities result = ZERO; result.probAddDeadFragColorWrites = 100; return result; } - public static final TransformationProbabilities onlyAddLiveFragColorWrites() { + public static TransformationProbabilities onlyAddLiveFragColorWrites() { TransformationProbabilities result = ZERO; result.probAddLiveFragColorWrites = 100; return result; } - public static final TransformationProbabilities onlyLiveCodeAlwaysSubstitute() { + public static TransformationProbabilities onlyLiveCodeAlwaysSubstitute() { TransformationProbabilities result = ZERO; result.probDonateLiveCodeAtStmt = 100; result.probSubstituteFreeVariable = 100; @@ -202,14 +202,14 @@ public static TransformationProbabilities closeToDefaultProbabilities(IRandom ge return closeTo(generator, DEFAULT_PROBABILITIES); } - public static final TransformationProbabilities likelyDonateDeadCode() { + public static TransformationProbabilities likelyDonateDeadCode() { TransformationProbabilities result = ZERO; result.probSubstituteFreeVariable = 50; result.probDonateDeadCodeAtStmt = 60; return result; } - public static final TransformationProbabilities likelyDonateLiveCode() { + public static TransformationProbabilities likelyDonateLiveCode() { TransformationProbabilities result = ZERO; result.probSubstituteFreeVariable = 50; result.probDonateLiveCodeAtStmt = 60; diff --git a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/FuzzerTest.java b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/FuzzerTest.java index c4c63f9e4..7d5177a51 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/FuzzerTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/FuzzerTest.java @@ -22,7 +22,7 @@ import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.ZeroCannedRandom; @@ -52,12 +52,12 @@ public void testStructExprFuzzing() throws Exception { TranslationUnit tu = ParseHelper.parse(shader); - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); if (variableIdentifierExpr.getName().equals("doitWhenYouReachMyUse")) { - Expr expr = new Fuzzer(new FuzzingContext(currentScope), ShadingLanguageVersion.ESSL_100, + Expr expr = new Fuzzer(new FuzzingContext(getCurrentScope()), ShadingLanguageVersion.ESSL_100, new ZeroCannedRandom(), GenerationParams.normal(ShaderKind.FRAGMENT, true), "prefix") .fuzzExpr(new StructNameType("B"), false, false, 0); assertTrue(expr instanceof TypeConstructorExpr); @@ -75,4 +75,4 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie }.visit(tu); } -} \ No newline at end of file +} diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java index a7bfe31c1..08f1e929a 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java @@ -39,7 +39,7 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; @@ -115,7 +115,7 @@ public void checkMutateSpecialCase() throws Exception { BlockInjectionPoint blockInjectionPoint = - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { BlockInjectionPoint blockInjectionPoint; @@ -123,8 +123,8 @@ public void checkMutateSpecialCase() throws Exception { public void visitBlockStmt(BlockStmt stmt) { super.visitBlockStmt(stmt); if (stmt.getNumStmts() == 0) { - blockInjectionPoint = new BlockInjectionPoint(stmt, null, enclosingFunction, - false, currentScope); + blockInjectionPoint = new BlockInjectionPoint(stmt, null, getEnclosingFunction(), + false, getCurrentScope()); } } @@ -424,13 +424,13 @@ public void testArrayAccessesAreInBounds() throws Exception { // // The following thus checks that if an array is indexed directly by a variable reference, // the initializer for that variable is not a function call expression. - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { @Override public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { super.visitArrayIndexExpr(arrayIndexExpr); if (arrayIndexExpr.getIndex() instanceof VariableIdentifierExpr) { - final ScopeEntry scopeEntry = currentScope.lookupScopeEntry( + final ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry( ((VariableIdentifierExpr) arrayIndexExpr.getIndex()).getName()); assertTrue(scopeEntry.hasVariableDeclInfo()); assertNotNull(scopeEntry.getVariableDeclInfo().getInitializer()); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/CheckAstFeatureVisitor.java b/reducer/src/main/java/com/graphicsfuzz/reducer/CheckAstFeatureVisitor.java index f5d70e432..201f8dc3a 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/CheckAstFeatureVisitor.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/CheckAstFeatureVisitor.java @@ -20,7 +20,7 @@ import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -33,7 +33,7 @@ * method to check for the feature of interest, and then invoke the trigger function * when the feature is found. */ -public abstract class CheckAstFeatureVisitor extends ScopeTreeBuilder { +public abstract class CheckAstFeatureVisitor extends ScopeTrackingVisitor { private Optional triggerFunction = Optional.empty(); private Map> callsIndirectly = new HashMap>(); @@ -49,7 +49,7 @@ public void visitFunctionPrototype(FunctionPrototype functionPrototype) { @Override public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { super.visitFunctionCallExpr(functionCallExpr); - final String enclosingFunctionName = enclosingFunction.getPrototype().getName(); + final String enclosingFunctionName = getEnclosingFunction().getPrototype().getName(); final String calledFunctionName = functionCallExpr.getCallee(); assert callsIndirectly.containsKey(enclosingFunctionName); if (!callsIndirectly.containsKey(calledFunctionName)) { @@ -72,7 +72,7 @@ boolean check(TranslationUnit tu) { /** * Use this method to register that the feature of interest has been found. */ - public void trigger() { - this.triggerFunction = Optional.of(enclosingFunction); + protected void trigger() { + this.triggerFunction = Optional.of(getEnclosingFunction()); } } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunities.java index 3d8d5fb85..41384cb3d 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunities.java @@ -19,13 +19,13 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import java.util.Arrays; import java.util.LinkedList; import java.util.List; -public class DestructifyReductionOpportunities extends ScopeTreeBuilder { +public class DestructifyReductionOpportunities extends ScopeTrackingVisitor { private final TranslationUnit tu; private final List opportunities; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java index 75076afee..38bbad503 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java @@ -33,7 +33,7 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.util.Constants; import java.util.Arrays; @@ -133,7 +133,7 @@ private void deStructify(StructifiedVariableInfo originalVariableInfo) { final IParentMap parentMap = IParentMap.createParentMap(tu); - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { @Override public void visitFunctionPrototype(FunctionPrototype functionPrototype) { @@ -170,7 +170,7 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { } VariableIdentifierExpr structVariable = ((VariableIdentifierExpr) memberLookupExpr .getStructure()); - ScopeEntry se = currentScope.lookupScopeEntry(structVariable.getName()); + ScopeEntry se = getCurrentScope().lookupScopeEntry(structVariable.getName()); if (se == null) { return; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java index 4fb36ae2e..23b093175 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java @@ -88,7 +88,7 @@ private void addOpportunities() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - final ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); if (se == null || !se.hasVariableDeclInfo()) { return; } @@ -103,7 +103,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie if (new CheckPredicateVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { - if (currentScope.isShadowed(variableIdentifierExpr.getName())) { + if (getCurrentScope().isShadowed(variableIdentifierExpr.getName())) { predicateHolds(); } } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunities.java index 04e39cd53..4388e7edf 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunities.java @@ -21,14 +21,14 @@ import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.util.Constants; import java.util.Arrays; import java.util.LinkedList; import java.util.List; -public class InlineStructifiedFieldReductionOpportunities extends ScopeTreeBuilder { +public class InlineStructifiedFieldReductionOpportunities extends ScopeTrackingVisitor { private final List opportunities; private final TranslationUnit tu; @@ -71,7 +71,7 @@ public void visitDeclarationStmt(DeclarationStmt declarationStmt) { public void findInliningOpportunities(StructNameType structType) { assert structType.getName().startsWith(Constants.STRUCTIFICATION_STRUCT_PREFIX); final StructDefinitionType structDefinitionType = - currentScope.lookupStructName(structType.getName()); + getCurrentScope().lookupStructName(structType.getName()); for (String f : structDefinitionType.getFieldNames()) { if (!f.startsWith(Constants.STRUCTIFICATION_FIELD_PREFIX)) { continue; @@ -81,7 +81,8 @@ public void findInliningOpportunities(StructNameType structType) { final StructNameType innerStructType = (StructNameType) structDefinitionType.getFieldType(f).getWithoutQualifiers(); opportunities.add(new InlineStructifiedFieldReductionOpportunity( - structDefinitionType, currentScope.lookupStructName(innerStructType.getName()), f, tu, + structDefinitionType, getCurrentScope().lookupStructName(innerStructType.getName()), f, + tu, getVistitationDepth())); findInliningOpportunities(innerStructType); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java index e18c427da..9cf4a4559 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java @@ -60,7 +60,7 @@ private InlineUniformReductionOpportunities(TranslationUnit tu, public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); final String name = variableIdentifierExpr.getName(); - final ScopeEntry se = currentScope.lookupScopeEntry(name); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(name); if (se == null) { return; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java index fb89522e2..d1885e63e 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java @@ -28,7 +28,7 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.util.Constants; import java.util.ArrayList; @@ -93,12 +93,12 @@ private boolean referencesBackup(Stmt stmt) { final VariableDeclInfo backupVdi = ((DeclarationStmt) block.getStmt(indexOfBackupDeclaration().get())) .getVariablesDeclaration().getDeclInfo(0); - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { private boolean found = false; @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - final ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); if (se != null && se.hasVariableDeclInfo() && se.getVariableDeclInfo() == backupVdi) { found = true; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java index b3becbdfd..513ce59e6 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java @@ -26,7 +26,7 @@ import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.util.Constants; import java.util.Arrays; @@ -34,7 +34,7 @@ import java.util.List; import java.util.Optional; -public class LoopMergeReductionOpportunities extends ScopeTreeBuilder { +public class LoopMergeReductionOpportunities extends ScopeTrackingVisitor { private final List opportunities; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java index 598904ce6..8ca3c3728 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java @@ -34,7 +34,7 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.type.Type; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.SideEffectChecker; @@ -44,7 +44,7 @@ public abstract class ReductionOpportunitiesBase - extends ScopeTreeBuilder { + extends ScopeTrackingVisitor { private final List opportunities; final InjectionTracker injectionTracker; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunities.java index 11d2ee2cc..eaae73780 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunities.java @@ -22,14 +22,14 @@ import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.util.Constants; import java.util.Arrays; import java.util.LinkedList; import java.util.List; -public class RemoveStructFieldReductionOpportunities extends ScopeTreeBuilder { +public class RemoveStructFieldReductionOpportunities extends ScopeTrackingVisitor { private final List opportunities; private final TranslationUnit translationUnit; @@ -76,7 +76,7 @@ private void getOpportunitiesForStruct(StructNameType structType, } final StructDefinitionType structDefinitionType = - currentScope.lookupStructName(structType.getName()); + getCurrentScope().lookupStructName(structType.getName()); for (String field : structDefinitionType.getFieldNames()) { if (!reachesOriginalVariable(structDefinitionType, field) @@ -109,7 +109,7 @@ private boolean reachesOriginalVariable(StructDefinitionType structDefinitionTyp final StructNameType fieldType = (StructNameType) structDefinitionType.getFieldType(field).getWithoutQualifiers(); final StructDefinitionType nestedStruct = - currentScope.lookupStructName(fieldType.getName()); + getCurrentScope().lookupStructName(fieldType.getName()); return nestedStruct.getFieldNames().stream() .anyMatch(item -> reachesOriginalVariable(nestedStruct, item)); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunities.java index 3ec88e2a2..d000fcb08 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunities.java @@ -125,7 +125,8 @@ public void visitFunctionDefinition(FunctionDefinition functionDefinition) { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - final ScopeEntry scopeEntry = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry(variableIdentifierExpr + .getName()); if (scopeEntry != null && scopeEntry.hasParameterDecl()) { unusedParametersForCurrentFunction.remove(scopeEntry.getParameterDecl()); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ShadowChecker.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ShadowChecker.java index 76ff52684..5e8b27f06 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ShadowChecker.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ShadowChecker.java @@ -20,9 +20,9 @@ import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; -class ShadowChecker extends ScopeTreeBuilder { +class ShadowChecker extends ScopeTrackingVisitor { private final BlockStmt blockOfInterest; private final String nameOfInterest; @@ -40,7 +40,7 @@ class ShadowChecker extends ScopeTreeBuilder { public void visitBlockStmt(BlockStmt stmt) { if (stmt == blockOfInterest) { inBlock = true; - possiblyShadowedScopeEntry = currentScope.lookupScopeEntry(nameOfInterest); + possiblyShadowedScopeEntry = getCurrentScope().lookupScopeEntry(nameOfInterest); } super.visitBlockStmt(stmt); if (stmt == blockOfInterest) { @@ -53,7 +53,7 @@ public void visitBlockStmt(BlockStmt stmt) { public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); if (inBlock && possiblyShadowedScopeEntry != null - && currentScope.lookupScopeEntry(variableIdentifierExpr.getName()) + && getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()) == possiblyShadowedScopeEntry) { ok = false; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java index 4b99de6b9..1c0a7ec23 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java @@ -118,7 +118,8 @@ protected void visitChildOfBlock(BlockStmt block, int childIndex) { final Set namesDeclaredDirectlyByParentOrInScopeAlready = new HashSet<>(); namesDeclaredDirectlyByParentOrInScopeAlready.addAll( UnwrapReductionOpportunity.getNamesDeclaredDirectlyByBlock(block)); - namesDeclaredDirectlyByParentOrInScopeAlready.addAll(currentScope.namesOfAllVariablesInScope()); + namesDeclaredDirectlyByParentOrInScopeAlready.addAll(getCurrentScope() + .namesOfAllVariablesInScope()); final Set namesDeclaredDirectlyByChild = UnwrapReductionOpportunity.getNamesDeclaredDirectlyByBlock(childAsBlock); if (Collections.disjoint(namesDeclaredDirectlyByParentOrInScopeAlready, diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java index aeddad550..c3c69dee8 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java @@ -46,8 +46,8 @@ private VariableDeclReductionOpportunities(TranslationUnit tu, } private void getReductionOpportunitiesForUnusedGlobals() { - for (String name : currentScope.keys()) { - ScopeEntry entry = currentScope.lookupScopeEntry(name); + for (String name : getCurrentScope().keys()) { + ScopeEntry entry = getCurrentScope().lookupScopeEntry(name); assert entry.hasVariableDeclInfo(); assert referencedScopeEntries.peek() != null; if (!referencedScopeEntries.peek().contains(entry)) { @@ -66,8 +66,8 @@ protected void pushScope() { @Override protected void popScope() { - for (String name : currentScope.keys()) { - ScopeEntry entry = currentScope.lookupScopeEntry(name); + for (String name : getCurrentScope().keys()) { + ScopeEntry entry = getCurrentScope().lookupScopeEntry(name); if (entry.hasVariableDeclInfo() && !referencedScopeEntries.peek().contains(entry)) { if (allowedToReduceLocalDecl(entry.getVariableDeclInfo())) { addOpportunity( @@ -92,9 +92,9 @@ protected void popScope() { // refer to a variable x in the parent scope, even though the current // scope also declares a variable x after the usage of x String name = entry.getVariableDeclInfo().getName(); - assert currentScope.lookupScopeEntry(name) == entry - || currentScope.getParent().lookupScopeEntry(name) == entry; - if (!currentScope.keys().contains(name)) { + assert getCurrentScope().lookupScopeEntry(name) == entry + || getCurrentScope().getParent().lookupScopeEntry(name) == entry; + if (!getCurrentScope().keys().contains(name)) { addReferencedScopeEntry(entry); } } @@ -110,7 +110,7 @@ private void addReferencedScopeEntry(ScopeEntry scopeEntry) { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - ScopeEntry scopeEntry = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); if (scopeEntry != null) { addReferencedScopeEntry(scopeEntry); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunities.java index 892bc7c50..c53811e0d 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunities.java @@ -80,7 +80,7 @@ public void visitVariableDeclInfo(VariableDeclInfo variableDeclInfo) { String name = variableDeclInfo.getName(); if (MergeSet.isMergedVariable(name)) { List componentsData = MergeSet.getComponentData(name); - assert inBlock(); + assert inSomeBlock(); for (MergedVariablesComponentData data : componentsData) { assert enclosingVariablesDeclaration != null; final VectorizationReductionOpportunity potentialOpportunity diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunity.java index 6f6fdcb13..4626c2cce 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunity.java @@ -31,7 +31,7 @@ import com.graphicsfuzz.common.transformreduce.MergedVariablesComponentData; import com.graphicsfuzz.common.typing.Scope; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import java.util.Arrays; import java.util.List; @@ -96,12 +96,12 @@ private void pullOutComponent() { // Look for cases where we are accessing the component we want to remove, and replace // those with the component variable. - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); - if (isComponentAccess(memberLookupExpr, currentScope)) { + if (isComponentAccess(memberLookupExpr, getCurrentScope())) { try { parentMap.getParent(memberLookupExpr).replaceChild(memberLookupExpr, new VariableIdentifierExpr(getComponentName())); @@ -153,13 +153,13 @@ public boolean preconditionHolds() { } private boolean componentIsUsed() { - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { private boolean isUsed = false; @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); - isUsed |= isComponentAccess(memberLookupExpr, currentScope); + isUsed |= isComponentAccess(memberLookupExpr, getCurrentScope()); } public boolean componentIsUsed() { @@ -206,7 +206,7 @@ private boolean incompatibleComponentVariableIsDeclaredInBlock() { private boolean vectorIsUsedWithoutFieldLookup() { return - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { private boolean vectorUsedDirectly = false; @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { @@ -217,7 +217,8 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie if (!variableIdentifierExpr.getName().equals(vectorVariableDeclInfo.getName())) { return; } - final ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr + .getName()); if (se != null && se.hasVariableDeclInfo() && se.getVariableDeclInfo() == vectorVariableDeclInfo) { vectorUsedDirectly = true; @@ -312,4 +313,4 @@ private boolean isComponentAccess(MemberLookupExpr memberLookupExpr, Scope curre return swizzleOffset == componentData.getOffset() && swizzleWidth == componentData.getWidth(); } -} \ No newline at end of file +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java deleted file mode 100755 index c99a3a5a6..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.tool; - -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.GlslParserException; -import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.common.util.ParseTimeoutException; -import com.graphicsfuzz.common.util.RandomWrapper; -import com.graphicsfuzz.common.util.ShaderJobFileOperations; -import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.reducer.ReductionDriver; -import com.graphicsfuzz.reducer.reductionopportunities.ReducerContext; -import com.graphicsfuzz.util.ArgsUtil; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Comparator; -import net.sourceforge.argparse4j.ArgumentParsers; -import net.sourceforge.argparse4j.impl.Arguments; -import net.sourceforge.argparse4j.inf.ArgumentParser; -import net.sourceforge.argparse4j.inf.ArgumentParserException; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.commons.io.FilenameUtils; - -public class ReducerBugPoint { - - private static final int STEP_LIMIT = 50; - - private static Namespace parse(String[] args) throws ArgumentParserException { - ArgumentParser parser = ArgumentParsers.newArgumentParser("ReducerBugPoint") - .defaultHelp(true) - .description("Find bugs in the reducer."); - - // Required arguments - parser.addArgument("shader") - .help("Path of shader job file (.json) to be analysed.") - .type(File.class); - - parser.addArgument("--output") - .help("Output directory.") - .setDefault(new File(".")) - .type(File.class); - - // Optional arguments - parser.addArgument("--max-iterations") - .help("Maximum number of times to iterate before giving up.") - .setDefault(30) - .type(Integer.class); - - parser.addArgument("--seed") - .help("Seed (unsigned 64 bit long integer) to initialize random number generator with.") - .type(String.class); - - parser.addArgument("--preserve-semantics") - .help("Only perform semantics-preserving reductions.") - .action(Arguments.storeTrue()); - - parser.addArgument("--verbose") - .help("Emit verbose info.") - .action(Arguments.storeTrue()); - - parser.addArgument("--exception-on-invalid") - .help("Throw exception when shader is invalid.") - .action(Arguments.storeTrue()); - - parser.addArgument("--expected-string") - .help("A string to look for in the exception message.") - .type(String.class); - - return parser.parseArgs(args); - } - - - public static void main(String[] args) - throws IOException, ParseTimeoutException, ArgumentParserException, InterruptedException, - GlslParserException { - - final Namespace ns = parse(args); - - IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); - - final int maxIterations = ns.get("max_iterations"); - - final boolean reduceEverywhere = !ns.getBoolean("preserve_semantics"); - - final boolean verbose = ns.get("verbose"); - - final String expectedString = ns.getString("expected_string") == null - ? "" - : ns.getString("expected_string"); - - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - - final File interestingShaderJobFile = new File("interesting.frag"); - - if (fileOps.doesShaderJobExist(interestingShaderJobFile)) { - fileOps.deleteShaderJobFile(interestingShaderJobFile); - } - - final File originalShaderJobFile = ns.get("shader"); - - fileOps.copyShaderJobFileTo(originalShaderJobFile, interestingShaderJobFile, false); - - final ShadingLanguageVersion shadingLanguageVersion = - ShadingLanguageVersion.getGlslVersionFromFirstTwoLines( - fileOps.getFirstTwoLinesOfShader(interestingShaderJobFile, ShaderKind.FRAGMENT)); - - ShaderJob initialState = fileOps.readShaderJobFile(interestingShaderJobFile); - - for (int i = 0; i < maxIterations; i++) { - - File workDir = Files.createDirectory(Paths.get("temp")).toFile(); - - System.err.println("Trying iteration " + i); - - try { - - new ReductionDriver( - new ReducerContext( - reduceEverywhere, - shadingLanguageVersion, - generator, - null, - 10, - 1), - verbose, - fileOps, - new RandomFileJudge( - generator, - 10, - ns.getBoolean("exception_on_invalid"), fileOps), - workDir - ).doReduction( - initialState, - FilenameUtils.removeExtension(interestingShaderJobFile.getAbsolutePath()), - 0, - STEP_LIMIT); - - } catch (Throwable throwable) { - if (!throwable.toString().contains(expectedString)) { - System.err.println("Exception does not contain required string:"); - System.err.println(throwable); - } else { - File[] files = - fileOps.listShaderJobFiles(workDir, (dir, name) -> name.endsWith("success.info")); - if (files.length == 0) { - continue; - } - final File maxSuccess = - Arrays.stream(files) - .max(Comparator.comparingInt(ReducerBugPoint::getStep)).get(); - - fileOps.deleteShaderJobFile(interestingShaderJobFile); - fileOps.copyShaderJobFileTo(maxSuccess, interestingShaderJobFile, true); - - initialState = fileOps.readShaderJobFile(interestingShaderJobFile); - - i = 0; - } - } finally { - fileOps.deleteDirectory(workDir); - } - } - } - - private static int getStep(File file) { - final String[] split = file.getName().split("_"); - return Integer.parseInt(split[split.length - 2]); - } - -} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java deleted file mode 100755 index b79331a89..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.tool; - -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.GlslParserException; -import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.common.util.IdGenerator; -import com.graphicsfuzz.common.util.ParseTimeoutException; -import com.graphicsfuzz.common.util.RandomWrapper; -import com.graphicsfuzz.common.util.ShaderJobFileOperations; -import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.reducer.reductionopportunities.Compatibility; -import com.graphicsfuzz.reducer.reductionopportunities.IReductionOpportunity; -import com.graphicsfuzz.reducer.reductionopportunities.ReducerContext; -import com.graphicsfuzz.reducer.reductionopportunities.ReductionOpportunities; -import com.graphicsfuzz.util.ArgsUtil; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import net.sourceforge.argparse4j.ArgumentParsers; -import net.sourceforge.argparse4j.impl.Arguments; -import net.sourceforge.argparse4j.inf.ArgumentParser; -import net.sourceforge.argparse4j.inf.ArgumentParserException; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.commons.lang3.exception.ExceptionUtils; - -public class ReducerBugPointBasic { - - private static Namespace parse(String[] args) throws ArgumentParserException { - ArgumentParser parser = ArgumentParsers.newArgumentParser("ReducerBugPoint") - .defaultHelp(true) - .description("Find bugs in the reducer."); - - // Required arguments - parser.addArgument("shader") - .help("Path of shader job file (.json) to be analysed.") - .type(File.class); - - // Optional arguments - parser.addArgument("--max-iterations") - .help("Maximum number of times to iterate before giving up.") - .setDefault(30) - .type(Integer.class); - - parser.addArgument("--seed") - .help("Seed (unsigned 64 bit long integer) to initialize random number generator with.") - .type(String.class); - - parser.addArgument("--preserve-semantics") - .help("Only perform semantics-preserving reductions.") - .action(Arguments.storeTrue()); - - parser.addArgument("--ignore-invalid") - .help("Do not log or fix on cases where the shader is invalid - " - + "ignore them and backtrack.") - .action(Arguments.storeTrue()); - - parser.addArgument("--expected-string") - .help("A string to look for in the exception message.") - .type(String.class); - - return parser.parseArgs(args); - } - - - public static void main(String[] args) - throws IOException, ParseTimeoutException, InterruptedException, ArgumentParserException, - GlslParserException { - - final Namespace ns = parse(args); - - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - - final File shaderJobFile = ns.get("shader"); - - final ShadingLanguageVersion shadingLanguageVersion = - ShadingLanguageVersion - .getGlslVersionFromFirstTwoLines( - fileOps.getFirstTwoLinesOfShader(shaderJobFile, ShaderKind.FRAGMENT)); - - IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); - - final ShaderJob originalShaderJob = fileOps.readShaderJobFile(shaderJobFile); - - final int maxIterations = ns.get("max_iterations"); - - final boolean reduceEverywhere = !ns.getBoolean("reduce_everywhere"); - - final boolean ignoreInvalid = ns.get("ignore_invalid"); - - final String expectedString = ns.getString("expected_string") == null - ? "" - : ns.getString("expected_string"); - - final IdGenerator idGenerator = new IdGenerator(); - - ShaderJob lastGoodButLeadingToBadShaderJob = originalShaderJob; - - int exceptionCount = 0; - int invalidCount = 0; - - int getOpsCounter = 0; - - for (int i = 0; i < maxIterations; i++) { - - ShaderJob current = lastGoodButLeadingToBadShaderJob.clone(); - - // TODO: this code was written pre vertex shader support, and does not take - // account of uniforms. - // If it still proves useful as a debugging tool, it could do with updating. - // Paul: partially updated during refactor, but not tested and still makes assumptions - // about frag files. - - while (true) { - final ShaderJob prev = current.clone(); - List ops; - try { - ops = ReductionOpportunities.getReductionOpportunities( - current, - new ReducerContext( - reduceEverywhere, - shadingLanguageVersion, - generator, - idGenerator), - fileOps); - - } catch (Exception exception) { - recordThrowsExceptionWhenGettingReductionOpportunities( - current, - exception, - fileOps); - break; - } - System.out.println(ops.size()); - if (ops.isEmpty()) { - break; - } - final List opsToApply = getOpsToApply(ops, generator); - //System.out.println("About to try applying " + op); - try { - for (IReductionOpportunity op : opsToApply) { - op.applyReduction(); - } - } catch (Exception exception) { - System.err.println("Exception occurred while applying a reduction opportunity."); - if (exception.toString().contains(expectedString)) { - lastGoodButLeadingToBadShaderJob = prev; - current = lastGoodButLeadingToBadShaderJob.clone(); - - fileOps.writeShaderJobFile( - lastGoodButLeadingToBadShaderJob, - new File("leads_to_exception_" + exceptionCount + ".json") - ); - - emitException(exception, "leads_to_exception_" + exceptionCount + ".txt", fileOps); - exceptionCount++; - } else { - System.err.println("The exception was not interesting as it did not contain \"" - + expectedString + "\""); - System.out.println(exception.toString()); - } - continue; - } - if (invalid(current, fileOps)) { - System.err.println("Invalid shader after reduction step."); - if (ignoreInvalid) { - System.err.println("Ignoring it and backtracking."); - current = prev; - } else { - fileOps.writeShaderJobFile( - prev, - new File("leads_to_invalid_" + invalidCount + "_before.json")); - fileOps.writeShaderJobFile( - current, - new File("leads_to_invalid_" + invalidCount + "_after.json")); - invalidCount++; - lastGoodButLeadingToBadShaderJob = prev; - current = lastGoodButLeadingToBadShaderJob.clone(); - } - } - } - } - } - - private static List getOpsToApply(List ops, - IRandom generator) { - List result = new ArrayList<>(); - final IReductionOpportunity initialOp = ops.remove(generator.nextInt(ops.size())); - result.add(initialOp); - for (int i = 0; i < 10; i++) { - if (generator.nextInt(10) < 10 - i) { - break; - } - while (!ops.isEmpty()) { - final IReductionOpportunity op = ops.remove(generator.nextInt(ops.size())); - if (!Compatibility.compatible(initialOp.getClass(), op.getClass())) { - continue; - } - result.add(op); - break; - } - if (ops.isEmpty()) { - break; - } - } - return result; - } - - private static boolean invalid( - ShaderJob shaderJob, - ShaderJobFileOperations fileOps) - throws IOException, InterruptedException { - File tempShaderJobFile = new File("temp_to_validate.json"); - fileOps.writeShaderJobFile( - shaderJob, - tempShaderJobFile); - return fileOps.areShadersValid(tempShaderJobFile, false); - } - - private static void emitException( - Exception exception, - String filename, - ShaderJobFileOperations fileOps) throws IOException { - String stacktrace = ExceptionUtils.getStackTrace(exception); - fileOps.writeStringToFile(new File(filename), stacktrace); - } - - private static void recordThrowsExceptionWhenGettingReductionOpportunities( - ShaderJob shaderJob, - Exception exception, - ShaderJobFileOperations fileOps) throws IOException, ParseTimeoutException, - InterruptedException, GlslParserException { - File tempShaderJobFile = new File("temp.json"); - - fileOps.writeShaderJobFile( - shaderJob, - tempShaderJobFile - ); - - ShaderJob reparsedShaderJob = fileOps.readShaderJobFile( - tempShaderJobFile - ); - - new ReportAstDifferences( - shaderJob.getShaders().get(0), - reparsedShaderJob.getShaders().get(0)); - - final File maybeExistingFile = new File("problem_getting_reduction_opportunities.json"); - - if (!fileOps.doesShaderJobExist(maybeExistingFile) - || (fileOps.getShaderLength(maybeExistingFile, ShaderKind.FRAGMENT) - > fileOps.getShaderLength(tempShaderJobFile, ShaderKind.FRAGMENT))) { - - if (fileOps.doesShaderJobExist(maybeExistingFile)) { - fileOps.deleteShaderJobFile(maybeExistingFile); - } - fileOps.moveShaderJobFileTo(tempShaderJobFile, maybeExistingFile, true); - final File maybeExistingExceptionFile = - new File("problem_getting_reduction_opportunities.txt"); - if (fileOps.isFile(maybeExistingExceptionFile)) { - fileOps.deleteFile(maybeExistingExceptionFile); - } - String stacktrace = ExceptionUtils.getStackTrace(exception); - fileOps.writeStringToFile(maybeExistingExceptionFile, stacktrace); - } - } - - -} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReportAstDifferences.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReportAstDifferences.java deleted file mode 100644 index a13cbe591..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReportAstDifferences.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.tool; - -import com.graphicsfuzz.common.ast.IAstNode; -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; -import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; -import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; -import java.util.ArrayList; -import java.util.List; - -public class ReportAstDifferences { - - private class TraversalVisitor extends StandardVisitor { - - private List nodes = new ArrayList<>(); - - TraversalVisitor(TranslationUnit tu) { - visit(tu); - } - - @Override - public void visit(IAstNode node) { - nodes.add(node); - super.visit(node); - } - - List getNodes() { - return nodes; - } - - } - - private class InformativeScopeTreeBuilder extends ScopeTreeBuilder { - - InformativeScopeTreeBuilder(TranslationUnit tu) { - visit(tu); - } - - @Override - protected void pushScope() { - System.out.println("Entering a scope"); - super.pushScope(); - } - - @Override - protected void popScope() { - super.popScope(); - System.out.println("Leaving a scope"); - } - - @Override - public void visitFunctionDefinition(FunctionDefinition functionDefinition) { - System.out.println("Entering function " + functionDefinition.getPrototype().getName()); - super.visitFunctionDefinition(functionDefinition); - System.out.println("Leaving function " + functionDefinition.getPrototype().getName()); - } - - @Override - public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { - System.out.println("Entering variables declaration"); - for (VariableDeclInfo vdi : variablesDeclaration.getDeclInfos()) { - System.out.println(" " + vdi.getName()); - } - super.visitVariablesDeclaration(variablesDeclaration); - System.out.println("Leaving variables declaration"); - } - } - - public ReportAstDifferences(TranslationUnit tu1, TranslationUnit tu2) { - try { - new InformativeScopeTreeBuilder(tu1); - } catch (Exception exception) { - System.out.println("Exception was thrown:"); - exception.printStackTrace(System.out); - } - try { - new InformativeScopeTreeBuilder(tu2); - } catch (Exception exception) { - System.out.println("Exception was thrown:"); - exception.printStackTrace(System.out); - } - - /*List nodes1 = new TraversalVisitor(tu1).getNodes(); - List nodes2 = new TraversalVisitor(tu2).getNodes(); - Map oneToTwo = new HashMap<>(); - Map twoToOne = new HashMap<>(); - IParentMap parentMap1 = IParentMap.createParentMap(tu1); - IParentMap parentMap2 = IParentMap.createParentMap(tu2); - for (int i = 0; i < Math.min(nodes1.size(), nodes2.size()); i++) { - IAstNode one = nodes1.get(i); - IAstNode two = nodes1.get(i); - if (oneToTwo.containsKey(one) || twoToOne.containsKey(two)) { - if (oneToTwo.get(one) != two || twoToOne.get(two) != one) { - throw new RuntimeException(); - } - } else { - oneToTwo.put(one, two); - twoToOne.put(two, one); - } - } - if (nodes1.size() != nodes2.size()) { - throw new RuntimeException(); - }*/ - } - -} From 71c8d4abf5989f4f7b0694380db8bfe2e763cf59 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Thu, 25 Jul 2019 03:25:46 -0500 Subject: [PATCH 078/180] Add additional uniform types to piglit converter (#638) --- .../drivers/graphicsfuzz_piglit_converter.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py index 6deae302d..e10f2e16e 100644 --- a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py +++ b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py @@ -23,7 +23,7 @@ # Note: We define a 'shader job' as the JSON file of a GraphicsFuzz shader. # Each shader job has a corresponding shader file that has the same base name # as the shader job - for example, a shader job file named 'variant_005.json' will -# have a corresponding shader file 'variant_005.frag' or 'variant_005.comp' in +# have a corresponding shader file 'variant_005.frag' or 'variant_005.comp' in # the same directory. # Piglit test section headers @@ -56,10 +56,23 @@ 'glUniform2i': 'ivec2', 'glUniform3i': 'ivec3', 'glUniform4i': 'ivec4', + 'glUniform1ui': 'uint', + 'glUniform2ui': 'uvec2', + 'glUniform3ui': 'uvec3', + 'glUniform4ui': 'uvec4', 'glUniform1f': 'float', 'glUniform2f': 'vec2', 'glUniform3f': 'vec3', 'glUniform4f': 'vec4', + 'glUniformMatrix2fv': 'mat2', + 'glUniformMatrix3fv': 'mat3', + 'glUniformMatrix4fv': 'mat4', + 'glUniformMatrix2x3fv': 'mat2x3', + 'glUniformMatrix3x2fv': 'mat3x2', + 'glUniformMatrix2x4fv': 'mat2x4', + 'glUniformMatrix4x2fv': 'mat4x2', + 'glUniformMatrix3x4fv': 'mat3x4', + 'glUniformMatrix4x3fv': 'mat4x3' } UNIFORM_DEC = 'uniform' @@ -222,7 +235,7 @@ def main_helper(args: List[str]) -> None: argparser = argparse.ArgumentParser(description=description) argparser.add_argument( - 'shader_job', + 'shader_job', help='Path to the GraphicsFuzz shader job JSON file.') argparser.add_argument( @@ -230,9 +243,9 @@ def main_helper(args: List[str]) -> None: action='store_true', help='Do not draw the shader output when running the test. Useful for crash testing.' ) - + args = argparser.parse_args(args) - + test_string = make_shader_test_string(args.shader_job, args.nodraw) with gfuzz_common.open_helper(get_shader_test_from_job(args.shader_job), 'w') as shader_test: From 6d888a57278042fb31edf366d08d2835caf07c45 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 31 Jul 2019 12:24:17 -0500 Subject: [PATCH 079/180] Piglit worker improvements (#632) --- .../src/main/python/drivers/gfuzz_common.py | 60 ++++++ .../src/main/python/drivers/piglit-worker.py | 171 +++++++++++------- 2 files changed, 168 insertions(+), 63 deletions(-) diff --git a/python/src/main/python/drivers/gfuzz_common.py b/python/src/main/python/drivers/gfuzz_common.py index a1f8e5a1e..9de9119e6 100644 --- a/python/src/main/python/drivers/gfuzz_common.py +++ b/python/src/main/python/drivers/gfuzz_common.py @@ -16,6 +16,7 @@ import os import shutil +import subprocess from typing import List import runspv @@ -70,6 +71,9 @@ def tool_on_path(tool: str) -> str: return runspv.tool_on_path(tool) +ToolNotOnPathError = runspv.ToolNotOnPathError + + def remove_end(str_in: str, str_end: str) -> str: """ Helper function to remove the end of a string. Useful for removing file extensions if you @@ -133,3 +137,59 @@ def remove_start(s: str, start: str): """ assert s.startswith(start) return s[len(start):] + + +def subprocess_helper(cmd: List[str], check=True, timeout=None, verbose=False) \ + -> subprocess.CompletedProcess: + """ + Helper function to execute a command in the shell. Wraps around runspv.subprocess_helper(). + :param cmd: the command (and its arguments) to execute. + :param check: whether to throw CalledProcessError if a non-zero returncode is issued from the + process. Defaults to True. + :param timeout: time to wait in seconds before killing process and throwing TimeoutExpired. + Defaults to None. + :param verbose: whether to log process stdout/stderr more extensively. Defaults to False. + :return: a subprocess.CompletedProcess object. + """ + return runspv.subprocess_helper(cmd, check, timeout, verbose) + + +def run_catchsegv(cmd: List[str], timeout=None, verbose=False) -> str: + """ + Helper function similar to subprocess_helper, but is able to handle the killing of child + processes as well. This is most useful when trying to capture segmentation faults, as you can + run 'catchsegv (command)' in shell to ensure that the segfault is logged. Wraps around + runspv.run_catchsegv(). + :param cmd: the command (and its arguments) to execute. + :param timeout: time to wait in seconds before killing process and throwing TimeoutExpired. + Defaults to None. + :param verbose: whether to log process stdout/stderr more extensively. Defaults to False. + :return: the status of the process after running - 'SUCCESS', 'TIMEOUT', or 'CRASH'. + """ + return runspv.run_catchsegv(cmd, timeout, verbose) + + +def log(message: str): + """ + Helper function to output a message to stdout, and optionally log to a file if set_logfile() + was used previously to set a global logfile. Wraps around runspv.log(). + :param message: the message to log to stdout/file. + """ + runspv.log(message) + + +def set_logfile(file): + """ + Helper function to set the global logfile. Wraps around runspv.log_to_file. Workaround for + runspv.log() using a global logfile. + :param file: the file to log to - must be opened already. + """ + runspv.log_to_file = file + + +def unset_logfile(): + """ + Helper function to unset the global logfile. Wraps around runspv.log_to_file. Workaround for + runspv.log() using a global logfile. + """ + runspv.log_to_file = None diff --git a/python/src/main/python/drivers/piglit-worker.py b/python/src/main/python/drivers/piglit-worker.py index b5fa16f98..20ab716a6 100644 --- a/python/src/main/python/drivers/piglit-worker.py +++ b/python/src/main/python/drivers/piglit-worker.py @@ -18,13 +18,12 @@ import json import os import sys -import subprocess +import filecmp import shutil -from subprocess import CalledProcessError import time +from typing import List import gfuzz_common -import runspv import graphicsfuzz_piglit_converter HERE = os.path.abspath(__file__) @@ -48,40 +47,44 @@ JSON_SUFFIX = '.json' PNG_SUFFIX = '.png' SHADER_TEST_SUFFIX = '.shader_test' -PNG_FILENAME = 'shader_runner_gles3000.png' -LOGFILE_NAME = 'piglit_log.txt' -STATUS_FILENAME = 'STATUS' SHADER_RUNNER_ARG_PNG = '-png' SHADER_RUNNER_ARG_AUTO = '-auto' SHADER_RUNNER_ARG_UNIFORMS = '-ignore-missing-uniforms' +SHADER_RUNNER_ARG_FBO = '-fbo' SHADER_RUNNER_ARG_SUBTESTS = '-report-subtests' + WORKER_INFO_FILE = 'worker_info.json' +PNG_FILENAME = 'shader_runner_gles3000.png' +COMPARE_PNG_FILENAME = 'shader_runner_gles3001.png' +NONDET0_PNG = 'nondet0.png' +NONDET1_PNG = 'nondet1.png' +LOGFILE_NAME = 'piglit_log.txt' +STATUS_FILENAME = 'STATUS' NO_DRAW_ARG = '--nodraw' -RETURNCODE_STR = 'Returncode: ' -STDOUT_STR = 'STDOUT: ' -STDERR_STR = 'STDERR: ' - STATUS_SUCCESS = 'SUCCESS' STATUS_CRASH = 'CRASH' STATUS_TIMEOUT = 'TIMEOUT' -STATUS_SANITYERROR = 'SANITY_ERROR' STATUS_UNEXPECTED = 'UNEXPECTED_ERROR' STATUS_NONDET = 'NONDET' TIMEOUT = 30 -def glxinfo_cmd(): +def glxinfo_cmd() -> List[str]: return [gfuzz_common.tool_on_path('glxinfo'), '-B'] -def shader_runner_cmd(): +def shader_runner_cmd() -> List[str]: return [gfuzz_common.tool_on_path('shader_runner_gles3')] +def catchsegv_cmd() -> str: + return gfuzz_common.tool_on_path('catchsegv') + + def thrift_connect(server: str, worker_name: str, worker_info: str) -> (FuzzerService, str): """ Helper function to initiate a connection from this worker to a Thrift server. @@ -102,18 +105,18 @@ def thrift_connect(server: str, worker_name: str, worker_info: str) -> (FuzzerSe transport.open() # Get worker name - runspv.log("Call getWorkerName()") + gfuzz_common.log("Call getWorkerName()") worker_res = service.getWorkerName(worker_info, worker_name) assert type(worker_res) is not None if worker_res.workerName is None: # noinspection PyProtectedMember - runspv.log('Worker error: ' + tt.WorkerNameError._VALUES_TO_NAMES[worker_res.error]) + gfuzz_common.log('Worker error: ' + tt.WorkerNameError._VALUES_TO_NAMES[worker_res.error]) exit(1) worker = worker_res.workerName - runspv.log("Got worker: " + worker) + gfuzz_common.log("Got worker: " + worker) assert (worker == worker_name) return service, worker @@ -133,7 +136,7 @@ def dump_glxinfo(filename: str) -> None: # into JSON. glxinfo_lines = filter( lambda glx_line: 'OpenGL' in glx_line, - runspv.subprocess_helper(glxinfo_cmd()).stdout.split('\n')) + gfuzz_common.subprocess_helper(glxinfo_cmd()).stdout.split('\n')) # We form keys out of the OpenGL info descriptors and values out of the hardware dependent # strings. For example, "OpenGL version string: 4.6.0 NVIDIA 430.14" would become # { "OpenGL version string": "4.6.0 NVIDIA 430.14" }. @@ -172,6 +175,8 @@ def do_image_job(image_job: tt.ImageJob, work_dir: str) -> tt.ImageJobResult: log_file = os.path.join(output_dir, LOGFILE_NAME) status_file = os.path.join(output_dir, STATUS_FILENAME) png_file = os.path.join(output_dir, name + PNG_SUFFIX) + nondet_0 = os.path.join(output_dir, NONDET0_PNG) + nondet_1 = os.path.join(output_dir, NONDET1_PNG) gfuzz_common.write_to_file(image_job.fragmentSource, frag_file) gfuzz_common.write_to_file(image_job.uniformsInfo, json_file) @@ -184,15 +189,14 @@ def do_image_job(image_job: tt.ImageJob, work_dir: str) -> tt.ImageJobResult: with gfuzz_common.open_helper(log_file, 'w') as f: try: - runspv.log_to_file = f - run_image_job(frag_file, json_file, status_file, png_file, - output_dir, image_job.skipRender) + gfuzz_common.set_logfile(f) + run_image_job(json_file, status_file, png_file, output_dir, image_job.skipRender) except Exception as ex: - runspv.log(str(ex)) - runspv.log('Removing status file and continuing...') + gfuzz_common.log(str(ex)) + gfuzz_common.log('Removing status file and continuing...') gfuzz_common.remove(status_file) finally: - runspv.log_to_file = None + gfuzz_common.unset_logfile() if os.path.isfile(log_file): with gfuzz_common.open_helper(log_file, 'r') as f: @@ -205,14 +209,20 @@ def do_image_job(image_job: tt.ImageJob, work_dir: str) -> tt.ImageJobResult: if os.path.isfile(status_file): with gfuzz_common.open_helper(status_file, 'r') as f: status = f.read().rstrip() - if status == 'SUCCESS': + if status == STATUS_SUCCESS: res.status = tt.JobStatus.SUCCESS - elif status == 'CRASH': + elif status == STATUS_CRASH: res.status = tt.JobStatus.CRASH - elif status == 'TIMEOUT': + elif status == STATUS_TIMEOUT: res.status = tt.JobStatus.TIMEOUT - elif status == 'UNEXPECTED_ERROR': + elif status == STATUS_UNEXPECTED: res.status = tt.JobStatus.UNEXPECTED_ERROR + elif status == STATUS_NONDET: + res.status = tt.JobStatus.NONDET + with gfuzz_common.open_bin_helper(nondet_0, 'rb') as f: + res.PNG = f.read() + with gfuzz_common.open_bin_helper(nondet_1, 'rb') as f: + res.PNG2 = f.read() else: res.log += '\nUnknown status value: ' + status + '\n' res.status = tt.JobStatus.UNEXPECTED_ERROR @@ -224,12 +234,11 @@ def do_image_job(image_job: tt.ImageJob, work_dir: str) -> tt.ImageJobResult: return res -def run_image_job(frag_file: str, json_file: str, status_file: str, +def run_image_job(json_file: str, status_file: str, png_file: str, output_dir: str, skip_render: bool): """ Runs an image job. Converts the shader job to a piglit shader_test file, then delegates to run_shader_test to render with shader_runner. Writes the status of the job to file. - :param frag_file: The fragment shader to convert and run. :param json_file: The JSON uniforms to use with the shader. :param status_file: The status file to write to. :param png_file: The PNG file to write to. @@ -237,8 +246,14 @@ def run_image_job(frag_file: str, json_file: str, status_file: str, :param skip_render: whether to skip rendering or not. """ + use_catchsegv = True + + try: + gfuzz_common.tool_on_path('catchsegv') + except gfuzz_common.ToolNotOnPathError: + use_catchsegv = False + assert os.path.isdir(output_dir) - assert os.path.isfile(frag_file) assert os.path.isfile(json_file) arglist = [json_file] @@ -248,34 +263,64 @@ def run_image_job(frag_file: str, json_file: str, status_file: str, shader_test_file = graphicsfuzz_piglit_converter.get_shader_test_from_job(json_file) try: - runspv.log('Creating shader_test file...') + gfuzz_common.log('Creating shader_test file...') graphicsfuzz_piglit_converter.main_helper(arglist) except Exception as ex: - runspv.log('Could not create shader_test from the given job.') + gfuzz_common.log('Could not create shader_test from the given job.') raise ex - - status = STATUS_SUCCESS - - shader_runner_cmd_list = shader_runner_cmd() + [shader_test_file, SHADER_RUNNER_ARG_AUTO, - SHADER_RUNNER_ARG_UNIFORMS, - SHADER_RUNNER_ARG_SUBTESTS] + shader_runner_cmd_list = shader_runner_cmd() + \ + [shader_test_file, SHADER_RUNNER_ARG_AUTO, + SHADER_RUNNER_ARG_UNIFORMS, SHADER_RUNNER_ARG_FBO, SHADER_RUNNER_ARG_SUBTESTS] + if use_catchsegv: + shader_runner_cmd_list.insert(0, catchsegv_cmd()) if not skip_render: shader_runner_cmd_list.append(SHADER_RUNNER_ARG_PNG) - try: - runspv.subprocess_helper(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) - except subprocess.TimeoutExpired: - status = STATUS_TIMEOUT - except subprocess.CalledProcessError: - status = STATUS_CRASH - runspv.log('STATUS: ' + status) + gfuzz_common.remove(PNG_FILENAME) + gfuzz_common.remove(COMPARE_PNG_FILENAME) + + status = \ + gfuzz_common.run_catchsegv(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) \ + if use_catchsegv else \ + gfuzz_common.subprocess_helper(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) # Piglit throws the output PNG render into whatever the current working directory is # (and there's no way to specify a location to write to) - we need to move it to wherever our # output is. - if os.path.isfile(PNG_FILENAME): - shutil.move(PNG_FILENAME, png_file) + if not skip_render and status == STATUS_SUCCESS: + try: + # An image was rendered, so we need to check for nondet. We do this by renaming the + # rendered image, rendering a second image, and using filecmp to compare the files. + assert os.path.isfile(PNG_FILENAME), \ + "Shader runner successfully rendered, but no image was dumped?" + gfuzz_common.log('An image was rendered - rendering again to check for nondet.') + os.rename(PNG_FILENAME, COMPARE_PNG_FILENAME) + status = \ + gfuzz_common.run_catchsegv(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) \ + if use_catchsegv else \ + gfuzz_common.subprocess_helper(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) + # Something is horribly wrong if shader crashes/timeouts are inconsistent per shader. + assert status == STATUS_SUCCESS, \ + "Shader inconsistently fails - check your graphics drivers?" + assert os.path.isfile(PNG_FILENAME), \ + "Shader runner successfully rendered, but no image was dumped?" + + gfuzz_common.log('Comparing dumped PNG images...') + if filecmp.cmp(PNG_FILENAME, COMPARE_PNG_FILENAME): + gfuzz_common.log('Images are identical.') + shutil.move(PNG_FILENAME, png_file) + else: + gfuzz_common.log('Images are different.') + status = STATUS_NONDET + shutil.move(COMPARE_PNG_FILENAME, os.path.join(output_dir, NONDET0_PNG)) + shutil.move(PNG_FILENAME, os.path.join(output_dir, NONDET1_PNG)) + finally: + gfuzz_common.log('Removing dumped images...') + gfuzz_common.remove(PNG_FILENAME) + gfuzz_common.remove(COMPARE_PNG_FILENAME) + + gfuzz_common.log('STATUS: ' + status) with gfuzz_common.open_helper(status_file, 'w') as f: f.write(status) @@ -304,33 +349,31 @@ def main(): args = parser.parse_args() gfuzz_common.tool_on_path('shader_runner_gles3') - gfuzz_common.tool_on_path('glxinfo') - - runspv.log('Worker: ' + args.worker_name) + gfuzz_common.log('Worker: ' + args.worker_name) server = args.server + '/request' - runspv.log('server: ' + server) + gfuzz_common.log('server: ' + server) # Get worker info worker_info_json_string = '{}' - runspv.log('Dumping glxinfo to file for worker info string...') + gfuzz_common.log('Dumping glxinfo to file for worker info string...') try: dump_glxinfo(WORKER_INFO_FILE) with gfuzz_common.open_helper(WORKER_INFO_FILE, 'r') as info_file: worker_info_json_string = info_file.read() except Exception as ex: - runspv.log(str(ex)) - runspv.log('Could not get worker info, continuing without it.') + gfuzz_common.log(str(ex)) + gfuzz_common.log('Could not get worker info, continuing without it.') service = None worker = None while True: if not service: - runspv.log('Connecting to server...') + gfuzz_common.log('Connecting to server...') service, worker = thrift_connect(server, args.worker_name, worker_info_json_string) if not service: - runspv.log('Failed to connect, retrying...') + gfuzz_common.log('Failed to connect, retrying...') time.sleep(1) continue @@ -341,25 +384,27 @@ def main(): try: job = service.getJob(worker) if job.noJob is not None: - runspv.log("No job") + gfuzz_common.log("No job") elif job.skipJob is not None: - runspv.log("Skip job") + gfuzz_common.log("Skip job") service.jobDone(worker, job) else: assert job.imageJob if job.imageJob.computeSource: - runspv.log("Got a compute job, but this worker " - "doesn't support compute shaders.") + gfuzz_common.log("Got a compute job, but this worker " + "doesn't support compute shaders.") job.imageJob.result = tt.ImageJobResult() job.imageJob.result.status = tt.JobStatus.UNEXPECTED_ERROR else: - runspv.log("#### Image job: " + job.imageJob.name) + gfuzz_common.log("#### Image job: " + job.imageJob.name) job.imageJob.result = do_image_job(job.imageJob, work_dir=worker) - runspv.log("Sending back, results status: {}".format(job.imageJob.result.status)) + gfuzz_common.log("Sending back, results status: {}" + .format(job.imageJob.result.status)) service.jobDone(worker, job) + gfuzz_common.remove(worker) continue except (TApplicationException, ConnectionError): - runspv.log("Connection to server lost. Re-initialising client.") + gfuzz_common.log("Connection to server lost. Re-initialising client.") service = None time.sleep(1) From 480e3794aac906edab952d41aecfc5c482db9a5e Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Mon, 5 Aug 2019 13:04:24 +0100 Subject: [PATCH 080/180] Improve reducer's ability to remove return statements (#624) The reducer needs to be a little conservative about removing non-void return statements, to avoid making a shader invalid. This change makes the reducer less conservative by allowing it to remove a statement that is or that contains a non-void return statement if either: - the enclosing function ends with a non-void return statement that is not the statement being considered for removal, or - the statement being considered for removal is in a block and is preceded by a return statement in the block. Some JavaDoc comments have been added along the way. Fixes #493 --- .../common/ast/stmt/BlockStmt.java | 14 +++ .../StmtReductionOpportunities.java | 43 ++------ .../StmtReductionOpportunity.java | 75 +++++++++++-- .../reducer/ReductionDriverTest.java | 75 +++++++++++++ .../StmtReductionOpportunitiesTest.java | 102 ++++++++++++++++-- 5 files changed, 256 insertions(+), 53 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/BlockStmt.java b/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/BlockStmt.java index abfa2b928..16e7de17b 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/BlockStmt.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/BlockStmt.java @@ -58,6 +58,16 @@ public int getNumStmts() { return stmts.size(); } + /** + * Requires the block to be non-empty. + * Yields the final statement in the block. + * @return The final statement of the block. + */ + public Stmt getLastStmt() { + assert !stmts.isEmpty(); + return stmts.get(stmts.size() - 1); + } + public boolean introducesNewScope() { return introducesNewScope; } @@ -154,6 +164,10 @@ public void insertBefore(Stmt originalStmt, Stmt insertedStmt) { throw new IllegalArgumentException("Should be unreachable."); } + /** + * Adds the given statement to the end of the block. + * @param stmt A statement to be added to the block + */ public void addStmt(Stmt stmt) { stmts.add(stmt); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java index af38009f0..b6f49085b 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java @@ -34,7 +34,6 @@ import com.graphicsfuzz.common.ast.stmt.IfStmt; import com.graphicsfuzz.common.ast.stmt.LoopStmt; import com.graphicsfuzz.common.ast.stmt.NullStmt; -import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; @@ -80,7 +79,8 @@ protected void visitChildOfBlock(BlockStmt block, int index) { final Stmt child = block.getStmt(index); if (isEmptyBlockStmt(child) || isDeadCodeInjection(child) || allowedToReduceStmt(block, index)) { - addOpportunity(new StmtReductionOpportunity(block, child, getVistitationDepth())); + addOpportunity(new StmtReductionOpportunity(enclosingFunction, block, child, + getVistitationDepth())); } } @@ -127,12 +127,11 @@ private boolean allowedToReduceStmt(BlockStmt block, int childIndex) { return true; } - // We cannot remove non-void return statements without special care, unless we are inside an - // injected block - if (containsNonVoidReturn(stmt)) { - if (!isReturnFollowedBySubsequentReturn(block, childIndex)) { - return false; - } + // Unless we are in an injected dead code block, we need to be careful about removing + // non-void return statements, so as to avoid making the shader invalid. + if (StmtReductionOpportunity + .removalCouldLeadToLackOfReturnFromNonVoidFunction(enclosingFunction, block, stmt)) { + return false; } return context.reduceEverywhere() @@ -141,23 +140,6 @@ private boolean allowedToReduceStmt(BlockStmt block, int childIndex) { } - /** - * Returns true if and only if the statement at childIndex is a return statement, and the block - * has another return statement at a later index. - */ - private boolean isReturnFollowedBySubsequentReturn(BlockStmt block, int childIndex) { - if (!(block.getStmt(childIndex) instanceof ReturnStmt)) { - return false; - } - for (int subsequentChildIndex = childIndex + 1; subsequentChildIndex < block.getNumStmts(); - subsequentChildIndex++) { - if (block.getStmt(subsequentChildIndex) instanceof ReturnStmt) { - return true; - } - } - return false; - } - private boolean isLoopLimiterBlock(Stmt stmt) { // Identifies when a block starts with a loop-limiter declaration, in which case the whole // block can go. We are really careful about otherwise removing loop-limiters, so this is @@ -324,17 +306,6 @@ private static boolean isLiveInjectionVariableReference(Expr lhs) { return isLiveInjectedVariableName(((VariableIdentifierExpr) lhs).getName()); } - private boolean containsNonVoidReturn(Stmt stmt) { - return new CheckPredicateVisitor() { - @Override - public void visitReturnStmt(ReturnStmt returnStmt) { - if (returnStmt.hasExpr()) { - predicateHolds(); - } - } - }.test(stmt); - } - private boolean isRedundantCopy(Stmt stmt) { if (!(stmt instanceof ExprStmt)) { return false; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunity.java index 2c63c9fee..0a7c9fa2d 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunity.java @@ -16,15 +16,18 @@ package com.graphicsfuzz.reducer.reductionopportunities; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.CaseLabel; +import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; -import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.util.StatsVisitor; public final class StmtReductionOpportunity extends AbstractReductionOpportunity { + private final FunctionDefinition enclosingFunction; private final BlockStmt blockStmt; private final Stmt childOfBlockStmt; @@ -33,19 +36,24 @@ public final class StmtReductionOpportunity extends AbstractReductionOpportunity // due to the effects of other opportunities). private final int numRemovableNodes; - public StmtReductionOpportunity(BlockStmt blockStmt, + /** + * Creates an opportunity to remove childOfBlockStmt from blockStmt. + * @param enclosingFunction A function that encloses the statement to be removed + * @param blockStmt A block that directly encloses the statement to be removed + * @param childOfBlockStmt The statement to be removed + * @param depth The depth at which this opportunity was found + */ + public StmtReductionOpportunity(FunctionDefinition enclosingFunction, + BlockStmt blockStmt, Stmt childOfBlockStmt, VisitationDepth depth) { super(depth); + this.enclosingFunction = enclosingFunction; this.blockStmt = blockStmt; this.childOfBlockStmt = childOfBlockStmt; this.numRemovableNodes = new StatsVisitor(childOfBlockStmt).getNumNodes(); } - public Stmt getChild() { - return childOfBlockStmt; - } - @Override public void applyReductionImpl() { for (int i = 0; i < blockStmt.getNumStmts(); i++) { @@ -94,6 +102,11 @@ public boolean preconditionHolds() { return false; } + if (removalCouldLeadToLackOfReturnFromNonVoidFunction(enclosingFunction, blockStmt, + childOfBlockStmt)) { + return false; + } + return true; } @@ -101,4 +114,54 @@ public int getNumRemovableNodes() { return numRemovableNodes; } + static boolean removalCouldLeadToLackOfReturnFromNonVoidFunction( + FunctionDefinition enclosingFunction, BlockStmt block, Stmt childOfBlockStmt) { + if (!containsNonVoidReturn(childOfBlockStmt)) { + return false; + } + if (statementIsPrecededByReturn(block, childOfBlockStmt)) { + return false; + } + if (functionEndsWithReturn(enclosingFunction) && !isLastStatementInFunction(enclosingFunction, + childOfBlockStmt)) { + return false; + } + return true; + } + + private static boolean containsNonVoidReturn(Stmt stmt) { + return new CheckPredicateVisitor() { + @Override + public void visitReturnStmt(ReturnStmt returnStmt) { + if (returnStmt.hasExpr()) { + predicateHolds(); + } + } + }.test(stmt); + } + + private static boolean statementIsPrecededByReturn(BlockStmt block, Stmt childOfBlockStmt) { + for (Stmt stmt : block.getStmts()) { + if (stmt == childOfBlockStmt) { + return false; + } + if (stmt instanceof ReturnStmt) { + return true; + } + } + throw new RuntimeException("Should be unreachable."); + } + + private static boolean functionEndsWithReturn(FunctionDefinition functionDefinition) { + return functionDefinition.getBody().getNumStmts() > 0 + && functionDefinition.getBody().getLastStmt() + instanceof ReturnStmt; + } + + private static boolean isLastStatementInFunction(FunctionDefinition functionDefinition, + Stmt stmt) { + return functionDefinition.getBody().getNumStmts() > 0 + && functionDefinition.getBody().getLastStmt() == stmt; + } + } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java index 4566f8462..3777bb9eb 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java @@ -24,6 +24,8 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; +import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; @@ -652,6 +654,79 @@ public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { } + @Test + public void testEliminationOfReturns() throws Exception { + final String shader = "#version 310 es\n" + + "int foo() {\n" + + " if (false) {\n" + + " return 1;\n" + + " }\n" + + " return 0;\n" + + "}\n" + + "void main() {\n" + + " if (true) {\n" + + " foo();\n" + + " return;\n" + + " } else {\n" + + " return;\n" + + " }\n" + + " return;\n" + + "}\n"; + + final String expected = "#version 310 es\n" + + "int foo() {\n" + + " return 1;\n" + + "}\n" + + "void main() {\n" + + " foo();\n" + + "}\n"; + + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), + ParseHelper.parse(shader)); + + final File workDir = testFolder.getRoot(); + final File tempShaderJobFile = new File(workDir, "temp.json"); + fileOps.writeShaderJobFile(shaderJob, tempShaderJobFile); + + final IFileJudge usesFooFileJudge = + new CheckAstFeaturesFileJudge(Arrays.asList( + () -> new CheckAstFeatureVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { + super.visitFunctionCallExpr(functionCallExpr); + if (functionCallExpr.getCallee().equals("foo")) { + trigger(); + } + } + }, + () -> new CheckAstFeatureVisitor() { + @Override + public void visitReturnStmt(ReturnStmt returnStmt) { + super.visitReturnStmt(returnStmt); + if (returnStmt.hasExpr() && returnStmt.getExpr() instanceof IntConstantExpr && + ((IntConstantExpr) returnStmt.getExpr()).getNumericValue() == 1) { + trigger(); + } + } + }), + ShaderKind.FRAGMENT, + fileOps); + + final String resultsPrefix = new ReductionDriver(new ReducerContext(true, + ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), + new IdGenerator()), + false, + fileOps, + usesFooFileJudge, + workDir) + .doReduction(shaderJob, "temp", 0, 100); + + CompareAsts.assertEqualAsts(expected, ParseHelper.parse(new File(testFolder.getRoot(), resultsPrefix + ".frag"))); + + } + private String getPrefix(File tempFile) { return FilenameUtils.removeExtension(tempFile.getName()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java index c48575c66..855ade1ea 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java @@ -25,6 +25,7 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; @@ -283,35 +284,54 @@ public void testDeadBasicReturnsPickedUp() throws Exception { @Test public void testRemoveDuplicateReturn1() throws Exception { final String program = "int foo() { return 0; return 1; return 2; return 3; }"; - final String expected = "int foo() { return 3; }"; + final String expected = "int foo() { return 1; }"; final TranslationUnit tu = ParseHelper.parse(program); - final List ops = StmtReductionOpportunities.findOpportunities( - MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), - new IdGenerator())); - assertEquals(3, ops.size()); + final ReducerContext context = new ReducerContext(true, ShadingLanguageVersion.ESSL_300, + new RandomWrapper(0), + new IdGenerator()); + final ShaderJob shaderJob = MakeShaderJobFromFragmentShader.make(tu); + List ops = StmtReductionOpportunities.findOpportunities( + shaderJob, + context); + assertEquals(4, ops.size()); ops.get(0).applyReduction(); - ops.get(1).applyReduction(); + ops.get(3).applyReduction(); ops.get(2).applyReduction(); + assertFalse(ops.get(1).preconditionHolds()); CompareAsts.assertEqualAsts(expected, tu); + ops = StmtReductionOpportunities.findOpportunities( + shaderJob, + context); + assertTrue(ops.isEmpty()); } @Test public void testRemoveDuplicateReturn2() throws Exception { - final String program = "int foo() { int x; return 0; x = 1; return 1; x = 2; return 2; return " - + "3; }"; - final String expected = "int foo() { int x; return 3; }"; + final String program = "int foo() {\n" + + " int x;\n" + + " return 0;\n" + + " x = 1;\n" + + " return 1;\n" + + " x = 2;\n" + + " return 2;\n" + + " return 3;\n" + + "}\n"; + final String expected = "int foo() {\n" + + " int x;\n" + + " return 3;\n" + + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), new IdGenerator())); - assertEquals(5, ops.size()); + assertEquals(6, ops.size()); ops.get(0).applyReduction(); ops.get(1).applyReduction(); ops.get(2).applyReduction(); ops.get(3).applyReduction(); ops.get(4).applyReduction(); + assertFalse(ops.get(5).preconditionHolds()); CompareAsts.assertEqualAsts(expected, tu); } @@ -326,6 +346,66 @@ public void testRemoveArbitraryVoidReturn() throws Exception { assertEquals(5, ops.size()); } + @Test + public void testRemoveNonVoidReturn() throws Exception { + // Both return statements are candidates for removal, but removing one disables removing the + // other. + final String program = "int foo() { return 1; return 0; }"; + final String expected = "int foo() { return 0; }"; + final TranslationUnit tu = ParseHelper.parse(program); + final ReducerContext context = new ReducerContext(true, ShadingLanguageVersion.ESSL_300, + new RandomWrapper(0), new IdGenerator()); + final ShaderJob shaderJob = MakeShaderJobFromFragmentShader.make(tu); + List ops = StmtReductionOpportunities.findOpportunities( + shaderJob, + context); + assertEquals(2, ops.size()); + ops.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + assertFalse(ops.get(1).preconditionHolds()); + ops = StmtReductionOpportunities.findOpportunities( + shaderJob, + context); + assertTrue(ops.isEmpty()); + } + + @Test + public void testDoNotRemoveReturnThatWouldChangeSemantics() throws Exception { + final String program = "layout(location = 0) out vec4 color;\n" + + "void main() {\n" + + " return;\n" + + " color = vec4(1.0);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = StmtReductionOpportunities.findOpportunities( + MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), + new IdGenerator())); + // We cannot remove the return, as this would make the write to 'color' reachable. + // We could in principle remove the color write itself, since it is unreachable. Right now we + // do not, so this test would have to be re-thought if we decided to add that facility. + assertEquals(0, ops.size()); + } + + @Test + public void testRemoveReturnsInIf() throws Exception { + final String program = "layout(location = 0) out vec4 color;\n" + + "int foo() {\n" + + " if (true) {\n" + + " return 1;\n" + + " } else {\n" + + " return 2;\n" + + " }\n" + + " return 0;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = StmtReductionOpportunities.findOpportunities( + MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), + new IdGenerator())); + assertTrue(ops.size() > 0); + } + @Test public void testIdentifyLiveCodeProperly() throws Exception { final String liveInjectedVariableName = Constants.LIVE_PREFIX + "somevar"; From e1342372ff2e58dddc00b06d094e02bf1240a637 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Mon, 5 Aug 2019 07:09:14 -0500 Subject: [PATCH 081/180] Revert shift opaque to use identities instead of fuzzed clamped exprs (#641) --- .../fuzzer/OpaqueExpressionGenerator.java | 62 ++++++------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index dc7b0ff91..d24cee0f7 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -185,14 +185,10 @@ private Optional opaqueZeroOrOneAbsolute(BasicType type, boolean constCont * This limits our maximum shift value to 7 bits (more on how we shift an opaque one below). * Possibilities for generating an opaque zero include: * Shifting an opaque zero by m bits: (opaque zero) >> n or (opaque zero) << n, where n is - * an expression of a clamped value between 0 and m, inclusive: - * n = clamp(fuzzedexpr, (opaque zero), identity(typeconstructor(m)), and m is an integer - * between 0 and 8, inclusive. + * an integer in [0, 8]. * Possibilities for generating an opaque zero include: * Shifting an opaque one to the left by n bits, then shifting it to the right by n bits: - * ((opaque one) << n) >> n, where n is an expression of a clamped value between 0 and m, - * inclusive: n = clamp(fuzzedexpr, (opaque zero), identity(typeconstructor(m)), and m is an - * integer between 0 and 7, inclusive. + * ((opaque one) << n) >> n, where n is an integer in [0, 7]. * * @param type - the base type of the opaque value being created. * @param constContext - true if we're in a constant expression context, false otherwise. @@ -220,62 +216,42 @@ private Optional opaqueZeroOrOneBitwiseShift(BasicType type, boolean const // that, we make sure not to shift our 1 bit out of the integer by limiting our maximum shift // to 7 bits. final int minBitsForLowpUnsignedInt = minBitsForLowpInt - 1; - final int maxValue = generator.nextInt(isZero ? minBitsForLowpInt : minBitsForLowpUnsignedInt); - // We pass true as constContext when fuzzing here because the expression will be evaluated, - // so we don't want any side effects. - final Expr shiftValue = makeClampedFuzzedExpr(type, constContext, depth, fuzzer, maxValue); + final int shiftValueConstant = generator.nextInt( + isZero ? minBitsForLowpInt : minBitsForLowpUnsignedInt); + final Expr shiftValueConstructor = + new TypeConstructorExpr(type.toString(), + BasicType.allUnsignedTypes().contains(type) + ? new UIntConstantExpr(String.valueOf(shiftValueConstant) + 'u') + : new IntConstantExpr(String.valueOf(shiftValueConstant))); + final Expr shiftValueWithIdentityApplied = identityConstructor( + shiftValueConstructor, + applyIdentityFunction(shiftValueConstructor, type, constContext, depth, fuzzer)); if (isZero) { final BinOp operator = generator.nextBoolean() ? BinOp.SHL : BinOp.SHR; return Optional.of( new ParenExpr( new BinaryExpr( makeOpaqueZero(type, constContext, depth, fuzzer), - shiftValue, + shiftValueWithIdentityApplied, operator))); } else { // We're going to shift twice in opposite directions by the same value. + final Expr backShiftValueWithIdentityApplied = identityConstructor( + shiftValueConstructor.clone(), + applyIdentityFunction(shiftValueConstructor.clone(), type, constContext, depth, fuzzer)); return Optional.of( new ParenExpr( new BinaryExpr( new ParenExpr( new BinaryExpr( makeOpaqueOne(type, constContext, depth, fuzzer), - shiftValue, + shiftValueWithIdentityApplied, BinOp.SHL)), - shiftValue.clone(), + backShiftValueWithIdentityApplied, BinOp.SHR))); } } - /** - * Utility function to clamp a fuzzed expression between an opaque zero and the identity of a - * type constructor of the given type, with the value of the bound argument. Note that this - * function only supports integer types currently - it could be extended to support floating - * point numbers as well if needed. Another note is that this function does not check its bound - * for validity - specifying a bound larger than 256 or a negative value could cause invalid GLSL - * to be generated depending on the precision and type of the integer. - * - * @param type - the type to make a clamped expression from. - * @param bound - the upper bound for the clamped expression. - * @return an expression of a clamped value between 0 and bound, inclusive: - * clamp(fuzzedexpr, (opaque zero), identity(typeconstructor(bound)), - */ - private Expr makeClampedFuzzedExpr(BasicType type, boolean constContext, - final int depth, Fuzzer fuzzer, int bound) { - assert BasicType.allIntegerTypes().contains(type); - return new FunctionCallExpr( - "clamp", - fuzzer.fuzzExpr(type, false, true, depth), - makeOpaqueZero(type, constContext, depth, fuzzer), - applyIdentityFunction( - new TypeConstructorExpr( - type.toString(), - type.getElementType() == BasicType.INT - ? new IntConstantExpr(String.valueOf(bound)) - : new UIntConstantExpr(bound + "u")), - type, constContext, depth, fuzzer)); - } - /** * Function to generate an opaque value by performing a bitwise operation on an opaque zero or * an opaque one. @@ -971,7 +947,7 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, BinOp.BOR)); } } - + /** * Identity transformation for integer types (both unsigned and signed, and their vectors) that * ORs an integer with zero, producing the same integer as output. From b1527408671b0bd0c1bfeef82b87d237f58dfef6 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Mon, 5 Aug 2019 17:34:13 +0100 Subject: [PATCH 082/180] Fix compilation problems (#651) Two seemingly independent PRs had clashed: one improved encapsulation of ScopeTrackingVisitor, the other depended on the existing less tight encapsulation. This change fixes the mismatch. --- .../reductionopportunities/StmtReductionOpportunities.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java index b6f49085b..a0fcd284d 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java @@ -79,7 +79,7 @@ protected void visitChildOfBlock(BlockStmt block, int index) { final Stmt child = block.getStmt(index); if (isEmptyBlockStmt(child) || isDeadCodeInjection(child) || allowedToReduceStmt(block, index)) { - addOpportunity(new StmtReductionOpportunity(enclosingFunction, block, child, + addOpportunity(new StmtReductionOpportunity(getEnclosingFunction(), block, child, getVistitationDepth())); } } @@ -130,7 +130,7 @@ private boolean allowedToReduceStmt(BlockStmt block, int childIndex) { // Unless we are in an injected dead code block, we need to be careful about removing // non-void return statements, so as to avoid making the shader invalid. if (StmtReductionOpportunity - .removalCouldLeadToLackOfReturnFromNonVoidFunction(enclosingFunction, block, stmt)) { + .removalCouldLeadToLackOfReturnFromNonVoidFunction(getEnclosingFunction(), block, stmt)) { return false; } From 56c0fc32009874a0c25f127afb4f221b7270ad0d Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 6 Aug 2019 10:06:02 -0500 Subject: [PATCH 083/180] Add function to generate opaque zero/one from dot product of two vectors (#642) --- .../fuzzer/OpaqueExpressionGenerator.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index d24cee0f7..83ce05cb0 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -100,6 +100,7 @@ private List waysToMakeZeroOrOne() { this::opaqueZeroOrOneFromInjectionSwitch, this::opaqueZeroOrOneSquareRoot, this::opaqueZeroOrOneAbsolute, + this::opaqueZeroOrOneDot, this::opaqueZeroOrOneBitwiseShift, this::opaqueZeroOrOneBitwiseOp ); @@ -172,6 +173,65 @@ private Optional opaqueZeroOrOneAbsolute(BasicType type, boolean constCont depth, fuzzer))); } + /** + * Function to generate an opaque zero or one by taking the dot product of two vectors. This + * function depends on two rules of the dot product. Let u and v be vectors of the same width, + * and let i be a valid index into those vectors. + * If u[i] != 0 <==> v[i] = 0 for all i, then dot(u, v) = 0. Example: u = (0, 1), v = (1, 0), + * then dot(u, v) = 1 * 0 + 0 * 1 = 0. + * If u[i] = v[i] = 0 for all i except for a single index, j, and if u[j] = v[j] = 1, then + * dot(u, v) = 0. Example: u = (0, 1), v = (0, 1), then dot(u, v) = 0 * 0 + 1 * 1 = 1. + * Because fuzzed expressions are not always well defined, we forgo those in favor of simply + * using opaque ones and zeroes when generating vectors for this function. + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque value + * made from the dot product of two vectors. + */ + private Optional opaqueZeroOrOneDot(BasicType type, boolean constContext, + final int depth, Fuzzer fuzzer, boolean isZero) { + if (type != BasicType.FLOAT) { + return Optional.empty(); + } + // If width is 1, type will be float - otherwise the type will be the corresponding vector type. + final int vectorWidth = generator.nextPositiveInt(BasicType.VEC4.getNumElements()) + 1; + final Expr dotProductExpr; + final List firstVectorArgs = new ArrayList(); + final List secondVectorArgs = new ArrayList(); + if (isZero) { + // We're basically producing inverse vectors: u[i] = 0 <==> v[i] != 0. + for (int i = 0; i < vectorWidth; i++) { + final boolean firstVectorArgIsZero = generator.nextBoolean(); + firstVectorArgs.add(makeOpaqueZeroOrOne( + firstVectorArgIsZero, type, constContext, depth, fuzzer)); + secondVectorArgs.add(makeOpaqueZeroOrOne( + !firstVectorArgIsZero, type, constContext, depth, fuzzer)); + } + } else { + // The two vectors will be exactly the same - all 0 except for one value, which will be 1. + final int nonZeroIndex = generator.nextInt(vectorWidth); + for (int i = 0; i < vectorWidth; i++) { + firstVectorArgs.add( + makeOpaqueZeroOrOne(i == nonZeroIndex, type, constContext, depth, fuzzer)); + secondVectorArgs.add( + makeOpaqueZeroOrOne(i == nonZeroIndex, type, constContext, depth, fuzzer)); + } + } + assert firstVectorArgs.size() == vectorWidth && secondVectorArgs.size() == vectorWidth; + dotProductExpr = new FunctionCallExpr("dot", + new TypeConstructorExpr( + BasicType.makeVectorType(type, vectorWidth).toString(), firstVectorArgs), + new TypeConstructorExpr( + BasicType.makeVectorType(type, vectorWidth).toString(), secondVectorArgs)); + return Optional.of( + identityConstructor( + dotProductExpr, + applyIdentityFunction(dotProductExpr.clone(), type, constContext, depth, fuzzer))); + } + /** * Function to generate an opaque zero or one by bitwise shifting left or right by an amount of * bits dependent on which opaque is being generated. From e0c6c355a63c52c25f1ae5188efc95aefea63610 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 6 Aug 2019 17:54:44 +0100 Subject: [PATCH 084/180] Add support for matrix uniforms in generation of AmberScript (#654) Matrix uniforms were not currently supported in 'runspv', neither for generation of AmberScript nor VkScript. This change adds support in the AmberScript case. --- python/src/main/python/drivers/runspv.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/src/main/python/drivers/runspv.py b/python/src/main/python/drivers/runspv.py index f6301e368..2682f2ec2 100755 --- a/python/src/main/python/drivers/runspv.py +++ b/python/src/main/python/drivers/runspv.py @@ -1027,6 +1027,15 @@ def amberscript_uniform_buffer_decl(uniform_json): 'glUniform2f': 'vec2', 'glUniform3f': 'vec3', 'glUniform4f': 'vec4', + 'glUniformMatrix2fv': 'mat2x2', + 'glUniformMatrix2x3fv': 'mat2x3', + 'glUniformMatrix2x4fv': 'mat2x4', + 'glUniformMatrix3x2fv': 'mat3x2', + 'glUniformMatrix3fv': 'mat3x3', + 'glUniformMatrix3x4fv': 'mat3x4', + 'glUniformMatrix4x2fv': 'mat4x2', + 'glUniformMatrix4x3fv': 'mat4x3', + 'glUniformMatrix4fv': 'mat4x4', } result = '' From a3118670af298fc5682a675bc08cdf86c32418f6 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 7 Aug 2019 13:16:49 -0500 Subject: [PATCH 085/180] Added identity to double transpose a matrix (#649) * Added identity to double transpose a matrix * Workaround for #653 - add constContext to preconditionHolds * Link to glslangValidator issue --- .../common/ast/type/BasicType.java | 34 ++++++++++++++ .../fuzzer/OpaqueExpressionGenerator.java | 44 ++++++++++++++++--- .../transformation/ExpressionIdentity.java | 2 +- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java index 5b37c2759..7c2588fad 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java @@ -447,6 +447,40 @@ public boolean isMatrix() { return allMatrixTypes().contains(this); } + /** + * Creates a matrix type that is the transpose of a given matrix type. + * + * @return the type that is the transpose of the given matrix type. + * @throws UnsupportedOperationException if the type this method is called on is not a matrix + * type (and thus has no transpose). + */ + public BasicType transposedMatrixType() { + if (!this.isMatrix()) { + throw new UnsupportedOperationException( + "Cannot transpose non-matrix type " + this); + } + if (BasicType.allSquareMatrixTypes().contains(this)) { + return this; // transpose of a square matrix is a square matrix. + } + if (this == MAT2X3) { + return MAT3X2; + } + if (this == MAT2X4) { + return MAT4X2; + } + if (this == MAT3X2) { + return MAT2X3; + } + if (this == MAT3X4) { + return MAT4X3; + } + if (this == MAT4X2) { + return MAT2X4; + } + assert this == MAT4X3; + return MAT3X4; + } + /** * Determines the vector type of the columns in the matrix. For example, accessing a column of a * mat2x2 would give you a variable of type vec2. Can only be invoked on a matrix type. diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 83ce05cb0..ae7e2f16c 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -87,10 +87,12 @@ public OpaqueExpressionGenerator(IRandom generator, GenerationParams generationP expressionIdentities.add(new IdentityClamp()); expressionIdentities.add(new IdentityRewriteComposite()); - if (shadingLanguageVersion.supportedMixNonfloatBool()) { expressionIdentities.add(new IdentityMixBvec()); } + if (shadingLanguageVersion.supportedTranspose()) { + expressionIdentities.add(new IdentityDoubleTranspose()); + } } @@ -517,7 +519,7 @@ public Expr applyIdentityFunction(Expr expr, BasicType type, boolean constContex } final List availableTransformations = expressionIdentities.stream() - .filter(item -> item.preconditionHolds(expr, type)) + .filter(item -> item.preconditionHolds(expr, type, constContext)) .collect(Collectors.toList()); return availableTransformations.isEmpty() ? expr : availableTransformations.get(generator.nextInt(availableTransformations.size())) @@ -796,7 +798,7 @@ private abstract class AbstractIdentityTransformation implements ExpressionIdent } @Override - public boolean preconditionHolds(Expr expr, BasicType basicType) { + public boolean preconditionHolds(Expr expr, BasicType basicType, boolean constContext) { if (!acceptableTypes.contains(basicType)) { return false; } @@ -1248,8 +1250,40 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, } @Override - public boolean preconditionHolds(Expr expr, BasicType basicType) { - return super.preconditionHolds(expr, basicType) && expr instanceof VariableIdentifierExpr; + public boolean preconditionHolds(Expr expr, BasicType basicType, boolean constContext) { + return super.preconditionHolds(expr, basicType, constContext) + && expr instanceof VariableIdentifierExpr; + } + } + + /** + * Identity function to transpose a matrix twice. When performed, transforms an expression of a + * matrix m -> transpose(identity(transpose(identity(m)))). + */ + private class IdentityDoubleTranspose extends AbstractIdentityTransformation { + private IdentityDoubleTranspose() { + super(BasicType.allMatrixTypes(), true); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert type.isMatrix(); + return identityConstructor( + expr, + new FunctionCallExpr("transpose", + applyIdentityFunction( + new FunctionCallExpr("transpose", + applyIdentityFunction(expr.clone(), type, constContext, depth, fuzzer)), + type.transposedMatrixType(), constContext, depth, fuzzer))); + } + + @Override + public boolean preconditionHolds(Expr expr, BasicType basicType, boolean constContext) { + // TODO(https://github.com/KhronosGroup/glslang/issues/1865): Workaround for glslangValidator + // issue, remove constContext check when fixed. + return super.preconditionHolds(expr, basicType, constContext) + && !constContext; } } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/ExpressionIdentity.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/ExpressionIdentity.java index ec613a8b1..d3290aeae 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/ExpressionIdentity.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/ExpressionIdentity.java @@ -24,6 +24,6 @@ public interface ExpressionIdentity { Expr apply(Expr expr, BasicType type, boolean constContext, final int depth, Fuzzer fuzzer); - boolean preconditionHolds(Expr expr, BasicType basicType); + boolean preconditionHolds(Expr expr, BasicType basicType, boolean constContext); } From 18b4975fa32fb38c561ac9443267eb3693f44824 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 7 Aug 2019 13:18:39 -0500 Subject: [PATCH 086/180] Add function to make opaque zero/one from determinant of a matrix (#648) * Add function to make opaque zero/one from determinant of a matrix * Workaround for glslang issue #653 --- .../common/ast/type/BasicType.java | 48 ++++++++ .../fuzzer/OpaqueExpressionGenerator.java | 111 +++++++++++++++++- 2 files changed, 158 insertions(+), 1 deletion(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java index 7c2588fad..b9af92f19 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java @@ -431,10 +431,58 @@ public static BasicType makeVectorType(BasicType elementType, int numElements) { return BVEC4; } } + // Should not be reachable. assert false; return null; } + /** + * Creates a matrix type of a specified size from the given dimensions. + * + * @return a matrix type of numColumns columns and numRows rows. + * @throws UnsupportedOperationException if numColumns or numRows are outside the bounds of + * possible GLSL matrix dimensions (numColumns < 2, numColumns > 4, numRows < 2, numRows > 4) + */ + public static BasicType makeMatrixType(int numColumns, int numRows) { + if (numColumns < 2 || numColumns > 4 || numRows < 2 || numRows > 4) { + throw new UnsupportedOperationException( + "Cannot make matrix type with " + numColumns + " columns and " + numRows + " rows."); + } + switch (numColumns) { + case 2: + switch (numRows) { + case 2: + return MAT2X2; + case 3: + return MAT2X3; + default: + return MAT2X4; + } + case 3: + switch (numRows) { + case 2: + return MAT3X2; + case 3: + return MAT3X3; + default: + return MAT3X4; + } + case 4: + switch (numRows) { + case 2: + return MAT4X2; + case 3: + return MAT4X3; + default: + return MAT4X4; + } + default: + // Should not be reachable. + assert false; + return null; + } + } + public boolean isScalar() { return allScalarTypes().contains(this); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index ae7e2f16c..a89420ab4 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -104,7 +104,8 @@ private List waysToMakeZeroOrOne() { this::opaqueZeroOrOneAbsolute, this::opaqueZeroOrOneDot, this::opaqueZeroOrOneBitwiseShift, - this::opaqueZeroOrOneBitwiseOp + this::opaqueZeroOrOneBitwiseOp, + this::opaqueZeroOrOneMatrixDet ); } @@ -175,6 +176,114 @@ private Optional opaqueZeroOrOneAbsolute(BasicType type, boolean constCont depth, fuzzer))); } + /** + * Function to generate an opaque zero or one from the determinant of a generated matrix. + * This function heavily relies on the fact that a triangular matrix - a matrix that only has + * values below the diagonal (lower triangular) or above the diagonal (upper triangular) - always + * has a determinant that is the product of its diagonal values. For example, + * [1 0 0] This is a lower triangular 3x3 matrix. Its determinant is the product of its + * [0 1 0] diagonals, so det(mat3) = 1 * 1 * 1 = 1. + * [1 0 1] + * -------------------------------------- + * [0 1 0 1] This is an upper triangular 4x4 matrix. Its determinant is the product of its + * [0 0 1 1] diagonals, so det(mat4) = 0 * 0 * 0 * 0 = 0. + * [0 0 0 1] + * [0 0 0 0] + * -------------------------------------- + * In general, a triangular matrix is of these forms for our purpose: + * [t x] [t 0] [t x x] [t 0 0] [t x x x] [t 0 0 0] + * [0 t] [x t] [0 t x] [x t 0] [0 t x x] [x t 0 0] + * [0 0 t] [x x t] [0 0 t x] [x x t 0] + * [0 0 0 t] [x x x t] + * where x is a "modifiable index" whose value is totally irrelevant for the determinant, + * and t is a "diagonal" that is zero if we're making an opaque zero, otherwise an opaque one. + * For simplicity, we choose not to consider the case where the diagonal has one zero and + * the rest ones (which would produce a determinant of zero). + * ------------------------------------- + * There are some other things to know about this algorithm: + * [0 1 2 3 ] - When constructing a square matrix in GLSL, the mat constructor takes + * [4 5 6 7 ] matrixDimension * matrixDimension number of arguments - this diagram shows + * [8 9 10 11] how the matrix is constructed from those arguments. + * [12 13 14 15] + * - Notice how in the above diagram, the diagonals for a 4x4 matrix are arguments 0, 5, 10, 15. + * This occurs similarly for 3x3 (0, 4, 8) and 2x2 (0, 3). The sequence of constructor indices + * that are diagonals: [0, (matrixDim + 1), 2 * (matrixDim + 1), 3 * (matrixDim + 1)...]. + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque value + * made from the determinant of a triangular matrix. + */ + private Optional opaqueZeroOrOneMatrixDet(BasicType type, boolean constContext, + final int depth, Fuzzer fuzzer, + boolean isZero) { + // TODO(https://github.com/KhronosGroup/glslang/issues/1865): Workaround for glslangvalidator. + // Remove when #653 is fixed. + if (constContext) { + return Optional.empty(); + } + if (type != BasicType.FLOAT) { + return Optional.empty(); + } + if (!shadingLanguageVersion.supportedDeterminant()) { + return Optional.empty(); + } + // If true, we will make an upper triangular matrix - otherwise it will be lower triangular. + final boolean isUpperTriangular = generator.nextBoolean(); + // Matrices sent to determinant() must be 2x2, 3x3, or 4x4. + final int matrixDimension = generator.nextInt(3) + 2; + // Indices of arguments in a matrix constructor that can be nonzero for triangular matrix, + // excluding the diagonal of the matrix. Zero-indexed. + List modifiableArgs; + switch (matrixDimension) { + case 2: + modifiableArgs = isUpperTriangular + ? Arrays.asList(1) // mat2, upper triangular + : Arrays.asList(2); // mat2, lower triangular + break; + case 3: + modifiableArgs = isUpperTriangular + ? Arrays.asList(1, 2, 5) // mat3, upper triangular + : Arrays.asList(3, 6, 7); // mat3, lower triangular + break; + case 4: + modifiableArgs = isUpperTriangular + ? Arrays.asList(1, 2, 3, 6, 7, 11) // mat4, upper triangular + : Arrays.asList(4, 8, 9, 12, 13, 14); // mat4, lower triangular + break; + default: + // Should not be reachable. + throw new UnsupportedOperationException("Generated matrix dimension was out of bounds?"); + } + final List matrixConstructorArgs = new ArrayList(); + // Diagonal indices are spaced by (matrixDimension + 1) - e.g. mat4 diagonals: 0, 5, 10, 15 + int nextDiagonalMatrixIndex = 0; + for (int i = 0; i < matrixDimension * matrixDimension; i++) { + if (i == nextDiagonalMatrixIndex) { + // We're in a diagonal, so the value depends on if we're making an opaque zero or one. + assert !modifiableArgs.contains(i); + matrixConstructorArgs.add(makeOpaqueZeroOrOne(isZero, type, constContext, depth, fuzzer)); + nextDiagonalMatrixIndex += matrixDimension + 1; + } else if (modifiableArgs.contains(i)) { + // We're in a modifiable index - the value doesn't matter for the determinant. + matrixConstructorArgs.add(makeOpaqueZeroOrOne( + generator.nextBoolean(), type, constContext, depth, fuzzer)); + } else { + // We're in a non-modifiable index - we need zero or we'll violate the triangular + // property of the matrix. + matrixConstructorArgs.add(makeOpaqueZero(type, constContext, depth, fuzzer)); + } + } + assert matrixConstructorArgs.size() == matrixDimension * matrixDimension; + return Optional.of( + new FunctionCallExpr("determinant", + new TypeConstructorExpr( + BasicType.makeMatrixType(matrixDimension, matrixDimension).toString(), + matrixConstructorArgs))); + } + /** * Function to generate an opaque zero or one by taking the dot product of two vectors. This * function depends on two rules of the dot product. Let u and v be vectors of the same width, From d0340f16c7aebb32fc443109cf937135f8458164 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Thu, 8 Aug 2019 09:46:30 -0500 Subject: [PATCH 087/180] Add householder_lattice shader (#647) --- .../samples/310es/householder_lattice.frag | 119 ++++++++++++++++++ .../samples/310es/householder_lattice.json | 11 ++ 2 files changed, 130 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/householder_lattice.frag create mode 100644 shaders/src/main/glsl/samples/310es/householder_lattice.json diff --git a/shaders/src/main/glsl/samples/310es/householder_lattice.frag b/shaders/src/main/glsl/samples/310es/householder_lattice.frag new file mode 100644 index 000000000..b7c36184f --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/householder_lattice.frag @@ -0,0 +1,119 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +int MATRIX_N = 4; + +uniform mat4 matrix_a_uni; + +void main() +{ + // We need to modify A in place, so we have to copy the uniform + mat4 matrix_a = mat4(matrix_a_uni); + vec4 matrix_b = gl_FragCoord.wxyz; + + vec4 matrix_u = vec4(0.0); + + float magnitudeX = 0.0; + float alpha1 = 0.0; + float alpha2 = 0.0; + float alpha3 = 0.0; + float beta = 0.0; + // QR factorization + for(int k = 0; k < MATRIX_N - 1; k++) + { + // define vector x as the column A(k) only with entries k - n + // creating our U vector at the same time we create X magnitude. + for(int x = MATRIX_N - 1; x >= k; x--) // start from the bottom and go up. + { + magnitudeX += pow(matrix_a[x][k], 2.0); + matrix_u[x] = matrix_a[x][k]; + } + magnitudeX = sqrt(magnitudeX); + // entry k is the top of our U vector for this iteration + // define U as x - sign(x1) (magnitude X) (elementary matrix 1) + matrix_u[k] -= (sign(matrix_u[k]) * magnitudeX); + + // alpha1 = U(T) * U gives us the dot product + for(int u = MATRIX_N - 1; u >= k; u--) + { + alpha1 += pow(matrix_u[u], 2.0); + } + + // alpha2 = 2.0 / (U(T) * U) = 2.0 / alpha1 + alpha2 = 2.0 / alpha1; + + for(int j = k; j < MATRIX_N; j++) + { + // evaluate alpha3 = U(T) * a(j) updated k - 1 times + for(int a = MATRIX_N - 1; a >= k; a--) + { + alpha3 += matrix_u[a] * matrix_a[a][j]; + } + + beta = alpha2 * alpha3; + + // evaluate a(j) updated k times = a(j) updated k - 1 times - (beta * U) + for(int a = MATRIX_N - 1; a >= k; a--) + { + matrix_a[a][j] = matrix_a[a][j] - (beta * matrix_u[a]); + } + alpha3 = 0.0; + beta = 0.0; + } + + // evaluate alpha3 = U(T) * a(j) updated k - 1 times + for(int b = MATRIX_N - 1; b >= k; b--) + { + alpha3 += matrix_u[b] * matrix_b[b]; + } + + // evaluate beta = alpha2 * alpha3 + beta = alpha2 * alpha3; + + // evaluate b(j) updated k times = b(j) updated k - 1 times - (beta * U) + for(int b = MATRIX_N - 1; b >= k; b--) + { + matrix_b[b] = matrix_b[b] - (beta * matrix_u[b]); + } + + magnitudeX = 0.0; + alpha1 = 0.0; + alpha2 = 0.0; + alpha3 = 0.0; + beta = 0.0; + } + + // back substitution algorithm + for(int i = (MATRIX_N - 1); i >= 0; i--) + { + for(int j = (MATRIX_N - 1); j >= (i + 1); j--) + { + matrix_b[i] -= (matrix_a[i][j] * matrix_b[j]); + } + + // final operation, using the diagonal; matrix_b becomes the solution matrix x + matrix_b[i] /= matrix_a[i][i]; + } + _GLF_color = tan(matrix_b); + _GLF_color.a = 1.0; +} + diff --git a/shaders/src/main/glsl/samples/310es/householder_lattice.json b/shaders/src/main/glsl/samples/310es/householder_lattice.json new file mode 100644 index 000000000..51d6c7a30 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/householder_lattice.json @@ -0,0 +1,11 @@ +{ + "matrix_a_uni": { + "func": "glUniformMatrix4fv", + "args": [ + 1.0, 5.0, 3.0, 7.0, + 9.0, 6.0, 7.0, 8.0, + 1.0, 2.0, 3.0, 4.0, + 5.0, 6.0, 7.0, 8.0 + ] + } +} From 3c70b264268baf0a507c5c19ccf44698f2a55fb3 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Sat, 10 Aug 2019 07:33:41 -0500 Subject: [PATCH 088/180] Refactor transposedMatrixType() to use makeMatrixType() (#668) --- .../common/ast/type/BasicType.java | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java index b9af92f19..af7ef4b33 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java @@ -496,7 +496,8 @@ public boolean isMatrix() { } /** - * Creates a matrix type that is the transpose of a given matrix type. + * Creates a matrix type that is the transpose of a given matrix type. A matrix type can be + * transposed by swapping its column and row widths. * * @return the type that is the transpose of the given matrix type. * @throws UnsupportedOperationException if the type this method is called on is not a matrix @@ -507,26 +508,7 @@ public BasicType transposedMatrixType() { throw new UnsupportedOperationException( "Cannot transpose non-matrix type " + this); } - if (BasicType.allSquareMatrixTypes().contains(this)) { - return this; // transpose of a square matrix is a square matrix. - } - if (this == MAT2X3) { - return MAT3X2; - } - if (this == MAT2X4) { - return MAT4X2; - } - if (this == MAT3X2) { - return MAT2X3; - } - if (this == MAT3X4) { - return MAT4X3; - } - if (this == MAT4X2) { - return MAT2X4; - } - assert this == MAT4X3; - return MAT3X4; + return BasicType.makeMatrixType(this.getNumRows(), this.getNumColumns()); } /** From 34f406cbc74475f81363f94916af7fe7410cc227 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Mon, 12 Aug 2019 13:53:10 +0700 Subject: [PATCH 089/180] Add tool for generating shaders with known expected outputs (#625) --- .../common/ast/stmt/ReturnStmt.java | 7 +- .../knownvaluegeneration/BooleanValue.java | 81 +++ .../knownvaluegeneration/CompositeValue.java | 98 +++ .../ExpressionGenerator.java | 684 ++++++++++++++++++ .../knownvaluegeneration/FactManager.java | 117 +++ .../knownvaluegeneration/FunctionFact.java | 51 ++ .../knownvaluegeneration/NumericValue.java | 94 +++ .../ParameterDeclFact.java | 42 ++ .../generator/knownvaluegeneration/Value.java | 50 ++ .../VariableDeclFact.java | 45 ++ .../knownvaluegeneration/VariableFact.java | 43 ++ .../semanticschanging/LiteralFuzzer.java | 9 +- .../graphicsfuzz/generator/tool/Generate.java | 2 +- .../tool/KnownValueShaderGenerator.java | 172 +++++ .../drivers/knownvalue-shader-generator | 25 + .../drivers/knownvalue-shader-generator.bat | 29 + .../drivers/knownvalue-shader-generator.py | 42 ++ .../java/com/graphicsfuzz/util/Constants.java | 7 + 18 files changed, 1591 insertions(+), 7 deletions(-) create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/BooleanValue.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/CompositeValue.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FunctionFact.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/NumericValue.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ParameterDeclFact.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/Value.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableDeclFact.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableFact.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java create mode 100755 python/src/main/python/drivers/knownvalue-shader-generator create mode 100644 python/src/main/python/drivers/knownvalue-shader-generator.bat create mode 100644 python/src/main/python/drivers/knownvalue-shader-generator.py diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/ReturnStmt.java b/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/ReturnStmt.java index 40453de2f..75cfd43b4 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/ReturnStmt.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/ReturnStmt.java @@ -25,17 +25,22 @@ public class ReturnStmt extends Stmt { private Expr expr; public ReturnStmt(Expr expr) { + assert expr != null; this.expr = expr; } public ReturnStmt() { - this(null); + this.expr = null; } public Expr getExpr() { return expr; } + public void setExpr(Expr expr) { + this.expr = expr; + } + public boolean hasExpr() { return getExpr() != null; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/BooleanValue.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/BooleanValue.java new file mode 100644 index 000000000..98f6a03a5 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/BooleanValue.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.expr.BoolConstantExpr; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; +import java.util.Objects; +import java.util.Optional; + +public class BooleanValue implements Value { + + private final Optional value; + + public BooleanValue(Optional value) { + this.value = value; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof BooleanValue)) { + return false; + } + + final BooleanValue thatBooleanValue = (BooleanValue) that; + return this.value.equals(thatBooleanValue.value); + } + + @Override + public int hashCode() { + return Objects.hash(getType(), value); + } + + @Override + public Type getType() { + return BasicType.BOOL; + } + + @Override + public boolean valueIsUnknown() { + return !value.isPresent(); + } + + public Optional getValue() { + return value; + } + + @Override + public Expr generateLiteral(LiteralFuzzer literalFuzzer) { + if (valueIsUnknown()) { + return literalFuzzer.fuzz(BasicType.BOOL).orElse(null); + } + return new BoolConstantExpr(value.get()); + } + + @Override + public String toString() { + return valueIsUnknown() ? "unknown_bool" : value.get().toString(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/CompositeValue.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/CompositeValue.java new file mode 100644 index 000000000..06e7ff508 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/CompositeValue.java @@ -0,0 +1,98 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +public class CompositeValue implements Value { + + private final Type type; + private final Optional> valueList; + + public CompositeValue(Type type, Optional> valueList) { + this.type = type; + this.valueList = valueList; + } + + public Optional> getValueList() { + return valueList; + } + + @Override + public Type getType() { + return type; + } + + @Override + public boolean valueIsUnknown() { + return !valueList.isPresent(); + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof CompositeValue)) { + return false; + } + + final CompositeValue thatCompositeValue = (CompositeValue) that; + return this.getType().equals(thatCompositeValue.getType()) + && this.getValueList().equals(thatCompositeValue.getValueList()); + } + + @Override + public int hashCode() { + return Objects.hash(type, valueList); + } + + @Override + public Expr generateLiteral(LiteralFuzzer literalFuzzer) { + if (valueIsUnknown()) { + return literalFuzzer.fuzz(type).orElse(null); + } + + if (type instanceof BasicType) { + final List args = valueList + .get() + .stream() + .map(item -> item.generateLiteral(literalFuzzer)) + .collect(Collectors.toList()); + return new TypeConstructorExpr(type.toString(), args); + } + + // TODO(https://github.com/google/graphicsfuzz/issues/664): we should also support array and + // struct types. + throw new RuntimeException("The given type is not supported"); + } + + @Override + public String toString() { + return valueIsUnknown() ? "unknown_composite" : valueList.get().toString(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java new file mode 100644 index 000000000..c8cf4a3b3 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java @@ -0,0 +1,684 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; +import com.graphicsfuzz.common.ast.decl.ParameterDecl; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; +import com.graphicsfuzz.common.ast.expr.UnOp; +import com.graphicsfuzz.common.ast.expr.UnaryExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.NullStmt; +import com.graphicsfuzz.common.ast.stmt.ReturnStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.PipelineInfo; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; +import com.graphicsfuzz.util.Constants; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +public class ExpressionGenerator { + + private static final int MAX_FUNCTION_PARAMS = 5; + private static final int NUM_WAYS_TO_GENERATE_EXPR = 7; + private static final int MAX_DEPTH = 5; + // Theses boundaries are taken from the LiteralFuzzer, we may want to consider + // modifying these values. + private static final int INT_MIN = 0; + private static final int INT_MAX = 1 << 17; + + private static final String NEGATIVE = "_negative"; + private static final String ID = "_id"; + + private final TranslationUnit translationUnit; + private final PipelineInfo pipelineInfo; + private final IdGenerator idGenerator; + private final IRandom generator; + private int currentDepth; + private final LiteralFuzzer literalFuzzer; + private final FactManager globalFactManager; + + public ExpressionGenerator(TranslationUnit translationUnit, PipelineInfo pipelineInfo, + IRandom generator, + FactManager globalFactManager) { + this.translationUnit = translationUnit; + this.pipelineInfo = pipelineInfo; + this.idGenerator = new IdGenerator(); + this.generator = generator; + this.currentDepth = 0; + this.literalFuzzer = new LiteralFuzzer(this.generator); + this.globalFactManager = globalFactManager; + } + + private Expr generateLiteralValue(Value value) { + return value.generateLiteral(literalFuzzer); + } + + /** + * This function applies various transformations and returns a expression that guarantees to + * compute the given value. If the value is unknown, fact manager would generate any + * arbitrary values that have the correct type. + * + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param functionDefinition a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @param value the value that will be computed by the expression generated by + * this method. + * @return the expression whose value is known by the fact manager. + */ + public Expr generateExpr(FactManager factManager, + FunctionDefinition functionDefinition, + Stmt stmtToInsertBefore, + Value value) { + if (currentDepth > MAX_DEPTH) { + // When current depth has reached the maximum limit, we don't want generation to go deeper. + // Thus, we consider generating a new expression from non-recursive approaches only. + // To do so, we will choose to either generating the expression by making the literal + // value or calling the already declared variables retrieved from the fact manager. + Expr result; + do { + if (generator.nextBoolean()) { + result = generateLiteralValue(value); + } else { + result = generateKnownVariableFact(value, factManager); + } + } while (result == null); + return result; + } + + currentDepth++; + Expr result; + + while (true) { + switch (generator.nextInt(NUM_WAYS_TO_GENERATE_EXPR)) { + case 0: + result = generateLiteralValue(value); + break; + case 1: + result = generateVariableFact(value, factManager, functionDefinition, stmtToInsertBefore); + break; + case 2: + result = generateKnownVariableFact(value, factManager); + break; + case 3: + result = generateFunctionFact(value, factManager, functionDefinition, stmtToInsertBefore); + break; + case 4: + result = generateKnownFunctionFact(factManager, value, functionDefinition, + stmtToInsertBefore); + break; + case 5: + result = generateAdditionValue(value, factManager, functionDefinition, + stmtToInsertBefore); + break; + case 6: + result = generateForLoopValue(value, factManager, functionDefinition, stmtToInsertBefore); + break; + default: + throw new RuntimeException("Should be unreachable as switch cases cover all cases"); + } + if (result != null) { + break; + } + } + currentDepth--; + return result; + } + + /** + * Assigns value to the newly-generated variable using a for loop. To do so, we first pick a + * random number (divisor) and add this number to the variable for each iteration. Then, we + * basically find a number of iterations by dividing the original value by the chosen divisor + * . We would also have to eventually add the remainder to the variable if needed. + * + *

For example, given a value x we can derive a for loop statement and a remainder from the + * equation x = (divisor * iterations) + remainder. + * + *

The code fragment below is an example result obtained by this method which returns an int 7. + * int _GLF_PRIMITIVE_int_7 = 0; + * // 3 is the number of iterations obtained by dividing the given value 7 by the random + * divisor 2. + * for(int i = 0; i < 3; i ++) { + * _GLF_PRIMITIVE_int_7 += 2; + * } + * _GLF_PRIMITIVE_int_7 += 1; // 1 is a remainder of 7 divided by 2. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @return a variable identifier to the new variable generated by this method. + */ + private Expr generateForLoopValue(Value value, + FactManager factManager, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + // TODO(https://github.com/google/graphicsfuzz/issues/661): we can also generate for loop + // value of UINT type. + if (value.getType() != BasicType.INT) { + return null; + } + // If the given fact manager is at global scope, the caller expects a expression generated by + // this method to be available at the global scope. However the new for loop statement + // generated by this method will be injected into the body of a given function making it + // available only in the local scope. Hence, if the global scope expression is requested, + // null is returned. + if (factManager.globalScope()) { + return null; + } + + // Declare and initialize a zero-value variable. + final String varName = genVarName(value, false); + final VariableDeclInfo variableDeclInfo = new VariableDeclInfo(varName, null, + new Initializer( + generateExpr(factManager, + currentFunction, + stmtToInsertBefore, + new NumericValue(BasicType.INT, Optional.of(0))) + )); + final VariablesDeclaration variablesDecl = new VariablesDeclaration(value.getType(), + variableDeclInfo); + currentFunction.getBody().insertBefore(stmtToInsertBefore, + new DeclarationStmt(variablesDecl)); + + // Decide whether the loop should be incremented or decremented for each iteration. + final boolean isIncrement = generator.nextBoolean(); + // If value is unknown, we could generate and return any number. + int original = (int) ((NumericValue) value).getValue().orElse(generator.nextInt(INT_MAX)); + + // if the original value is zero, calling random wrapper will thrown an error illegal argument + // exception. + if (original < 1) { + return null; + } + // A random number that will be added to the variable on each iteration. + // We use max function here to prevent the division by zero error. + int divisor = Math.max(1, generator.nextInt(original)); + + // TODO(https://github.com/google/graphicsfuzz/issues/659): we should be able to set the max + // limit as we don't want to end up having a large number of iterations. + final int iterations = original / divisor; + // A left over number that will be added after for loop is executed. The binary + // expression responsible to add a remainder to the variable will be inserted after for + // loop statement. + final int remainder = original % divisor; + + // Values of numbers which will be used for the for loop. + final Value divisorValue = new NumericValue(BasicType.INT, Optional.of(divisor)); + final Value iterationValue = new NumericValue(BasicType.INT, + Optional.of(isIncrement ? 0 : iterations)); + final Value conditionValue = new NumericValue(BasicType.INT, + Optional.of(isIncrement ? iterations : 0)); + + final Stmt init = new DeclarationStmt(new VariablesDeclaration(BasicType.INT, + new VariableDeclInfo("i", null, + new Initializer(generateExpr(factManager, currentFunction, stmtToInsertBefore, + iterationValue)))) + ); + final Expr condition = new BinaryExpr(new VariableIdentifierExpr("i"), + generateExpr(factManager, currentFunction, stmtToInsertBefore, conditionValue), + isIncrement ? BinOp.LT : BinOp.GE); + final Expr increment = new UnaryExpr(new VariableIdentifierExpr("i"), + isIncrement ? UnOp.POST_INC : UnOp.POST_DEC); + final Stmt body = new BlockStmt(Arrays.asList(new ExprStmt(new BinaryExpr( + new VariableIdentifierExpr(varName), + generateExpr( + factManager, + currentFunction, + stmtToInsertBefore, + divisorValue), + BinOp.ADD_ASSIGN + ))), false); + final ForStmt forStmt = new ForStmt(init, condition, increment, body); + currentFunction.getBody().insertBefore(stmtToInsertBefore, forStmt); + + if (remainder > 0) { + final Value remainderValue = new NumericValue((BasicType) value.getType(), + Optional.of(remainder)); + currentFunction.getBody().insertBefore(stmtToInsertBefore, new ExprStmt(new BinaryExpr( + new VariableIdentifierExpr(varName), + generateExpr(factManager, currentFunction, stmtToInsertBefore, remainderValue), + BinOp.ADD_ASSIGN + ))); + } + final VariableDeclFact variableDeclFact = new VariableDeclFact(variablesDecl, + variableDeclInfo, value); + factManager.addVariableFact(value, variableDeclFact); + return new VariableIdentifierExpr(varName); + } + + /** + * This method generates the expression that performs the addition of two values which result is + * equal to the given value. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @return a binary expression performing the addition of two numbers whose sum is equal to Value. + */ + private Expr generateAdditionValue(Value value, + FactManager factManager, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + if (!(value instanceof NumericValue)) { + return null; + } + // TODO(https://github.com/google/graphicsfuzz/issues/661): we can also generate addition + // value of UINT type. + if (value.getType() != BasicType.INT && value.getType() != BasicType.FLOAT) { + return null; + } + + final Pair, Optional> pair = getPairSum(value); + return new BinaryExpr( + generateExpr(factManager, currentFunction, stmtToInsertBefore, + new NumericValue((BasicType) value.getType(), pair.getLeft())), + generateExpr(factManager, currentFunction, stmtToInsertBefore, + new NumericValue((BasicType) value.getType(), pair.getRight())), + BinOp.ADD + ); + } + + /** + * Retrieves known value from the function facts hold by fact manager and returns the + * function call to that function with the appropriate parameters if there is a function fact that + * represents the given value. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @return a function call expression to the already declared function representing the Value. + */ + private Expr generateKnownFunctionFact(FactManager factManager, + Value value, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + final List availableFacts = factManager.getFunctionFacts(value); + if (availableFacts.isEmpty()) { + return null; + } + final FunctionFact functionFact = availableFacts.get(generator.nextInt(availableFacts.size())); + final List argValues = functionFact.getArguments(); + final List args = generateArgsForFunctionCall(factManager, currentFunction, + stmtToInsertBefore, + argValues); + return new FunctionCallExpr(functionFact.getFunctionName(), args); + } + + /** + * Retrieves known value from the variable facts hold by fact manager and returns the + * variable identifier to that variable if there is a variable fact that represents the given + * value. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @return a variable identifier expression to the already declared variable representing value. + */ + private Expr generateKnownVariableFact(Value value, FactManager factManager) { + final List availableFacts = factManager.getVariableFacts(value); + if (availableFacts.isEmpty()) { + return null; + } + final VariableFact variableFact = availableFacts.get(generator.nextInt(availableFacts.size())); + return new VariableIdentifierExpr(variableFact.getVariableName()); + } + + /** + * Utility function to generate a set of expressions used as the arguments of the function call + * expression. + * + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param functionDefinition a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @param argValues values of function arguments from which the new values being + * generated are derived. + * @return a list of function argument expressions. + */ + private List generateArgsForFunctionCall(FactManager factManager, + FunctionDefinition functionDefinition, + Stmt stmtToInsertBefore, + List argValues) { + return argValues.stream() + .map(item -> generateExpr( + factManager, + functionDefinition, + stmtToInsertBefore, + item + )).collect(Collectors.toList()); + } + + private String freshId() { + return ID + "_" + idGenerator.freshId(); + } + + private String genVarName(Value value, boolean isGlobal) { + // Provides name for a new variable based on the given value. + // For example: + // _GLF_PRIMITIVE_int_negative_103_0_id_18: a variable of a value -103. + // _GLF_PRIMITIVE_float_unknown_numeric_id_69: a variable that can be any value of float type. + return (isGlobal ? Constants.GLF_PRIMITIVE_GLOBAL : Constants.GLF_PRIMITIVE) + + "_" + value.getType().toString() + + parseNameFromValue(value) + + freshId(); + } + + private String genFunctionName(Value value) { + // Provides name for a new function based on the given value. + // For example: + // _GLF_COMPUTE_float_1_0_id_0: a function that returns a value 1.0 of float type. + // _GLF_COMPUTE_vec4_UNKNOWN_id_1: a function that returns randomly generated value of vec4 + // type. + return Constants.GLF_COMPUTE + + "_" + value.getType().toString() + + parseNameFromValue(value) + + freshId(); + } + + private String genParamName(Type type, boolean unknownValue) { + // Provides name for a function arguments based on the given value. + // For example: + // _GLF_UNKNOWN_PARAM_vec4_id_1: a parameter of unknown value of vec4 type. + // _GLF_PARAM_int_id_62: a parameter of integer type. + return (unknownValue ? Constants.GLF_UNKNOWN_PARAM : Constants.GLF_PARAM) + + "_" + type.toString() + + freshId(); + } + + /** + * Utility function to parse name from the given value, for example, -0.45 will be parsed as + * negative_0_45. + * + * @param value value that will be converted to the name of variables or functions. + * @return a string derived from the given value which will be used as function or variable names. + */ + private String parseNameFromValue(Value value) { + if (value.valueIsUnknown()) { + return "_" + value.toString(); + } + final StringBuilder name = new StringBuilder(); + if (value instanceof NumericValue) { + final NumericValue numericValue = (NumericValue) value; + float floatValue = numericValue.getValue().get().floatValue(); + if (floatValue < 0.0) { + name.append(NEGATIVE); + floatValue = Math.abs(floatValue); + } + name.append("_"); + // Replace dot with underscore, i.e., 0.45 will be converted to 0_45. + name.append(Float.toString(floatValue).replace(".", "_")); + + } + if (value instanceof BooleanValue) { + name.append("_").append(value.toString()); + } + return name.toString(); + } + + /** + * Generates a new variable to which the expected value is assigned. The variable being + * generated is also randomly chosen whether it would be available at the local or global scope. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before.* + * @return the variable identifier expression of the declared variable generated by this method. + */ + private Expr generateVariableFact(Value value, + FactManager factManager, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + boolean atGlobalScope = factManager.globalScope() || generator.nextBoolean(); + final String varName = genVarName(value, atGlobalScope); + + final VariableDeclInfo variableDeclInfo = new VariableDeclInfo(varName, null, + new Initializer(generateExpr(atGlobalScope ? globalFactManager : factManager, + currentFunction, + stmtToInsertBefore, + value + ))); + + final VariablesDeclaration variablesDecl = new VariablesDeclaration(value.getType(), + variableDeclInfo); + final VariableDeclFact variableDeclFact = new VariableDeclFact(variablesDecl, + variableDeclInfo, value); + if (atGlobalScope) { + translationUnit.addDeclarationBefore(variablesDecl, currentFunction); + globalFactManager.addVariableFact(value, variableDeclFact); + } else { + currentFunction.getBody().insertBefore(stmtToInsertBefore, + new DeclarationStmt(variablesDecl)); + factManager.addVariableFact(value, variableDeclFact); + } + return new VariableIdentifierExpr(varName); + } + + /** + * Generates a new function that guarantees to return the value representing the expected Value. + * A number and type of the function arguments for the function being generated are randomized. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @return function call expression of the function generated by this method. + */ + private Expr generateFunctionFact(Value value, + FactManager factManager, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + final String functionName = genFunctionName(value); + final FactManager newFunctionScope = globalFactManager.newScope(); + final List argumentValues = new ArrayList<>(); + final List parameterDecls = new ArrayList<>(); + + final int noOfParams = generator.nextInt(MAX_FUNCTION_PARAMS); + for (int i = 0; i < noOfParams; i++) { + final Type paramType = getAvailableTypes().get(generator.nextInt(getAvailableTypes().size())); + // Decide whether the value generated should be known by the fact manager. + // If the fact manager is generating an unknown parameter(value is Optional.empty), + // when calling this function the fact manager will generate any arbitrary value that + // matches the parameter type. + final Value paramValue = fuzzValue(paramType); + final String paramName = genParamName(paramType, paramValue.valueIsUnknown()); + + argumentValues.add(paramValue); + final ParameterDecl parameterDecl = new ParameterDecl( + paramName, + paramType, + null + ); + parameterDecls.add(parameterDecl); + newFunctionScope.addVariableFact(paramValue, new ParameterDeclFact(parameterDecl, + paramValue)); + } + final BlockStmt body = new BlockStmt(Collections.emptyList(), false); + final FunctionPrototype functionPrototype = new FunctionPrototype(functionName, + value.getType(), parameterDecls); + final FunctionDefinition newFunction = new FunctionDefinition(functionPrototype, + body); + translationUnit.addDeclarationBefore(newFunction, currentFunction); + + // Since the new function has an empty body, we first need to add a placeholder statement + // into the body to be the point in a function where the new expressions can be injected + // before. + final Stmt placeholderStmt = new NullStmt(); + body.addStmt(placeholderStmt); + + // We then replace the placeholder statement with a new return statement, returning a + // newly-generated expression. + body.replaceChild(placeholderStmt, new ReturnStmt( + generateExpr( + newFunctionScope, + newFunction, + placeholderStmt, + value + ) + )); + + globalFactManager.addFunctionFact(value, new FunctionFact(functionPrototype, argumentValues, + value)); + final List args = generateArgsForFunctionCall(factManager, currentFunction, + stmtToInsertBefore, + argumentValues); + return new FunctionCallExpr(functionName, args); + } + + private List getAvailableTypes() { + // We will have to consider supporting more types but at the moment as we are using + // LiteralFuzzer to generate literal expressions so we are unable to cover all basic type now. + // The following are types currently supported by LiteralFuzzer. + return Arrays.asList(BasicType.BOOL, BasicType.INT, BasicType.FLOAT, BasicType.VEC2, + BasicType.VEC3, BasicType.VEC4); + } + + private Value fuzzValue(Type type) { + // An Unknown value variable is the variable whose value could be anything. The fact manager + // could generate any arbitrary value but with the correct type. + final boolean isUnknown = generator.nextBoolean(); + if (type == BasicType.BOOL) { + return new BooleanValue( + isUnknown ? Optional.empty() : Optional.of(generator.nextBoolean())); + } + if (BasicType.allScalarTypes().contains(type)) { + if (isUnknown) { + return new NumericValue((BasicType) type, Optional.empty()); + } + if (type == BasicType.INT) { + final String intString = String.valueOf(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN); + return new NumericValue(BasicType.INT, Optional.of(Integer.valueOf(intString))); + } + if (type == BasicType.FLOAT) { + final String floatString = LiteralFuzzer.randomFloatString(generator); + return new NumericValue(BasicType.FLOAT, Optional.of(Float.valueOf(floatString))); + } + if (type == BasicType.UINT) { + final String intString = String.valueOf(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN); + return new NumericValue(BasicType.UINT, Optional.of(Math.abs(Integer.valueOf(intString)))); + } + throw new RuntimeException("Not implemented yet!"); + } + + if (isUnknown) { + return new CompositeValue(type, Optional.empty()); + } + if (BasicType.allBasicTypes().contains(type)) { + final List values = new ArrayList<>(); + for (int i = 0; i < ((BasicType) type).getNumElements(); i++) { + values.add(fuzzValue(((BasicType) type).getElementType())); + } + return new CompositeValue(type, Optional.of(values)); + } + + // TODO(https://github.com/google/graphicsfuzz/issues/664): we should also support array and + // struct types as well. + throw new RuntimeException("Not implemented yet!"); + } + + /** + * A utility method that returns a pair of two numbers whose sum is equal to the given value. + * + * @param value the original value that will be split into two numbers. + * @return if value is unknown, returns a pair of empty value. Otherwise find and return two + * numbers that will add up to the given value. + */ + public Pair, Optional> getPairSum(Value value) { + assert value instanceof NumericValue; + final NumericValue numericValue = (NumericValue) value; + + if (numericValue.valueIsUnknown()) { + return new ImmutablePair<>(Optional.empty(), Optional.empty()); + } + + // Following the equation a = (a/b) + (a - a/b), we first need to pick a random integer b + // in range 1-10. We then derive the left number by dividing the original number a with the + // random number b. We then generate the right number by subtracting the original value a + // with the left number obtained from the previous step. + // + // For example, if a number 7 is an input and we pick a random integer 3. The left number + // is (7/3) which equals to 2. Next we subtract the original value with the left number + // to find the right number: 7 - 2 = 5. We finally have two numbers 2 and 5 that can add + // up to 7. + + // TODO(https://github.com/google/graphicsfuzz/issues/663): we can derive a number based on the + // known facts hold by FactManager here. + + if (numericValue.getType() == BasicType.FLOAT) { + final float a = numericValue.getValue().get().floatValue(); + final float b = Math.max(1, generator.nextInt(10)); + final float left = a / b; + final float right = a - left; + return new ImmutablePair<>(Optional.of(left), Optional.of(right)); + } + + if (numericValue.getType() == BasicType.INT) { + final int a = numericValue.getValue().get().intValue(); + final int b = Math.max(1, generator.nextInt(10)); + final int left = a / b; + final int right = a - left; + return new ImmutablePair<>(Optional.of(left), Optional.of(right)); + } + + throw new RuntimeException("Should be unreachable"); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java new file mode 100644 index 000000000..58cc4f712 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java @@ -0,0 +1,117 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class FactManager { + + // A parent fact manager whose facts this fact manager extends. + private final FactManager prototype; + private final Map> variableFacts; + private final Map> functionFacts; + + public FactManager(FactManager prototype) { + this.prototype = prototype; + variableFacts = new HashMap>(); + functionFacts = new HashMap>(); + } + + /** + * Creates a new fact manager where this fact manager is used as a prototype. We basically call + * this method only when generating a new function where the return fact manager is non global. + * + * @return new fact manager. + */ + public FactManager newScope() { + return new FactManager(this); + } + + /** + * Determines whether the fact manager is at the global scope. + * + * @return true if this manager has a prototype and false otherwise. + */ + public boolean globalScope() { + return prototype == null; + } + + /** + * Adds a new variable fact to the variable fact map of this fact manager. + * + * @param value value which the given variable fact is representing. + * @param variableFact a new variable fact that will be added into the map of variable facts. + */ + public void addVariableFact(Value value, VariableFact variableFact) { + final List newFacts = variableFacts.getOrDefault(value, new ArrayList<>()); + newFacts.add(variableFact); + variableFacts.put(value, newFacts); + } + + /** + * As function facts can only exist at the global scope, this method adds a new function + * fact into the map of the function facts of the global scope fact manager. + * + * @param value value which the given function fact is representing. + * @param functionFact a new function fact that will be added into the map of function facts. + */ + public void addFunctionFact(Value value, FunctionFact functionFact) { + if (globalScope()) { + final List newFacts = functionFacts.getOrDefault(value, new ArrayList<>()); + newFacts.add(functionFact); + functionFacts.put(value, newFacts); + } else { + prototype.addFunctionFact(value, functionFact); + } + } + + /** + * Retrieves a list of variable facts representing the value. + * + * @param value a value which the fact manager is asked to search in the variable fact map. + * @return if value does not exist in the variable fact map, an empty list is returned. Otherwise, + * returns a list of variable facts that guarantees to provide the given value. + */ + public List getVariableFacts(Value value) { + final List result = new ArrayList<>(); + result.addAll(variableFacts.getOrDefault(value, Collections.emptyList())); + if (!globalScope()) { + result.addAll(prototype.getVariableFacts(value)); + } + return result; + } + + /** + * Retrieves a list of function facts representing the value from the global scope fact + * manager. If the value does not exist in the map, an empty list is returned. + * + * @param value a value which the fact manager is asked to search in the function fact map. + * @return if value does not exist in the function fact map, an empty list is returned. + * Otherwise, returns a list of function facts that guarantees to provide the given value. + */ + public List getFunctionFacts(Value value) { + if (globalScope()) { + return functionFacts.getOrDefault(value, Collections.emptyList()); + } + return prototype.getFunctionFacts(value); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FunctionFact.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FunctionFact.java new file mode 100644 index 000000000..dfbdb15a5 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FunctionFact.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import java.util.List; + +/** + * This class holds the information of the newly-generated function and its associated Value. + * Each time we generate a new function, we create a function fact and keep it in Fact Manager + * which later will be used by the Expression Generator when generating known value expression. + */ +public class FunctionFact { + + private final FunctionPrototype prototype; + private final List arguments; + private final Value value; + + public FunctionFact(FunctionPrototype prototype, List arguments, Value value) { + this.prototype = prototype; + this.arguments = arguments; + this.value = value; + } + + public String getFunctionName() { + return prototype.getName(); + } + + public List getArguments() { + return arguments; + } + + public Value getValue() { + return value; + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/NumericValue.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/NumericValue.java new file mode 100644 index 000000000..983a8451f --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/NumericValue.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.FloatConstantExpr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; +import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; +import java.util.Objects; +import java.util.Optional; + +public class NumericValue implements Value { + + private final BasicType basicType; + private final Optional value; + + public NumericValue(BasicType basicType, Optional value) { + this.basicType = basicType; + this.value = value; + } + + @Override + public Type getType() { + return basicType; + } + + @Override + public boolean valueIsUnknown() { + return !value.isPresent(); + } + + @Override + public Expr generateLiteral(LiteralFuzzer literalFuzzer) { + if (valueIsUnknown()) { + return literalFuzzer.fuzz(basicType).orElse(null); + } + if (basicType == BasicType.FLOAT) { + return new FloatConstantExpr(value.get().toString()); + } + if (basicType == BasicType.INT) { + return new IntConstantExpr(value.get().toString()); + } + if (basicType == BasicType.UINT) { + return new UIntConstantExpr(value.get().toString() + "u"); + } + throw new RuntimeException("Numeric value does not support the given type"); + } + + public Optional getValue() { + return value; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof NumericValue)) { + return false; + } + final NumericValue thatNumericValue = (NumericValue) that; + return this.basicType.equals(thatNumericValue.basicType) + && this.value.equals(thatNumericValue.getValue()); + } + + @Override + public int hashCode() { + return Objects.hash(basicType, value); + } + + @Override + public String toString() { + return valueIsUnknown() ? "unknown_numeric" : value.get().toString(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ParameterDeclFact.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ParameterDeclFact.java new file mode 100644 index 000000000..f990d26ab --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ParameterDeclFact.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.decl.ParameterDecl; + +/** + * This class holds the information of a parameter of generated functions and its associated value. + * For any generated function, we randomly decide how parameters should be and for each parameter + * we create a parameter declaration fact and keep it in Fact Manager which later will be used by + * the Expression Generator when generating known value expression. + */ +public class ParameterDeclFact extends VariableFact { + + private final ParameterDecl parameterDecl; + + public ParameterDeclFact(ParameterDecl parameterDecl, + Value value) { + super(value); + this.parameterDecl = parameterDecl; + } + + @Override + public String getVariableName() { + return parameterDecl.getName(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/Value.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/Value.java new file mode 100644 index 000000000..da9197605 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/Value.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; + +/** + * This interface defines an expected Value used by {@link ExpressionGenerator} when + * generating an expression. It represents a possibly unknown value of some type. + */ +public interface Value { + + /** + * Indicates whether or not this is an unknown value. + * @return true if the value is unknown and false otherwise. + */ + boolean valueIsUnknown(); + + /** + * Gets the type of the underlying value. + * @return the basic type of the value. + */ + Type getType(); + + /** + * Provides a literal with the same type as the Value's type, such that all parts of the value + * that are known will have the expected values, and all other parts will be randomized. + * + * @param literalFuzzer a util class used to generate fuzzed expressions. + * @return the expression that represents the value. + */ + Expr generateLiteral(LiteralFuzzer literalFuzzer); + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableDeclFact.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableDeclFact.java new file mode 100644 index 000000000..8b49f1a5c --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableDeclFact.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; + +/** + * This class holds the information of the newly-generated variable and its associated Value. + * Each time we declare a new variable, we create a variable declaration fact and keep it in Fact + * Manager which later will be used by the Expression Generator when generating known value + * expression. + */ +public class VariableDeclFact extends VariableFact { + + private final VariablesDeclaration variablesDeclaration; + private final VariableDeclInfo variableDeclInfo; + + public VariableDeclFact(VariablesDeclaration variablesDeclaration, + VariableDeclInfo variableDeclInfo, + Value value) { + super(value); + this.variablesDeclaration = variablesDeclaration; + this.variableDeclInfo = variableDeclInfo; + } + + public String getVariableName() { + return variableDeclInfo.getName(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableFact.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableFact.java new file mode 100644 index 000000000..62b5c1e5d --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableFact.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +public abstract class VariableFact { + + private final Value value; + + /** + * @param value which a new VariableDeclFact or ParameterDeclFact is representing. + */ + VariableFact(Value value) { + this.value = value; + } + + /** + * @return a known value of the variable fact. + */ + public Value getValue() { + return value; + } + + /** + * @return a variable name of this variable fact. This is used by the generator + * when it is generating a new variable identifier expression. + */ + public abstract String getVariableName(); + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java index 8a6c46de2..e678835e1 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java @@ -47,14 +47,13 @@ public Optional fuzz(Type type) { } if (type == BasicType.INT) { return Optional.of(new IntConstantExpr( - String.valueOf(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN))); + String.valueOf(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN))); } if (type == BasicType.FLOAT) { - return Optional.of(new FloatConstantExpr( - randomFloatString())); + return Optional.of(new FloatConstantExpr(LiteralFuzzer.randomFloatString(generator))); } if (type == BasicType.VEC2 || type == BasicType.VEC3 || type == BasicType.VEC4 - || BasicType.allMatrixTypes().contains(type)) { + || BasicType.allMatrixTypes().contains(type)) { final List args = new ArrayList<>(); for (int i = 0; i < ((BasicType) type).getNumElements(); i++) { args.add(fuzz(((BasicType) type).getElementType()).get()); @@ -64,7 +63,7 @@ public Optional fuzz(Type type) { return Optional.empty(); } - private String randomFloatString() { + public static String randomFloatString(IRandom generator) { final int maxDigitsEitherSide = 5; StringBuilder sb = new StringBuilder(); sb.append(generator.nextBoolean() ? "-" : ""); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java index c59710edd..5a40c701b 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java @@ -90,7 +90,7 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .type(File.class); parser.addArgument("output") - .help("Output shader job file file (.json.") + .help("Output shader job file file (.json).") .type(File.class); addGeneratorCommonArguments(parser); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java new file mode 100644 index 000000000..30f4c827a --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java @@ -0,0 +1,172 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.tool; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.NullStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.LayoutQualifierSequence; +import com.graphicsfuzz.common.ast.type.LocationLayoutQualifier; +import com.graphicsfuzz.common.ast.type.QualifiedType; +import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.ast.type.VoidType; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.PipelineInfo; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; +import com.graphicsfuzz.generator.knownvaluegeneration.ExpressionGenerator; +import com.graphicsfuzz.generator.knownvaluegeneration.FactManager; +import com.graphicsfuzz.generator.knownvaluegeneration.NumericValue; +import com.graphicsfuzz.util.ArgsUtil; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import java.util.stream.Stream; +import net.sourceforge.argparse4j.ArgumentParsers; +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.ArgumentParserException; +import net.sourceforge.argparse4j.inf.Namespace; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KnownValueShaderGenerator { + + private static final Logger LOGGER = LoggerFactory.getLogger(KnownValueShaderGenerator.class); + + private static Namespace parse(String[] args) throws ArgumentParserException { + ArgumentParser parser = ArgumentParsers.newArgumentParser("KnownValueShaderGenerator") + .defaultHelp(true) + .description("Generate a known value shader with the desired RGBA color."); + + parser.addArgument("output") + .help("Output shader job file file (.json).") + .type(File.class); + + parser.addArgument("r") + .help("Floating point number of red value.") + .type(Float.class); + + parser.addArgument("g") + .help("Floating point number of green value.") + .type(Float.class); + + parser.addArgument("b") + .help("Floating point number of blue value.") + .type(Float.class); + + parser.addArgument("a") + .help("Floating point number of alpha value.") + .type(Float.class); + + parser.addArgument("--version") + .help("Shading language version of a generated fragment shader.") + .setDefault("410") + .type(String.class); + + parser.addArgument("--seed") + .help("Seed (unsigned 64 bit long integer) for the random number generator.") + .type(String.class); + + return parser.parseArgs(args); + } + + public static void mainHelper(String[] args) throws ArgumentParserException, IOException { + final Namespace ns = parse(args); + final float rFloat = ns.getFloat("r"); + final float gFloat = ns.getFloat("g"); + final float bFloat = ns.getFloat("b"); + final float aFloat = ns.getFloat("a"); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); + final String version = ns.getString("version"); + final File shaderJobFile = ns.get("output"); + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + final TranslationUnit tu = + new TranslationUnit(Optional.of(ShadingLanguageVersion.fromVersionString(version)), + Arrays.asList( + new PrecisionDeclaration("precision highp float;"), + new PrecisionDeclaration("precision highp int;"), + new VariablesDeclaration(new QualifiedType(BasicType.VEC4, + Arrays.asList(new LayoutQualifierSequence( + new LocationLayoutQualifier(0)), TypeQualifier.SHADER_OUTPUT)), + new VariableDeclInfo("_GLF_color", null, null)), + new FunctionDefinition( + new FunctionPrototype("main", VoidType.VOID, Collections.emptyList()), + new BlockStmt(new ArrayList<>(), false)))); + + final PipelineInfo pipelineInfo = new PipelineInfo(); + final FactManager globalFactManager = new FactManager(null); + + // A placeholder statement for what will eventually be the color assignment. + final Stmt placeholderForColorAssignment = new NullStmt(); + tu.getMainFunction().getBody().addStmt(placeholderForColorAssignment); + + LOGGER.info("About to generate the known value fragment shader" + + " with the parameters R = " + rFloat + ", G = " + gFloat + ", B = " + bFloat + " and" + + " A = " + aFloat + "."); + + final ExpressionGenerator expressionGenerator = new + ExpressionGenerator(tu, pipelineInfo, generator, globalFactManager); + final FactManager mainFactManager = globalFactManager.newScope(); + final Expr[] rgbaExprs = Stream.of(rFloat, gFloat, bFloat, aFloat) + .map(item -> expressionGenerator.generateExpr( + mainFactManager, + tu.getMainFunction(), + placeholderForColorAssignment, + new NumericValue(BasicType.FLOAT, Optional.of(item)))).toArray(Expr[]::new); + + tu.getMainFunction().getBody().replaceChild(placeholderForColorAssignment, + new ExprStmt(new BinaryExpr(new VariableIdentifierExpr("_GLF_color"), + new TypeConstructorExpr("vec4", rgbaExprs), BinOp.ASSIGN))); + + final ShaderJob newShaderJob = new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu); + fileOps.writeShaderJobFile(newShaderJob, shaderJobFile); + LOGGER.info("Generation complete."); + } + + public static void main(String[] args) { + try { + mainHelper(args); + } catch (ArgumentParserException exception) { + exception.getParser().handleError(exception); + System.exit(1); + } catch (IOException exception) { + exception.printStackTrace(); + System.exit(1); + } + } + +} diff --git a/python/src/main/python/drivers/knownvalue-shader-generator b/python/src/main/python/drivers/knownvalue-shader-generator new file mode 100755 index 000000000..4ec96a5bb --- /dev/null +++ b/python/src/main/python/drivers/knownvalue-shader-generator @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if test -n "${PYTHON_GF}"; then + "${PYTHON_GF}" ${BASH_SOURCE}.py "$@" +elif type -P python3 >/dev/null; then + python3 ${BASH_SOURCE}.py "$@" +elif type -P py >/dev/null; then + py -3 ${BASH_SOURCE}.py "$@" +else + python ${BASH_SOURCE}.py "$@" +fi diff --git a/python/src/main/python/drivers/knownvalue-shader-generator.bat b/python/src/main/python/drivers/knownvalue-shader-generator.bat new file mode 100644 index 000000000..aded83e44 --- /dev/null +++ b/python/src/main/python/drivers/knownvalue-shader-generator.bat @@ -0,0 +1,29 @@ +@echo off + +@REM +@REM Copyright 2019 The GraphicsFuzz Project Authors +@REM +@REM Licensed under the Apache License, Version 2.0 (the "License"); +@REM you may not use this file except in compliance with the License. +@REM You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. +@REM + +where /q py +IF ERRORLEVEL 0 ( + py -3 "%~dpn0.py" %* +) ELSE ( + where /q python3 + IF ERRORLEVEL 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) +) diff --git a/python/src/main/python/drivers/knownvalue-shader-generator.py b/python/src/main/python/drivers/knownvalue-shader-generator.py new file mode 100644 index 000000000..aa773ba57 --- /dev/null +++ b/python/src/main/python/drivers/knownvalue-shader-generator.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +HERE = os.path.abspath(__file__) + +sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) + +import cmd_helpers + + +def go(argv): + java_tool_path = cmd_helpers.get_tool_path() + print(java_tool_path) + + cmd = ["java", "-ea", "-cp", java_tool_path, "com.graphicsfuzz.generator.tool.KnownValueShaderGenerator"] \ + + argv + print(cmd) + + generate_proc = subprocess.Popen(cmd) + generate_proc.communicate() + return generate_proc.returncode + + +if __name__ == "__main__": + sys.exit(go(sys.argv[1:])) diff --git a/util/src/main/java/com/graphicsfuzz/util/Constants.java b/util/src/main/java/com/graphicsfuzz/util/Constants.java index 2336a9679..71c74fdb8 100755 --- a/util/src/main/java/com/graphicsfuzz/util/Constants.java +++ b/util/src/main/java/com/graphicsfuzz/util/Constants.java @@ -45,6 +45,13 @@ private Constants() { public static final String GLF_IDENTITY = "_GLF_IDENTITY"; public static final String GLF_SWITCH = "_GLF_SWITCH"; + public static final String GLF_PRIMITIVE = "_GLF_PRIMITIVE"; + public static final String GLF_PRIMITIVE_GLOBAL = "_GLF_PRIMITIVE_GLOBAL"; + public static final String GLF_COMPUTE = "_GLF_COMPUTE"; + public static final String GLF_PARAM = "_GLF_PARAM"; + public static final String GLF_UNKNOWN_PARAM = "GLF_UNKNOWN_PARAM"; + + public static final String INJECTED_LOOP_COUNTER = "_injected_loop_counter"; public static final String INJECTION_SWITCH = "injectionSwitch"; From 5937a06005f778b532917f3ce199744fec34992f Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Mon, 12 Aug 2019 14:31:05 +0100 Subject: [PATCH 090/180] Gracefully handle scenario where no compatible donors are available (#656) It is possible to get into a situation with live or dead code injection where none of the available donors are compatible with the reference. For example, they might all share a named struct in common (which at present is a source of incompatibility). When such a situation arises the generator would throw an exception. This change causes the generator, instead, to inject a null statement (;). Fixes #645. --- .../DonateCodeTransformation.java | 106 +++++++++++++----- .../generator/util/GenerationParams.java | 14 +-- .../tester/GeneratorUnitTest.java | 31 +++++ 3 files changed, 117 insertions(+), 34 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java index 5991c2fae..8cb945990 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java @@ -72,6 +72,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -80,11 +81,20 @@ public abstract class DonateCodeTransformation implements ITransformation { private final Function probabilityOfDonation; + // During a single donation pass, this is populated on demand with the donors that are used. private Map donorsToTranslationUnits; + private final List functionPrototypes; private final Map globalVariables; private final Set structNames; - private final List donorFiles; + + // 'donorFiles' contains those donors that have not yet been used for a code donation. Once a + // donor has been used, it is moved to 'usedDonorFiles' (unless it is found to be incompatible, + // in which case it is discarded). If 'donorFiles' becomes empty, it and 'usedDonorFiles' are + // swapped, so that previously used donors can be used again. + private List donorFiles; + private List usedDonorFiles; + final GenerationParams generationParams; private int translationUnitCount; @@ -95,13 +105,14 @@ public DonateCodeTransformation(Function probabilityOfDonation this.functionPrototypes = new ArrayList<>(); this.globalVariables = new HashMap<>(); this.structNames = new HashSet<>(); + assert donorsDirectory.exists(); this.donorFiles = new ArrayList<>(); + this.donorFiles.addAll(Arrays.asList(donorsDirectory.listFiles( + pathname -> pathname.getName().endsWith(".frag")))); + this.donorFiles.sort(Comparator.naturalOrder()); + this.usedDonorFiles = new ArrayList<>(); this.generationParams = generationParams; this.translationUnitCount = 0; - assert donorsDirectory.exists(); - donorFiles.addAll(Arrays.asList(donorsDirectory.listFiles( - pathname -> pathname.getName().endsWith(".frag")))); - donorFiles.sort(Comparator.naturalOrder()); } /** @@ -208,11 +219,19 @@ private Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, final int maxTries = 10; int tries = 0; while (true) { - DonationContext donationContext = new DonationContextFinder(chooseDonor(generator), generator) + final Optional maybeDonor = chooseDonor(generator); + if (!maybeDonor.isPresent()) { + // No compatible donors were found, thus we cannot do serious code donation here; + // we return a null statement instead. + return new NullStmt(); + } + DonationContext donationContext = new DonationContextFinder(maybeDonor.get(), generator) .getDonationContext(); if (incompatible(injectionPoint, donationContext, shadingLanguageVersion)) { tries++; if (tries == maxTries) { + // We have tried and tried to find something compatible to inject but not managed; + // return a null statement instead of a real piece of code to inject. return new NullStmt(); } } else { @@ -257,10 +276,18 @@ public boolean apply(TranslationUnit tu, } private void eliminateUsedDonors() { - // Having done donation using a particular donor, we remove it so that we do not use it - // again in a future pass. + // Having done donation using a particular donor, we move it to the list of used donors so that + // we only use it again in a future pass once all other donors have been tried. for (File donor : donorsToTranslationUnits.keySet()) { + assert donorFiles.contains(donor); donorFiles.remove(donor); + usedDonorFiles.add(donor); + } + // If there are no donors left -- i.e., if all have been used, then recycle the list of used + // donors. + if (donorFiles.isEmpty()) { + donorFiles = usedDonorFiles; + usedDonorFiles = new ArrayList<>(); } donorsToTranslationUnits = new HashMap<>(); } @@ -580,25 +607,49 @@ private boolean prototypeMatches(FunctionPrototype fp, List f return !fs.stream().filter(item -> fp.matches(item)).collect(Collectors.toList()).isEmpty(); } - TranslationUnit chooseDonor(IRandom generator) { + Optional chooseDonor(IRandom generator) { + // The donors that we have previously selected during this donation pass are captured via + // 'donorsToTranslationUnits'. Furthermore, there is a maximum number of distinct donors we + // are allowed to use per donation pass. So first check whether the donors we have already + // used have hit this maximum. + + final String previouslyUsedDonorFoundToBeIncompatibleMessage = "A donor that was previously " + + "used has now been found to be incompatible. This should not occur."; + + if (donorsToTranslationUnits.size() >= generationParams.getMaxDonorsPerDonationPass()) { + // We have used the maximum number of donors we are allowed to use in this pass, so choose + // one of them again. + // The limit should be reached but never exceeded. + assert donorsToTranslationUnits.size() == generationParams.getMaxDonorsPerDonationPass(); + final List sortedKeys = new ArrayList<>(donorsToTranslationUnits.keySet()); + sortedKeys.sort(Comparator.naturalOrder()); + try { + return Optional.of(getDonorTranslationUnit(sortedKeys.get(generator.nextInt(sortedKeys + .size())), + generator)); + } catch (IncompatibleDonorException exception) { + throw new RuntimeException(previouslyUsedDonorFoundToBeIncompatibleMessage); + } + } + + // We have not reached the limit. So we can choose any one of the available donor files. It + // might turn out to be a donor we tried before (in which case it will already be a key to + // the 'donorsToTranslationUnits' map), or it may be new. while (!donorFiles.isEmpty()) { - int index = generator.nextInt(donorFiles.size()); - File donorFile = donorFiles.get(index); + final File candidateDonorFile = donorFiles + .get(generator.nextInt(donorFiles.size())); try { - if (donorsToTranslationUnits.keySet().size() >= generationParams.getMaxDonors() - && !donorsToTranslationUnits.containsKey(donorFile)) { - // We have reached the donor limit; try again until we find a donor that we've - // already used - throw new IncompatibleDonorException(); - } - return getDonorTranslationUnit(donorFile, generator); + return Optional.of(getDonorTranslationUnit(candidateDonorFile, generator)); } catch (IncompatibleDonorException exception) { - assert index >= 0; - assert index < donorFiles.size(); - donorFiles.remove(index); + if (donorsToTranslationUnits.containsKey(candidateDonorFile)) { + throw new RuntimeException(previouslyUsedDonorFoundToBeIncompatibleMessage); + } + donorFiles.remove(candidateDonorFile); } } - throw new RuntimeException("Could not find any compatible donors."); + // We did not manage to find any suitable donor. This happens if all donors prove to be + // incompatible. + return Optional.empty(); } private TranslationUnit getDonorTranslationUnit(File donorFile, IRandom generator) @@ -622,10 +673,11 @@ private TranslationUnit getDonorTranslationUnit(File donorFile, IRandom generato } private boolean compatibleDonor(TranslationUnit donor) { - List usedFunctionNames = functionPrototypes.stream().map(item -> item.getName()) + final List usedFunctionNames = functionPrototypes.stream() + .map(FunctionPrototype::getName) .collect(Collectors.toList()); - Set usedGlobalVariableNames = globalVariables.keySet(); - Set usedStructNames = structNames; + final Set usedGlobalVariableNames = globalVariables.keySet(); + final Set usedStructNames = structNames; for (FunctionPrototype donorPrototype : AstUtil.getFunctionPrototypesFromShader(donor)) { if (usedGlobalVariableNames.contains(donorPrototype.getName()) @@ -637,8 +689,8 @@ private boolean compatibleDonor(TranslationUnit donor) { } } for (Map.Entry global : getGlobalVariablesFromShader(donor).entrySet()) { - String name = global.getKey(); - Type type = global.getValue(); + final String name = global.getKey(); + final Type type = global.getValue(); if (usedFunctionNames.contains(name) || usedStructNames.contains(name)) { return false; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/GenerationParams.java b/generator/src/main/java/com/graphicsfuzz/generator/util/GenerationParams.java index f80b22c3a..df8c494e6 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/GenerationParams.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/GenerationParams.java @@ -29,7 +29,7 @@ public static GenerationParams small(ShaderKind shaderKind, boolean injectionSwi result.maxDepthForGeneratedExpr = 2; result.maxStructNestingDepth = 1; result.maxStructFields = 3; - result.maxDonors = 2; + result.maxDonorsPerDonationPass = 2; return result; } @@ -38,7 +38,7 @@ public static GenerationParams normal(ShaderKind shaderKind, boolean injectionSw result.maxDepthForGeneratedExpr = 3; result.maxStructNestingDepth = 2; result.maxStructFields = 5; - result.maxDonors = 4; + result.maxDonorsPerDonationPass = 4; return result; } @@ -47,7 +47,7 @@ public static GenerationParams large(ShaderKind shaderKind, boolean injectionSwi result.maxDepthForGeneratedExpr = 5; result.maxStructNestingDepth = 4; result.maxStructFields = 7; - result.maxDonors = 6; + result.maxDonorsPerDonationPass = 6; return result; } @@ -67,8 +67,8 @@ private GenerationParams(ShaderKind shaderKind, boolean injectionSwitchIsAvailab private int maxStructNestingDepth = 3; private int maxStructFields = 8; - // Donors - private int maxDonors = 5; + // The maximum number of distinct donors that can be used during one donation pass. + private int maxDonorsPerDonationPass = 5; private final boolean injectionSwitchIsAvailable; @@ -84,8 +84,8 @@ public int getMaxStructFields() { return maxStructFields; } - public int getMaxDonors() { - return maxDonors; + public int getMaxDonorsPerDonationPass() { + return maxDonorsPerDonationPass; } public ShaderKind getShaderKind() { diff --git a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java index fb573ab5e..1bad33a66 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java @@ -364,5 +364,36 @@ private void testTransformationForSpecificShader(List t new File[] { referenceJson }); } + @Test + public void testDonateDeadCodeNoCompatibleDonors() throws Exception { + // By design, we set up a situation where the reference and donors are all identical + // and all declare a struct, which makes them incompatible (because at the moment we + // conservatively say that a donor and reference are incompatible if they declare structs + // with the same name). + // + // Dead code donation will not be able to succeed, but should have no effect rather than + // aborting. + final File donors = temporaryFolder.newFolder(); + final ShaderJob declaresSingleStruct = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), ParseHelper.parse("#version 300 es\n" + + "precision highp float;\n" + + "struct S {\n" + + " int x;\n" + + "};\n" + + "void main() {\n" + + "}\n")); + for (int i = 0; i < 3; i++) { + fileOps.writeShaderJobFile(declaresSingleStruct, new File(donors, "donor_" + i + ".json")); + } + final File reference = temporaryFolder.newFile("reference.json"); + fileOps.writeShaderJobFile(declaresSingleStruct, reference); + testTransformation(Collections.singletonList(() -> new DonateDeadCodeTransformation( + TransformationProbabilities.likelyDonateDeadCode()::donateDeadCodeAtStmt, + donors, + GenerationParams.normal(ShaderKind.FRAGMENT, true))), + TransformationProbabilities.likelyDonateDeadCode(), + "donatedead", Collections.emptyList(), + new File[] { reference }); + } } From a18b8f8e4ae796f77ed738202079a412be038bb9 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Tue, 13 Aug 2019 18:02:08 +0700 Subject: [PATCH 091/180] New 310es shader - binary search tree (#676) --- .../glsl/samples/310es/binarysearch_tree.frag | 176 ++++++++++++++++++ .../glsl/samples/310es/binarysearch_tree.json | 16 ++ 2 files changed, 192 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/binarysearch_tree.frag create mode 100644 shaders/src/main/glsl/samples/310es/binarysearch_tree.json diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag b/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag new file mode 100644 index 000000000..273458383 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag @@ -0,0 +1,176 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct BST{ + int data; + int leftIndex; + int rightIndex; +}; + +BST tree[10]; + +void makeTreeNode(inout BST tree, int data) +{ + tree.data = data; + tree.leftIndex = -1; + tree.rightIndex = -1; +} + +void insert(int treeIndex, int data) +{ + int baseIndex = 0; + while (baseIndex <= treeIndex) { + // If new value is smaller thatn the current node, we known that we will have + // add this element in the left side. + if (data <= tree[baseIndex].data) { + // If a left subtree of the current node is empty, the new node is added as + // a left subtree of the current node. + if (tree[baseIndex].leftIndex == -1) { + tree[baseIndex].leftIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].leftIndex; + continue; + } + } else { + // If a right subtree of the current node is empty, the new node is added as + // a right subtree of the current node. + if (tree[baseIndex].rightIndex == -1) { + tree[baseIndex].rightIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].rightIndex; + continue; + } + } + } +} + +// Return element data if the given target exists in a tree. Otherwise, we simply return -1. +int search(int target){ + BST currentNode; + int index = 0; + while (index != -1) { + currentNode = tree[index]; + if (currentNode.data == target) { + return target; + } + index = target > currentNode.data ? currentNode.rightIndex : currentNode.leftIndex; + } + return -1; +} + +vec3 hueColor(float angle) { + float nodeData = float(search(15)); + vec3 color; + color = clamp(fract(angle * vec3(1.0, 5.0, nodeData)), 0.0, 1.0); + color.x *= cosh(isnan(float(search(30))) ? injectionSwitch.x : injectionSwitch.y); + return color; +} + +float makeFrame(float v) { + v *= 6.5; + if (v < 1.5) { + return float(search(100)); + } + if (v < 4.0) { + return injectionSwitch.x; + } + if (v < float(search(6))) { + return 1.0; + } + return 10.0 + float(search(30)); +} + +/* +* This shader implements binary search tree using an array data structure. The elements of +* tree are kept in the array that contains a list of BST object holding indices of left and +* right subtree in the array. +* +* - Tree representation of the number used in this shader: +* 9 +* / \ +* 5 12 +* / \ \ +* 2 7 15 +* / \ / \ +* 6 8 13 17 +* +* - Array representation: +* [9, 5, 12, 15, 7, 8, 2, 6, 17, 13] +* +*/ + +void main() { + int treeIndex = int(injectionSwitch.x); + // Initialize root node. + makeTreeNode(tree[int(injectionSwitch.x)], 9); + // Each time we insert a new node into the tree, we increment one. + treeIndex++; + + insert(treeIndex, 5); + treeIndex++; + insert(treeIndex, 12); + treeIndex++; + insert(treeIndex, 15); + treeIndex++; + insert(treeIndex, 7); + treeIndex++; + insert(treeIndex, 8); + treeIndex++; + insert(treeIndex, 2); + treeIndex++; + insert(treeIndex, 6); + treeIndex++; + insert(treeIndex, 17); + treeIndex++; + insert(treeIndex, 13); + + vec2 z = (gl_FragCoord.yx / resolution); + float x = makeFrame(z.x); + float y = makeFrame(z.y); + + int sum = -100; + for (int target = 0; target < 20; target ++) { + int result = search(target); + if (result > 0) { + sum += result; + } else { + switch (result) { + case -1: + sum += int(injectionSwitch.y); + break; + case 0: + return; + } + } + } + float a = sinh(x + y * float(sum)); + _GLF_color = vec4(hueColor(a), 1.); + +} diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree.json b/shaders/src/main/glsl/samples/310es/binarysearch_tree.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/binarysearch_tree.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From 75b352e9c6228b8d4d57106a3fca2c6e112bee87 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Tue, 13 Aug 2019 18:38:51 +0700 Subject: [PATCH 092/180] Expression Generator: Add global initializer method (#677) Adds a function that initializes global variables to known values, invoked at the start of main. --- .../ExpressionGenerator.java | 55 ++++++++++++++++--- .../java/com/graphicsfuzz/util/Constants.java | 1 + 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java index c8cf4a3b3..5b486edb2 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java @@ -17,6 +17,7 @@ package com.graphicsfuzz.generator.knownvaluegeneration; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.Declaration; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; import com.graphicsfuzz.common.ast.decl.Initializer; @@ -39,6 +40,7 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.PipelineInfo; @@ -486,19 +488,58 @@ private Expr generateVariableFact(Value value, Stmt stmtToInsertBefore) { boolean atGlobalScope = factManager.globalScope() || generator.nextBoolean(); final String varName = genVarName(value, atGlobalScope); - + final Expr initializer = generateExpr(atGlobalScope ? globalFactManager : factManager, + currentFunction, stmtToInsertBefore, value); + // If generating global scope variables, initialisation would be performed later in a global + // initializer function. final VariableDeclInfo variableDeclInfo = new VariableDeclInfo(varName, null, - new Initializer(generateExpr(atGlobalScope ? globalFactManager : factManager, - currentFunction, - stmtToInsertBefore, - value - ))); - + atGlobalScope ? null : new Initializer(initializer)); final VariablesDeclaration variablesDecl = new VariablesDeclaration(value.getType(), variableDeclInfo); final VariableDeclFact variableDeclFact = new VariableDeclFact(variablesDecl, variableDeclInfo, value); + if (atGlobalScope) { + // Search for an existing function for initialization of global variables. If not found, we + // have to create one and add a call to the top of 'main' to invoke it. + Optional maybeInitGlobalsFunction = + translationUnit.getTopLevelDeclarations() + .stream() + .filter(item -> item instanceof FunctionDefinition) + .map(item -> (FunctionDefinition) item) + .filter(item -> item.getPrototype().getName().equals(Constants.GLF_INIT_GLOBALS)) + .findFirst(); + + FunctionDefinition initGlobalsFunction; + if (maybeInitGlobalsFunction.isPresent()) { + initGlobalsFunction = maybeInitGlobalsFunction.get(); + } else { + // Function prototype of the globals initializer, this must be declared at the top level of + // the shader being generated. + final FunctionPrototype functionPrototype = + new FunctionPrototype(Constants.GLF_INIT_GLOBALS, VoidType.VOID, + new ArrayList<>()); + translationUnit.addDeclarationBefore(functionPrototype, translationUnit.getMainFunction()); + + // The invocation to the globals initializer which needs to be inserted as the first + // statement in main function. + translationUnit.getMainFunction().getBody().insertStmt(0, + new ExprStmt(new FunctionCallExpr(Constants.GLF_INIT_GLOBALS, new ArrayList<>()))); + + // A function into which the assignments of values for global variables are injected. + initGlobalsFunction = new FunctionDefinition(functionPrototype, + new BlockStmt(new ArrayList<>(), false)); + final List newTopLevelDeclarations = new ArrayList<>(); + newTopLevelDeclarations.addAll(translationUnit.getTopLevelDeclarations()); + newTopLevelDeclarations.add(initGlobalsFunction); + translationUnit.setTopLevelDeclarations(newTopLevelDeclarations); + } + + // Inject a statement assigning value to the global variable into function body. + initGlobalsFunction.getBody().addStmt(new ExprStmt(new BinaryExpr( + new VariableIdentifierExpr(varName), initializer, BinOp.ASSIGN + ))); + translationUnit.addDeclarationBefore(variablesDecl, currentFunction); globalFactManager.addVariableFact(value, variableDeclFact); } else { diff --git a/util/src/main/java/com/graphicsfuzz/util/Constants.java b/util/src/main/java/com/graphicsfuzz/util/Constants.java index 71c74fdb8..97e323f54 100755 --- a/util/src/main/java/com/graphicsfuzz/util/Constants.java +++ b/util/src/main/java/com/graphicsfuzz/util/Constants.java @@ -50,6 +50,7 @@ private Constants() { public static final String GLF_COMPUTE = "_GLF_COMPUTE"; public static final String GLF_PARAM = "_GLF_PARAM"; public static final String GLF_UNKNOWN_PARAM = "GLF_UNKNOWN_PARAM"; + public static final String GLF_INIT_GLOBALS = "_GLF_init_globals"; public static final String INJECTED_LOOP_COUNTER = "_injected_loop_counter"; From 856f477c4a282c5b3442db93ad8a71c0ff135e67 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Wed, 14 Aug 2019 03:34:51 +0700 Subject: [PATCH 093/180] New 310es shader: quicksort (#660) --- .../glsl/samples/310es/quicksort_palette.frag | 131 ++++++++++++++++++ .../glsl/samples/310es/quicksort_palette.json | 16 +++ 2 files changed, 147 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/quicksort_palette.frag create mode 100644 shaders/src/main/glsl/samples/310es/quicksort_palette.json diff --git a/shaders/src/main/glsl/samples/310es/quicksort_palette.frag b/shaders/src/main/glsl/samples/310es/quicksort_palette.frag new file mode 100644 index 000000000..af897b24c --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/quicksort_palette.frag @@ -0,0 +1,131 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct QuicksortObject{ + int numbers[10]; +}; + +QuicksortObject obj; + + void swap(int i, int j) { + int temp = obj.numbers[i]; + obj.numbers[i] = obj.numbers[j]; + obj.numbers[j] = temp; + } + + // Since "partition" is the preserved word, we add prefix to this function name to prevent an error. + int performPartition(int l, int h) { + // The rightmost element is chosen as a pivot. + int pivot = obj.numbers[h]; + int i = (l - 1); + + for (int j = l; j <= h - 1; j++) { + if (obj.numbers[j] <= pivot) { + i++; + swap(i, j); + } + } + swap(i + 1, h); + return (i + 1); + } + + void quicksort() { + int l = 0, h = 9; + int stack[10]; + int top = -1; + + stack[++top] = l; + stack[++top] = h; + + while (top >= 0) { + h = stack[top--]; + l = stack[top--]; + + int p = performPartition(l, h); + if (p - 1 > l) { + stack[++top] = l; + stack[++top] = p - 1; + } + if (p + 1 < h) { + stack[++top] = p + 1; + stack[++top] = h; + } + } + } + +vec3 palette(vec3 a, vec3 b, vec3 c, vec3 d) { + return fract(mix(d * a, a * c, b + d - c)); +} + +float randomize(vec2 co) +{ + return float(floor(fract(sin(dot(co.xy ,vec2(12.5, 3.))) * 4250.) + 0.5)); +} + +bool puzzlelize(vec2 pos) +{ + return randomize(pos.xy) < .12; +} + +void main() { + // Initialize decreasing values to an array starting from 10. + for (int i = int(injectionSwitch.x); i < 10; i ++) { + obj.numbers[i] = (10 - i) * int(injectionSwitch.y); + } + quicksort(); + vec2 grid = vec2(20, 20); + vec2 uv = gl_FragCoord.xy / resolution; + vec3 color = palette(vec3(float(obj.numbers[4]) * 0.1), vec3(0.9, float(obj.numbers[8]) * 0.1, 0.8), trunc(vec3(injectionSwitch.y)), vec3(injectionSwitch.x, 0.3, 0.7)); + if (uv.x > (1.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color += palette(vec3(0.5, float(obj.numbers[8]) * 0.1, 0.2), vec3(0.5), trunc(vec3(injectionSwitch.y)), vec3(float(obj.numbers[4]) * 0.1, injectionSwitch.x, 0.6)); + count++; + } while (count != obj.numbers[int(injectionSwitch.x)]); + grid = vec2(count + obj.numbers[8], count + obj.numbers[6]); + } + if (uv.x > (2.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color -= palette(vec3(float(obj.numbers[4]) * 0.1), trunc(vec3(0.1)), vec3(float(obj.numbers[int(injectionSwitch.y)]) * 0.1), vec3(injectionSwitch.x, float(obj.numbers[2]) * 0.1 , float(obj.numbers[8]) * 0.1)); + count++; + } while (count != obj.numbers[1]); + grid += vec2(count + int(injectionSwitch.y), count + int(injectionSwitch.x)); + } + if(uv.x > (3.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color -= palette(vec3(float(obj.numbers[int(injectionSwitch.y)]) * 0.1), vec3(0.8), trunc(vec3(0.1)), vec3(injectionSwitch.x, 0.6, float(obj.numbers[int(injectionSwitch.x)]) * 0.1)); + count++; + } while (count != obj.numbers[2]); + grid += vec2(count + obj.numbers[3], count + obj.numbers[3]); + } + + vec2 position = vec2(gl_FragCoord.x, resolution.x - gl_FragCoord.y); + position = floor(position / grid); + _GLF_color = vec4(color, injectionSwitch.y) + vec4(!puzzlelize(position)); + +} diff --git a/shaders/src/main/glsl/samples/310es/quicksort_palette.json b/shaders/src/main/glsl/samples/310es/quicksort_palette.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/quicksort_palette.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From 95ad83362e657a81f1487fe55c8808d5c3b5c858 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 14 Aug 2019 01:41:17 -0500 Subject: [PATCH 094/180] Add identity to multiply rectangular matrix/vector by identity matrix (#683) Fixes #674. --- .../fuzzer/OpaqueExpressionGenerator.java | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index a89420ab4..5db3f07ce 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -48,6 +48,7 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; public final class OpaqueExpressionGenerator { @@ -93,6 +94,7 @@ public OpaqueExpressionGenerator(IRandom generator, GenerationParams generationP if (shadingLanguageVersion.supportedTranspose()) { expressionIdentities.add(new IdentityDoubleTranspose()); } + expressionIdentities.add(new IdentityMatrixMultIdentity()); } @@ -1366,8 +1368,8 @@ public boolean preconditionHolds(Expr expr, BasicType basicType, boolean constCo } /** - * Identity function to transpose a matrix twice. When performed, transforms an expression of a - * matrix m -> transpose(identity(transpose(identity(m)))). + * Identity transformation to transpose a matrix twice. When performed, transforms an expression + * of a matrix m -> transpose(identity(transpose(identity(m)))). */ private class IdentityDoubleTranspose extends AbstractIdentityTransformation { private IdentityDoubleTranspose() { @@ -1395,4 +1397,48 @@ public boolean preconditionHolds(Expr expr, BasicType basicType, boolean constCo && !constContext; } } + + /** + * Identity transformation to multiply a vector or a rectangular matrix P by an identity matrix I, + * producing the same vector/matrix P as output. This relies on two properties of the identity + * matrix: + * If P is a matrix of size n x m, where n is the number of columns and m the number of rows, + * and I is the identity matrix of size n x n, then P * I = P. + * If P is a matrix of size n x m, where n is the number of columns and m the number of rows, + * and I is the identity matrix of size m x m, then I * P = P. + * Note that matrices of size 1 x m or n x 1 are vecm or vecn, respectively. + * When performed, transforms an expression of a vector or rectangular matrix P -> + * identity(identityMatrix * P) or identity(P * identityMatrix). + */ + private class IdentityMatrixMultIdentity extends AbstractIdentityTransformation { + private IdentityMatrixMultIdentity() { + super(Stream.concat(BasicType.allNonSquareMatrixTypes().stream(), + Stream.of(BasicType.VEC2, BasicType.VEC3, BasicType.VEC4)) + .collect(Collectors.toList()), true); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, Fuzzer fuzzer) { + // We use isVector()/isMatrix() in this function for brevity and self-documentation instead of + // concatenating streams. + assert type.isVector() || type.isMatrix(); + final boolean usingRightMultiplication = generator.nextBoolean(); + final int identityMatrixSize = + type.isVector() + ? type.getNumElements() + : usingRightMultiplication + ? type.getNumRows() + : type.getNumColumns(); + final Expr identityMatrix = makeOpaqueIdentityMatrix( + BasicType.makeMatrixType(identityMatrixSize, identityMatrixSize), + constContext, depth, fuzzer); + // We wrap the original expr in parentheses to prevent issues with ternary operators. + final Expr binaryMultExpr = usingRightMultiplication + ? new BinaryExpr(identityMatrix, new ParenExpr(expr.clone()), BinOp.MUL) + : new BinaryExpr(new ParenExpr(expr.clone()), identityMatrix, BinOp.MUL); + return identityConstructor( + expr, + applyIdentityFunction(binaryMultExpr, type, constContext, depth, fuzzer)); + } + } } From 0855f56125d87e5ecfe6f36ecaf281d2501d7a72 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 14 Aug 2019 01:42:39 -0500 Subject: [PATCH 095/180] Fix making array accesses inbounds with uint index accesses (#670) Fixes #669. --- .../donation/MakeArrayAccessesInBounds.java | 52 ++++++++++++++----- .../MakeArrayAccessesInBoundsTest.java | 51 ++++++++++++++++++ 2 files changed, 89 insertions(+), 14 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java index 672280532..8e114a39a 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java @@ -24,6 +24,7 @@ import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.TernaryExpr; +import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; import com.graphicsfuzz.common.ast.type.ArrayType; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.Type; @@ -51,20 +52,37 @@ public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { type = type.getWithoutQualifiers(); assert isArrayVectorOrMatrix(type); if (!staticallyInBounds(arrayIndexExpr.getIndex(), type)) { - arrayIndexExpr.setIndex(new TernaryExpr( + Type indexType = typer.lookupType(arrayIndexExpr.getIndex()); + if (indexType == null) { + return; + } + indexType = indexType.getWithoutQualifiers(); + if (indexType == BasicType.UINT) { + arrayIndexExpr.setIndex(new TernaryExpr( new BinaryExpr( - new BinaryExpr( - new ParenExpr(arrayIndexExpr.getIndex().clone()), - new IntConstantExpr("0"), - BinOp.GE), - new BinaryExpr( - new ParenExpr(arrayIndexExpr.getIndex().clone()), - new IntConstantExpr(getSize(type).toString()), - BinOp.LT), - BinOp.LAND), + new ParenExpr(arrayIndexExpr.getIndex().clone()), + new UIntConstantExpr(getSize(type).toString() + "u"), + BinOp.LT), + arrayIndexExpr.getIndex(), + new UIntConstantExpr("0u")) + ); + } else { + assert indexType == BasicType.INT; + arrayIndexExpr.setIndex(new TernaryExpr( + new BinaryExpr( + new BinaryExpr( + new ParenExpr(arrayIndexExpr.getIndex().clone()), + new IntConstantExpr("0"), + BinOp.GE), + new BinaryExpr( + new ParenExpr(arrayIndexExpr.getIndex().clone()), + new IntConstantExpr(getSize(type).toString()), + BinOp.LT), + BinOp.LAND), arrayIndexExpr.getIndex(), new IntConstantExpr("0")) - ); + ); + } } super.visitArrayIndexExpr(arrayIndexExpr); } @@ -87,11 +105,17 @@ private static boolean isArrayVectorOrMatrix(Type type) { } private static boolean staticallyInBounds(Expr index, Type type) { - if (!(index instanceof IntConstantExpr)) { + if (!(index instanceof IntConstantExpr || index instanceof UIntConstantExpr)) { return false; } - Integer indexValue = Integer.parseInt(((IntConstantExpr) index).getValue()); - return indexValue >= 0 && indexValue < getSize(type); + Integer indexValue; + if (index instanceof IntConstantExpr) { + indexValue = ((IntConstantExpr) index).getNumericValue(); + return indexValue >= 0 && indexValue < getSize(type); + } else { // index instanceof UIntConstantExpr + indexValue = ((UIntConstantExpr) index).getNumericValue(); + return indexValue < getSize(type); + } } } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java index 32fcba6bd..fc239df28 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java @@ -83,4 +83,55 @@ public void testMatrixVector2() throws Exception { PrettyPrinterVisitor.prettyPrintAsString(tu)); } + @Test + public void testUIntConstantExprIndex() throws Exception { + final String shader = "#version 300 es\n" + + "void main() { vec3 stuff[16];" + + " uint x = 19u;" + + " vec3 f = stuff[x];" + + "}"; + final String expected = "#version 300 es\n" + + "void main() { vec3 stuff[16];" + + " uint x = 19u;" + + " vec3 f = stuff[(x) < 16u ? x : 0u];" + + "}"; + final TranslationUnit tu = ParseHelper.parse(shader); + final Typer typer = new Typer(tu); + MakeArrayAccessesInBounds.makeInBounds(tu, typer); + assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), + PrettyPrinterVisitor.prettyPrintAsString(tu)); + } + + @Test + public void testUIntFunctionCallReturnIndex() throws Exception { + final String shader = "#version 310 es\n" + + "void main() { vec3 stuff[16];" + + " uint uselessOut;" + + " vec3 f = stuff[uaddCarry(19u, 15u, uselessOut)];" + + "}"; + final String expected = "#version 310 es\n" + + "void main() { vec3 stuff[16];" + + " uint uselessOut;" + + " vec3 f = stuff[(uaddCarry(19u, 15u, uselessOut)) < 16u ? uaddCarry(19u, 15u, uselessOut) : 0u];" + + "}"; + final TranslationUnit tu = ParseHelper.parse(shader); + final Typer typer = new Typer(tu); + MakeArrayAccessesInBounds.makeInBounds(tu, typer); + assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), + PrettyPrinterVisitor.prettyPrintAsString(tu)); + } + + @Test + public void testUIntStaticallyInBounds() throws Exception { + final String shader = "#version 310 es\n" + + "void main() { float stuff[16];" + + " stuff[3u] = 1.0;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(shader); + final Typer typer = new Typer(tu); + MakeArrayAccessesInBounds.makeInBounds(tu, typer); + assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(shader)), + PrettyPrinterVisitor.prettyPrintAsString(tu)); + } + } From 732a67e96825adf4400c9878135151926d7b0971 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Wed, 14 Aug 2019 11:27:34 +0100 Subject: [PATCH 096/180] Add simpler version of binary search tree shader (#680) Adds a version of 'binarysearch_tree.frag' that is potentially less floating point-sensitive, e.g. by not making use of trigonometric functions nor functions such as 'fract'. --- .../310es/binarysearch_tree_simple.frag | 173 ++++++++++++++++++ .../310es/binarysearch_tree_simple.json | 16 ++ 2 files changed, 189 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.frag create mode 100644 shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.json diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.frag b/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.frag new file mode 100644 index 000000000..0df0eee8b --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.frag @@ -0,0 +1,173 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct BST{ + int data; + int leftIndex; + int rightIndex; +}; + +BST tree[10]; + +void makeTreeNode(inout BST tree, int data) +{ + tree.data = data; + tree.leftIndex = -1; + tree.rightIndex = -1; +} + +void insert(int treeIndex, int data) +{ + int baseIndex = 0; + while (baseIndex <= treeIndex) { + // If new value is smaller thatn the current node, we known that we will have + // add this element in the left side. + if (data <= tree[baseIndex].data) { + // If a left subtree of the current node is empty, the new node is added as + // a left subtree of the current node. + if (tree[baseIndex].leftIndex == -1) { + tree[baseIndex].leftIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].leftIndex; + continue; + } + } else { + // If a right subtree of the current node is empty, the new node is added as + // a right subtree of the current node. + if (tree[baseIndex].rightIndex == -1) { + tree[baseIndex].rightIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].rightIndex; + continue; + } + } + } +} + +// Return element data if the given target exists in a tree. Otherwise, we simply return -1. +int search(int target){ + BST currentNode; + int index = 0; + while (index != -1) { + currentNode = tree[index]; + if (currentNode.data == target) { + return target; + } + index = target > currentNode.data ? currentNode.rightIndex : currentNode.leftIndex; + } + return -1; +} + +vec3 hueColor(float angle) { + float nodeData = float(search(15)); + return (30.0 + angle * vec3(1.0, 5.0, nodeData)) / 50.0; +} + +float makeFrame(float v) { + v *= 6.5; + if (v < 1.5) { + return float(search(100)); + } + if (v < 4.0) { + return injectionSwitch.x; + } + if (v < float(search(6))) { + return 1.0; + } + return 10.0 + float(search(30)); +} + +/* +* This shader implements binary search tree using an array data structure. The elements of +* tree are kept in the array that contains a list of BST object holding indices of left and +* right subtree in the array. +* +* - Tree representation of the number used in this shader: +* 9 +* / \ +* 5 12 +* / \ \ +* 2 7 15 +* / \ / \ +* 6 8 13 17 +* +* - Array representation: +* [9, 5, 12, 15, 7, 8, 2, 6, 17, 13] +* +*/ + +void main() { + int treeIndex = int(injectionSwitch.x); + // Initialize root node. + makeTreeNode(tree[int(injectionSwitch.x)], 9); + // Each time we insert a new node into the tree, we increment one. + treeIndex++; + + insert(treeIndex, 5); + treeIndex++; + insert(treeIndex, 12); + treeIndex++; + insert(treeIndex, 15); + treeIndex++; + insert(treeIndex, 7); + treeIndex++; + insert(treeIndex, 8); + treeIndex++; + insert(treeIndex, 2); + treeIndex++; + insert(treeIndex, 6); + treeIndex++; + insert(treeIndex, 17); + treeIndex++; + insert(treeIndex, 13); + + vec2 z = (gl_FragCoord.yx / resolution); + float x = makeFrame(z.x); + float y = makeFrame(z.y); + + int sum = -100; + for (int target = 0; target < 20; target ++) { + int result = search(target); + if (result > 0) { + sum += result; + } else { + switch (result) { + case -1: + sum += int(injectionSwitch.y); + break; + case 0: + return; + } + } + } + float a = x + y * float(sum); + _GLF_color = vec4(hueColor(a), 1.); + +} diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.json b/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From 29a98697a7e2071b02419dc79ca038d69b4c3034 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Wed, 14 Aug 2019 11:28:12 +0100 Subject: [PATCH 097/180] Avoid injecting switch case labels into inappropriate places (#666) Adds checks to make sure that we don't inject top-level 'case' or 'default' labels into code, by being aware of switch statements during donation. Does allow the insertion of top-level break statements into switch statements (which was previously disallowed due to switch statements not being considered, and loops being regarded as the only acceptable place to add break statements). Fixes #665. --- .../DonateDeadCodeTransformation.java | 22 +- .../DonateLiveCodeTransformation.java | 10 +- .../injection/BlockInjectionPoint.java | 4 +- .../injection/IInjectionPoint.java | 7 + .../injection/IfInjectionPoint.java | 4 +- .../injection/InjectionPoint.java | 8 + .../injection/InjectionPoints.java | 57 ++- .../injection/LoopInjectionPoint.java | 5 +- .../util/RemoveImmediateBreakStatements.java | 57 +++ .../util/RemoveImmediateCaseLabels.java | 40 ++ ...=> RemoveImmediateContinueStatements.java} | 15 +- .../DonateDeadCodeTransformationTest.java | 375 ++++++++++++++++++ .../DonateLiveCodeTransformationTest.java | 344 ++++++++++++++-- .../SplitForLoopTransformationTest.java | 5 + .../injection/InjectionPointTest.java | 6 +- 15 files changed, 886 insertions(+), 73 deletions(-) create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java create mode 100644 generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java rename generator/src/main/java/com/graphicsfuzz/generator/util/{RemoveImmediateBreakAndContinueStatements.java => RemoveImmediateContinueStatements.java} (65%) create mode 100644 generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformationTest.java diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java index 2750aba22..a66111e79 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java @@ -39,7 +39,9 @@ import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.RemoveDiscardStatements; -import com.graphicsfuzz.generator.util.RemoveImmediateBreakAndContinueStatements; +import com.graphicsfuzz.generator.util.RemoveImmediateBreakStatements; +import com.graphicsfuzz.generator.util.RemoveImmediateCaseLabels; +import com.graphicsfuzz.generator.util.RemoveImmediateContinueStatements; import com.graphicsfuzz.generator.util.RemoveReturnStatements; import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.util.Constants; @@ -125,8 +127,24 @@ Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, DonationContext do shadingLanguageVersion, generator, generationParams)), new BlockStmt(donatedStmts, true), null); + // We cannot have 'case' and 'default' labels occurring in the donated statement, unless they + // are nested in their own 'switch' statement. In principle we could allow these if the + // injection point happens to be in a 'switch' already, but this would require some fiddly + // checks that do not seem worth doing. + new RemoveImmediateCaseLabels(donatedStmt); + + // If the injection point is in a loop then it's fine for the injected code to have continue + // statements (this is a dead code injection, so semantics will not be changed). But otherwise + // they would make the shader invalid. if (!injectionPoint.inLoop()) { - new RemoveImmediateBreakAndContinueStatements(donatedStmt); + new RemoveImmediateContinueStatements(donatedStmt); + } + + // If the injection point is in a loop or switch then it's fine for the injected code to have + // break statements (this is a dead code injection, so semantics will not be changed). But + // otherwise they would make the shader invalid. + if (!(injectionPoint.inLoop() || injectionPoint.inSwitch())) { + new RemoveImmediateBreakStatements(donatedStmt); } if (generationParams.getShaderKind() != ShaderKind.FRAGMENT) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java index 1dbbda5ea..db5cf1df5 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java @@ -33,7 +33,9 @@ import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.RemoveDiscardStatements; -import com.graphicsfuzz.generator.util.RemoveImmediateBreakAndContinueStatements; +import com.graphicsfuzz.generator.util.RemoveImmediateBreakStatements; +import com.graphicsfuzz.generator.util.RemoveImmediateCaseLabels; +import com.graphicsfuzz.generator.util.RemoveImmediateContinueStatements; import com.graphicsfuzz.generator.util.RemoveReturnStatements; import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.util.Constants; @@ -58,7 +60,7 @@ public DonateLiveCodeTransformation(Function probabilityOfDona } @Override - public Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, + Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, DonationContext donationContext, TransformationProbabilities probabilities, IRandom generator, ShadingLanguageVersion shadingLanguageVersion) { @@ -87,7 +89,9 @@ public Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, } donatedStmts.add(donationContext.getDonorFragment()); BlockStmt donatedStmt = new BlockStmt(donatedStmts, true); - new RemoveImmediateBreakAndContinueStatements(donatedStmt); + new RemoveImmediateBreakStatements(donatedStmt); + new RemoveImmediateContinueStatements(donatedStmt); + new RemoveImmediateCaseLabels(donatedStmt); new RemoveReturnStatements(donatedStmt); new RemoveDiscardStatements(donatedStmt); return donatedStmt; diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/BlockInjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/BlockInjectionPoint.java index 614c7367c..206161f22 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/BlockInjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/BlockInjectionPoint.java @@ -27,8 +27,8 @@ public class BlockInjectionPoint extends InjectionPoint { private Stmt nextStmt; // null if there is no next statement public BlockInjectionPoint(BlockStmt blockStmt, Stmt nextStmt, - FunctionDefinition enclosingFunction, boolean inLoop, Scope scope) { - super(enclosingFunction, inLoop, scope); + FunctionDefinition enclosingFunction, boolean inLoop, boolean inSwitch, Scope scope) { + super(enclosingFunction, inLoop, inSwitch, scope); this.blockStmt = blockStmt; this.nextStmt = nextStmt; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IInjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IInjectionPoint.java index d22e201bc..58487e05b 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IInjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IInjectionPoint.java @@ -59,6 +59,13 @@ public interface IInjectionPoint { */ boolean inLoop(); + /** + * Determines whether the injection point is located inside a switch statement. + * + * @return true if and only if the injection point is inside a switch statement. + */ + boolean inSwitch(); + /** * Returns the function enclosing the injection point. * diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IfInjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IfInjectionPoint.java index 6fe24ab2a..788f781f9 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IfInjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IfInjectionPoint.java @@ -30,8 +30,8 @@ public class IfInjectionPoint extends InjectionPoint { private boolean chooseThen; public IfInjectionPoint(IfStmt ifStmt, boolean chooseThen, FunctionDefinition enclosingFunction, - boolean inLoop, Scope scope) { - super(enclosingFunction, inLoop, scope); + boolean inLoop, boolean inSwitch, Scope scope) { + super(enclosingFunction, inLoop, inSwitch, scope); this.ifStmt = ifStmt; this.chooseThen = chooseThen; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoint.java index f4bb95412..b9dfd773b 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoint.java @@ -23,12 +23,15 @@ abstract class InjectionPoint implements IInjectionPoint { private final FunctionDefinition enclosingFunction; private final boolean inLoop; + private final boolean inSwitch; private final Scope scope; InjectionPoint(FunctionDefinition enclosingFunction, boolean inLoop, + boolean inSwitch, Scope scope) { this.enclosingFunction = enclosingFunction; this.inLoop = inLoop; + this.inSwitch = inSwitch; this.scope = scope.shallowClone(); } @@ -42,6 +45,11 @@ public final boolean inLoop() { return inLoop; } + @Override + public final boolean inSwitch() { + return inSwitch; + } + @Override public final Scope scopeAtInjectionPoint() { return scope; diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java index ded76e845..d3a8863c2 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java @@ -23,7 +23,9 @@ import com.graphicsfuzz.common.ast.stmt.DoStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.IfStmt; +import com.graphicsfuzz.common.ast.stmt.LoopStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.stmt.WhileStmt; import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.IRandom; @@ -39,6 +41,7 @@ public class InjectionPoints extends ScopeTrackingVisitor { private final List injectionPoints; private FunctionDefinition currentFunction; private int loopNestingDepth; + private int switchNestingDepth; private final IRandom generator; private final Predicate suitable; @@ -47,6 +50,7 @@ public InjectionPoints(TranslationUnit tu, IRandom generator, this.injectionPoints = new ArrayList<>(); this.currentFunction = null; this.loopNestingDepth = 0; + this.switchNestingDepth = 0; this.generator = generator; this.suitable = suitable; visit(tu); @@ -73,10 +77,12 @@ public void visitBlockStmt(BlockStmt stmt) { continue; } maybeAddInjectionPoint(new BlockInjectionPoint(stmt, innerStmt, currentFunction, inLoop(), + inSwitch(), getCurrentScope())); visit(innerStmt); } maybeAddInjectionPoint(new BlockInjectionPoint(stmt, null, currentFunction, inLoop(), + inSwitch(), getCurrentScope())); leaveBlockStmt(stmt); } @@ -85,12 +91,21 @@ private boolean inLoop() { return loopNestingDepth > 0; } - @Override - public void visitDoStmt(DoStmt doStmt) { - if (!(doStmt.getBody() instanceof BlockStmt)) { - maybeAddInjectionPoint(new LoopInjectionPoint(doStmt, currentFunction, + private boolean inSwitch() { + return switchNestingDepth > 0; + } + + private void considerLoopInjectionPoint(LoopStmt loopStmt) { + if (!(loopStmt.getBody() instanceof BlockStmt)) { + maybeAddInjectionPoint(new LoopInjectionPoint(loopStmt, currentFunction, + inSwitch(), getCurrentScope())); } + } + + @Override + public void visitDoStmt(DoStmt doStmt) { + considerLoopInjectionPoint(doStmt); loopNestingDepth++; super.visitDoStmt(doStmt); loopNestingDepth--; @@ -98,24 +113,33 @@ public void visitDoStmt(DoStmt doStmt) { @Override public void visitForStmt(ForStmt forStmt) { - if (!(forStmt.getBody() instanceof BlockStmt)) { - maybeAddInjectionPoint(new LoopInjectionPoint(forStmt, currentFunction, - getCurrentScope())); - } + considerLoopInjectionPoint(forStmt); loopNestingDepth++; super.visitForStmt(forStmt); loopNestingDepth--; } + @Override + public void visitWhileStmt(WhileStmt whileStmt) { + considerLoopInjectionPoint(whileStmt); + loopNestingDepth++; + super.visitWhileStmt(whileStmt); + loopNestingDepth--; + } + @Override public void visitIfStmt(IfStmt ifStmt) { if (!(ifStmt.getThenStmt() instanceof BlockStmt)) { - maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, true, currentFunction, inLoop(), + maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, true, currentFunction, + inLoop(), + inSwitch(), getCurrentScope())); } if (ifStmt.hasElseStmt()) { if (!(ifStmt.getElseStmt() instanceof BlockStmt)) { - maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, false, currentFunction, inLoop(), + maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, false, currentFunction, + inLoop(), + inSwitch(), getCurrentScope())); } } @@ -123,14 +147,11 @@ public void visitIfStmt(IfStmt ifStmt) { } @Override - public void visitWhileStmt(WhileStmt whileStmt) { - if (!(whileStmt.getBody() instanceof BlockStmt)) { - maybeAddInjectionPoint(new LoopInjectionPoint(whileStmt, currentFunction, - getCurrentScope())); - } - loopNestingDepth++; - super.visitWhileStmt(whileStmt); - loopNestingDepth--; + public void visitSwitchStmt(SwitchStmt switchStmt) { + // We cannot inject immediately inside a switch statement. + switchNestingDepth++; + super.visitSwitchStmt(switchStmt); + switchNestingDepth--; } private void maybeAddInjectionPoint(IInjectionPoint injectionPoint) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/LoopInjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/LoopInjectionPoint.java index 3a66f757b..9a8eed652 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/LoopInjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/LoopInjectionPoint.java @@ -29,8 +29,9 @@ public class LoopInjectionPoint extends InjectionPoint { private LoopStmt loopStmt; public LoopInjectionPoint(LoopStmt loopStmt, FunctionDefinition enclosingFunction, - Scope scope) { - super(enclosingFunction, true, scope); + boolean inSwitch, + Scope scope) { + super(enclosingFunction, true, inSwitch, scope); this.loopStmt = loopStmt; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java new file mode 100644 index 000000000..810d181fd --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.util; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.BreakStmt; +import com.graphicsfuzz.common.ast.stmt.DoStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.NullStmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.ast.stmt.WhileStmt; +import java.util.Optional; + +/** + * This class removes break statements that are not nested inside loop or switch statements. + */ +public class RemoveImmediateBreakStatements extends RemoveStatements { + + public RemoveImmediateBreakStatements(IAstNode node) { + super(item -> item instanceof BreakStmt, + item -> Optional.of(new NullStmt()), node); + } + + @Override + public void visitDoStmt(DoStmt doStmt) { + // Block visitation: we don't want to remove break statements from inside a loop + } + + @Override + public void visitForStmt(ForStmt forStmt) { + // Block visitation: we don't want to remove break statements from inside a loop + } + + @Override + public void visitWhileStmt(WhileStmt whileStmt) { + // Block visitation: we don't want to remove break statements from inside a loop + } + + @Override + public void visitSwitchStmt(SwitchStmt switchStmt) { + // Block visitation: we don't want to remove break statements from inside a switch + } +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java new file mode 100644 index 000000000..0c320be22 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.util; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.CaseLabel; +import com.graphicsfuzz.common.ast.stmt.NullStmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import java.util.Optional; + +/** + * This class removes case labels (including default) that are not nested inside switch statements. + */ +public class RemoveImmediateCaseLabels extends RemoveStatements { + + public RemoveImmediateCaseLabels(IAstNode node) { + super(item -> item instanceof CaseLabel, + item -> Optional.of(new NullStmt()), node); + } + + @Override + public void visitSwitchStmt(SwitchStmt switchStmt) { + // Block visitation: we don't want to remove labels from inside switch statements + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakAndContinueStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java similarity index 65% rename from generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakAndContinueStatements.java rename to generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java index f7dc4ac87..6d2b5d5e5 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakAndContinueStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java @@ -17,7 +17,6 @@ package com.graphicsfuzz.generator.util; import com.graphicsfuzz.common.ast.IAstNode; -import com.graphicsfuzz.common.ast.stmt.BreakStmt; import com.graphicsfuzz.common.ast.stmt.ContinueStmt; import com.graphicsfuzz.common.ast.stmt.DoStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; @@ -26,28 +25,28 @@ import java.util.Optional; /** - * This class removes break and continue statements that are not nested inside loops. + * This class removes continue statements that are not nested inside loops. */ -public class RemoveImmediateBreakAndContinueStatements extends RemoveStatements { +public class RemoveImmediateContinueStatements extends RemoveStatements { - public RemoveImmediateBreakAndContinueStatements(IAstNode node) { - super(item -> item instanceof BreakStmt || item instanceof ContinueStmt, + public RemoveImmediateContinueStatements(IAstNode node) { + super(item -> item instanceof ContinueStmt, item -> Optional.of(new NullStmt()), node); } @Override public void visitDoStmt(DoStmt doStmt) { - // Block visitation: we don't want to remove break and continue statements from inside a loop + // Block visitation: we don't want to remove continue statements from inside a loop } @Override public void visitForStmt(ForStmt forStmt) { - // Block visitation: we don't want to remove break and continue statements from inside a loop + // Block visitation: we don't want to remove continue statements from inside a loop } @Override public void visitWhileStmt(WhileStmt whileStmt) { - // Block visitation: we don't want to remove break and continue statements from inside a loop + // Block visitation: we don't want to remove continue statements from inside a loop } } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformationTest.java new file mode 100644 index 000000000..d92425aed --- /dev/null +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformationTest.java @@ -0,0 +1,375 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.transformation; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.stmt.BreakStmt; +import com.graphicsfuzz.common.ast.stmt.ContinueStmt; +import com.graphicsfuzz.common.ast.stmt.DefaultCaseLabel; +import com.graphicsfuzz.common.ast.stmt.ExprCaseLabel; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.IfStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderKind; +import com.graphicsfuzz.generator.transformation.donation.DonationContext; +import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; +import com.graphicsfuzz.generator.transformation.injection.InjectionPoints; +import com.graphicsfuzz.generator.util.GenerationParams; +import com.graphicsfuzz.generator.util.TransformationProbabilities; +import java.util.ArrayList; +import java.util.HashMap; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class DonateDeadCodeTransformationTest { + + @Rule + public TemporaryFolder testFolder = new TemporaryFolder(); + + private DonateDeadCodeTransformation getDummyTransformationObject() { + return new DonateDeadCodeTransformation(IRandom::nextBoolean, testFolder.getRoot(), + GenerationParams.normal(ShaderKind.FRAGMENT, true)); + } + + @Test + public void prepareStatementToDonateTopLevelBreakRemovedWhenNecessary() throws Exception { + + // Checks that a top-level 'break' gets removed, unless injecting into a loop or switch. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) break;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " switch (i) {\n" + + " case 0:\n" + + " i++;\n" + + " default:\n" + + " i++;\n" + + " }\n" + + " }" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + final Stmt toDonate = ((ForStmt) donor.getMainFunction().getBody().getStmt(0)).getBody() + .clone(); + assert toDonate instanceof IfStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + final boolean containsBreak = new CheckPredicateVisitor() { + @Override + public void visitBreakStmt(BreakStmt breakStmt) { + predicateHolds(); + } + }.test(donated); + assertEquals(containsBreak, injectionPoint.inLoop() || injectionPoint.inSwitch()); + } + } + + @Test + public void prepareStatementToDonateTopLevelContinueRemovedWhenNecessary() throws Exception { + + // Checks that a top-level 'continue' gets removed, unless injecting into a loop. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) continue;\n" + + "\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " switch(0) {\n" + + " case 1:\n" + + " break;\n" + + " default:\n" + + " 1;\n" + + " }\n" + + " for(int i = 0; i < 100; i++) {\n" + + " ;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = ((ForStmt) donor.getMainFunction().getBody().getStmt(0)).getBody() + .clone(); + assert toDonate instanceof IfStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + + final boolean containsContinue = new CheckPredicateVisitor() { + @Override + public void visitContinueStmt(ContinueStmt continueStmt) { + predicateHolds(); + } + }.test(donated); + assertEquals(containsContinue, injectionPoint.inLoop()); + } + + } + + @Test + public void prepareStatementToDonateTopLevelCaseAndDefaultRemoved() throws Exception { + // Checks that top-level 'case' and 'default' labels get removed, even when injecting into + // a switch. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " int x = 3;\n" + + " switch (x) {\n" + + " case 0:\n" + + " x++;\n" + + " default:\n" + + " x++;\n" + + " }\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " switch (0) {\n" + + " case 1:\n" + + " 1;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = ((SwitchStmt) donor.getMainFunction().getBody().getStmt(1)).getBody() + .clone(); + + DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + + new StandardVisitor() { + @Override + public void visitDefaultCaseLabel(DefaultCaseLabel defaultCaseLabel) { + // 'default' labels should have been removed. + fail(); + } + + @Override + public void visitExprCaseLabel(ExprCaseLabel exprCaseLabel) { + // 'case' labels should have been removed. + fail(); + } + }.visit(donated); + } + } + + @Test + public void prepareStatementToDonateBreakFromLoopKept() throws Exception { + // Checks that a 'break' in a loop gets kept if the whole loop is donated. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) break;\n" + + "\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof ForStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + assertTrue(new CheckPredicateVisitor() { + @Override + public void visitBreakStmt(BreakStmt breakStmt) { + predicateHolds(); + } + }.test(donated)); + } + } + + @Test + public void prepareStatementToDonateSwitchWithBreakAndDefaultKept() throws Exception { + // Checks that 'case', 'default' and 'break' occurring in a switch are kept if the whole + // switch statement is donated. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " switch (0) {\n" + + " case 0:\n" + + " 1;\n" + + " break;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " switch (0) {\n" + + " case 1:\n" + + " 1;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof SwitchStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + + // Check that the donated statement contains exactly one each of 'break', 'case' and + // 'default'. + new StandardVisitor() { + + private boolean foundBreak = false; + private boolean foundCase = false; + private boolean foundDefault = false; + + @Override + public void visitBreakStmt(BreakStmt breakStmt) { + assertFalse(foundBreak); + foundBreak = true; + } + + @Override + public void visitExprCaseLabel(ExprCaseLabel exprCaseLabel) { + assertFalse(foundCase); + foundCase = true; + } + + @Override + public void visitDefaultCaseLabel(DefaultCaseLabel defaultCaseLabel) { + assertFalse(foundDefault); + foundDefault = true; + } + + private void check(Stmt stmt) { + visit(stmt); + assertTrue(foundBreak); + assertTrue(foundCase); + assertTrue(foundDefault); + } + }; + } + } + + @Test + public void prepareStatementToDonateContinueInLoopKept() throws Exception { + // Checks that a 'continue' in a loop gets kept if the whole loop is donated. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) continue;\n" + + "\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof ForStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + + // Check that the 'continue' statement is retained. + assertTrue(new CheckPredicateVisitor() { + @Override + public void visitContinueStmt(ContinueStmt continueStmt) { + predicateHolds(); + } + }.test(donated)); + } + } + +} diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java index 08f1e929a..19a095e12 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java @@ -31,7 +31,10 @@ import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; import com.graphicsfuzz.common.ast.stmt.DiscardStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.IfStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.VoidType; @@ -47,6 +50,8 @@ import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.transformation.donation.DonationContext; import com.graphicsfuzz.generator.transformation.injection.BlockInjectionPoint; +import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; +import com.graphicsfuzz.generator.transformation.injection.InjectionPoints; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.server.thrift.ImageJob; @@ -59,6 +64,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -68,12 +74,15 @@ public class DonateLiveCodeTransformationTest { @Rule public TemporaryFolder testFolder = new TemporaryFolder(); + private DonateLiveCodeTransformation getDummyTransformationObject() { + return new DonateLiveCodeTransformation(IRandom::nextBoolean, testFolder.getRoot(), GenerationParams.normal(ShaderKind.FRAGMENT, true), + false); + } + @Test - public void prepareStatementToDonate() throws Exception { + public void prepareStatementToDonateDiscardRemoved() throws Exception { - final DonateLiveCodeTransformation dlc = - new DonateLiveCodeTransformation(IRandom::nextBoolean, testFolder.getRoot(), GenerationParams.normal(ShaderKind.FRAGMENT, true), - false); + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); DonationContext dc = new DonationContext(new DiscardStmt(), new HashMap<>(), new ArrayList<>(), null); @@ -89,6 +98,278 @@ public void visitDiscardStmt(DiscardStmt discardStmt) { } + @Test + public void prepareStatementToDonateTopLevelBreakRemoved() throws Exception { + + // Checks that a top-level 'break' gets removed, even when injecting into a loop or + // switch. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) break;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for(int i = 0; i < 100; i++) {\n" + + " switch (i) {\n" + + " case 0:\n" + + " i++;\n" + + " default:\n" + + " i++;\n" + + " }\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = ((ForStmt) donor.getMainFunction().getBody().getStmt(0)).getBody() + .clone(); + assert toDonate instanceof IfStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + assertEquals("{\n" + + " if(i > 5)\n" + + " ;\n" + + "}\n", donated.getText()); + } + + } + + @Test + public void prepareStatementToDonateTopLevelContinueRemoved() throws Exception { + + // Checks that a top-level 'continue' gets removed, even when injecting into a loop. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) continue;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for(int i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + final Stmt toDonate = ((ForStmt) donor.getMainFunction().getBody().getStmt(0)).getBody() + .clone(); + assert toDonate instanceof IfStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + assertEquals("{\n" + + " if(i > 5)\n" + + " ;\n" + + "}\n", donated.getText()); + } + + } + + @Test + public void prepareStatementToDonateTopLevelCaseAndDefaultRemoved() throws Exception { + // Checks that top-level 'case' and 'default' labels get removed, even when injecting into + // a switch. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " int x = 3;\n" + + " switch (x) {\n" + + " case 0:\n" + + " x++;\n" + + " default:\n" + + " x++;\n" + + " }\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " switch (0) {\n" + + " case 1:\n" + + " 1;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = ((SwitchStmt) donor.getMainFunction().getBody().getStmt(1)).getBody() + .clone(); + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + assertEquals("{\n" + + " {\n" + + " ;\n" + + " x ++;\n" + + " ;\n" + + " x ++;\n" + + " }\n" + + "}\n", donated.getText()); + } + } + + @Test + public void prepareStatementToDonateBreakFromLoopKept() throws Exception { + // Checks that a 'break' in a loop gets kept if the whole loop is donated. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) break;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0) + .clone(); + assert toDonate instanceof ForStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + assertEquals("{\n" + + " for(\n" + + " int i = 0;\n" + + " i < 10;\n" + + " i ++\n" + + " )\n" + + " if(i > 5)\n" + + " break;\n" + + "}\n", donated.getText()); + } + } + + @Test + public void prepareStatementToDonateSwitchWithBreakAndDefaultKept() throws Exception { + // Checks that 'case', 'default' and 'break' occurring in a switch are kept if the whole + // switch statement is donated. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " switch (0) {\n" + + " case 0:\n" + + " 1;\n" + + " break;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " switch (0) {\n" + + " case 1:\n" + + " 1;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof SwitchStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + assertEquals("{\n" + + " switch(0)\n" + + " {\n" + + " case 0:\n" + + " 1;\n" + + " break;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n", donated.getText()); + } + } + + @Test + public void prepareStatementToDonateContinueInLoopKept() throws Exception { + // Checks that a 'continue' in a loop gets kept if the whole loop is donated. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) continue;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof ForStmt; + + DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + assertEquals("{\n" + + " for(\n" + + " int i = 0;\n" + + " i < 10;\n" + + " i ++\n" + + " )\n" + + " if(i > 5)\n" + + " continue;\n" + + "}\n", donated.getText()); + } + } + @Test public void checkMutateSpecialCase() throws Exception { @@ -96,12 +377,12 @@ public void checkMutateSpecialCase() throws Exception { // here in the spirit of "why delete a test?" final String reference = "#version 300 es\n" - + "void main() {" - + " int t;" - + " {" - + " }" - + " gl_FragColor = vec4(float(t));" - + "}"; + + "void main() {\n" + + " int t;\n" + + " {\n" + + " }\n" + + " gl_FragColor = vec4(float(t));\n" + + "}\n"; final IRandom generator = new RandomWrapper(0); @@ -124,7 +405,7 @@ public void visitBlockStmt(BlockStmt stmt) { super.visitBlockStmt(stmt); if (stmt.getNumStmts() == 0) { blockInjectionPoint = new BlockInjectionPoint(stmt, null, getEnclosingFunction(), - false, getCurrentScope()); + false, false, getCurrentScope()); } } @@ -229,9 +510,8 @@ public void checkFunctionDeclsUnique() throws Exception { { final String referenceSource = "#version 300 es\n" - + "void main() {" - + " " - + "}"; + + "void main() {\n" + + "}\n"; fileOps.writeShaderJobFileFromImageJob( new ImageJob() @@ -296,9 +576,8 @@ public void verySimpleDonorAndSourceNoPrecision() throws Exception { { final String referenceSource = "#version 300 es\n" - + "void main() {" - + " " - + "}"; + + "void main() {\n" + + "}\n"; fileOps.writeShaderJobFileFromImageJob( new ImageJob() @@ -349,19 +628,19 @@ public void testArrayAccessesAreInBounds() throws Exception { final String donorSource = "#version 300 es\n" + "void main() {\n" - + " int x = 0;" - + " {" - + " int A[1];" - + " A[x] = 42;" - + " {" - + " int B[1];" - + " B[x] = 42;" - + " {" - + " int C[1];" - + " C[x] = 42;" - + " }" - + " }" - + " }" + + " int x = 0;\n" + + " {\n" + + " int A[1];\n" + + " A[x] = 42;\n" + + " {\n" + + " int B[1];\n" + + " B[x] = 42;\n" + + " {\n" + + " int C[1];\n" + + " C[x] = 42;\n" + + " }\n" + + " }\n" + + " }\n" + "}\n"; fileOps.writeShaderJobFileFromImageJob( @@ -374,9 +653,8 @@ public void testArrayAccessesAreInBounds() throws Exception { { final String referenceSource = "#version 300 es\n" - + "void main() {" - + " " - + "}"; + + "void main() {\n" + + "}\n"; fileOps.writeShaderJobFileFromImageJob( new ImageJob() diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java index beaaa8eed..5cddc4694 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java @@ -93,6 +93,11 @@ public boolean inLoop() { throw new RuntimeException(); } + @Override + public boolean inSwitch() { + throw new RuntimeException(); + } + @Override public FunctionDefinition getEnclosingFunction() { throw new RuntimeException(); diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/InjectionPointTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/InjectionPointTest.java index 4caeba47c..6b0f0033d 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/InjectionPointTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/InjectionPointTest.java @@ -34,7 +34,7 @@ public void testScopeAtInjectionPoint() { final Scope scope = new Scope(null); assertNotNull( - new InjectionPoint(null, false, scope) { + new InjectionPoint(null, false, false, scope) { @Override public void inject(Stmt stmt) { @@ -64,7 +64,7 @@ public void replaceNext(Stmt stmt) { public void testThatScopeIsCloned() { Scope s = new Scope(null); s.add("v", BasicType.INT, Optional.empty()); - IInjectionPoint injectionPoint = new InjectionPoint(null, false, s) { + IInjectionPoint injectionPoint = new InjectionPoint(null, false, false, s) { @Override public void inject(Stmt stmt) { throw new RuntimeException(); @@ -92,4 +92,4 @@ public void replaceNext(Stmt stmt) { } -} \ No newline at end of file +} From 16fed03fe2d21d0a9fba344b1a18b0ebca92c0cb Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Wed, 14 Aug 2019 20:01:00 +0700 Subject: [PATCH 098/180] New 310es reference shaders (#643) - binarysearch_bw - mergesort_mosaic --- .../glsl/samples/310es/binarysearch_bw.frag | 147 +++++++++++++++ .../glsl/samples/310es/binarysearch_bw.json | 16 ++ .../glsl/samples/310es/mergesort_mosaic.frag | 167 ++++++++++++++++++ .../glsl/samples/310es/mergesort_mosaic.json | 16 ++ 4 files changed, 346 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/binarysearch_bw.frag create mode 100644 shaders/src/main/glsl/samples/310es/binarysearch_bw.json create mode 100644 shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag create mode 100644 shaders/src/main/glsl/samples/310es/mergesort_mosaic.json diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_bw.frag b/shaders/src/main/glsl/samples/310es/binarysearch_bw.frag new file mode 100644 index 000000000..1368e7a5b --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/binarysearch_bw.frag @@ -0,0 +1,147 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct BinarySearchObject{ + int prime_numbers[10]; +}; + +vec2 brick(vec2 uv) { + int a = 4; + do { + uv.y -= step(injectionSwitch.y, uv.x); + uv.x -= fract(tanh(uv.x)) / ldexp(injectionSwitch.y, findMSB(a)); + a--; + } while (a > int(injectionSwitch.x)); + int b = 3; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a); + uv.x *= (isnan(uv.y) ? cosh(gl_FragCoord.y) : tanh(gl_FragCoord.x)); + b--; + } while (b > int(injectionSwitch.x)); + int c = 2; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a) + float(b); + uv.x += ldexp(injectionSwitch.y, isinf(uv.y + uv.x) ? findMSB(b) : findMSB(a)); + c--; + } while (c > int(injectionSwitch.x)); + int d = 1; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a) + float(b) + float(c); + d--; + } while (d > int(injectionSwitch.x)); + return fract(uv); +} + +float patternize(vec2 uv) { + vec2 size = vec2(0.45); + vec2 st = smoothstep(size, size, uv); + switch (int(mod(gl_FragCoord.y, 5.0))) { + case 0: + return mix(pow(st.x, injectionSwitch.y), st.x, size.y); + break; + case 1: + return mix(pow(uv.y, injectionSwitch.y), st.y, size.x); + break; + case 2: + discard; + break; + case 3: + return mix(pow(uv.y, injectionSwitch.y), uv.y, size.y); + break; + case 4: + return mix(pow(st.y, injectionSwitch.y), st.x, size.x); + break; + } +} + +int binarySearch(BinarySearchObject obj, int x) { + int l = 0, r = 9; + while (l <= r) { + int m = (l + r) / 2; + if (obj.prime_numbers[m] == x) { + return m; + } + + if (obj.prime_numbers[m] < x) { + l = m + 1; + } else { + r = m - 1; + } + } + // If an element is not present in the array we return -1. + return -1; +} + +void main() { + BinarySearchObject obj; + // Initialize first 10 prime numbers to the array. + for (int i = 0; i < 10; i++) { + if (i == 0) { + obj.prime_numbers[i] = 2; + } else if (i == 1) { + obj.prime_numbers[i] = 3; + } else if (i == 2) { + obj.prime_numbers[i] = 5; + } else if (i == 3) { + obj.prime_numbers[i] = 7; + } else if (i == 4) { + obj.prime_numbers[i] = 11; + } else if (i == 5) { + obj.prime_numbers[i] = 13; + } else if (i == 6) { + obj.prime_numbers[i] = 17; + } else if (i == 7) { + obj.prime_numbers[i] = 19; + } else if (i == 8) { + obj.prime_numbers[i] = 23; + } else if (i == 9) { + obj.prime_numbers[i] = 29; + } + } + + vec2 uv = (gl_FragCoord.xy / resolution.x) * vec2(resolution.x / resolution.y, 1.0); + vec2 b = brick(uv * 7.0); + vec3 color = vec3(patternize(b)); + + if (gl_FragCoord.y < resolution.y / 1.1) { + // We are going to search the item in array by giving the value of the item index 4 and 0 in an array. + if (binarySearch(obj, obj.prime_numbers[4]) != -(int(resolution.y)) && binarySearch(obj, obj.prime_numbers[0]) >= -(int(resolution.x))) { + color.yz -= dot(float(binarySearch(obj, obj.prime_numbers[4])), float(binarySearch(obj, obj.prime_numbers[0]))); + } else { + discard; + } + } else { + // The following condition is true as there is no value 1 in the array. + if (binarySearch(obj, 1) == -1) { + discard; + } else { + color.yz += color.yz; + } + } + + _GLF_color = vec4(color, injectionSwitch.y); + +} diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_bw.json b/shaders/src/main/glsl/samples/310es/binarysearch_bw.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/binarysearch_bw.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag new file mode 100644 index 000000000..a57cc536e --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag @@ -0,0 +1,167 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +// Size of an array. +const int N = 10; +// An array and its temperary array whose elements will be sorted. +int data[10], temp[10]; + +// Merge two sorted subarrays data[from ... mid] and data[mid+1 ... to]. +void merge(int from, int mid, int to) { + int k = from, i = from, j = mid +1; + + while (i <= mid && j <= to) { + if (data[i] < data[j]) { + temp[k++] = data[i++]; + } else { + temp[k++] = data[j++]; + } + } + // Copy remaining elements. + while (i < N && i <= mid) { + temp[k++] = data[i++]; + } + // Copy back to the original array. + for (int i = from; i <= to; i++) { + data[i] = temp[i]; + } +} + +// Sort array using the iterative approach. +void mergeSort() { + int low = 0; + int high = N - 1; + + // Devide the array into blocks of size m. + // m = [1, 2, 4 ,8, 16...]. + for (int m = 1; m <= high; m = 2 * m) { + + // For m = 1, i = [0, 2, 4, 6, 8]. + // For m = 2, i = [0, 4, 8]. + // For m = 4, i = [0, 8]. + for (int i = low; i< high; i += 2 * m) { + int from = i; + int mid = i + m - 1; + int to = min (i + 2 * m - 1, high); + merge(from, mid, to); + } + } +} + +void main() { + int i = int(injectionSwitch.x); + do { + switch(i) { + case 0: + data[i] = 4; + break; + case 1: + data[i] = 3 ; + break; + case 2: + data[i] = 2 ; + break; + case 3: + data[i] = 1 ; + break; + case 4: + data[i] = 0 ; + break; + case 5: + data[i] = -1; + break; + case 6: + data[i] = -2; + break; + case 7: + data[i] = -3; + break; + case 8: + data[i] = -4; + break; + case 9: + data[i] = -5; + break; + } + i++; + } while (i < 10); + + for (int j = 0; j < 10; j++) { + temp[j] = data[j]; + } + mergeSort(); + + mat3 pos = mat3( + vec3(4, 0, int(injectionSwitch.y)) * ldexp(0.5, 2), + vec3(0, -5, int(injectionSwitch.y)) * ldexp(0.2, 5), + vec3(1, 8, int(injectionSwitch.y)) * ldexp(injectionSwitch.y, 0) + ); + vec3 vecCoor = roundEven(pos * vec3(gl_FragCoord.xx / resolution.yx, 1)); + vec2 color; + + do { + if (int(gl_FragCoord[1]) < 30) { + color = fract(sin(vecCoor.yx - trunc(float(data[0])))); + color[0] = dFdy(gl_FragCoord.y); + break; + } else if (int(gl_FragCoord[1]) < 60) { + color = fract(sin(vecCoor.yx - trunc(float(data[1])))); + color[1] *= atan(faceforward(injectionSwitch, color.xx, vecCoor.yx).y); + break; + } else if (int(gl_FragCoord[1]) < 90) { + color = fract(sin(vecCoor.yx - trunc(float(data[2])))); + color.x += atanh(color.x) * cosh(injectionSwitch.y) * smoothstep(color, injectionSwitch, gl_FragCoord.yy).x; + break; + } else if (int(gl_FragCoord[1]) < 120) { + color = fract(acosh(vecCoor.yx - trunc(float(data[3])))); + color.x += (isnan(gl_FragCoord.x) ? log2(gl_FragCoord.x) : log2(gl_FragCoord.y)); + break; + } else if (int(gl_FragCoord[1]) < 150) { + discard; + } else if (int(gl_FragCoord[1]) < 180) { + color = fract(sin(vecCoor.yx - trunc(float(data[4])))); + color[1] += asinh(gl_FragCoord.y * ldexp(color.y, -i)); + break; + } else if (int(gl_FragCoord[1]) < 210) { + color = fract(sin(vecCoor.yx - trunc(float(data[5])))); + color.y -= tanh(color.x) / cosh(color.y); + break; + } else if (int(gl_FragCoord[1]) < 240) { + color = fract(asinh(vecCoor.yx - trunc(float(data[6])))); + color.y -= isnan(float(i)) ? tanh(gl_FragCoord.x): atanh(gl_FragCoord.y); + break; + } else if (int(gl_FragCoord[1]) < 270) { + color = fract(sin(vecCoor.yx - trunc(float(data[7])))); + color.y *= mix(normalize(vecCoor), normalize(vec3(color, degrees(color.x))), injectionSwitch.x).y; + break; + } else { + discard; + } + } while(0 == int(injectionSwitch.x)); + + _GLF_color = vec4(vec3(color, trunc(injectionSwitch.y)), injectionSwitch.y); + +} diff --git a/shaders/src/main/glsl/samples/310es/mergesort_mosaic.json b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From f0e9120104b3161f9112695739aa31ff8d0014ce Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Wed, 14 Aug 2019 14:27:21 +0100 Subject: [PATCH 099/180] Add gfauto (#684) GraphicsFuzz auto (gfauto) provides scripts for fuzzing using GraphicsFuzz (glsl-fuzz) and spirv-fuzz with minimal interaction. --- build/travis/check_headers.py | 23 +- gfauto/.editorconfig | 20 + gfauto/.flake8 | 63 ++ gfauto/.gitignore | 17 + .../inspectionProfiles/gfauto_inspections.xml | 16 + .../inspectionProfiles/profiles_settings.xml | 7 + gfauto/.idea/mypy.xml | 8 + gfauto/.idea/runConfigurations/Run_fuzz.xml | 24 + gfauto/.idea/scopes/python_files.xml | 3 + gfauto/.idea/watcherTasks.xml | 25 + gfauto/.isort.cfg | 23 + gfauto/Pipfile | 84 +++ gfauto/Pipfile.lock | 699 ++++++++++++++++++ gfauto/README.md | 141 ++++ gfauto/azure-pipelines.yml | 45 ++ gfauto/check_all.sh | 28 + gfauto/dev_shell.sh.template | 47 ++ gfauto/fix_all.sh | 26 + gfauto/gfauto/__init__.py | 15 + gfauto/gfauto/add_amber_tests_to_cts.py | 353 +++++++++ gfauto/gfauto/amber_converter.py | 450 +++++++++++ gfauto/gfauto/android_device.py | 277 +++++++ gfauto/gfauto/artifact.proto | 54 ++ gfauto/gfauto/artifact_pb2.py | 173 +++++ gfauto/gfauto/artifact_pb2.pyi | 86 +++ gfauto/gfauto/artifact_util.py | 253 +++++++ gfauto/gfauto/binaries_util.py | 513 +++++++++++++ gfauto/gfauto/common.proto | 61 ++ gfauto/gfauto/common_pb2.py | 192 +++++ gfauto/gfauto/common_pb2.pyi | 82 ++ gfauto/gfauto/device.proto | 60 ++ gfauto/gfauto/device_pb2.py | 316 ++++++++ gfauto/gfauto/device_pb2.pyi | 126 ++++ gfauto/gfauto/devices_util.py | 95 +++ gfauto/gfauto/fuzz.py | 290 ++++++++ gfauto/gfauto/fuzz_glsl_test.py | 677 +++++++++++++++++ gfauto/gfauto/fuzz_spirv_test.py | 166 +++++ gfauto/gfauto/generate_cts_test_template.py | 57 ++ gfauto/gfauto/gfauto_interestingness_test.py | 146 ++++ gfauto/gfauto/gflogging.py | 53 ++ gfauto/gfauto/glsl_generate_util.py | 59 ++ gfauto/gfauto/glslang_validator_util.py | 86 +++ gfauto/gfauto/host_device_util.py | 129 ++++ gfauto/gfauto/proto_util.py | 62 ++ gfauto/gfauto/recipe.proto | 36 + ...recipe_download_and_extract_archive_set.py | 84 +++ gfauto/gfauto/recipe_pb2.py | 119 +++ gfauto/gfauto/recipe_pb2.pyi | 57 ++ gfauto/gfauto/recipe_wrap.py | 33 + gfauto/gfauto/result_util.py | 42 ++ gfauto/gfauto/settings.proto | 36 + gfauto/gfauto/settings_pb2.py | 89 +++ gfauto/gfauto/settings_pb2.pyi | 52 ++ gfauto/gfauto/settings_util.py | 48 ++ gfauto/gfauto/shader_job_util.py | 124 ++++ gfauto/gfauto/signature_util.py | 288 ++++++++ gfauto/gfauto/spirv_dis_util.py | 77 ++ gfauto/gfauto/spirv_fuzz_util.py | 101 +++ gfauto/gfauto/spirv_opt_util.py | 131 ++++ gfauto/gfauto/subprocess_util.py | 164 ++++ gfauto/gfauto/test.proto | 42 ++ gfauto/gfauto/test_create_readme.py | 103 +++ gfauto/gfauto/test_pb2.py | 185 +++++ gfauto/gfauto/test_pb2.pyi | 87 +++ gfauto/gfauto/test_update_binaries.py | 76 ++ gfauto/gfauto/test_util.py | 98 +++ gfauto/gfauto/tool.py | 279 +++++++ gfauto/gfauto/util.py | 212 ++++++ gfauto/mypy.ini | 16 + gfauto/pylintrc | 519 +++++++++++++ gfauto/pyproject.toml | 29 + gfauto/run_protoc.sh | 23 + gfauto/setup.cfg | 26 + gfauto/setup.py | 54 ++ gfauto/whitelist.dic | 166 +++++ 75 files changed, 9517 insertions(+), 9 deletions(-) create mode 100644 gfauto/.editorconfig create mode 100644 gfauto/.flake8 create mode 100644 gfauto/.gitignore create mode 100644 gfauto/.idea/inspectionProfiles/gfauto_inspections.xml create mode 100644 gfauto/.idea/inspectionProfiles/profiles_settings.xml create mode 100644 gfauto/.idea/mypy.xml create mode 100644 gfauto/.idea/runConfigurations/Run_fuzz.xml create mode 100644 gfauto/.idea/scopes/python_files.xml create mode 100644 gfauto/.idea/watcherTasks.xml create mode 100644 gfauto/.isort.cfg create mode 100644 gfauto/Pipfile create mode 100644 gfauto/Pipfile.lock create mode 100644 gfauto/README.md create mode 100644 gfauto/azure-pipelines.yml create mode 100755 gfauto/check_all.sh create mode 100755 gfauto/dev_shell.sh.template create mode 100755 gfauto/fix_all.sh create mode 100644 gfauto/gfauto/__init__.py create mode 100644 gfauto/gfauto/add_amber_tests_to_cts.py create mode 100644 gfauto/gfauto/amber_converter.py create mode 100644 gfauto/gfauto/android_device.py create mode 100644 gfauto/gfauto/artifact.proto create mode 100644 gfauto/gfauto/artifact_pb2.py create mode 100644 gfauto/gfauto/artifact_pb2.pyi create mode 100644 gfauto/gfauto/artifact_util.py create mode 100644 gfauto/gfauto/binaries_util.py create mode 100644 gfauto/gfauto/common.proto create mode 100644 gfauto/gfauto/common_pb2.py create mode 100644 gfauto/gfauto/common_pb2.pyi create mode 100644 gfauto/gfauto/device.proto create mode 100644 gfauto/gfauto/device_pb2.py create mode 100644 gfauto/gfauto/device_pb2.pyi create mode 100644 gfauto/gfauto/devices_util.py create mode 100644 gfauto/gfauto/fuzz.py create mode 100644 gfauto/gfauto/fuzz_glsl_test.py create mode 100644 gfauto/gfauto/fuzz_spirv_test.py create mode 100644 gfauto/gfauto/generate_cts_test_template.py create mode 100644 gfauto/gfauto/gfauto_interestingness_test.py create mode 100644 gfauto/gfauto/gflogging.py create mode 100644 gfauto/gfauto/glsl_generate_util.py create mode 100644 gfauto/gfauto/glslang_validator_util.py create mode 100644 gfauto/gfauto/host_device_util.py create mode 100644 gfauto/gfauto/proto_util.py create mode 100644 gfauto/gfauto/recipe.proto create mode 100644 gfauto/gfauto/recipe_download_and_extract_archive_set.py create mode 100644 gfauto/gfauto/recipe_pb2.py create mode 100644 gfauto/gfauto/recipe_pb2.pyi create mode 100644 gfauto/gfauto/recipe_wrap.py create mode 100644 gfauto/gfauto/result_util.py create mode 100644 gfauto/gfauto/settings.proto create mode 100644 gfauto/gfauto/settings_pb2.py create mode 100644 gfauto/gfauto/settings_pb2.pyi create mode 100644 gfauto/gfauto/settings_util.py create mode 100644 gfauto/gfauto/shader_job_util.py create mode 100644 gfauto/gfauto/signature_util.py create mode 100644 gfauto/gfauto/spirv_dis_util.py create mode 100644 gfauto/gfauto/spirv_fuzz_util.py create mode 100644 gfauto/gfauto/spirv_opt_util.py create mode 100644 gfauto/gfauto/subprocess_util.py create mode 100644 gfauto/gfauto/test.proto create mode 100644 gfauto/gfauto/test_create_readme.py create mode 100644 gfauto/gfauto/test_pb2.py create mode 100644 gfauto/gfauto/test_pb2.pyi create mode 100644 gfauto/gfauto/test_update_binaries.py create mode 100644 gfauto/gfauto/test_util.py create mode 100644 gfauto/gfauto/tool.py create mode 100644 gfauto/gfauto/util.py create mode 100644 gfauto/mypy.ini create mode 100644 gfauto/pylintrc create mode 100644 gfauto/pyproject.toml create mode 100755 gfauto/run_protoc.sh create mode 100644 gfauto/setup.cfg create mode 100644 gfauto/setup.py create mode 100644 gfauto/whitelist.dic diff --git a/build/travis/check_headers.py b/build/travis/check_headers.py index d9ec51051..7ba3794e5 100644 --- a/build/travis/check_headers.py +++ b/build/travis/check_headers.py @@ -24,7 +24,9 @@ def exclude_dirname(f: str): - return f in [ + return ( + f.endswith(".egg-info") or + f in [ "target", ".git", ".idea", @@ -38,8 +40,9 @@ def exclude_dirname(f: str): "cmake-build-debug", "cmake-build-release", ".pytest_cache", - - ] + ".venv", + ] + ) def exclude_dirpath(f: str): @@ -110,6 +113,7 @@ def exclude_filename(f: str): f.endswith(".primitives") or \ f.endswith(".jar") or \ f.endswith(".spv") or \ + f.endswith(".dic") or \ f in [ ".editorconfig", ".gitmodules", @@ -128,13 +132,14 @@ def exclude_filename(f: str): "gradlew.bat", "dependency-reduced-pom.xml", "gradle-wrapper.properties", - + "Pipfile.lock", ] def go(): fail = False copyright_pattern = re.compile(r"Copyright (2018|2019) The GraphicsFuzz Project Authors") + generated_pattern = re.compile(r"(g|G)enerated") for (dirpath, dirnames, filenames) in os.walk(os.curdir, topdown=True): @@ -153,13 +158,13 @@ def go(): with io.open(file, "r") as fin: contents = fin.read() - # Generated files don't need a header. - if contents.split("\n")[0].find("generated") > -1: - # This file is OK. Continue to the next file. - continue first_lines = "\n".join(contents.split("\n")[:10]) - # Other files must contain a header for any year within the first few lines. + # OK if the generated pattern is found within the first few lines. + if generated_pattern.search(first_lines): + continue + + # Must contain a header for any year within the first few lines. if copyright_pattern.search(first_lines) is None: fail = True print("Missing license header " + file) diff --git a/gfauto/.editorconfig b/gfauto/.editorconfig new file mode 100644 index 000000000..d9eb82f9b --- /dev/null +++ b/gfauto/.editorconfig @@ -0,0 +1,20 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true +insert_final_newline = true +tab_width = 2 +# IDEA specific. +continuation_indent_size = 4 + +[*.py] +indent_size = 4 +tab_width = 4 +# IDEA specific. +continuation_indent_size = 4 diff --git a/gfauto/.flake8 b/gfauto/.flake8 new file mode 100644 index 000000000..fb740b865 --- /dev/null +++ b/gfauto/.flake8 @@ -0,0 +1,63 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is not a TOML file; no quoting. + +[flake8] +filename = */gfauto/*.py +exclude = *_pb2.py* +max-line-length = 88 +ignore = + E203 + W503 + E501 + + # Missing doc strings. + D100 + D101 + D103 + D104 + D107 + D102 + + # Audit url open for permitted schemes. + S310 + + # Standard pseudo-random generators are not suitable for security/cryptographic purposes. + S311 + + # Use of subprocess call and run. + S404 + S603 + + # Warning: First line should be in imperative mood. + # This is the opposite of the Google style. + D401 + + # Missing trailing comma. + C812 + + # Missing trailing comma. + S101 + + # Allow unindexed string format parameters + P101 + + +# For flake8-quotes: +inline-quotes = " + +# For flake8-spellcheck. Default is whitelist.txt but using whitelist.dic means +# we can add the file to PyCharm/IntelliJ (albeit, globally, not per project). +whitelist=whitelist.dic diff --git a/gfauto/.gitignore b/gfauto/.gitignore new file mode 100644 index 000000000..a082d4075 --- /dev/null +++ b/gfauto/.gitignore @@ -0,0 +1,17 @@ + +.idea/misc.xml +.idea/modules.xml +.idea/vcs.xml +.idea/workspace.xml +.idea/dictionaries/* + +.venv/ +*.egg-info/ +__pycache__/ +.mypy_cache/ + + +.pytest_cache/ +.vscode/ + +pip-wheel-metadata/ diff --git a/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml b/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml new file mode 100644 index 000000000..eec3863dc --- /dev/null +++ b/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml @@ -0,0 +1,16 @@ + + + + \ No newline at end of file diff --git a/gfauto/.idea/inspectionProfiles/profiles_settings.xml b/gfauto/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 000000000..333a4e80e --- /dev/null +++ b/gfauto/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,7 @@ + + + + diff --git a/gfauto/.idea/mypy.xml b/gfauto/.idea/mypy.xml new file mode 100644 index 000000000..877b548bc --- /dev/null +++ b/gfauto/.idea/mypy.xml @@ -0,0 +1,8 @@ + + + + + \ No newline at end of file diff --git a/gfauto/.idea/runConfigurations/Run_fuzz.xml b/gfauto/.idea/runConfigurations/Run_fuzz.xml new file mode 100644 index 000000000..34423a980 --- /dev/null +++ b/gfauto/.idea/runConfigurations/Run_fuzz.xml @@ -0,0 +1,24 @@ + + + + + \ No newline at end of file diff --git a/gfauto/.idea/scopes/python_files.xml b/gfauto/.idea/scopes/python_files.xml new file mode 100644 index 000000000..a2c0629a3 --- /dev/null +++ b/gfauto/.idea/scopes/python_files.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/gfauto/.idea/watcherTasks.xml b/gfauto/.idea/watcherTasks.xml new file mode 100644 index 000000000..5f6740146 --- /dev/null +++ b/gfauto/.idea/watcherTasks.xml @@ -0,0 +1,25 @@ + + + + + + + + \ No newline at end of file diff --git a/gfauto/.isort.cfg b/gfauto/.isort.cfg new file mode 100644 index 000000000..d0ad0b236 --- /dev/null +++ b/gfauto/.isort.cfg @@ -0,0 +1,23 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is not a TOML file; no quoting. + +[isort] +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +use_parentheses = True +line_length = 88 +skip_glob = *_pb2.py* diff --git a/gfauto/Pipfile b/gfauto/Pipfile new file mode 100644 index 000000000..bf68d9ce4 --- /dev/null +++ b/gfauto/Pipfile @@ -0,0 +1,84 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[packages] +# Add "this" package as a dependency; this ensures all packages listed in +# setup.py are installed into the venv, as well as gfauto itself. +gfauto = {path = ".", editable = true} + +[dev-packages] +# autoflake = "*" # Tool to auto-remove unused imports; does not seem to work +# well. + +# iPython shell is useful for modifying artifacts interactively. +ipython = "*" +jedi = "*" + +# Code formatter. Explicit version given because it is a pre-release and +# otherwise we get errors. +black = "==19.3b0" + +# Plugin to protoc that outputs .pyi files (type information). +mypy-protobuf = "*" + +# Provides protoc. +grpcio-tools = "*" + +# Type checking. +mypy = "*" + +# PyLint linter. +pylint = "*" + +# Flake8 linter. +flake8 = "*" +# cohesion = "*" # A tool for measuring Python class cohesion +# flake8-alfred = "*" # Can be used to blacklist symbols. +# flake8-copyright = "*" # Not maintained. +# flake8-return = "*" # Does not look that good. +# flake8-if-expr = "*" # Disallows ternary expressions. +# flake8-strict = "*" # Just contains two redundant checks. +# flake8-eradicate = "*" # Disallows commented out code, but has false-positives. +flake8-bandit = "*" +flake8-black = "*" +flake8-breakpoint = "*" +flake8-broken-line = "*" +flake8-bugbear = "*" +flake8-builtins = "*" +flake8-coding = "*" # Checks for a utf-8 comment at top of every file. +flake8-comprehensions = "*" +flake8-commas = "*" +flake8-debugger = "*" +flake8-docstrings = "*" +# TODO: Remove once bug in flake8-docstrings is fixed: https://gitlab.com/pycqa/flake8-docstrings/issues/36 +pydocstyle = "==3.0.0" +flake8-isort = "*" +flake8-logging-format = "*" +flake8-mock = "*" +flake8-mutable = "*" +# flake8-mypy = "*" # We run the full mypy; this plugin gives false-positives. +flake8-pep3101 = "*" +flake8-print = "*" +flake8-quotes = "*" +flake8-spellcheck = "*" +flake8-string-format = "*" +flake8-type-annotations = "*" +flake8-variables-names = "*" +mccabe = "*" +pep8-naming = "*" diff --git a/gfauto/Pipfile.lock b/gfauto/Pipfile.lock new file mode 100644 index 000000000..3fc9134ba --- /dev/null +++ b/gfauto/Pipfile.lock @@ -0,0 +1,699 @@ +{ + "_meta": { + "hash": { + "sha256": "a4ba469d9d6dc05205708ab9462b64a7d9436c5d1146fdf821e4a39a69b75993" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "gfauto": { + "editable": true, + "path": "." + }, + "protobuf": { + "hashes": [ + "sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8", + "sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538", + "sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e", + "sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a", + "sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6", + "sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0", + "sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc", + "sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47", + "sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01", + "sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115", + "sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277", + "sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c", + "sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea", + "sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87", + "sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7", + "sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126", + "sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a", + "sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832" + ], + "version": "==3.8.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + } + }, + "develop": { + "appdirs": { + "hashes": [ + "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", + "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" + ], + "version": "==1.4.3" + }, + "astroid": { + "hashes": [ + "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", + "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" + ], + "version": "==2.2.5" + }, + "attrs": { + "hashes": [ + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + ], + "version": "==19.1.0" + }, + "backcall": { + "hashes": [ + "sha256:38ecd85be2c1e78f77fd91700c76e14667dc21e2713b63876c0eb901196e01e4", + "sha256:bbbf4b1e5cd2bdb08f915895b51081c041bac22394fdfcfdfbe9f14b77c08bf2" + ], + "version": "==0.1.0" + }, + "bandit": { + "hashes": [ + "sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952", + "sha256:41e75315853507aa145d62a78a2a6c5e3240fe14ee7c601459d0df9418196065" + ], + "version": "==1.6.2" + }, + "black": { + "hashes": [ + "sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf", + "sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c" + ], + "index": "pypi", + "version": "==19.3b0" + }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "version": "==7.0" + }, + "decorator": { + "hashes": [ + "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", + "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" + ], + "version": "==4.4.0" + }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, + "flake8": { + "hashes": [ + "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", + "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" + ], + "index": "pypi", + "version": "==3.7.7" + }, + "flake8-bandit": { + "hashes": [ + "sha256:52a67f453ba765398098b39a2273e138983b3499465ef39a12e8ee544c4598fc" + ], + "index": "pypi", + "version": "==2.1.1" + }, + "flake8-black": { + "hashes": [ + "sha256:6b5fe2a609fa750170da8d5b1ed7c11029bceaff025660be7f19307ec6fa0c35" + ], + "index": "pypi", + "version": "==0.1.0" + }, + "flake8-breakpoint": { + "hashes": [ + "sha256:27e0cb132647f9ef348b4a3c3126e7350bedbb22e8e221cd11712a223855ea0b", + "sha256:5bc70d478f0437a3655d094e1d2fca81ddacabaa84d99db45ad3630bf2004064" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "flake8-broken-line": { + "hashes": [ + "sha256:3eb823f48b4ec67735ebbe8e282319826c8e8be7dfc185c9e259228084c21de2", + "sha256:84147d38a1562d011a8de0bb1de299a715f7dea1a7332bd6946db04a1e4c3ddd" + ], + "index": "pypi", + "version": "==0.1.0" + }, + "flake8-bugbear": { + "hashes": [ + "sha256:5070774b668be92c4312e5ca82748ddf4ecaa7a24ff062662681bb745c7896eb", + "sha256:fef9c9826d14ec23187ae1edeb3c6513c4e46bf0e70d86bac38f7d9aabae113d" + ], + "index": "pypi", + "version": "==19.3.0" + }, + "flake8-builtins": { + "hashes": [ + "sha256:8d806360767947c0035feada4ddef3ede32f0a586ef457e62d811b8456ad9a51", + "sha256:cd7b1b7fec4905386a3643b59f9ca8e305768da14a49a7efb31fe9364f33cd04" + ], + "index": "pypi", + "version": "==1.4.1" + }, + "flake8-coding": { + "hashes": [ + "sha256:79704112c44d09d4ab6c8965e76a20c3f7073d52146db60303bce777d9612260", + "sha256:b8f4d5157a8f74670e6cfea732c3d9f4291a4e994c8701d2c55f787c6e6cb741" + ], + "index": "pypi", + "version": "==1.3.2" + }, + "flake8-commas": { + "hashes": [ + "sha256:d3005899466f51380387df7151fb59afec666a0f4f4a2c6a8995b975de0f44b7", + "sha256:ee2141a3495ef9789a3894ed8802d03eff1eaaf98ce6d8653a7c573ef101935e" + ], + "index": "pypi", + "version": "==2.0.0" + }, + "flake8-comprehensions": { + "hashes": [ + "sha256:35f826956e87f230415cde9c3b8b454e785736cf5ff0be551c441b41b937f699", + "sha256:f0b61d983d608790abf3664830d68efd3412265c2d10f6a4ba1a353274dbeb64" + ], + "index": "pypi", + "version": "==2.1.0" + }, + "flake8-debugger": { + "hashes": [ + "sha256:be4fb88de3ee8f6dd5053a2d347e2c0a2b54bab6733a2280bb20ebd3c4ca1d97" + ], + "index": "pypi", + "version": "==3.1.0" + }, + "flake8-docstrings": { + "hashes": [ + "sha256:4e0ce1476b64e6291520e5570cf12b05016dd4e8ae454b8a8a9a48bc5f84e1cd", + "sha256:8436396b5ecad51a122a2c99ba26e5b4e623bf6e913b0fea0cb6c2c4050f91eb" + ], + "index": "pypi", + "version": "==1.3.0" + }, + "flake8-isort": { + "hashes": [ + "sha256:1e67b6b90a9b980ac3ff73782087752d406ce0a729ed928b92797f9fa188917e", + "sha256:81a8495eefed3f2f63f26cd2d766c7b1191e923a15b9106e6233724056572c68" + ], + "index": "pypi", + "version": "==2.7.0" + }, + "flake8-logging-format": { + "hashes": [ + "sha256:ca5f2b7fc31c3474a0aa77d227e022890f641a025f0ba664418797d979a779f8" + ], + "index": "pypi", + "version": "==0.6.0" + }, + "flake8-mock": { + "hashes": [ + "sha256:2fa775e7589f4e1ad74f35d60953eb20937f5d7355235e54bf852c6837f2bede" + ], + "index": "pypi", + "version": "==0.3" + }, + "flake8-mutable": { + "hashes": [ + "sha256:38fd9dadcbcda6550a916197bc40ed76908119dabb37fbcca30873666c31d2d5", + "sha256:ee9b77111b867d845177bbc289d87d541445ffcc6029a0c5c65865b42b18c6a6" + ], + "index": "pypi", + "version": "==1.2.0" + }, + "flake8-pep3101": { + "hashes": [ + "sha256:493821d6bdd083794eb0691ebe5b68e5c520b622b269d60e54308fb97440e21a", + "sha256:b661ab718df42b87743dde266ef5de4f9e900b56c67dbccd45d24cf527545553" + ], + "index": "pypi", + "version": "==1.2.1" + }, + "flake8-plugin-utils": { + "hashes": [ + "sha256:1ac5eb19773d5c7fdde60b0d901ae86be9c751bf697c61fdb6609b86872f3c6e", + "sha256:24b4a3b216ad588951d3d7adef4645dcb3b32a33b878e03baa790b5a66bf3a73" + ], + "version": "==1.0.0" + }, + "flake8-polyfill": { + "hashes": [ + "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9", + "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda" + ], + "version": "==1.0.2" + }, + "flake8-print": { + "hashes": [ + "sha256:5010e6c138b63b62400da4b06afa33becc5e08bd1fcce9af3752445cf3342f54" + ], + "index": "pypi", + "version": "==3.1.0" + }, + "flake8-quotes": { + "hashes": [ + "sha256:10c9af6b472d4302a8e721c5260856c3f985c5c082b04841aefd2f808ac02038" + ], + "index": "pypi", + "version": "==2.0.1" + }, + "flake8-spellcheck": { + "hashes": [ + "sha256:adbf59c4491d18a20809b8bec452245b7aaf0c0e7d9e2db646a3b0a2867570ab", + "sha256:c33e097e4abd0586cb30dcd08af0744a8787d8cfaa0dae61e8cef5a50556d696" + ], + "index": "pypi", + "version": "==0.8.1" + }, + "flake8-string-format": { + "hashes": [ + "sha256:68ea72a1a5b75e7018cae44d14f32473c798cf73d75cbaed86c6a9a907b770b2", + "sha256:774d56103d9242ed968897455ef49b7d6de272000cfa83de5814273a868832f1" + ], + "index": "pypi", + "version": "==0.2.3" + }, + "flake8-type-annotations": { + "hashes": [ + "sha256:88775455792ad7bbd63a71bc94e8a077deb5608eacb5add7e5a7a648c7636426", + "sha256:de64de5efef3277d7b6012e8618c37d35b21465fb16292e46e6eec5b87e47a8c" + ], + "index": "pypi", + "version": "==0.1.0" + }, + "flake8-variables-names": { + "hashes": [ + "sha256:728cfe7ca01fd2458fde22563e0bf53ff88018ada96471c73f6072f782218597" + ], + "index": "pypi", + "version": "==0.0.1" + }, + "gitdb2": { + "hashes": [ + "sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2", + "sha256:e3a0141c5f2a3f635c7209d56c496ebe1ad35da82fe4d3ec4aaa36278d70648a" + ], + "version": "==2.0.5" + }, + "gitpython": { + "hashes": [ + "sha256:563221e5a44369c6b79172f455584c9ebbb122a13368cc82cb4b5addff788f82", + "sha256:8237dc5bfd6f1366abeee5624111b9d6879393d84745a507de0fda86043b65a8" + ], + "version": "==2.1.11" + }, + "grpcio": { + "hashes": [ + "sha256:03b78b4e7dcdfe3e257bb528cc93923f9cbbab6d5babf15a60d21e9a4a70b1a2", + "sha256:1ce0ccfbdfe84387dbcbf44adb4ae16ec7ae70e166ffab478993eb1ea1cba3ce", + "sha256:22e167a9406d73dd19ffe8ed6a485f17e6eac82505be8c108897f15e68badcbb", + "sha256:31d0aeca8d8ee2301c62c5c340e0889d653b1280d68f9fa203982cb6337b050e", + "sha256:44c7f99ca17ebbcc96fc54ed00b454d8313f1eac28c563098d8b901025aff941", + "sha256:5471444f53f9db6a1f1f11f5dbc173228881df8446380b6b98f90afb8fd8348e", + "sha256:561bca3b1bde6d6564306eb05848fd155136e9c3a25d2961129b1e2edba22fce", + "sha256:5bf58e1d2c2f55365c06e8cb5abe067b88ca2e5550fb62009c41df4b54505acf", + "sha256:6b7163d1e85d76b0815df63fcc310daec02b44532bb433f743142d4febcb181f", + "sha256:766d79cddad95f5f6020037fe60ea8b98578afdf0c59d5a60c106c1bdd886303", + "sha256:770b7372d5ca68308ff66d7baee53369fa5ce985f84bcb6aa1948c1f2f7b02f2", + "sha256:7ab178da777fc0f55b6aef5a755f99726e8e4b75e3903954df07b27059b54fcf", + "sha256:8078305e77c2f6649d36b24d8778096413e474d9d7892c6f92cfb589c9d71b2e", + "sha256:85600b63a386d860eeaa955e9335e18dd0d7e5477e9214825abf2c2884488369", + "sha256:857d9b939ae128be1c0c792eb885c7ff6a386b9dea899ac4b06f4d90a31f9d87", + "sha256:87a41630c90c179fa5c593400f30a467c498972c702f348d41e19dafeb1d319e", + "sha256:8805d486c6128cc0fcc8ecf16c4095d99a8693a541ef851429ab334e028a4a97", + "sha256:8d71b7a89c306a41ccc7741fc9409b14f5b86727455c2a1c0c7cfcb0f784e1f2", + "sha256:9e1b80bd65f8f160880cb4dad7f55697f6d37b2d7f251fc0c2128e811928f369", + "sha256:9e290c84a145ae2411ee0ec9913c41cd7500e2e7485fe93632434d84ef4fda67", + "sha256:9ec9f88b5bc94bd99372f27cdd53af1c92ba06717380b127733b953cfb181174", + "sha256:a0a02a8b4ba6deadf706d5f849539b3685b72b186a3c9ef5d43e8972ed60fb6f", + "sha256:a4059c59519f5940e01a071f74ae2a60ea8f6185b03d22a09d40c7959a36b16b", + "sha256:a6e028c2a6da2ebfa2365a5b32531d311fbfec0e3600fc27e901b64f0ff7e54e", + "sha256:adcdebf9f8463df4120c427cf6c9aed39258bccd03ed37b6939e7a145d64d6e0", + "sha256:bdec982610259d07156a58f80b8c3e69be7751a9208bc577b059c5193d087fad", + "sha256:cefc4d4251ffb73feb303d4b7e9d6c367cb60f2db16d259ea28b114045f965aa", + "sha256:d4145c8aa6afbac10ad27e408f7ce15992fe89ba5d0b4abca31c0c2729864c03", + "sha256:da76dc5ad719ee99de5ea28a5629ff92172cbb4a70d8a6ae3a5b7a53c7382ce1", + "sha256:dde2452c08ef8b6426ccab6b5b6de9f06d836d9937d6870e68153cbf8cb49348", + "sha256:e3d88091d2539a4868750914a6fe7b9ec50e42b913851fc1b77423b5bd918530", + "sha256:f9c67cfe6278499d7f83559dc6322a8bbb108e307817a3d7acbfea807b3603cc" + ], + "version": "==1.22.0" + }, + "grpcio-tools": { + "hashes": [ + "sha256:0d8e85d7e62ea4e04ff75eace98dd0b53b22bd57dc1efbed0f8398db884e1cf6", + "sha256:160e6fd9c44a3f6c76a8c2f84014e367d41a3cae27c451406b616f9a08ba2b1f", + "sha256:337554658b4a4c0a6e1ee75acd6681ec45007e1d6d1975e885f92b665881e33a", + "sha256:36390281f31861b8b685f7e8366ff8abbcdcedf3dda7b73133be56db71d3283b", + "sha256:3dd362bcd1db5247760b7f8b8157abd53e47210a17df9de994f064819ba64c04", + "sha256:4381f3722f3c5d02626796f9db6dd0dcb75c13245ea5655b54880ab7bd239a73", + "sha256:4b5a5fe3e949bd03e068c05070f3cdc6a01b68323efb667199d3c67a30ef515c", + "sha256:4f43d03e3e1f5f7bb14b8bba35a607e35ea4589fed778b8321c97810bc0a76ac", + "sha256:67d1a8d71b2572250124ebdebbfac7248563bb97ab139916fb265705d1b0dbe2", + "sha256:6e6fab6e8e92aadb70b4167e5eadd843e8c850d97213fdf4a39f6e4d3596e6f0", + "sha256:75fe16b642564e47c65cf6d23362079cdbbbc5c544b59c7bc1a3a7c8865d73e3", + "sha256:78bc91e38fe6a6c80de084014e22365c788811773829f28621878a803dd6af48", + "sha256:8dde6f660e8390380e987b0661b2ced31bafa2d3dde9845bd918a007d998e7a8", + "sha256:928de5a03fa2664d20433d3023e512fa1f8bc0588ff3eceee33c2d93b82e0110", + "sha256:9a4348b0309c444e5b7bfd3a2cf38bdb354c56e5f4b966eb88b249aff46b0a4c", + "sha256:9af68a58dc1a339e5735425a66b0a7146114b6fbc860d1f8b74b277d151f7bf1", + "sha256:a3447b842ab2964a834776be0a38b93d1977887d43cb69a2ce9941388923d8a9", + "sha256:a83b82b39e32a3110e45a11f0df9aceacefd201c25da79a66edda60d37a9f2e6", + "sha256:b05de8f16752b50851cd1caa3e63a49b35f8adb3eee0cec8991b231fac0c158d", + "sha256:b659d2306dd79818b2fae0543377e199bcf656404032e24f2af53a0e5338b5c9", + "sha256:b688dd1d83fdb46ab84d14a0711a0033984add9717619513f21e00f1b005f986", + "sha256:bbe9c13774ecaf99beb39d05c3ee659771c903add0c5e7a761c79387f653f69f", + "sha256:c23f59a04ed9db331922e5f02425958a907a019e157f14a6b95dd0ca1bcf05a6", + "sha256:cf0c35fe4a8b0dc78d9bf181317d9fe9cd21a397429d9c0c01c0178cf20e4442", + "sha256:cfd2320a0b4233ec4b9d3ac8d19df69194548bd32473fdba366cb50930f59dc2", + "sha256:d5c82c63ec24f0de5151e478cabfca23afd39005b6d0ba62dccf81cf19b0ae7f", + "sha256:d8d3dcb3832c209c66c67bd0f62289358b2cb942a68c00ac94c2ebf6870f731e", + "sha256:d98ed0a731d6c4cee39e76ba0bc2225455da097f41fa11ebfa27fe3854b6cc98", + "sha256:dbb2dd5367bfd71a6a779f6b237de015d6e508df129a126fc2a39da8a675457b", + "sha256:f3c135ad51ec95667bb945be107d0176f3048e615283104bf30027fe8bc06a8e", + "sha256:f8d75ead0ef7d060699b6c87b11f3c54e0c4b8e24065cbd262b64d300558a420" + ], + "index": "pypi", + "version": "==1.22.0" + }, + "ipython": { + "hashes": [ + "sha256:11067ab11d98b1e6c7f0993506f7a5f8a91af420f7e82be6575fcb7a6ca372a0", + "sha256:60bc55c2c1d287161191cc2469e73c116d9b634cff25fe214a43cba7cec94c79" + ], + "index": "pypi", + "version": "==7.6.1" + }, + "ipython-genutils": { + "hashes": [ + "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", + "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8" + ], + "version": "==0.2.0" + }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, + "jedi": { + "hashes": [ + "sha256:49ccb782651bb6f7009810d17a3316f8867dde31654c750506970742e18b553d", + "sha256:79d0f6595f3846dffcbe667cc6dc821b96e5baa8add125176c31a3917eb19d58" + ], + "index": "pypi", + "version": "==0.14.0" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661", + "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f", + "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13", + "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821", + "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71", + "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e", + "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea", + "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229", + "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4", + "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e", + "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20", + "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16", + "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b", + "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7", + "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c", + "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a", + "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e", + "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1" + ], + "version": "==1.4.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "index": "pypi", + "version": "==0.6.1" + }, + "mypy": { + "hashes": [ + "sha256:12d18bd7fc642c5d54b1bb62dde813a7e2ab79b32ee11ff206ac387c68fc2ad4", + "sha256:23e24bc1683a36f39dee67d8ac74ea414654642eee26d420bada95b8ee8c9095", + "sha256:2b38e64c52a8968df4ebcae0ddba4a54eb94d184695dd4e54e14509a9389b78c", + "sha256:3d4f551466a76e278187ec3a5b26cfb50f72f6760b749aa00ac69a6f9c99898d", + "sha256:53d5dacb8d844e50be698830509aa592b093547e7ab90aee63eb23db61109007", + "sha256:56f981d246010ba21cac6b2455eaecfaf68fc8a5663d865b26c8e579c36f751d", + "sha256:8c57f6f59f1e8479d9fc6e1bf034353e54626ed64e32394c613afc493a441dc1", + "sha256:bbed4a593d87476b592d52867ef86da2155ccd0becf0c4c02e6567d842e43368", + "sha256:d6ff850e2ba18b2db7704897c8f2f1384478e3b75ad292ec06196bf7794f3a40", + "sha256:e13b1bb8785d7f785e0b88873f1c21cda58ceba9ce1153b58cbfa24b09a111d5", + "sha256:e2b9ee6f648ce72d6741925a47c88c2391168ef973b6f74f17969450c5b1ffdd" + ], + "index": "pypi", + "version": "==0.711" + }, + "mypy-extensions": { + "hashes": [ + "sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812", + "sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e" + ], + "version": "==0.4.1" + }, + "mypy-protobuf": { + "hashes": [ + "sha256:feff06ebda0462d663761f135840c1d0e2d1470499efd5b4f847f4b60585368c" + ], + "index": "pypi", + "version": "==1.10" + }, + "parso": { + "hashes": [ + "sha256:5052bb33be034cba784193e74b1cde6ebf29ae8b8c1e4ad94df0c4209bfc4826", + "sha256:db5881df1643bf3e66c097bfd8935cf03eae73f4cb61ae4433c9ea4fb6613446" + ], + "version": "==0.5.0" + }, + "pbr": { + "hashes": [ + "sha256:36ebd78196e8c9588c972f5571230a059ff83783fabbbbedecc07be263ccd7e6", + "sha256:5a03f59455ad54f01a94c15829b8b70065462b7bd8d5d7e983306b59127fc841" + ], + "version": "==5.4.0" + }, + "pep8-naming": { + "hashes": [ + "sha256:01cb1dab2f3ce9045133d08449f1b6b93531dceacb9ef04f67087c11c723cea9", + "sha256:0ec891e59eea766efd3059c3d81f1da304d858220678bdc351aab73c533f2fbb" + ], + "index": "pypi", + "version": "==0.8.2" + }, + "pexpect": { + "hashes": [ + "sha256:2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1", + "sha256:9e2c1fd0e6ee3a49b28f95d4b33bc389c89b20af6a1255906e90ff1262ce62eb" + ], + "markers": "sys_platform != 'win32'", + "version": "==4.7.0" + }, + "pickleshare": { + "hashes": [ + "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", + "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56" + ], + "version": "==0.7.5" + }, + "prompt-toolkit": { + "hashes": [ + "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", + "sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1", + "sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55" + ], + "version": "==2.0.9" + }, + "protobuf": { + "hashes": [ + "sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8", + "sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538", + "sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e", + "sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a", + "sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6", + "sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0", + "sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc", + "sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47", + "sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01", + "sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115", + "sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277", + "sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c", + "sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea", + "sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87", + "sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7", + "sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126", + "sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a", + "sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832" + ], + "version": "==3.8.0" + }, + "ptyprocess": { + "hashes": [ + "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", + "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f" + ], + "version": "==0.6.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + ], + "version": "==2.5.0" + }, + "pydocstyle": { + "hashes": [ + "sha256:2258f9b0df68b97bf3a6c29003edc5238ff8879f1efb6f1999988d934e432bd8", + "sha256:5741c85e408f9e0ddf873611085e819b809fca90b619f5fd7f34bd4959da3dd4", + "sha256:ed79d4ec5e92655eccc21eb0c6cf512e69512b4a97d215ace46d17e4990f2039" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "pyflakes": { + "hashes": [ + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" + ], + "version": "==2.1.1" + }, + "pygments": { + "hashes": [ + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + ], + "version": "==2.4.2" + }, + "pylint": { + "hashes": [ + "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", + "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" + ], + "index": "pypi", + "version": "==2.3.1" + }, + "pyyaml": { + "hashes": [ + "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", + "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", + "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", + "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", + "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", + "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", + "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", + "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", + "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", + "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", + "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" + ], + "version": "==5.1.1" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "smmap2": { + "hashes": [ + "sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde", + "sha256:29a9ffa0497e7f2be94ca0ed1ca1aa3cd4cf25a1f6b4f5f87f74b46ed91d609a" + ], + "version": "==2.0.5" + }, + "snowballstemmer": { + "hashes": [ + "sha256:9f3b9ffe0809d174f7047e121431acf99c89a7040f0ca84f94ba53a498e6d0c9" + ], + "version": "==1.9.0" + }, + "stevedore": { + "hashes": [ + "sha256:7be098ff53d87f23d798a7ce7ae5c31f094f3deb92ba18059b1aeb1ca9fec0a0", + "sha256:7d1ce610a87d26f53c087da61f06f9b7f7e552efad2a7f6d2322632b5f932ea2" + ], + "version": "==1.30.1" + }, + "testfixtures": { + "hashes": [ + "sha256:665a298976c8d77f311b65c46f16b7cda7229a47dff5ad7c822e5b3371a439e2", + "sha256:9d230c5c80746f9f86a16a1f751a5cf5d8e317d4cc48243a19fb180d22303bce" + ], + "version": "==6.10.0" + }, + "toml": { + "hashes": [ + "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + ], + "version": "==0.10.0" + }, + "traitlets": { + "hashes": [ + "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835", + "sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9" + ], + "version": "==4.3.2" + }, + "typed-ast": { + "hashes": [ + "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", + "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", + "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", + "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", + "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", + "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", + "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", + "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", + "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", + "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", + "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", + "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + ], + "markers": "implementation_name == 'cpython'", + "version": "==1.4.0" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + } + } +} diff --git a/gfauto/README.md b/gfauto/README.md new file mode 100644 index 000000000..5fd1a18dc --- /dev/null +++ b/gfauto/README.md @@ -0,0 +1,141 @@ +# GraphicsFuzz auto + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Build Status](https://paulthomson.visualstudio.com/gfauto/_apis/build/status/google.graphicsfuzz?branchName=master)](https://paulthomson.visualstudio.com/gfauto/_build/latest?definitionId=2&branchName=master) + + +## GraphicsFuzz auto is a set of Python scripts for running GraphicsFuzz + +[GraphicsFuzz](https://github.com/google/graphicsfuzz) provides tools that automatically find and simplify bugs in graphics shader compilers. +GraphicsFuzz auto (this project) provides scripts for running these tools with minimal interaction. + +## Development setup + +> Optional: if you have just done `git pull` to get a more recent version of GraphicsFuzz auto, consider deleting `.venv/` to start from a fresh virtual environment. This is rarely needed. + +Execute `./dev_shell.sh.template`. If the default settings don't work, make a copy of the file called `dev_shell.sh` and modify according to the comments before executing. `pip` must be installed for the version of Python you wish to use. + +The script generates and activates a Python virtual environment (located at `.venv/`) with all dependencies installed. + +* Execute `./check_all.sh` to run various presubmit checks, linters, etc. +* Execute `./fix_all.sh` to automatically fix certain issues, such as formatting. + + +### PyCharm + +Use PyCharm to open the top-level `gfauto` directory. +It should pick up the Python virtual environment (at `.venv/`) automatically +for both the code +and when you open a `Terminal` or `Python Console` tab. + +Install and configure plugins: + +* Protobuf Support +* File Watchers (may already be installed) + * The watcher task should already be under version control. +* Mypy: the built-in PyCharm type checking uses Mypy behinds the scenes, but this plugin enhances it by using the latest version and allowing the use of stricter settings, matching the settings used by the `./check_all.sh` script. + +Add `whitelist.dic` as a custom dictionary (search for "Spelling" in Actions). Do not add words via PyCharm's "Quick Fixes" feature, as the word will only be added to your personal dictionary. Instead, manually add the word to `whitelist.dic`. + +## Imports + +We use the `black` Python code formatter and `isort` for sorting imports. + +We also use the following import style, which is not automatically checked: use the package name (e.g. `gfauto`) and only import modules, not functions. For example: + + +```python +# Good: importing a module +from gfauto import binaries_util + +binaries_util.add_common_tags_from_platform_suffix(...) # using a function +b = binaries_util.BinaryManager() # using a class +d = binaries_util.DEFAULT_BINARIES # using a variable + + +# Bad: directly importing a function, class, variable +from gfauto.binaries_util import add_common_tags_from_platform_suffix, BinaryManager, DEFAULT_BINARIES + +add_common_tags_from_platform_suffix(...) +b = BinaryManager() +d = DEFAULT_BINARIES + + +# Bad: importing from "." +from . import binaries_util + + +# Bad: import from "." AND importing a function +from .binaries_util import add_common_tags_from_platform_suffix + + +# OK: importing "check" and "log" functions directly +from gfauto.gflogging import log +from gfauto.util import check + +log("Running") +check(1 + 1 == 2, AssertionError("1 + 1 should be 2")) + + +# OK: importing types for type annotations +from typing import Dict, List, Optional, Union + +def prepend_catchsegv_if_available(cmd: List[str]) -> List[str]: + ... + + +# OK: importing Path +from pathlib import Path + +def tool_path(tool: str) -> Path: + return Path(tool) + + +# OK: importing generated protobuf types +from gfauto.settings_pb2 import Settings + +DEFAULT_SETTINGS = Settings() +``` + +## Symlinking other scripts + +GraphicsFuzz auto moves fast and so it is useful to add symlinks to other repositories that contain Python scripts that depend on GraphicsFuzz auto. This allows you to search for all references before changing a function. A `temp/` directory exists for this purpose. For example: + +```sh +cd temp +ln -s /path/to/shader-generation shader-generation +``` + +Now any scripts in the `shader-generation` repository are visible in PyCharm. + +You can execute scripts in this repository by opening a Terminal in PyCharm. + +## Terminal + +The `Terminal` tab in PyCharm is useful and will use the project's Python virtual environment. In any other terminal, use `source .venv/bin/activate`. You can alternatively execute the `./dev_shell.sh` script, but this is fairly slow as it checks and reinstalls all dependencies + +## Fuzzing + +To start fuzzing, create and change to a directory outside the `gfauto/` directory. E.g. `/data/temp/gfauto_fuzzing/2019_06_24`. From here, create a `donors/` directory containing GLSL shader jobs as used by GraphicsFuzz. +You can get some samples from the GraphicsFuzz project. + +```sh +mkdir donors/ +cp /data/graphicsfuzz_zip/samples/310es/* donors/ +``` + +Now run the fuzzer. + +```sh +gfauto_fuzz +``` + +It will fail because there is no settings file, but a default `settings.json` file will be created for you. +Review this file. +All plugged Android devices should have been added to the list of devices. +The `active_device_names` list should be modified to include only the unique devices that you care about: + +* Including multiple, identical devices (with the same GPU and drivers) is not recommended, as it will currently result in redundant testing. +* `host_preprocessor` should always be included first, as this virtual device detects failures in tools such as `glslangValidator` and `spirv-opt`, and will ensure such failures will not be logged for real devices. +* You can use `--settings SETTINGS.json` to use a settings file other than `settings.json` (the default). In this way, you can run multiple instances of `gfauto_fuzz` in parallel to test *different* devices (`host_preprocessor` and `swift_shader` can be included in parallel instances). The code that downloads the binaries is not safe to run concurrently at the time of writing; running one instance for a few minutes to download the latest binaries is usually sufficient to then allow additional instances to be started without conflicts. + diff --git a/gfauto/azure-pipelines.yml b/gfauto/azure-pipelines.yml new file mode 100644 index 000000000..096efbfc9 --- /dev/null +++ b/gfauto/azure-pipelines.yml @@ -0,0 +1,45 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +trigger: +- gfauto + +pr: +- dev +- master + +pool: + vmImage: 'ubuntu-latest' +strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + cd gfauto + export PYTHON=python + export PIPENV_IGNORE_PIPFILE=1 + export SKIP_SHELL=1 + ./dev_shell.sh.template + source .venv/bin/activate + ./check_all.sh + displayName: 'Install dependencies' diff --git a/gfauto/check_all.sh b/gfauto/check_all.sh new file mode 100755 index 000000000..26f6050d9 --- /dev/null +++ b/gfauto/check_all.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -u + +if [ -z ${VIRTUAL_ENV+x} ]; then + source .venv/bin/activate +fi + +mypy --strict gfauto +pylint gfauto +# Flake checks formatting via black. +flake8 . diff --git a/gfauto/dev_shell.sh.template b/gfauto/dev_shell.sh.template new file mode 100755 index 000000000..ed3ace4d8 --- /dev/null +++ b/gfauto/dev_shell.sh.template @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -u + +# Modify if needed; this should be a Python 3.6 binary. +# E.g. python, python3, python3.6. +PYTHON=${PYTHON-python3.6} + +# Upgrade/install pip and pipenv if needed. +"${PYTHON}" -m pip install --upgrade --user 'pip>=19.1.1' 'pipenv>=2018.11.26' + +# The following (optional) line causes the virtual environment to be placed at +# `gfauto/.venv`. You may wish to add this environment variable permanently, +# such as by adding to your .bashrc file. +export PIPENV_VENV_IN_PROJECT=1 + +# The following (optional) line causes the hard-coded versions of packages (in +# Pipfile.lock) to be used, for better reproducibility. Enabled during CI but +# disabled otherwise so that packages are updated automatically during +# development. +# export PIPENV_IGNORE_PIPFILE=1 + +# Install project dependencies, including development dependencies, into a +# virtual environment using pipenv. +"${PYTHON}" -m pipenv install --dev + +if [ -z ${SKIP_SHELL+x} ]; then + # Enter the virtual environment. + # `python` should now point to the correct version of Python. + "${PYTHON}" -m pipenv shell +fi diff --git a/gfauto/fix_all.sh b/gfauto/fix_all.sh new file mode 100755 index 000000000..390539620 --- /dev/null +++ b/gfauto/fix_all.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -u + +if [ -z ${VIRTUAL_ENV+x} ]; then + source .venv/bin/activate +fi + +isort -rc gfauto +black gfauto diff --git a/gfauto/gfauto/__init__.py b/gfauto/gfauto/__init__.py new file mode 100644 index 000000000..f7bad95d3 --- /dev/null +++ b/gfauto/gfauto/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/gfauto/gfauto/add_amber_tests_to_cts.py b/gfauto/gfauto/add_amber_tests_to_cts.py new file mode 100644 index 000000000..970c52ee3 --- /dev/null +++ b/gfauto/gfauto/add_amber_tests_to_cts.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add amber tests to CTS. + +This module/script adds Amber test files to the VK-GL-CTS project. +This file is self-contained so it can be provided alongside Amber test files. +""" + +import argparse +import os +import shutil +import sys +from typing import TextIO, cast + +SHORT_DESCRIPTION_LINE_PREFIX = "# Short description: " + +MUST_PASS_PATHS = [ + os.path.join("android", "cts", "master", "vk-master.txt"), + os.path.join( + "external", "vulkancts", "mustpass", "master", "vk-default-no-waivers.txt" + ), + os.path.join("external", "vulkancts", "mustpass", "master", "vk-default.txt"), +] + + +def check(condition: bool, exception: Exception) -> None: + if not condition: + raise exception + + +def log(message: str = "") -> None: + print(message, flush=True) # noqa T001 + + +def remove_start(string: str, start: str) -> str: + check( + string.startswith(start), AssertionError("|string| does not start with |start|") + ) + + return string[len(start) :] + + +def open_helper(file: str, mode: str) -> TextIO: # noqa VNE002 + check("b" not in mode, AssertionError(f"|mode|(=={mode}) should not contain 'b'")) + return cast(TextIO, open(file, mode, encoding="utf-8", errors="ignore")) + + +def check_dir_exists(directory: str) -> None: + log("Checking that directory {} exists".format(directory)) + if not os.path.isdir(directory): + raise FileNotFoundError("Directory not found: {}".format(directory)) + + +def check_file_exists(file: str) -> None: # noqa VNE002 + log("Checking that file {} exists".format(file)) + if not os.path.isfile(file): + raise FileNotFoundError("File not found: {}".format(file)) + + +def get_amber_test_file_path(vk_gl_cts: str, amber_test_name: str) -> str: + return os.path.join( + vk_gl_cts, + "external", + "vulkancts", + "data", + "vulkan", + "amber", + "graphicsfuzz", + amber_test_name + ".amber", + ) + + +def get_graphics_fuzz_tests_cpp_file_path(vk_gl_cts: str) -> str: + return os.path.join( + vk_gl_cts, + "external", + "vulkancts", + "modules", + "vulkan", + "amber", + "vktAmberGraphicsFuzzTests.cpp", + ) + + +def get_amber_test_short_description(amber_test_file_path: str) -> str: + with open_helper(amber_test_file_path, "r") as f: + for line in f: + if line.startswith(SHORT_DESCRIPTION_LINE_PREFIX): + line = remove_start(line, SHORT_DESCRIPTION_LINE_PREFIX) + # Remove \n + line = line[:-1] + return line + return "" + + +def check_and_add_tabs( + line: str, string_name: str, string_value: str, field_index: int, tab_size: int +) -> str: + + # Field index starts at 1. Change it to start at 0. + field_index -= 1 + + check( + len(line.expandtabs(tab_size)) <= field_index, + AssertionError('{} "{}" is too long!'.format(string_name, string_value)), + ) + + while len(line.expandtabs(tab_size)) < field_index: + line += "\t" + + check( + len(line.expandtabs(tab_size)) == field_index, + AssertionError( + "Field index {} is incorrect; Python script needs fixing".format( + field_index + ) + ), + ) + + return line + + +def get_cpp_line_to_write(amber_test_name: str, short_description: str) -> str: + + # A test line has the following form, except with tabs aligning each field. + # { "name.amber", "name", "description" }, + # | | | | + # 1 2 3 4 + + # 1 + test_file_name_start_index = 13 + # 2 + test_name_start_index = 61 + # 3 + test_description_start_index = 101 + # 4 + test_close_bracket_index = 189 + + tab_size = 4 + + line = "\t\t{" + + line = check_and_add_tabs( + line, "internal", "internal", test_file_name_start_index, tab_size + ) + + line += '"{}.amber",'.format(amber_test_name) + + line = check_and_add_tabs( + line, "amber test name", amber_test_name, test_name_start_index, tab_size + ) + + line += '"{}",'.format(amber_test_name) + + line = check_and_add_tabs( + line, "amber test name", amber_test_name, test_description_start_index, tab_size + ) + + line += '"{}"'.format(short_description) + + line = check_and_add_tabs( + line, "short description", short_description, test_close_bracket_index, tab_size + ) + + line += "},\n" + + return line + + +def add_amber_test_to_cpp( + vk_gl_cts: str, amber_test_name: str, input_amber_test_file_path: str +) -> None: + log("Adding Amber test to the C++.") + + short_description = get_amber_test_short_description(input_amber_test_file_path) + + cpp_file = get_graphics_fuzz_tests_cpp_file_path(vk_gl_cts) + cpp_file_bak = cpp_file + ".bak" + + copyfile(cpp_file, cpp_file_bak) + + line_to_write = get_cpp_line_to_write(amber_test_name, short_description) + + log("Writing from {} to {}.".format(cpp_file_bak, cpp_file)) + + with open_helper(cpp_file_bak, "r") as cpp_in: + with open_helper(cpp_file, "w") as cpp_out: + + # The start of the tests look like this (except with tabs!): + # tests[] = + # { + # { "continue-and-merge.amber",... + # { "control-flow-switch.amber",... + + # Get to just before the first test line, writing lines as we go. + for line in cpp_in: + cpp_out.write(line) + if line.startswith("\ttests[] ="): + break + cpp_out.write(cpp_in.readline()) + + # Get to the point where we should insert our line. + line = "" + for line in cpp_in: + if not line.startswith(" {"): + break + elif line >= line_to_write: + break + else: + cpp_out.write(line) + + # Write our line and then the previously read line. + + # Don't write the line if it already exists; idempotent. + if line != line_to_write: + log("Writing line: {}".format(line_to_write[:-1])) + cpp_out.write(line_to_write) + else: + log("Line already exists.") + cpp_out.write(line) + + # Write the remaining lines. + for line in cpp_in: + cpp_out.write(line) + + remove(cpp_file_bak) + + +def add_amber_test_to_must_pass(amber_test_name: str, must_pass_file_path: str) -> None: + log("Adding the Amber test to {}".format(must_pass_file_path)) + + must_pass_file_path_bak = must_pass_file_path + ".bak" + copyfile(must_pass_file_path, must_pass_file_path_bak) + + line_to_write = "dEQP-VK.graphicsfuzz.{}\n".format(amber_test_name) + + log("Writing from {} to {}.".format(must_pass_file_path_bak, must_pass_file_path)) + + with open_helper(must_pass_file_path_bak, "r") as pass_in: + with open_helper(must_pass_file_path, "w") as pass_out: + # Get to just before the first GraphicsFuzz test. + line = "" + for line in pass_in: + if line.startswith("dEQP-VK.graphicsfuzz."): + break + pass_out.write(line) + + # |line| contains an unwritten line. + # Get to the point where we need to write line_to_write. + while True: + if (not len) or line >= line_to_write: + break + pass_out.write(line) + line = pass_in.readline() + + # Don't write the line if it already exists; idempotent. + if line != line_to_write: + log("Writing line: {}".format(line_to_write[:-1])) + pass_out.write(line_to_write) + else: + log("Line already exists.") + pass_out.write(line) + + # Write remaining lines. + for line in pass_in: + pass_out.write(line) + + remove(must_pass_file_path_bak) + + +def copyfile(source: str, dest: str) -> None: + log("Copying {} to {}".format(source, dest)) + shutil.copyfile(source, dest) + + +def remove(file: str) -> None: # noqa VNE002 + log("Deleting {}".format(file)) + os.remove(file) + + +def copy_amber_test_file( + vk_gl_cts: str, amber_test_name: str, input_amber_test_file_path: str +) -> None: + log("Copying Amber test file") + + amber_test_file_path = get_amber_test_file_path(vk_gl_cts, amber_test_name) + + check_dir_exists(os.path.dirname(amber_test_file_path)) + + copyfile(input_amber_test_file_path, amber_test_file_path) + + +def add_amber_test(input_amber_test_file_path: str, vk_gl_cts: str) -> None: + log('Adding Amber test "{}" to "{}"'.format(input_amber_test_file_path, vk_gl_cts)) + # E.g. "continue-and-merge" + amber_test_name = os.path.basename(input_amber_test_file_path) + amber_test_name = os.path.splitext(amber_test_name)[0] + + log('Using test name "{}"'.format(amber_test_name)) + + add_amber_test_to_cpp(vk_gl_cts, amber_test_name, input_amber_test_file_path) + + copy_amber_test_file(vk_gl_cts, amber_test_name, input_amber_test_file_path) + + for must_pass_file_path in MUST_PASS_PATHS: + add_amber_test_to_must_pass( + amber_test_name, os.path.join(vk_gl_cts, must_pass_file_path) + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="A script to add Amber tests to the CTS." + ) + + parser.add_argument("vk_gl_cts", help="Path to a checkout of VK-GL-CTS") + + parser.add_argument( + "amber_files", + help="One or more Amber test files (often ending in .amber_script, .amber, .vkscript)", + nargs="+", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + vk_gl_cts = parsed_args.vk_gl_cts + amber_files = parsed_args.amber_files + + check_dir_exists(vk_gl_cts) + check_file_exists(get_graphics_fuzz_tests_cpp_file_path(vk_gl_cts)) + + for amber_file in amber_files: + add_amber_test(amber_file, vk_gl_cts) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py new file mode 100644 index 000000000..fa66ff76e --- /dev/null +++ b/gfauto/gfauto/amber_converter.py @@ -0,0 +1,450 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Amber shader job converter module. + +Converts a SPIR-V assembly shader job (all shaders are already disassembled) to an Amber script file. +""" + +import json +import pathlib +from copy import copy +from typing import List, Optional + +from gfauto import shader_job_util, util +from gfauto.gflogging import log +from gfauto.util import check + +AMBER_FENCE_TIMEOUT_MS = 60000 + + +class AmberfySettings: # pylint: disable=too-many-instance-attributes + def __init__( + self, + copyright_header_text: Optional[str] = None, + add_generated_comment: bool = False, + add_graphics_fuzz_comment: bool = False, + short_description: Optional[str] = None, + comment_text: Optional[str] = None, + use_default_fence_timeout: bool = False, + extra_commands: Optional[str] = None, + spirv_opt_args: Optional[List[str]] = None, + spirv_opt_hash: Optional[str] = None, + ): + self.copyright_header_text = copyright_header_text + self.add_generated_comment = add_generated_comment + self.add_graphics_fuzz_comment = add_graphics_fuzz_comment + self.short_description = short_description + self.comment_text = comment_text + self.use_default_fence_timeout = use_default_fence_timeout + self.extra_commands = extra_commands + self.spirv_opt_args = spirv_opt_args + self.spirv_opt_hash = spirv_opt_hash + + def copy(self: "AmberfySettings") -> "AmberfySettings": + # A shallow copy is adequate. + return copy(self) + + +def get_spirv_opt_args_comment( + spirv_opt_args: List[str], spirv_opt_hash: Optional[str] +) -> str: + if not spirv_opt_args: + return "" + result = "# Optimized using spirv-opt with the following arguments:\n" + args = [f"# '{arg}'" for arg in spirv_opt_args] + result += "\n".join(args) + if spirv_opt_hash: + result += f"\n# spirv-opt commit hash: {spirv_opt_hash}" + result += "\n\n" + return result + + +def get_text_as_comment(text: str) -> str: + lines = text.split("\n") + + # Remove empty lines from start and end. + while not lines[0]: + lines.pop(0) + while not lines[-1]: + lines.pop() + + lines = [("# " + line).rstrip() for line in lines] + return "\n".join(lines) + + +def uniform_json_to_amberscript(uniform_json_contents: str) -> str: + """ + Returns the string representing VkScript version of uniform declarations. + + Skips the special '$compute' key, if present. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + "$compute": { ... will be ignored ... } + } + + becomes: + + # myuniform + uniform ubo 0:3 float 0 42.0 + """ + uniform_types = { + "glUniform1i": "int", + "glUniform2i": "ivec2", + "glUniform3i": "ivec3", + "glUniform4i": "ivec4", + "glUniform1f": "float", + "glUniform2f": "vec2", + "glUniform3f": "vec3", + "glUniform4f": "vec4", + } + + descriptor_set = 0 # always 0 in our tests + offset = 0 # We never have uniform offset in our tests + + result = "" + uniform_json = json.loads(uniform_json_contents) + for name, entry in uniform_json.items(): + + if name == "$compute": + continue + + func = entry["func"] + binding = entry["binding"] + + check( + func in uniform_types.keys(), + AssertionError(f"unknown uniform type for function{func}"), + ) + + uniform_type = uniform_types[func] + + result += "# " + name + "\n" + result += f"uniform ubo {descriptor_set}:{binding}" + result += " " + uniform_type + result += f" {offset}" + for arg in entry["args"]: + result += f" {arg}" + result += "\n" + + return result + + +def translate_type_for_amber(type_name: str) -> str: + if type_name == "bool": + return "uint" + return type_name + + +def comp_json_to_amberscript(shader_json_contents: str) -> str: + """ + Returns the string representing VkScript version of compute shader setup. + + The compute shader setup is found under the special "$compute" key in the JSON. + + { + "my_uniform_name": { ... ignored by this function ... }, + + "$compute": { + "num_groups": [12, 13, 14]; + "buffer": { + "binding": 123, + "fields": [ + { + "type": "int", + "data": [42, 43, 44, 45] + } + ] + } + } + + } + + becomes: + offset + | + ssbo 123 subdata int 0 42 43 44 45 + + compute 12 13 14 + + """ + shader_json = json.loads(shader_json_contents) + check( + "$compute" in shader_json.keys(), + AssertionError('Cannot find "$compute" key in JSON file'), + ) + compute_json = shader_json["$compute"] + + result = "## SSBO\n" + + binding = compute_json["buffer"]["binding"] + offset = 0 + for field_info in compute_json["buffer"]["fields"]: + result += ( + "ssbo " + + str(binding) + + " subdata " + + translate_type_for_amber(field_info["type"]) + + " " + + str(offset) + ) + for datum in field_info["data"]: + result += " " + str(datum) + offset += 4 + result += "\n" + result += "\n" + + result += "compute" + result += " " + str(compute_json["num_groups"][0]) + result += " " + str(compute_json["num_groups"][1]) + result += " " + str(compute_json["num_groups"][2]) + result += "\n" + + return result + + +def amberscriptify_image( # pylint: disable=too-many-branches,too-many-locals + vert_asm_contents: Optional[str], + frag_asm_contents: str, + shader_job_json_contents: str, + amberfy_settings: AmberfySettings, + vert_glsl_contents: Optional[str] = None, + frag_glsl_contents: Optional[str] = None, + add_red_pixel_probe: bool = False, +) -> str: + """Generates Amberscript representation of an image test.""" + result = "" + + if amberfy_settings.copyright_header_text: + result += get_text_as_comment(amberfy_settings.copyright_header_text) + "\n\n" + + if amberfy_settings.add_generated_comment: + result = "# Generated.\n\n" + + if amberfy_settings.add_graphics_fuzz_comment: + result += "# A test for a bug found by GraphicsFuzz.\n\n" + + if amberfy_settings.short_description: + result += f"# Short description: {amberfy_settings.short_description}\n\n" + + if amberfy_settings.comment_text: + result += get_text_as_comment(amberfy_settings.comment_text) + "\n\n" + + if amberfy_settings.spirv_opt_args: + result += get_spirv_opt_args_comment( + amberfy_settings.spirv_opt_args, amberfy_settings.spirv_opt_hash + ) + + if vert_glsl_contents or frag_glsl_contents: + result += "# Derived from the following GLSL.\n\n" + + if vert_glsl_contents: + result += "# Vertex shader GLSL:\n" + result += get_text_as_comment(vert_glsl_contents) + result += "\n\n" + + if frag_glsl_contents: + result += "# Fragment shader GLSL:\n" + result += get_text_as_comment(frag_glsl_contents) + result += "\n\n" + + result += "[require]\n" + result += "fbsize 256 256\n" + + if not amberfy_settings.use_default_fence_timeout: + result += "fence_timeout " + str(AMBER_FENCE_TIMEOUT_MS) + "\n" + + result += "\n" + + if vert_asm_contents: + result += "[vertex shader spirv]\n" + result += vert_asm_contents + else: + result += "[vertex shader passthrough]" + result += "\n\n" + + result += "[fragment shader spirv]\n" + result += frag_asm_contents + result += "\n\n" + + result += "[test]\n" + + uniforms_text = uniform_json_to_amberscript(shader_job_json_contents) + if uniforms_text: + result += "## Uniforms\n" + result += uniforms_text + result += "\n" + result += "draw rect -1 -1 2 2\n" + + if add_red_pixel_probe: + result += "probe rgba (0, 0) (1, 0, 0, 1)\n" + + if amberfy_settings.extra_commands: + result += amberfy_settings.extra_commands + + return result + + +def is_compute_job(input_asm_spirv_job_json_path: pathlib.Path) -> bool: + comp_files = shader_job_util.get_related_files( + input_asm_spirv_job_json_path, + [shader_job_util.EXT_COMP], + [shader_job_util.SUFFIX_ASM_SPIRV], + ) + check( + len(comp_files) <= 1, + AssertionError(f"Expected 1 or 0 compute shader files: {comp_files}"), + ) + return len(comp_files) == 1 + + +def amberscriptify_comp( + comp_asm_contents: str, + shader_job_json_contents: str, + amberfy_settings: AmberfySettings, + comp_glsl_contents: Optional[str], +) -> str: + result = "" + + if amberfy_settings.copyright_header_text: + result += get_text_as_comment(amberfy_settings.copyright_header_text) + "\n\n" + + if amberfy_settings.add_generated_comment: + result = "# Generated.\n\n" + + if amberfy_settings.add_graphics_fuzz_comment: + result += "# A test for a bug found by GraphicsFuzz.\n\n" + + if amberfy_settings.short_description: + result += f"# Short description: {amberfy_settings.short_description}\n\n" + + if amberfy_settings.comment_text: + result += get_text_as_comment(amberfy_settings.comment_text) + "\n\n" + + if amberfy_settings.spirv_opt_args: + result += get_spirv_opt_args_comment( + amberfy_settings.spirv_opt_args, amberfy_settings.spirv_opt_hash + ) + + if comp_glsl_contents: + result += "# Derived from the following GLSL.\n\n" + result += "# Compute shader GLSL:\n" + result += get_text_as_comment(comp_glsl_contents) + result += "\n\n" + + if not amberfy_settings.use_default_fence_timeout: + result += "[require]\n" + result += "fence_timeout " + str(AMBER_FENCE_TIMEOUT_MS) + "\n\n" + + result += "[compute shader spirv]\n" + result += comp_asm_contents + result += "\n\n" + + result += "[test]\n" + uniforms_text = uniform_json_to_amberscript(shader_job_json_contents) + if uniforms_text: + result += "## Uniforms\n" + result += uniforms_text + + # This also includes the "compute" command that runs the shader. + result += comp_json_to_amberscript(shader_job_json_contents) + + if amberfy_settings.extra_commands: + result += amberfy_settings.extra_commands + + return result + + +def run_spirv_asm_shader_job_to_amber_script( # pylint: disable=too-many-locals + input_asm_spirv_job_json_path: pathlib.Path, + output_amber_script_file_path: pathlib.Path, + amberfy_settings: AmberfySettings, + input_glsl_source_json_path: Optional[pathlib.Path] = None, + add_red_pixel_probe: bool = False, +) -> pathlib.Path: + + log( + f"Amberfy: {str(input_asm_spirv_job_json_path)} to {str(output_amber_script_file_path)}" + ) + + # Get the JSON contents. + json_contents = util.file_read_text(input_asm_spirv_job_json_path) + + if is_compute_job(input_asm_spirv_job_json_path): # pylint:disable=no-else-raise + glsl_comp_contents = None + if input_glsl_source_json_path: + glsl_comp_contents = shader_job_util.get_shader_contents( + input_glsl_source_json_path, shader_job_util.EXT_COMP + ) + comp_asm_contents = shader_job_util.get_shader_contents( + input_asm_spirv_job_json_path, + shader_job_util.EXT_COMP, + shader_job_util.SUFFIX_ASM_SPIRV, + must_exist=True, + ) + + assert comp_asm_contents # noqa + + result = amberscriptify_comp( + comp_asm_contents, json_contents, amberfy_settings, glsl_comp_contents + ) + + else: + # Get GLSL contents + glsl_vert_contents = None + glsl_frag_contents = None + if input_glsl_source_json_path: + glsl_vert_contents = shader_job_util.get_shader_contents( + input_glsl_source_json_path, shader_job_util.EXT_VERT + ) + glsl_frag_contents = shader_job_util.get_shader_contents( + input_glsl_source_json_path, shader_job_util.EXT_FRAG + ) + + # Get spirv asm contents + vert_contents = shader_job_util.get_shader_contents( + input_asm_spirv_job_json_path, + shader_job_util.EXT_VERT, + shader_job_util.SUFFIX_ASM_SPIRV, + ) + + frag_contents = shader_job_util.get_shader_contents( + input_asm_spirv_job_json_path, + shader_job_util.EXT_FRAG, + shader_job_util.SUFFIX_ASM_SPIRV, + must_exist=True, + ) + + # Guaranteed. + assert frag_contents # noqa + + result = amberscriptify_image( + vert_contents, + frag_contents, + json_contents, + amberfy_settings, + glsl_vert_contents, + glsl_frag_contents, + add_red_pixel_probe, + ) + + util.file_write_text(output_amber_script_file_path, result) + return output_amber_script_file_path diff --git a/gfauto/gfauto/android_device.py b/gfauto/gfauto/android_device.py new file mode 100644 index 000000000..c0db103e5 --- /dev/null +++ b/gfauto/gfauto/android_device.py @@ -0,0 +1,277 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Android devices. + +Provides functions for interacting with Android devices. +""" + +import os +import shutil +import subprocess +import time +from pathlib import Path +from subprocess import CompletedProcess +from typing import List, Optional + +from gfauto import fuzz, gflogging, result_util, subprocess_util, util +from gfauto.device_pb2 import Device, DeviceAndroid +from gfauto.gflogging import log +from gfauto.util import check, file_open_text, file_write_text + +ANDROID_DEVICE_DIR = "/data/local/tmp" +ANDROID_AMBER_NDK = "amber_ndk" +ANDROID_DEVICE_AMBER = ANDROID_DEVICE_DIR + "/" + ANDROID_AMBER_NDK + +ANDROID_DEVICE_GRAPHICSFUZZ_DIR = ANDROID_DEVICE_DIR + "/graphicsfuzz" +ANDROID_DEVICE_RESULT_DIR = ANDROID_DEVICE_GRAPHICSFUZZ_DIR + "/result" +ANDROID_DEVICE_AMBER_SCRIPT_FILE = ANDROID_DEVICE_GRAPHICSFUZZ_DIR + "/test.amber" + +BUSY_WAIT_SLEEP_SLOW = 1.0 + + +def adb_path() -> Path: + if "ANDROID_HOME" in os.environ: + platform_tools_path = Path(os.environ["ANDROID_HOME"]) / "platform-tools" + adb = shutil.which("adb", path=str(platform_tools_path)) + if adb: + return Path(adb) + return util.tool_on_path("adb") + + +def adb_helper( + serial: Optional[str], + adb_args: List[str], + check_exit_code: bool, + verbose: bool = False, +) -> subprocess.CompletedProcess: + + adb_cmd = [str(adb_path())] + if serial: + adb_cmd.append("-s") + adb_cmd.append(serial) + + adb_cmd.extend(adb_args) + + return subprocess_util.run( + adb_cmd, + check_exit_code=check_exit_code, + timeout=fuzz.AMBER_RUN_TIME_LIMIT, + verbose=verbose, + ) + + +def adb_check( + serial: Optional[str], adb_args: List[str], verbose: bool = False +) -> subprocess.CompletedProcess: + + return adb_helper(serial, adb_args, check_exit_code=True, verbose=verbose) + + +def adb_can_fail( + serial: Optional[str], adb_args: List[str], verbose: bool = False +) -> subprocess.CompletedProcess: + + return adb_helper(serial, adb_args, check_exit_code=False, verbose=verbose) + + +def stay_awake_warning(serial: Optional[str] = None) -> None: + res = adb_check(serial, ["shell", "settings get global stay_on_while_plugged_in"]) + if str(res.stdout).strip() == "0": + log('\nWARNING: please enable "Stay Awake" from developer settings\n') + + +def is_screen_off_or_locked(serial: Optional[str] = None) -> bool: + """:return: True: the screen is off or locked. False: unknown.""" + res = adb_can_fail(serial, ["shell", "dumpsys nfc"]) + if res.returncode != 0: + return False + + stdout = str(res.stdout) + # You will often find "mScreenState=OFF_LOCKED", but this catches OFF too, which is good. + if "mScreenState=OFF" in stdout: + return True + if "mScreenState=ON_LOCKED" in stdout: + return True + + return False + + +def get_all_android_devices() -> List[Device]: + result: List[Device] = [] + + log("Getting the list of connected Android devices via adb") + + adb_devices = adb_check(None, ["devices", "-l"], verbose=True) + stdout: str = adb_devices.stdout + lines: List[str] = stdout.splitlines() + # Remove empty lines. + lines = [l for l in lines if l] + check( + lines[0].startswith("List of devices"), + AssertionError("Could find list of devices from 'adb devices'"), + ) + for i in range(1, len(lines)): + fields = lines[i].split() + device_serial = fields[0] + device_state = fields[1] + if device_state != "device": + log( + f'Skipping adb device with serial {device_serial} as its state "{device_state}" is not "device".' + ) + # Set a simple model name, but then try to find the actual model name. + device_model = "android_device" + for field_index in range(2, len(fields)): + if fields[field_index].startswith("model:"): + device_model = util.remove_start(fields[field_index], "model:") + break + + device = Device( + name=f"{device_model}_{device_serial}", + android=DeviceAndroid(serial=device_serial, model=device_model), + ) + log(f"Found Android device:\n{str(device)}") + result.append(device) + + return result + + +def prepare_device(wait_for_screen: bool, serial: Optional[str] = None) -> None: + adb_check(serial, ["logcat", "-c"]) + res = adb_can_fail(serial, ["shell", "test -e " + ANDROID_DEVICE_AMBER]) + check( + res.returncode == 0, + AssertionError("Failed to find amber on device at: " + ANDROID_DEVICE_AMBER), + ) + + adb_check( + serial, + [ + "shell", + # One string: + # Remove graphicsfuzz directory. + f"rm -rf {ANDROID_DEVICE_GRAPHICSFUZZ_DIR} && " + # Make result directory. + f"mkdir -p {ANDROID_DEVICE_RESULT_DIR}", + ], + ) + + if wait_for_screen: + stay_awake_warning(serial) + # We cannot reliably know if the screen is on, but this function definitely knows if it is + # off or locked. So we wait here while we definitely know there is an issue. + count = 0 + while is_screen_off_or_locked(serial): + log( + "\nWARNING: The screen appears to be off or locked. Please unlock the device and ensure 'Stay Awake' is enabled in developer settings.\n" + ) + time.sleep(BUSY_WAIT_SLEEP_SLOW) + count += 1 + if count > 1 and (count % 10) == 0: + log("Pressing the menu key.") + adb_can_fail(serial, ["shell", "input keyevent 82"], verbose=True) + + +def run_amber_on_device( + amber_script_file: Path, + output_dir: Path, + dump_image: bool, + dump_buffer: bool, + skip_render: bool = False, + serial: Optional[str] = None, +) -> Path: + + with file_open_text(result_util.get_amber_log_path(output_dir), "w") as log_file: + try: + gflogging.push_stream_for_logging(log_file) + + run_amber_on_device_helper( + amber_script_file, + output_dir, + dump_image, + dump_buffer, + skip_render, + serial, + ) + finally: + gflogging.pop_stream_for_logging() + + return output_dir + + +def run_amber_on_device_helper( + amber_script_file: Path, + output_dir: Path, + dump_image: bool, + dump_buffer: bool, + skip_render: bool = False, + serial: Optional[str] = None, +) -> Path: + + prepare_device(wait_for_screen=True, serial=serial) + + adb_check( + serial, ["push", str(amber_script_file), ANDROID_DEVICE_AMBER_SCRIPT_FILE] + ) + + amber_flags = "--log-graphics-calls-time --disable-spirv-val" + if skip_render: + # -ps tells amber to stop after pipeline creation + amber_flags += " -ps" + else: + if dump_image: + amber_flags += f" -i {fuzz.IMAGE_FILE_NAME}" + if dump_buffer: + amber_flags += f" -b {fuzz.BUFFER_FILE_NAME} -B 0" + + cmd = [ + "shell", + # The following is a single string: + f"cd {ANDROID_DEVICE_RESULT_DIR} && " + # -d disables Vulkan validation layers. + f"{ANDROID_DEVICE_AMBER} -d {ANDROID_DEVICE_AMBER_SCRIPT_FILE} {amber_flags}", + ] + + status = "UNEXPECTED_ERROR" + + result: Optional[CompletedProcess] = None + + try: + result = adb_can_fail(serial, cmd, verbose=True) + except subprocess.TimeoutExpired: + status = fuzz.STATUS_TIMEOUT + + if result: + if result.returncode != 0: + status = fuzz.STATUS_CRASH + else: + status = fuzz.STATUS_SUCCESS + + if not skip_render: + adb_check( + serial, + # The /. syntax means the contents of the results directory will be copied into output_dir. + ["pull", ANDROID_DEVICE_RESULT_DIR + "/.", str(output_dir)], + ) + + # Grab log: + adb_check(serial, ["logcat", "-d"], verbose=True) + + log("\nSTATUS " + status + "\n") + + file_write_text(result_util.get_status_path(output_dir), status) + + return output_dir diff --git a/gfauto/gfauto/artifact.proto b/gfauto/gfauto/artifact.proto new file mode 100644 index 000000000..a76fcfffe --- /dev/null +++ b/gfauto/gfauto/artifact.proto @@ -0,0 +1,54 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; + + +// An artifact is a directory containing an ArtifactMetadata proto stored in "artifact.json" and/or a Recipe proto +// stored in "recipe.json", plus any other files and directories within. The recipe can be executed to produce the files +// that make up the artifact, which must include the "artifact.json" file. An artifact is referred to via an +// "artifact path" that starts from a directory containing a file named "ROOT" (the contents is not used), written +// as "//", followed by the path to the directory, using "/" as the directory separator. E.g. +// +// //binaries/my_binary +// +// Artifacts were planned to be used more frequently, but are now mainly used just for downloading and extracting +// binaries. +// See recipe.proto. +message ArtifactMetadata { + + // The artifact's type-specific data. + Data data = 1; + + // A list of artifact paths from which this artifact is derived. Typically, this is similar to + // following the input artifacts in the recipe transitively, but some artifacts may not have a + // recipe because they were created manually so it is still useful to have this field. + repeated string derived_from = 2; + + string comment = 3; + + message Data { + oneof data { + ArtifactMetadataExtractedArchiveSet extracted_archive_set = 1; + } + } +} + +message ArtifactMetadataExtractedArchiveSet { + ArchiveSet archive_set = 1; +} diff --git a/gfauto/gfauto/artifact_pb2.py b/gfauto/gfauto/artifact_pb2.py new file mode 100644 index 000000000..5b8b9a396 --- /dev/null +++ b/gfauto/gfauto/artifact_pb2.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/artifact.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/artifact.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x15gfauto/artifact.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"\xc4\x01\n\x10\x41rtifactMetadata\x12+\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32\x1d.gfauto.ArtifactMetadata.Data\x12\x14\n\x0c\x64\x65rived_from\x18\x02 \x03(\t\x12\x0f\n\x07\x63omment\x18\x03 \x01(\t\x1a\\\n\x04\x44\x61ta\x12L\n\x15\x65xtracted_archive_set\x18\x01 \x01(\x0b\x32+.gfauto.ArtifactMetadataExtractedArchiveSetH\x00\x42\x06\n\x04\x64\x61ta\"N\n#ArtifactMetadataExtractedArchiveSet\x12\'\n\x0b\x61rchive_set\x18\x01 \x01(\x0b\x32\x12.gfauto.ArchiveSetb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,]) + + + + +_ARTIFACTMETADATA_DATA = _descriptor.Descriptor( + name='Data', + full_name='gfauto.ArtifactMetadata.Data', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='extracted_archive_set', full_name='gfauto.ArtifactMetadata.Data.extracted_archive_set', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='data', full_name='gfauto.ArtifactMetadata.Data.data', + index=0, containing_type=None, fields=[]), + ], + serialized_start=159, + serialized_end=251, +) + +_ARTIFACTMETADATA = _descriptor.Descriptor( + name='ArtifactMetadata', + full_name='gfauto.ArtifactMetadata', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='data', full_name='gfauto.ArtifactMetadata.data', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='derived_from', full_name='gfauto.ArtifactMetadata.derived_from', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='comment', full_name='gfauto.ArtifactMetadata.comment', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_ARTIFACTMETADATA_DATA, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=55, + serialized_end=251, +) + + +_ARTIFACTMETADATAEXTRACTEDARCHIVESET = _descriptor.Descriptor( + name='ArtifactMetadataExtractedArchiveSet', + full_name='gfauto.ArtifactMetadataExtractedArchiveSet', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='archive_set', full_name='gfauto.ArtifactMetadataExtractedArchiveSet.archive_set', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=253, + serialized_end=331, +) + +_ARTIFACTMETADATA_DATA.fields_by_name['extracted_archive_set'].message_type = _ARTIFACTMETADATAEXTRACTEDARCHIVESET +_ARTIFACTMETADATA_DATA.containing_type = _ARTIFACTMETADATA +_ARTIFACTMETADATA_DATA.oneofs_by_name['data'].fields.append( + _ARTIFACTMETADATA_DATA.fields_by_name['extracted_archive_set']) +_ARTIFACTMETADATA_DATA.fields_by_name['extracted_archive_set'].containing_oneof = _ARTIFACTMETADATA_DATA.oneofs_by_name['data'] +_ARTIFACTMETADATA.fields_by_name['data'].message_type = _ARTIFACTMETADATA_DATA +_ARTIFACTMETADATAEXTRACTEDARCHIVESET.fields_by_name['archive_set'].message_type = gfauto_dot_common__pb2._ARCHIVESET +DESCRIPTOR.message_types_by_name['ArtifactMetadata'] = _ARTIFACTMETADATA +DESCRIPTOR.message_types_by_name['ArtifactMetadataExtractedArchiveSet'] = _ARTIFACTMETADATAEXTRACTEDARCHIVESET +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +ArtifactMetadata = _reflection.GeneratedProtocolMessageType('ArtifactMetadata', (_message.Message,), { + + 'Data' : _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), { + 'DESCRIPTOR' : _ARTIFACTMETADATA_DATA, + '__module__' : 'gfauto.artifact_pb2' + # @@protoc_insertion_point(class_scope:gfauto.ArtifactMetadata.Data) + }) + , + 'DESCRIPTOR' : _ARTIFACTMETADATA, + '__module__' : 'gfauto.artifact_pb2' + # @@protoc_insertion_point(class_scope:gfauto.ArtifactMetadata) + }) +_sym_db.RegisterMessage(ArtifactMetadata) +_sym_db.RegisterMessage(ArtifactMetadata.Data) + +ArtifactMetadataExtractedArchiveSet = _reflection.GeneratedProtocolMessageType('ArtifactMetadataExtractedArchiveSet', (_message.Message,), { + 'DESCRIPTOR' : _ARTIFACTMETADATAEXTRACTEDARCHIVESET, + '__module__' : 'gfauto.artifact_pb2' + # @@protoc_insertion_point(class_scope:gfauto.ArtifactMetadataExtractedArchiveSet) + }) +_sym_db.RegisterMessage(ArtifactMetadataExtractedArchiveSet) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/artifact_pb2.pyi b/gfauto/gfauto/artifact_pb2.pyi new file mode 100644 index 000000000..ac7a3528c --- /dev/null +++ b/gfauto/gfauto/artifact_pb2.pyi @@ -0,0 +1,86 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + ArchiveSet as gfauto___common_pb2___ArchiveSet, +) + +from google.protobuf.internal.containers import ( + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class ArtifactMetadata(google___protobuf___message___Message): + class Data(google___protobuf___message___Message): + + @property + def extracted_archive_set(self) -> ArtifactMetadataExtractedArchiveSet: ... + + def __init__(self, + extracted_archive_set : typing___Optional[ArtifactMetadataExtractedArchiveSet] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ArtifactMetadata.Data: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"data",u"extracted_archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"data",u"extracted_archive_set"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"data",b"data",u"extracted_archive_set",b"extracted_archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"data",b"extracted_archive_set"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"data",b"data"]) -> typing_extensions___Literal["extracted_archive_set"]: ... + + derived_from = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + comment = ... # type: typing___Text + + @property + def data(self) -> ArtifactMetadata.Data: ... + + def __init__(self, + data : typing___Optional[ArtifactMetadata.Data] = None, + derived_from : typing___Optional[typing___Iterable[typing___Text]] = None, + comment : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ArtifactMetadata: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"data"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"comment",u"data",u"derived_from"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"data",b"data"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"comment",b"data",b"derived_from"]) -> None: ... + +class ArtifactMetadataExtractedArchiveSet(google___protobuf___message___Message): + + @property + def archive_set(self) -> gfauto___common_pb2___ArchiveSet: ... + + def __init__(self, + archive_set : typing___Optional[gfauto___common_pb2___ArchiveSet] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ArtifactMetadataExtractedArchiveSet: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"archive_set"]) -> None: ... diff --git a/gfauto/gfauto/artifact_util.py b/gfauto/gfauto/artifact_util.py new file mode 100644 index 000000000..5d715cd16 --- /dev/null +++ b/gfauto/gfauto/artifact_util.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Artifacts utility module. + +See artifact.proto for information about artifacts. +""" + +import pathlib +from typing import List, Optional, Tuple + +from gfauto import ( + binaries_util, + gflogging, + proto_util, + recipe_download_and_extract_archive_set, + util, +) +from gfauto.artifact_pb2 import ArtifactMetadata +from gfauto.common_pb2 import ArchiveSet +from gfauto.recipe_pb2 import Recipe +from gfauto.util import check + +ARTIFACT_METADATA_FILE_NAME = "artifact.json" +ARTIFACT_RECIPE_FILE_NAME = "recipe.json" +ARTIFACT_RECIPE_LOG_FILE_NAME = "recipe.log" +ARTIFACT_ROOT_FILE_NAME = "ROOT" + + +class ArtifactWrap: + def __init__(self, path: str, metadata: Optional[ArtifactMetadata] = None): + self.path = path + self.metadata = ( + metadata if metadata is not None else artifact_read_metadata(path) + ) + + +def recipes_write_built_in() -> None: + for recipe_wrap in binaries_util.BUILT_IN_BINARY_RECIPES: + if not artifact_get_metadata_file_path(recipe_wrap.path).exists(): + recipe_wrap.write() + + +def binary_artifacts_find(artifact_path_prefix: str) -> List[Tuple[ArchiveSet, str]]: + artifact_paths = artifacts_find(artifact_path_prefix) + result: List[Tuple[ArchiveSet, str]] = [] + for artifact_path in artifact_paths: + archive_set = None + if artifact_get_metadata_file_path(artifact_path).exists(): + metadata = artifact_read_metadata(artifact_path) + if metadata.data.HasField("extracted_archive_set"): + archive_set = metadata.data.extracted_archive_set.archive_set + elif artifact_get_recipe_file_path(artifact_path).exists(): + recipe = artifact_read_recipe(artifact_path) + if recipe.HasField("download_and_extract_archive_set"): + archive_set = recipe.download_and_extract_archive_set.archive_set + if archive_set: + result.append((archive_set, artifact_path)) + + return result + + +def artifacts_find(artifact_path_prefix: str) -> List[str]: + path_prefix = artifact_get_directory_path(artifact_path_prefix) + + metadata_files = path_prefix.rglob("*.json") + metadata_files = ( + file + for file in metadata_files + if file.name in (ARTIFACT_METADATA_FILE_NAME, ARTIFACT_RECIPE_FILE_NAME) + ) + artifact_paths = {path_to_artifact_path(file.parent) for file in metadata_files} + return sorted(artifact_paths) + + +def artifact_path_get_root() -> pathlib.Path: + root_file_suffix = pathlib.Path(ARTIFACT_ROOT_FILE_NAME) + fake_file = util.norm_path(pathlib.Path("fake").absolute()) + for parent in fake_file.parents: + if (parent / root_file_suffix).exists(): + return parent + raise FileNotFoundError( + "Could not find root file {}".format(ARTIFACT_ROOT_FILE_NAME) + ) + + +def path_to_artifact_path(path: pathlib.Path) -> str: + from_root = path.relative_to(artifact_path_get_root()).as_posix() + return "//" + from_root + + +def artifact_path_absolute(artifact_path: str) -> str: + """ + Returns the absolute |artifact_path| starting with '//'. + + Artifact paths should almost always begin with '//', but for convenience it can be useful to + use relative paths, especially when calling functions from an IPython shell. + :param artifact_path: An artifact path. + :return: absolute |artifact_path| starting with '//'. + """ + if artifact_path.startswith("//"): + return artifact_path + path = util.norm_path(pathlib.Path(artifact_path)).absolute() + return path_to_artifact_path(path) + + +def artifact_path_to_path(artifact_path: str) -> pathlib.Path: + """ + Returns |artifact_path| converted to an OS specific path. + + Artifact paths always use '/' to separate paths. + Artifact paths usually begin with '//' which is the artifact root directory, marked via a ROOT + file. + + :param artifact_path: an artifact path. + :return: + """ + if artifact_path.startswith("//"): + return util.norm_path( + artifact_path_get_root() / pathlib.Path(artifact_path[2:]) + ) + + return util.norm_path(pathlib.Path(artifact_path)) + + +def artifact_get_directory_path(artifact_path: str = "") -> pathlib.Path: + return artifact_path_to_path(artifact_path) + + +def artifact_get_recipe_file_path(artifact_path: str = "") -> pathlib.Path: + return artifact_get_inner_file_path(ARTIFACT_RECIPE_FILE_NAME, artifact_path) + + +def artifact_get_metadata_file_path(artifact_path: str = "") -> pathlib.Path: + return artifact_get_inner_file_path(ARTIFACT_METADATA_FILE_NAME, artifact_path) + + +def artifact_get_recipe_log_file_path(artifact_path: str) -> pathlib.Path: + return artifact_get_inner_file_path(ARTIFACT_RECIPE_LOG_FILE_NAME, artifact_path) + + +def artifact_write_recipe_and_execute(recipe: Recipe, artifact_path: str = "") -> str: + artifact_path_full = artifact_path_absolute(artifact_path) + + artifact_write_recipe(recipe, artifact_path_full) + artifact_execute_recipe(artifact_path_full) + return artifact_path_full + + +def artifact_write_recipe( + recipe: Optional[Recipe] = None, artifact_path: str = "" +) -> str: + if recipe is None: + recipe = Recipe() + + artifact_path = artifact_path_absolute(artifact_path) + + json_text = proto_util.message_to_json(recipe, including_default_value_fields=True) + json_file_path = artifact_get_recipe_file_path(artifact_path) + util.file_write_text(json_file_path, json_text) + return artifact_path + + +def artifact_read_recipe(artifact_path: str = "") -> Recipe: + recipe = Recipe() + json_file_path = artifact_get_recipe_file_path(artifact_path) + json_text = util.file_read_text(json_file_path) + proto_util.json_to_message(json_text, recipe) + return recipe + + +def artifact_write_metadata( + artifact_metadata: ArtifactMetadata, artifact_path: str +) -> str: + artifact_path = artifact_path_absolute(artifact_path) + + json_text = proto_util.message_to_json( + artifact_metadata, including_default_value_fields=True + ) + json_file_path = artifact_get_metadata_file_path(artifact_path) + util.file_write_text(json_file_path, json_text) + return artifact_path + + +def artifact_read_metadata(artifact_path: str = "") -> ArtifactMetadata: + artifact_metadata = ArtifactMetadata() + json_file_path = artifact_get_metadata_file_path(artifact_path) + json_contents = util.file_read_text(json_file_path) + proto_util.json_to_message(json_contents, artifact_metadata) + return artifact_metadata + + +def artifact_execute_recipe_if_needed(artifact_path: str = "") -> None: + artifact_execute_recipe(artifact_path, only_if_artifact_json_missing=True) + + +def artifact_execute_recipe( + artifact_path: str = "", only_if_artifact_json_missing: bool = False +) -> None: + + artifact_path = artifact_path_absolute(artifact_path) + + if ( + only_if_artifact_json_missing + and artifact_get_metadata_file_path(artifact_path).exists() + ): + return + + recipe = artifact_read_recipe(artifact_path) + + with util.file_open_text( + artifact_get_recipe_log_file_path(artifact_path), "w" + ) as f: + gflogging.push_stream_for_logging(f) + try: + if recipe.HasField("download_and_extract_archive_set"): + recipe_download_and_extract_archive_set.recipe_download_and_extract_archive_set( + recipe.download_and_extract_archive_set, artifact_path + ) + else: + raise NotImplementedError( + "Artifact {} has recipe type {} and this is not implemented".format( + artifact_path, recipe.WhichOneof("recipe") + ) + ) + finally: + gflogging.pop_stream_for_logging() + + +def artifact_get_inner_file_path(inner_file: str, artifact_path: str) -> pathlib.Path: + check( + not inner_file.startswith("//"), + AssertionError( + "bad inner_file argument passed to artifact_get_inner_file_path" + ), + ) + # TODO: Consider absolute paths that we might want to support for quick hacks. + return util.norm_path( + artifact_get_directory_path(artifact_path) / pathlib.Path(inner_file) + ) diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py new file mode 100644 index 000000000..6cb296037 --- /dev/null +++ b/gfauto/gfauto/binaries_util.py @@ -0,0 +1,513 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Binary utilities module. + +Defines the latest binaries (name and version) that will be used by default for new fuzzing sessions. +Defines the recipes (see recipe.proto) for all built-in binaries, including old versions of binaries. +Defines BinaryManager; see below. +""" + +import abc +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import attr + +from gfauto import artifact_util, recipe_wrap, test_util, util +from gfauto.common_pb2 import Archive, ArchiveSet, Binary +from gfauto.gflogging import log +from gfauto.recipe_pb2 import Recipe, RecipeDownloadAndExtractArchiveSet + +LATEST_GRAPHICSFUZZ_ARTIFACT = "//binaries/graphicsfuzz_v1.2.1" + +GLSLANG_VALIDATOR_NAME = "glslangValidator" +SPIRV_OPT_NAME = "spirv-opt" +SPIRV_VAL_NAME = "spirv-val" +SPIRV_DIS_NAME = "spirv-dis" +SWIFT_SHADER_NAME = "swift_shader_icd" + +SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG = "no-validate-after-all" + +BUILT_IN_BINARY_RECIPES_PATH_PREFIX = "//binaries" + +PLATFORM_SUFFIXES_DEBUG = ["Linux_x64_Debug", "Windows_x64_Debug", "Mac_x64_Debug"] +PLATFORM_SUFFIXES_RELEASE = [ + "Linux_x64_Release", + "Windows_x64_Release", + "Mac_x64_Release", +] +PLATFORM_SUFFIXES_RELWITHDEBINFO = [ + "Linux_x64_RelWithDebInfo", + "Windows_x64_RelWithDebInfo", + "Mac_x64_RelWithDebInfo", +] + +DEFAULT_SPIRV_TOOLS_VERSION = "9559cdbdf011c487f67f89e2d694bd4a18d5c1e0" + +DEFAULT_BINARIES = [ + Binary( + name="glslangValidator", + tags=["Debug"], + version="f04f1f93a70f4608ffa9903b20bfb95f20a063f5", + ), + Binary(name="spirv-opt", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-dis", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-val", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-fuzz", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary( + name="swift_shader_icd", + tags=["Debug"], + version="fa0175c0988dd542f008257232207a8b87ad6c63", + ), +] + + +@attr.dataclass +class BinaryPathAndInfo: + path: Path + binary: Binary + + +class BinaryGetter(abc.ABC): + def get_binary_path_by_name(self, name: str) -> BinaryPathAndInfo: + pass + + +class BinaryNotFound(Exception): + pass + + +class BinaryPathNotFound(Exception): + def __init__(self, binary: Binary): + super().__init__(f"Could not find binary path for binary: \n{binary}") + + +class BinaryManager(BinaryGetter): + """ + Implements BinaryGetter. + + An instance of BinaryManager is the main way that code accesses binaries. BinaryManger allows certain tests and/or + devices to override binaries by passing a list of binary versions that take priority, so the correct versions are + always used. Plus, the current platform will be used when deciding which binary to download and return. + + See the Binary proto. + + _binary_list: A list of Binary with name, version, configuration. This is used to map a binary name to a Binary. + _resolved_paths: A map containing: Binary (serialized bytes) -> Path + _binary_artifacts: A list of all available binary artifacts/recipes stored as tuples: (ArchiveSet, artifact_path). + """ + + _binary_list: List[Binary] + _resolved_paths: Dict[bytes, Path] + _binary_artifacts: List[Tuple[ArchiveSet, str]] + + def __init__( + self, + binary_list: Optional[List[Binary]] = None, + platform: Optional[str] = None, + binary_artifacts_prefix: Optional[str] = BUILT_IN_BINARY_RECIPES_PATH_PREFIX, + ): + self._binary_list = binary_list or DEFAULT_BINARIES + self._resolved_paths = {} + self._platform = platform or util.get_platform() + self._binary_artifacts = [] + + if binary_artifacts_prefix: + self._binary_artifacts.extend( + artifact_util.binary_artifacts_find(binary_artifacts_prefix) + ) + + @staticmethod + def get_binary_list_from_test_metadata(test_json_path: Path) -> List[Binary]: + test_metadata = test_util.metadata_read_from_path(test_json_path) + result: List[Binary] = [] + if test_metadata.device: + result.extend(test_metadata.device.binaries) + result.extend(test_metadata.binaries) + return result + + def get_binary_path(self, binary: Binary) -> Path: + result = self._resolved_paths.get(binary.SerializePartialToString()) + if result: + return result + log(f"Finding path of binary:\n{binary}") + binary_tags = set(binary.tags) + binary_tags.add(self._platform) + for (archive_set, artifact_path) in self._binary_artifacts: + for artifact_binary in archive_set.binaries: # type: Binary + if artifact_binary.name != binary.name: + continue + if artifact_binary.version != binary.version: + continue + recipe_binary_tags = set(artifact_binary.tags) + if not binary_tags.issubset(recipe_binary_tags): + continue + artifact_util.artifact_execute_recipe_if_needed(artifact_path) + result = artifact_util.artifact_get_inner_file_path( + artifact_binary.path, artifact_path + ) + self._resolved_paths[binary.SerializePartialToString()] = result + return result + raise BinaryPathNotFound(binary) + + @staticmethod + def get_binary_by_name_from_list(name: str, binary_list: List[Binary]) -> Binary: + for binary in binary_list: + if binary.name == name: + return binary + raise BinaryNotFound( + f"Could not find binary named {name} in list:\n{binary_list}" + ) + + def get_binary_path_by_name(self, name: str) -> BinaryPathAndInfo: + binary = self.get_binary_by_name(name) + return BinaryPathAndInfo(self.get_binary_path(binary), binary) + + def get_binary_by_name(self, name: str) -> Binary: + return self.get_binary_by_name_from_list(name, self._binary_list) + + def get_child_binary_manager(self, binary_list: List[Binary]) -> "BinaryManager": + result = BinaryManager( + binary_list + self._binary_list, + self._platform, + binary_artifacts_prefix=None, + ) + # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. + result._resolved_paths = self._resolved_paths + # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. + result._binary_artifacts = self._binary_artifacts + return result + + +@attr.dataclass +class ToolNameAndPath: + name: str + subpath: str + add_exe_on_windows: bool = True + + +def get_platform_from_platform_suffix(platform_suffix: str) -> str: + platforms = ("Linux", "Mac", "Windows") + for platform in platforms: + if platform in platform_suffix: + return platform + raise AssertionError(f"Could not guess platform of {platform_suffix}") + + +def add_common_tags_from_platform_suffix(tags: List[str], platform_suffix: str) -> None: + platform = get_platform_from_platform_suffix(platform_suffix) + tags.append(platform) + common_tags = ["Release", "Debug", "RelWithDebInfo", "x64"] + for common_tag in common_tags: + if common_tag in platform_suffix: + tags.append(common_tag) + + +def _get_built_in_binary_recipe_from_build_github_repo( + project_name: str, + version_hash: str, + build_version_hash: str, + platform_suffixes: List[str], + tools: List[ToolNameAndPath], +) -> List[recipe_wrap.RecipeWrap]: + + result: List[recipe_wrap.RecipeWrap] = [] + + for platform_suffix in platform_suffixes: + tags: List[str] = [] + add_common_tags_from_platform_suffix(tags, platform_suffix) + binaries = [ + Binary( + name=binary.name, + tags=tags, + path=( + f"{project_name}/{(binary.subpath + '.exe') if 'Windows' in tags and binary.add_exe_on_windows else binary.subpath}" + ), + version=version_hash, + ) + for binary in tools + ] + + result.append( + recipe_wrap.RecipeWrap( + f"//binaries/{project_name}_{version_hash}_{platform_suffix}", + Recipe( + download_and_extract_archive_set=RecipeDownloadAndExtractArchiveSet( + archive_set=ArchiveSet( + archives=[ + Archive( + url=f"https://github.com/paulthomson/build-{project_name}/releases/download/github/paulthomson/build-{project_name}/{build_version_hash}/build-{project_name}-{build_version_hash}-{platform_suffix}.zip", + output_file=f"{project_name}.zip", + output_directory=project_name, + ) + ], + binaries=binaries, + ) + ) + ), + ) + ) + + return result + + +def _get_built_in_swift_shader_version( + version_hash: str, build_version_hash: str +) -> List[recipe_wrap.RecipeWrap]: + return _get_built_in_binary_recipe_from_build_github_repo( + project_name="swiftshader", + version_hash=version_hash, + build_version_hash=build_version_hash, + platform_suffixes=PLATFORM_SUFFIXES_RELEASE + + PLATFORM_SUFFIXES_DEBUG + + PLATFORM_SUFFIXES_RELWITHDEBINFO, + tools=[ + ToolNameAndPath( + name="swift_shader_icd", + subpath="lib/vk_swiftshader_icd.json", + add_exe_on_windows=False, + ) + ], + ) + + +def _get_built_in_spirv_tools_version( + version_hash: str, build_version_hash: str, includes_spirv_fuzz: bool = True +) -> List[recipe_wrap.RecipeWrap]: + return _get_built_in_binary_recipe_from_build_github_repo( + project_name="SPIRV-Tools", + version_hash=version_hash, + build_version_hash=build_version_hash, + platform_suffixes=PLATFORM_SUFFIXES_RELEASE + PLATFORM_SUFFIXES_DEBUG, + tools=[ + ToolNameAndPath(name="spirv-as", subpath="bin/spirv-as"), + ToolNameAndPath(name="spirv-dis", subpath="bin/spirv-dis"), + ToolNameAndPath(name="spirv-opt", subpath="bin/spirv-opt"), + ToolNameAndPath(name="spirv-val", subpath="bin/spirv-val"), + ] + + ( + [ToolNameAndPath(name="spirv-fuzz", subpath="bin/spirv-fuzz")] + if includes_spirv_fuzz + else [] + ), + ) + + +def _get_built_in_glslang_version( + version_hash: str, build_version_hash: str +) -> List[recipe_wrap.RecipeWrap]: + return _get_built_in_binary_recipe_from_build_github_repo( + project_name="glslang", + version_hash=version_hash, + build_version_hash=build_version_hash, + platform_suffixes=PLATFORM_SUFFIXES_RELEASE + PLATFORM_SUFFIXES_DEBUG, + tools=[ + ToolNameAndPath(name="glslangValidator", subpath="bin/glslangValidator") + ], + ) + + +def get_graphics_fuzz_121() -> List[recipe_wrap.RecipeWrap]: + return [ + recipe_wrap.RecipeWrap( + "//binaries/graphicsfuzz_v1.2.1", + Recipe( + download_and_extract_archive_set=RecipeDownloadAndExtractArchiveSet( + archive_set=ArchiveSet( + archives=[ + Archive( + url="https://github.com/google/graphicsfuzz/releases/download/v1.2.1/graphicsfuzz.zip", + output_file="graphicsfuzz.zip", + output_directory="graphicsfuzz", + ) + ], + binaries=[ + # + # glslangValidator + Binary( + name="glslangValidator", + tags=["Linux", "x64", "Release"], + path="graphicsfuzz/bin/Linux/glslangValidator", + version="40c16ec0b3ad03fc170f1369a58e7bbe662d82cd", + ), + Binary( + name="glslangValidator", + tags=["Windows", "x64", "Release"], + path="graphicsfuzz/bin/Windows/glslangValidator.exe", + version="40c16ec0b3ad03fc170f1369a58e7bbe662d82cd", + ), + Binary( + name="glslangValidator", + tags=["Mac", "x64", "Release"], + path="graphicsfuzz/bin/Mac/glslangValidator", + version="40c16ec0b3ad03fc170f1369a58e7bbe662d82cd", + ), + # + # spirv-opt + Binary( + name="spirv-opt", + tags=[ + "Linux", + "x64", + "Release", + SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG, + ], + path="graphicsfuzz/bin/Linux/spirv-opt", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-opt", + tags=[ + "Windows", + "x64", + "Release", + SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG, + ], + path="graphicsfuzz/bin/Windows/spirv-opt.exe", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-opt", + tags=[ + "Mac", + "x64", + "Release", + SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG, + ], + path="graphicsfuzz/bin/Mac/spirv-opt", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + # + # spirv-dis + Binary( + name="spirv-dis", + tags=["Linux", "x64", "Release"], + path="graphicsfuzz/bin/Linux/spirv-dis", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-dis", + tags=["Windows", "x64", "Release"], + path="graphicsfuzz/bin/Windows/spirv-dis.exe", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-dis", + tags=["Mac", "x64", "Release"], + path="graphicsfuzz/bin/Mac/spirv-dis", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + # + # spirv-as + Binary( + name="spirv-as", + tags=["Linux", "x64", "Release"], + path="graphicsfuzz/bin/Linux/spirv-as", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-as", + tags=["Windows", "x64", "Release"], + path="graphicsfuzz/bin/Windows/spirv-as.exe", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-as", + tags=["Mac", "x64", "Release"], + path="graphicsfuzz/bin/Mac/spirv-as", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + # + # spirv-val + Binary( + name="spirv-val", + tags=["Linux", "x64", "Release"], + path="graphicsfuzz/bin/Linux/spirv-val", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-val", + tags=["Windows", "x64", "Release"], + path="graphicsfuzz/bin/Windows/spirv-val.exe", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-val", + tags=["Mac", "x64", "Release"], + path="graphicsfuzz/bin/Mac/spirv-val", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + ], + ) + ) + ), + ) + ] + + +BUILT_IN_BINARY_RECIPES: List[recipe_wrap.RecipeWrap] = ( + _get_built_in_spirv_tools_version( + version_hash="4a00a80c40484a6f6f72f48c9d34943cf8f180d4", + build_version_hash="422f2fe0f0f32494fa687a12ba343d24863b330a", + includes_spirv_fuzz=False, + ) + + _get_built_in_glslang_version( + version_hash="9866ad9195cec8f266f16191fb4ec2ce4896e5c0", + build_version_hash="1586e566f4949b1957e7c32454cbf27e501ed632", + ) + + _get_built_in_swift_shader_version( + version_hash="a0b3a02601da8c48012a4259d335be04d00818da", + build_version_hash="08fb8d429272ef8eedb4d610943b9fe59d336dc6", + ) + + get_graphics_fuzz_121() + + _get_built_in_spirv_tools_version( + version_hash="1c1e749f0b51603032ed573acb5ee4cd6fee8d01", + build_version_hash="7663d620a7fbdccb330d2baec138d0e3e096457c", + includes_spirv_fuzz=False, + ) + + _get_built_in_spirv_tools_version( + version_hash="55adf4cf707bb12c29fc12f784ebeaa29a819e9b", + build_version_hash="f2170cc791d0eaa5789ec7528862ae00b984b3b8", + includes_spirv_fuzz=False, + ) + + _get_built_in_glslang_version( + version_hash="e383c5f55defdb884a77820483d3360617391d78", + build_version_hash="f3df04d4f582af6b54989d7da86f58f8f38423ba", + ) + + _get_built_in_spirv_tools_version( + version_hash="230c9e437146e48ec58adb4433890403c23c98fa", + build_version_hash="288b0f57443e221df530b705085df59f2da93843", + includes_spirv_fuzz=False, + ) + + _get_built_in_spirv_tools_version( + version_hash="76b75c40a1e27939957e6a598292e9f32b4e98d4", + build_version_hash="9debf645007ef2807ba68f4497d50638c4c57878", + includes_spirv_fuzz=False, + ) + + _get_built_in_spirv_tools_version( + version_hash="9559cdbdf011c487f67f89e2d694bd4a18d5c1e0", + build_version_hash="693b9805d162d5a49592912f6b9bb2d0b4868ec8", + ) + + _get_built_in_glslang_version( + version_hash="f04f1f93a70f4608ffa9903b20bfb95f20a063f5", + build_version_hash="211afd921a2b354ee579cd4b60f761bfe27c1003", + ) + + _get_built_in_swift_shader_version( + version_hash="fa0175c0988dd542f008257232207a8b87ad6c63", + build_version_hash="ea3b929604da6873ace48988b8d4651bbcd2e573", + ) +) diff --git a/gfauto/gfauto/common.proto b/gfauto/gfauto/common.proto new file mode 100644 index 000000000..91b97509a --- /dev/null +++ b/gfauto/gfauto/common.proto @@ -0,0 +1,61 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + + +// Defines a binary (or really, any file). +message Binary { + + // E.g. glslangValidator. + string name = 1; + + // This should be a list of strings (tags) containing the platform and other details. + // E.g. "Linux", "x64", "Debug", "malloc", "msan". + // When being used to specify how to *find* a binary, the tags must be found in the binary definition. + // E.g. "Debug", "msan". + // Typically, the current platform will also need to be found (e.g. "Linux") before the Binary can be chosen for use. + repeated string tags = 2; + + // Used when defining a binary within a recipe or artifact. Gives the path (always using "/" as the directory + // separator) relative from the artifact.json/recipe.json file to the binary, after the ArchiveSet has been extracted. + // Should be empty in Settings. + // Also used internally (but never written to disk) to store the resolved path of a binary. + string path = 3; + + // This should typically be a hash of some kind. + string version = 4; +} + +message Archive { + // The URL from which to download the archive. + string url = 1; + + // The output location for the downloaded archive. + string output_file = 2; + + // The directory in which the archive will be extracted. + string output_directory = 3; +} + +message ArchiveSet { + // A list of Archives; all of these will be downloaded and extracted when used in a recipe. + // See RecipeDownloadAndExtractArchiveSet proto. + repeated Archive archives = 1; + + // A list of Binaries that can be found in the artifact. See ArtifactMetadataExtractedArchiveSet proto. + repeated Binary binaries = 2; +} diff --git a/gfauto/gfauto/common_pb2.py b/gfauto/gfauto/common_pb2.py new file mode 100644 index 000000000..d1e59c91d --- /dev/null +++ b/gfauto/gfauto/common_pb2.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/common.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/common.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x13gfauto/common.proto\x12\x06gfauto\"C\n\x06\x42inary\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\t\"E\n\x07\x41rchive\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x13\n\x0boutput_file\x18\x02 \x01(\t\x12\x18\n\x10output_directory\x18\x03 \x01(\t\"Q\n\nArchiveSet\x12!\n\x08\x61rchives\x18\x01 \x03(\x0b\x32\x0f.gfauto.Archive\x12 \n\x08\x62inaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binaryb\x06proto3') +) + + + + +_BINARY = _descriptor.Descriptor( + name='Binary', + full_name='gfauto.Binary', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='gfauto.Binary.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tags', full_name='gfauto.Binary.tags', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='path', full_name='gfauto.Binary.path', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='version', full_name='gfauto.Binary.version', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=31, + serialized_end=98, +) + + +_ARCHIVE = _descriptor.Descriptor( + name='Archive', + full_name='gfauto.Archive', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='url', full_name='gfauto.Archive.url', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='output_file', full_name='gfauto.Archive.output_file', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='output_directory', full_name='gfauto.Archive.output_directory', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=100, + serialized_end=169, +) + + +_ARCHIVESET = _descriptor.Descriptor( + name='ArchiveSet', + full_name='gfauto.ArchiveSet', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='archives', full_name='gfauto.ArchiveSet.archives', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='binaries', full_name='gfauto.ArchiveSet.binaries', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=171, + serialized_end=252, +) + +_ARCHIVESET.fields_by_name['archives'].message_type = _ARCHIVE +_ARCHIVESET.fields_by_name['binaries'].message_type = _BINARY +DESCRIPTOR.message_types_by_name['Binary'] = _BINARY +DESCRIPTOR.message_types_by_name['Archive'] = _ARCHIVE +DESCRIPTOR.message_types_by_name['ArchiveSet'] = _ARCHIVESET +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Binary = _reflection.GeneratedProtocolMessageType('Binary', (_message.Message,), { + 'DESCRIPTOR' : _BINARY, + '__module__' : 'gfauto.common_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Binary) + }) +_sym_db.RegisterMessage(Binary) + +Archive = _reflection.GeneratedProtocolMessageType('Archive', (_message.Message,), { + 'DESCRIPTOR' : _ARCHIVE, + '__module__' : 'gfauto.common_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Archive) + }) +_sym_db.RegisterMessage(Archive) + +ArchiveSet = _reflection.GeneratedProtocolMessageType('ArchiveSet', (_message.Message,), { + 'DESCRIPTOR' : _ARCHIVESET, + '__module__' : 'gfauto.common_pb2' + # @@protoc_insertion_point(class_scope:gfauto.ArchiveSet) + }) +_sym_db.RegisterMessage(ArchiveSet) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/common_pb2.pyi b/gfauto/gfauto/common_pb2.pyi new file mode 100644 index 000000000..8dd92eaa2 --- /dev/null +++ b/gfauto/gfauto/common_pb2.pyi @@ -0,0 +1,82 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Binary(google___protobuf___message___Message): + name = ... # type: typing___Text + tags = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + path = ... # type: typing___Text + version = ... # type: typing___Text + + def __init__(self, + name : typing___Optional[typing___Text] = None, + tags : typing___Optional[typing___Iterable[typing___Text]] = None, + path : typing___Optional[typing___Text] = None, + version : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Binary: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"name",u"path",u"tags",u"version"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"name",b"path",b"tags",b"version"]) -> None: ... + +class Archive(google___protobuf___message___Message): + url = ... # type: typing___Text + output_file = ... # type: typing___Text + output_directory = ... # type: typing___Text + + def __init__(self, + url : typing___Optional[typing___Text] = None, + output_file : typing___Optional[typing___Text] = None, + output_directory : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Archive: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"output_directory",u"output_file",u"url"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"output_directory",b"output_file",b"url"]) -> None: ... + +class ArchiveSet(google___protobuf___message___Message): + + @property + def archives(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Archive]: ... + + @property + def binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Binary]: ... + + def __init__(self, + archives : typing___Optional[typing___Iterable[Archive]] = None, + binaries : typing___Optional[typing___Iterable[Binary]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ArchiveSet: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"archives",u"binaries"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"archives",b"binaries"]) -> None: ... diff --git a/gfauto/gfauto/device.proto b/gfauto/gfauto/device.proto new file mode 100644 index 000000000..8bd000e97 --- /dev/null +++ b/gfauto/gfauto/device.proto @@ -0,0 +1,60 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; + + +// A list of devices. Also includes the |active_device_names|, which are the ones that will be used when fuzzing +// (so this is really more than just a list). +message DeviceList { + repeated string active_device_names = 1; + repeated Device devices = 2; +} + +// A device, such as an Android device or the host machine with its GPU and drivers. +message Device { + string name = 1; + oneof device { + DevicePreprocess preprocess = 2; + DeviceSwiftShader swift_shader = 3; + DeviceHost host = 4; + DeviceAndroid android = 5; + } + // A device can specify/override some binaries. E.g. you may want several SwiftShader "devices" with different + // versions or configurations of SwiftShader. + repeated Binary binaries = 6; +} + +message DeviceSwiftShader { +} + +// A virtual device that just executes all steps up until the actual graphics hardware or simulator is needed. This +// helps to find bugs in, say, spirv-opt. +message DevicePreprocess { + +} + +// The host machine on which we are running gfauto. +message DeviceHost { + +} + +message DeviceAndroid { + string serial = 1; + string model = 2; +} diff --git a/gfauto/gfauto/device_pb2.py b/gfauto/gfauto/device_pb2.py new file mode 100644 index 000000000..9cf837838 --- /dev/null +++ b/gfauto/gfauto/device_pb2.py @@ -0,0 +1,316 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/device.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/device.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x13gfauto/device.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"J\n\nDeviceList\x12\x1b\n\x13\x61\x63tive_device_names\x18\x01 \x03(\t\x12\x1f\n\x07\x64\x65vices\x18\x02 \x03(\x0b\x32\x0e.gfauto.Device\"\xf3\x01\n\x06\x44\x65vice\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\npreprocess\x18\x02 \x01(\x0b\x32\x18.gfauto.DevicePreprocessH\x00\x12\x31\n\x0cswift_shader\x18\x03 \x01(\x0b\x32\x19.gfauto.DeviceSwiftShaderH\x00\x12\"\n\x04host\x18\x04 \x01(\x0b\x32\x12.gfauto.DeviceHostH\x00\x12(\n\x07\x61ndroid\x18\x05 \x01(\x0b\x32\x15.gfauto.DeviceAndroidH\x00\x12 \n\x08\x62inaries\x18\x06 \x03(\x0b\x32\x0e.gfauto.BinaryB\x08\n\x06\x64\x65vice\"\x13\n\x11\x44\x65viceSwiftShader\"\x12\n\x10\x44\x65vicePreprocess\"\x0c\n\nDeviceHost\".\n\rDeviceAndroid\x12\x0e\n\x06serial\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\tb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,]) + + + + +_DEVICELIST = _descriptor.Descriptor( + name='DeviceList', + full_name='gfauto.DeviceList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='active_device_names', full_name='gfauto.DeviceList.active_device_names', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='devices', full_name='gfauto.DeviceList.devices', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=52, + serialized_end=126, +) + + +_DEVICE = _descriptor.Descriptor( + name='Device', + full_name='gfauto.Device', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='gfauto.Device.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='preprocess', full_name='gfauto.Device.preprocess', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='swift_shader', full_name='gfauto.Device.swift_shader', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='host', full_name='gfauto.Device.host', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='android', full_name='gfauto.Device.android', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='binaries', full_name='gfauto.Device.binaries', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='device', full_name='gfauto.Device.device', + index=0, containing_type=None, fields=[]), + ], + serialized_start=129, + serialized_end=372, +) + + +_DEVICESWIFTSHADER = _descriptor.Descriptor( + name='DeviceSwiftShader', + full_name='gfauto.DeviceSwiftShader', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=374, + serialized_end=393, +) + + +_DEVICEPREPROCESS = _descriptor.Descriptor( + name='DevicePreprocess', + full_name='gfauto.DevicePreprocess', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=395, + serialized_end=413, +) + + +_DEVICEHOST = _descriptor.Descriptor( + name='DeviceHost', + full_name='gfauto.DeviceHost', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=415, + serialized_end=427, +) + + +_DEVICEANDROID = _descriptor.Descriptor( + name='DeviceAndroid', + full_name='gfauto.DeviceAndroid', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='serial', full_name='gfauto.DeviceAndroid.serial', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='model', full_name='gfauto.DeviceAndroid.model', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=429, + serialized_end=475, +) + +_DEVICELIST.fields_by_name['devices'].message_type = _DEVICE +_DEVICE.fields_by_name['preprocess'].message_type = _DEVICEPREPROCESS +_DEVICE.fields_by_name['swift_shader'].message_type = _DEVICESWIFTSHADER +_DEVICE.fields_by_name['host'].message_type = _DEVICEHOST +_DEVICE.fields_by_name['android'].message_type = _DEVICEANDROID +_DEVICE.fields_by_name['binaries'].message_type = gfauto_dot_common__pb2._BINARY +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['preprocess']) +_DEVICE.fields_by_name['preprocess'].containing_oneof = _DEVICE.oneofs_by_name['device'] +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['swift_shader']) +_DEVICE.fields_by_name['swift_shader'].containing_oneof = _DEVICE.oneofs_by_name['device'] +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['host']) +_DEVICE.fields_by_name['host'].containing_oneof = _DEVICE.oneofs_by_name['device'] +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['android']) +_DEVICE.fields_by_name['android'].containing_oneof = _DEVICE.oneofs_by_name['device'] +DESCRIPTOR.message_types_by_name['DeviceList'] = _DEVICELIST +DESCRIPTOR.message_types_by_name['Device'] = _DEVICE +DESCRIPTOR.message_types_by_name['DeviceSwiftShader'] = _DEVICESWIFTSHADER +DESCRIPTOR.message_types_by_name['DevicePreprocess'] = _DEVICEPREPROCESS +DESCRIPTOR.message_types_by_name['DeviceHost'] = _DEVICEHOST +DESCRIPTOR.message_types_by_name['DeviceAndroid'] = _DEVICEANDROID +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +DeviceList = _reflection.GeneratedProtocolMessageType('DeviceList', (_message.Message,), { + 'DESCRIPTOR' : _DEVICELIST, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceList) + }) +_sym_db.RegisterMessage(DeviceList) + +Device = _reflection.GeneratedProtocolMessageType('Device', (_message.Message,), { + 'DESCRIPTOR' : _DEVICE, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Device) + }) +_sym_db.RegisterMessage(Device) + +DeviceSwiftShader = _reflection.GeneratedProtocolMessageType('DeviceSwiftShader', (_message.Message,), { + 'DESCRIPTOR' : _DEVICESWIFTSHADER, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceSwiftShader) + }) +_sym_db.RegisterMessage(DeviceSwiftShader) + +DevicePreprocess = _reflection.GeneratedProtocolMessageType('DevicePreprocess', (_message.Message,), { + 'DESCRIPTOR' : _DEVICEPREPROCESS, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DevicePreprocess) + }) +_sym_db.RegisterMessage(DevicePreprocess) + +DeviceHost = _reflection.GeneratedProtocolMessageType('DeviceHost', (_message.Message,), { + 'DESCRIPTOR' : _DEVICEHOST, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceHost) + }) +_sym_db.RegisterMessage(DeviceHost) + +DeviceAndroid = _reflection.GeneratedProtocolMessageType('DeviceAndroid', (_message.Message,), { + 'DESCRIPTOR' : _DEVICEANDROID, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceAndroid) + }) +_sym_db.RegisterMessage(DeviceAndroid) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/device_pb2.pyi b/gfauto/gfauto/device_pb2.pyi new file mode 100644 index 000000000..2d4120652 --- /dev/null +++ b/gfauto/gfauto/device_pb2.pyi @@ -0,0 +1,126 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + Binary as gfauto___common_pb2___Binary, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class DeviceList(google___protobuf___message___Message): + active_device_names = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + @property + def devices(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Device]: ... + + def __init__(self, + active_device_names : typing___Optional[typing___Iterable[typing___Text]] = None, + devices : typing___Optional[typing___Iterable[Device]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"active_device_names",u"devices"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"active_device_names",b"devices"]) -> None: ... + +class Device(google___protobuf___message___Message): + name = ... # type: typing___Text + + @property + def preprocess(self) -> DevicePreprocess: ... + + @property + def swift_shader(self) -> DeviceSwiftShader: ... + + @property + def host(self) -> DeviceHost: ... + + @property + def android(self) -> DeviceAndroid: ... + + @property + def binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... + + def __init__(self, + name : typing___Optional[typing___Text] = None, + preprocess : typing___Optional[DevicePreprocess] = None, + swift_shader : typing___Optional[DeviceSwiftShader] = None, + host : typing___Optional[DeviceHost] = None, + android : typing___Optional[DeviceAndroid] = None, + binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Device: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"android",u"device",u"host",u"preprocess",u"swift_shader"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"android",u"binaries",u"device",u"host",u"name",u"preprocess",u"swift_shader"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"android",b"android",u"device",b"device",u"host",b"host",u"preprocess",b"preprocess",u"swift_shader",b"swift_shader"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"android",b"binaries",b"device",b"host",b"name",b"preprocess",b"swift_shader"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"device",b"device"]) -> typing_extensions___Literal["preprocess","swift_shader","host","android"]: ... + +class DeviceSwiftShader(google___protobuf___message___Message): + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceSwiftShader: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + +class DevicePreprocess(google___protobuf___message___Message): + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DevicePreprocess: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + +class DeviceHost(google___protobuf___message___Message): + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceHost: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + +class DeviceAndroid(google___protobuf___message___Message): + serial = ... # type: typing___Text + model = ... # type: typing___Text + + def __init__(self, + serial : typing___Optional[typing___Text] = None, + model : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceAndroid: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"model",u"serial"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"model",b"serial"]) -> None: ... diff --git a/gfauto/gfauto/devices_util.py b/gfauto/gfauto/devices_util.py new file mode 100644 index 000000000..b5b4b4536 --- /dev/null +++ b/gfauto/gfauto/devices_util.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Devices utility module. + +Used to enumerate the available devices and work with device lists. +""" + +from typing import Dict, List, Optional + +from gfauto import android_device +from gfauto.device_pb2 import ( + Device, + DeviceHost, + DeviceList, + DevicePreprocess, + DeviceSwiftShader, +) +from gfauto.gflogging import log +from gfauto.util import ToolNotOnPathError + + +def swift_shader_device() -> Device: + # TODO: version hash, maybe a better name. + return Device(name="swift_shader", swift_shader=DeviceSwiftShader()) + + +def device_preprocessor() -> Device: + # TODO: think about the fact that the versions of glslang, spirv-opt etc. are stored in the test and not the device. + # This might make it rather strange if you want to test many different versions of, say, spirv-opt. + # We could store version hashes in both the test and the "preprocessor device", and let the "preprocessor device" + # have higher priority. + return Device(name="host_preprocessor", preprocess=DevicePreprocess()) + + +def device_host() -> Device: + # TODO: add details to host (and indeed, to all devices). + return Device(name="host", host=DeviceHost()) + + +def get_device_list(device_list: Optional[DeviceList] = None) -> DeviceList: + + if not device_list: + device_list = DeviceList() + + # We use |extend| below (instead of |append|) because you cannot append to a list of non-scalars in protobuf. + # |extend| copies the elements from the list and appends them. + + # Host preprocessor. + device = device_preprocessor() + device_list.devices.extend([device]) + device_list.active_device_names.append(device.name) + + # SwiftShader. + device = swift_shader_device() + device_list.devices.extend([device]) + device_list.active_device_names.append(device.name) + + # Host device. + device = device_host() + device_list.devices.extend([device]) + device_list.active_device_names.append(device.name) + + try: + # Android devices. + android_devices = android_device.get_all_android_devices() + device_list.devices.extend(android_devices) + device_list.active_device_names.extend([d.name for d in android_devices]) + except ToolNotOnPathError: + log( + "WARNING: adb was not found on PATH nor was ANDROID_HOME set; " + "Android devices will not be added to settings.json" + ) + + return device_list + + +def get_active_devices(device_list: DeviceList) -> List[Device]: + device_map: Dict[str, Device] = {} + for device in device_list.devices: + device_map[device.name] = device + return [device_map[device] for device in device_list.active_device_names] diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py new file mode 100644 index 000000000..2f33f5411 --- /dev/null +++ b/gfauto/gfauto/fuzz.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fuzzing module. + +The main entry point to GraphicsFuzz Auto. +""" + +import argparse +import random +import secrets +import shutil +import sys +import uuid +from pathlib import Path +from typing import Optional + +from gfauto import ( + artifact_util, + binaries_util, + devices_util, + fuzz_glsl_test, + fuzz_spirv_test, + gflogging, + settings_util, + shader_job_util, + test_util, + util, +) +from gfauto.device_pb2 import Device +from gfauto.gflogging import log + +# Root: +# - donors/ (contains GLSL shader jobs) +# - temp/ (contains staging directories with random names) +# - reports/ (contains reports) + +# Staging directory. +# - source_template/ (source directory but with no test metadata yet) +# - test_1/, test_2/, etc. (test directories) + +# Test directory: +# - source/ (source directory, with test.json and other files) +# - results/ (results) + +# Report: a test directory with a reduction for a specific device. +# E.g. v signature v device name added +# reports/crashes/Null_point/123_opt_pixel/ +# - source/ +# - results/ +# - laptop/ +# - reference/ variant/ +# - ... (see below for a more detailed example) +# - pixel/ +# - reference/ variant/ +# - ... (see below for a more detailed example) +# - reductions/ (since this is a report for a pixel device, we have reductions) +# - ... (see below for a more detailed example) + + +# - temp/123/ (a staging directory; not a proper test_dir, as it only has "source_template", not "source".) +# - source_template/ +# - --test.json-- this will NOT be present because this is just a source template directory. +# - reference/ variant/ +# - shader.json, shader.{comp,frag} +# - 123_no_opt/ 123_opt_O/ 123_opt_Os/ 123_opt_rand_1/ etc. (Proper test_dirs, as they have "source". These may be +# copied to become a report if a bug is found.) +# - source/ (same as source_template, but with test.json) +# - results/ +# - pixel/ other_phone/ laptop/ etc. +# - reference/ variant/ +# - test.amber +# - image.png +# - STATUS +# - log.txt +# - (all other result files and intermediate files for running the shader on the device) +# - reductions/ (reductions are only added once the staging directory is copied to the reports directory) +# - reduction_1/ reduction_blah/ etc. (reduction name; also a test_dir) +# - source/ (same as other source dirs, but with the final reduced shader source) +# - reduction_work/ +# - reference/ variant/ +# - shader.json, shader_reduction_001_success.json, +# shader_reduction_002_failed.json, etc., shader_reduced_final.json +# - shader/ shader_reduction_001/ +# (these are the result directories for each step, containing STATUS, etc.) +# + + +IMAGE_FILE_NAME = "image.png" +BUFFER_FILE_NAME = "buffer.bin" + +BEST_REDUCTION_NAME = "best" + +AMBER_RUN_TIME_LIMIT = 30 + +STATUS_TOOL_CRASH = "TOOL_CRASH" +STATUS_CRASH = "CRASH" +STATUS_TOOL_TIMEOUT = "TOOL_TIMEOUT" +STATUS_TIMEOUT = "TIMEOUT" +STATUS_SUCCESS = "SUCCESS" + +# Number of bits for seeding the RNG. +# Python normally uses 256 bits internally when seeding its RNG, hence this choice. +ITERATION_SEED_BITS = 256 + +SPIRV_FUZZ = False + + +def get_random_name() -> str: + # TODO: could change to human-readable random name or the date. + return uuid.uuid4().hex + + +class NoSettingsFile(Exception): + pass + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fuzz") + + parser.add_argument( + "--settings", + help="Path to the settings JSON file for this fuzzing instance.", + default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), + ) + + parser.add_argument( + "--iteration_seed", + help="The seed to use for one fuzzing iteration (useful for reproducing an issue).", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + settings_path = Path(parsed_args.settings) + iteration_seed: Optional[int] = int( + parsed_args.iteration_seed + ) if parsed_args.iteration_seed else None + + with util.file_open_text(Path(f"log_{get_random_name()}.txt"), "w") as log_file: + gflogging.push_stream_for_logging(log_file) + try: + main_helper(settings_path, iteration_seed) + finally: + gflogging.pop_stream_for_logging() + + +def main_helper( # pylint: disable=too-many-locals, too-many-branches; + settings_path: Path, iteration_seed_override: Optional[int] +) -> None: + + try: + settings = settings_util.read(settings_path) + except FileNotFoundError as exception: + message = f"Could not find settings file at: {settings_path}" + if not settings_util.DEFAULT_SETTINGS_FILE_PATH.exists(): + settings_util.write_default(settings_util.DEFAULT_SETTINGS_FILE_PATH) + message += ( + f"; a default settings file has been created for you at {str(settings_util.DEFAULT_SETTINGS_FILE_PATH)}. " + f"Please review it and then run fuzz again. " + ) + raise NoSettingsFile(message) from exception + + active_devices = devices_util.get_active_devices(settings.device_list) + + reports_dir = Path() / "reports" + temp_dir = Path() / "temp" + donors_dir = Path() / "donors" + spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders" + + if donors_dir.exists(): + try: + artifact_util.artifact_path_get_root() + except FileNotFoundError: + log( + "Could not find ROOT file (in the current directory or above) to mark where binaries should be stored. " + "Creating a ROOT file in the current directory." + ) + util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") + + artifact_util.recipes_write_built_in() + + # TODO: make GraphicsFuzz find donors recursively. + references = sorted(donors_dir.rglob("*.json")) + + # Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file. + references = [ref for ref in references if shader_job_util.get_related_files(ref)] + + spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json")) + + binary_manager = binaries_util.BinaryManager( + list(settings.custom_binaries) + binaries_util.DEFAULT_BINARIES, + util.get_platform(), + binary_artifacts_prefix=binaries_util.BUILT_IN_BINARY_RECIPES_PATH_PREFIX, + ) + + # For convenience, we add the default (i.e. newest) SwiftShader ICD (binary) to any swift_shader devices + # so that we don't need to specify it and update it in the device list (on disk). + # Thus, when we save the test, the device will contain the version of SwiftShader we used. + for device in active_devices: + if device.HasField("swift_shader"): + swift_binaries = [ + binary + for binary in device.binaries + if "swift" not in binary.name.lower() + ] + if not swift_binaries: + device.binaries.extend( + [binary_manager.get_binary_by_name("swift_shader_icd")] + ) + + while True: + + if iteration_seed_override: + iteration_seed = iteration_seed_override + else: + iteration_seed = secrets.randbits(ITERATION_SEED_BITS) + + log(f"Iteration seed: {iteration_seed}") + random.seed(iteration_seed) + + staging_name = get_random_name() + staging_dir = temp_dir / staging_name + + util.mkdirs_p(staging_dir) + + # Pseudocode: + # - Create test_dir(s) in staging directory. + # - Run test_dir(s) on all active devices (stop early if appropriate). + # - For each test failure on each device, copy the test to reports_dir, adding the device and crash signature. + # - Reduce each report (on the given device). + # - Produce a summary for each report. + + if SPIRV_FUZZ: + fuzz_spirv_test.fuzz_spirv( + staging_dir, + reports_dir, + active_devices, + spirv_fuzz_shaders, + settings, + binary_manager, + ) + else: + fuzz_glsl_test.fuzz_glsl( + staging_dir, + reports_dir, + active_devices, + references, + donors_dir, + settings, + binary_manager, + ) + + shutil.rmtree(staging_dir) + + if iteration_seed_override: + log("Stopping due to iteration_seed") + break + + +def create_summary_and_reproduce( + test_dir: Path, + binary_manager: binaries_util.BinaryManager, + device: Optional[Device] = None, +) -> None: + util.mkdirs_p(test_dir / "summary") + test_metadata = test_util.metadata_read(test_dir) + if test_metadata.HasField("glsl"): + fuzz_glsl_test.create_summary_and_reproduce_glsl( + test_dir, binary_manager, device + ) + else: + raise AssertionError("Unrecognized test type") + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py new file mode 100644 index 000000000..64e7f824d --- /dev/null +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -0,0 +1,677 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GLSL fuzzing module. + +Functions for handling GLSL shader job tests. +""" + +import random +import subprocess +from pathlib import Path +from typing import List, Optional + +from gfauto import ( + amber_converter, + android_device, + binaries_util, + fuzz, + gflogging, + glsl_generate_util, + host_device_util, + result_util, + shader_job_util, + signature_util, + spirv_opt_util, + subprocess_util, + test_util, + tool, + util, +) +from gfauto.device_pb2 import Device +from gfauto.gflogging import log +from gfauto.settings_pb2 import Settings +from gfauto.test_pb2 import Test, TestGlsl +from gfauto.util import check + + +class ReductionFailedError(Exception): + def __init__(self, message: str, reduction_name: str, reduction_work_dir: Path): + super().__init__(message) + self.reduction_name = reduction_name + self.reduction_work_dir = reduction_work_dir + + +def fuzz_glsl( + staging_dir: Path, + reports_dir: Path, + active_devices: List[Device], + references: List[Path], + donors_dir: Path, + settings: Settings, + binary_manager: binaries_util.BinaryManager, +) -> None: + test_dirs = create_staging_tests( + staging_dir, references, donors_dir, binary_manager + ) + + for test_dir in test_dirs: + if handle_test(test_dir, reports_dir, active_devices, binary_manager, settings): + # If we generated a report, don't bother trying other optimization combinations. + break + + +def make_test( + base_source_dir: Path, + subtest_dir: Path, + spirv_opt_args: Optional[List[str]], + binary_manager: binaries_util.BinaryManager, +) -> Path: + # Create the subtest by copying the base source. + util.copy_dir(base_source_dir, test_util.get_source_dir(subtest_dir)) + + test = Test(glsl=TestGlsl(spirv_opt_args=spirv_opt_args)) + + test.binaries.extend([binary_manager.get_binary_by_name(name="glslangValidator")]) + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-dis")]) + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-val")]) + if spirv_opt_args: + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-opt")]) + + # Write the test metadata. + test_util.metadata_write(test, subtest_dir) + + return subtest_dir + + +def create_staging_tests( + staging_dir: Path, + references: List[Path], + donors_dir: Path, + binary_manager: binaries_util.BinaryManager, +) -> List[Path]: + + staging_name = staging_dir.name + template_source_dir = staging_dir / "source_template" + + # Copy in a randomly chosen reference. + reference_glsl_shader_job = shader_job_util.copy( + random.choice(references), + template_source_dir / test_util.REFERENCE_DIR / test_util.SHADER_JOB, + ) + + # TODO: Allow GraphicsFuzz to be downloaded. + + glsl_generate_util.run_generate( + util.tool_on_path("graphicsfuzz-tool"), + reference_glsl_shader_job, + donors_dir, + template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, + seed=str(random.getrandbits(glsl_generate_util.GENERATE_SEED_BITS)), + ) + + test_dirs = [ + make_test( + template_source_dir, + staging_dir / f"{staging_name}_no_opt_test", + spirv_opt_args=None, + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_O_test", + spirv_opt_args=["-O"], + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_Os_test", + spirv_opt_args=["-Os"], + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand1_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand2_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand3_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + ] + + return test_dirs + + +def run( + test_dir: Path, + binary_manager: binaries_util.BinaryManager, + device: Optional[Device] = None, +) -> str: + + test: Test = test_util.metadata_read(test_dir) + if not device: + device = test.device + + log(f"Running test on device:\n{device.name}") + + result_output_dir = run_shader_job( + shader_job=test_util.get_shader_job_path(test_dir, is_variant=True), + output_dir=test_util.get_results_directory( + test_dir, device.name, is_variant=True + ), + test=test, + device=device, + binary_manager=binary_manager, + ) + + return result_util.get_status(result_output_dir) + + +def maybe_add_report( + test_dir: Path, reports_dir: Path, device: Device, settings: Settings +) -> Optional[Path]: + + result_output_dir = test_util.get_results_directory( + test_dir, device.name, is_variant=True + ) + + status = result_util.get_status(result_output_dir) + + report_subdirectory_name = "" + + if status == fuzz.STATUS_CRASH: + report_subdirectory_name = "crashes" + elif status == fuzz.STATUS_TOOL_CRASH: + report_subdirectory_name = "tool_crashes" + + if not report_subdirectory_name: + return None + log_path = result_util.get_log_path(result_output_dir) + + log_contents = util.file_read_text(log_path) + signature = signature_util.get_signature_from_log_contents(log_contents) + + signature_dir = reports_dir / report_subdirectory_name / signature + + util.mkdirs_p(signature_dir) + + # If the signature_dir contains a NOT_INTERESTING file, then don't bother creating a report. + if (signature_dir / "NOT_INTERESTING").exists(): + return None + + # If we have reached the maximum number of crashes per signature for this device, don't create a report. + num_duplicates = [ + report_dir + for report_dir in signature_dir.iterdir() + if report_dir.is_dir() and report_dir.name.endswith(f"_{device.name}") + ] + if len(num_duplicates) >= settings.maximum_duplicate_crashes: + return None + + # We include the device name in the directory name because it is possible that this test crashes on two + # different devices but gives the same crash signature in both cases (e.g. for generic signatures + # like "compile_error"). This would lead to two test copies having the same path. + # It also means we can limit duplicates per device using the directory name. + test_dir_in_reports = signature_dir / f"{test_dir.name}_{device.name}" + + util.copy_dir(test_dir, test_dir_in_reports) + + test_metadata = test_util.metadata_read(test_dir_in_reports) + test_metadata.crash_signature = signature + test_metadata.device.CopyFrom(device) + test_util.metadata_write(test_metadata, test_dir_in_reports) + + return test_dir_in_reports + + +def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: + test = test_util.metadata_read(test_dir) + + try: + part_1_reduced_test = run_reduction( + test_dir_reduction_output=test_dir, + test_dir_to_reduce=test_dir, + preserve_semantics=True, + reduction_name="part_1_preserve_semantics", + ) + + part_2_reduced_test = run_reduction( + test_dir_reduction_output=test_dir, + test_dir_to_reduce=part_1_reduced_test, + preserve_semantics=False, + reduction_name="part_2_change_semantics", + ) + + device_name = test.device.name + + # Create a symlink to the "best" reduction. + best_reduced_test_link = test_util.get_reduced_test_dir( + test_dir, device_name, fuzz.BEST_REDUCTION_NAME + ) + util.make_directory_symlink( + new_symlink_file_path=best_reduced_test_link, + existing_dir=part_2_reduced_test, + ) + except ReductionFailedError as ex: + # Create a symlink to the failed reduction so it is easy to investigate failed reductions. + link_to_failed_reduction_path = ( + reports_dir / "failed_reductions" / f"{test_dir.name}_{ex.reduction_name}" + ) + util.make_directory_symlink( + new_symlink_file_path=link_to_failed_reduction_path, + existing_dir=ex.reduction_work_dir, + ) + + +def handle_test( + test_dir: Path, + reports_dir: Path, + active_devices: List[Device], + binary_manager: binaries_util.BinaryManager, + settings: Settings, +) -> bool: + + report_paths: List[Path] = [] + issue_found = False + + # Run on all devices. + for device in active_devices: + status = run(test_dir, binary_manager, device) + if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): + issue_found = True + if status == fuzz.STATUS_TOOL_CRASH: + # No need to run further on real devices if the pre-processing step failed. + break + + # For each device that saw a crash, copy the test to reports_dir, adding the signature and device info to the test + # metadata. + for device in active_devices: + report_dir = maybe_add_report(test_dir, reports_dir, device, settings) + if report_dir: + report_paths.append(report_dir) + + # For each report, run a reduction on the target device with the device-specific crash signature. + for test_dir_in_reports in report_paths: + run_reduction_on_report(test_dir_in_reports, reports_dir) + + # For each report, create a summary and reproduce the bug. + for test_dir_in_reports in report_paths: + fuzz.create_summary_and_reproduce(test_dir_in_reports, binary_manager) + + return issue_found + + +def run_shader_job( + shader_job: Path, + output_dir: Path, + test: Test, + device: Device, + binary_manager: binaries_util.BinaryManager, + use_default_binaries: bool = False, +) -> Path: + + with util.file_open_text(output_dir / "log.txt", "w") as log_file: + try: + gflogging.push_stream_for_logging(log_file) + + child_binary_manager = binary_manager + if not use_default_binaries: + child_binary_manager = child_binary_manager.get_child_binary_manager( + list(device.binaries) + list(test.binaries) + ) + + # TODO: Find amber path. NDK or host. + + # TODO: If Amber is going to be used, check if Amber can use Vulkan debug layers now, and if not, pass that + # info down via a bool. + + try: + + spirv_opt_hash: Optional[str] = None + if test.glsl.spirv_opt_args: + spirv_opt_hash = child_binary_manager.get_binary_by_name( + binaries_util.SPIRV_OPT_NAME + ).version + + amber_script_file = tool.glsl_shader_job_to_amber_script( + shader_job, + output_dir / "test.amber", + output_dir, + child_binary_manager, + amber_converter.AmberfySettings( + spirv_opt_args=list(test.glsl.spirv_opt_args), + spirv_opt_hash=spirv_opt_hash, + ), + spirv_opt_args=list(test.glsl.spirv_opt_args), + ) + except subprocess.CalledProcessError: + util.file_write_text( + result_util.get_status_path(output_dir), fuzz.STATUS_TOOL_CRASH + ) + return output_dir + except subprocess.TimeoutExpired: + util.file_write_text( + result_util.get_status_path(output_dir), fuzz.STATUS_TOOL_TIMEOUT + ) + return output_dir + + is_compute = bool( + shader_job_util.get_related_files( + shader_job, [shader_job_util.EXT_COMP] + ) + ) + + # Consider device type. + + if device.HasField("preprocess"): + # The "preprocess" device type just needs to get this far, so this is a success. + util.file_write_text( + result_util.get_status_path(output_dir), fuzz.STATUS_SUCCESS + ) + return output_dir + + if device.HasField("host") or device.HasField("swift_shader"): + icd: Optional[Path] = None + + if device.HasField("swift_shader"): + icd = child_binary_manager.get_binary_path_by_name( + binaries_util.SWIFT_SHADER_NAME + ).path + + # Run the shader on the host using Amber. + host_device_util.run_amber( + amber_script_file, + output_dir, + dump_image=(not is_compute), + dump_buffer=is_compute, + icd=icd, + ) + return output_dir + + if device.HasField("android"): + + android_device.run_amber_on_device( + amber_script_file, + output_dir, + dump_image=(not is_compute), + dump_buffer=is_compute, + serial=device.android.serial, + ) + return output_dir + + # TODO: For a remote device (which we will probably need to support), use log_a_file to output the + # "amber_log.txt" file. + + raise AssertionError(f"Unhandled device type:\n{str(device)}") + + finally: + gflogging.pop_stream_for_logging() + + +def get_final_reduced_shader_job_path(reduction_work_shader_dir: Path) -> Path: + return reduction_work_shader_dir / "shader_reduced_final.json" + + +def run_reduction( + test_dir_reduction_output: Path, + test_dir_to_reduce: Path, + preserve_semantics: bool, + reduction_name: str = "reduction1", + device_name: Optional[str] = None, +) -> Path: + test = test_util.metadata_read(test_dir_to_reduce) + + if not device_name and not test.device: + raise AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)}; device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}" + ) + + if not device_name: + device_name = test.device.name + + if not test.crash_signature: + raise AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)} because there is no crash string specified; " + f"for now, only crash reductions are supported" + ) + + reduced_test_dir_1 = test_util.get_reduced_test_dir( + test_dir_reduction_output, device_name, reduction_name + ) + + reduction_work_variant_dir = run_glsl_reduce( + input_shader_job=test_util.get_shader_job_path( + test_dir_to_reduce, is_variant=True + ), + test_metadata_path=test_util.get_metadata_path(test_dir_to_reduce), + output_dir=test_util.get_reduction_work_directory( + reduced_test_dir_1, is_variant=True + ), + preserve_semantics=preserve_semantics, + ) + + final_reduced_shader_job_path = get_final_reduced_shader_job_path( + reduction_work_variant_dir + ) + + check( + final_reduced_shader_job_path.exists(), + ReductionFailedError( + "Reduction failed; not yet handled", + reduction_name, + reduction_work_variant_dir, + ), + ) + + # Finally, write the test metadata and shader job, so the returned directory can be used as a test_dir. + + test_util.metadata_write(test, reduced_test_dir_1) + + shader_job_util.copy( + final_reduced_shader_job_path, + test_util.get_shader_job_path(reduced_test_dir_1, is_variant=True), + ) + + return reduced_test_dir_1 + + +def run_glsl_reduce( + input_shader_job: Path, + test_metadata_path: Path, + output_dir: Path, + preserve_semantics: bool = False, +) -> Path: + + cmd = [ + "glsl-reduce", + str(input_shader_job), + "--output", + str(output_dir), + "--", + "gfauto_interestingness_test", + str(test_metadata_path), + ] + + if preserve_semantics: + cmd.insert(1, "--preserve-semantics") + + # Log the reduction. + with util.file_open_text(output_dir / "command.log", "w") as f: + gflogging.push_stream_for_logging(f) + try: + # The reducer can fail, but it will typically output an exception file, so we can ignore the exit code. + subprocess_util.run(cmd, verbose=True, check_exit_code=False) + finally: + gflogging.pop_stream_for_logging() + + return output_dir + + +def create_summary_and_reproduce_glsl( + test_dir: Path, + binary_manager: binaries_util.BinaryManager, + device: Optional[Device] = None, +) -> None: + test_metadata = test_util.metadata_read(test_dir) + if not device: + device = test_metadata.device + + summary_dir = test_dir / "summary" + + unreduced_glsl = util.copy_dir( + test_util.get_source_dir(test_dir), summary_dir / "unreduced_glsl" + ) + + reduced_test_dir = test_util.get_reduced_test_dir( + test_dir, test_metadata.device.name, fuzz.BEST_REDUCTION_NAME + ) + reduced_source_dir = test_util.get_source_dir(reduced_test_dir) + reduced_glsl: Optional[Path] = None + if reduced_source_dir.exists(): + reduced_glsl = util.copy_dir(reduced_source_dir, summary_dir / "reduced_glsl") + + run_shader_job( + unreduced_glsl / test_util.VARIANT_DIR / test_util.SHADER_JOB, + summary_dir / "unreduced_glsl_result" / test_util.VARIANT_DIR, + test_metadata, + device, + binary_manager, + ) + + variant_reduced_glsl_result: Optional[Path] = None + if reduced_glsl: + variant_reduced_glsl_result = run_shader_job( + reduced_glsl / test_util.VARIANT_DIR / test_util.SHADER_JOB, + summary_dir / "reduced_glsl_result" / test_util.VARIANT_DIR, + test_metadata, + device, + binary_manager, + ) + + # Some post-processing for common error types. + + if variant_reduced_glsl_result: + status = result_util.get_status(variant_reduced_glsl_result) + if status == fuzz.STATUS_TOOL_CRASH: + tool_crash_summary_bug_report_dir( + reduced_source_dir, + variant_reduced_glsl_result, + summary_dir, + binary_manager, + ) + + +def tool_crash_summary_bug_report_dir( # pylint: disable=too-many-locals; + reduced_glsl_source_dir: Path, + variant_reduced_glsl_result_dir: Path, + output_dir: Path, + binary_manager: binaries_util.BinaryManager, +) -> Path: + # Create a simple script and README. + + shader_job = reduced_glsl_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB + + test_metadata: Test = test_util.metadata_read_from_path( + reduced_glsl_source_dir / test_util.TEST_METADATA + ) + + shader_files = shader_job_util.get_related_files( + shader_job, shader_job_util.EXT_ALL + ) + check( + len(shader_files) > 0, + AssertionError(f"Need at least one shader for {shader_job}"), + ) + + shader_extension = shader_files[0].suffix + + bug_report_dir = util.copy_dir( + variant_reduced_glsl_result_dir, output_dir / "bug_report" + ) + + shader_files = sorted(bug_report_dir.rglob("shader.*")) + + glsl_files = [ + shader_file + for shader_file in shader_files + if shader_file.suffix == shader_extension + ] + + asm_files = [ + shader_file + for shader_file in shader_files + if shader_file.name.endswith( + shader_extension + shader_job_util.SUFFIX_ASM_SPIRV + ) + ] + + spv_files = [ + shader_file + for shader_file in shader_files + if shader_file.name.endswith(shader_extension + shader_job_util.SUFFIX_SPIRV) + ] + + readme = "\n\n" + readme += ( + "Issue found using [GraphicsFuzz](https://github.com/google/graphicsfuzz).\n\n" + ) + readme += "Tool versions:\n\n" + readme += f"* glslangValidator commit hash: {binary_manager.get_binary_by_name(binaries_util.GLSLANG_VALIDATOR_NAME).version}\n" + + if test_metadata.glsl.spirv_opt_args: + readme += f"* spirv-opt commit hash: {binary_manager.get_binary_by_name(binaries_util.SPIRV_OPT_NAME).version}\n" + + readme += "\nTo reproduce:\n\n" + readme += f"`glslangValidator -V shader{shader_extension} -o shader{shader_extension}.spv`\n\n" + + if spv_files and not test_metadata.glsl.spirv_opt_args: + # There was an .spv file and no spirv-opt, so validate the SPIR-V. + readme += f"`spirv-val shader{shader_extension}.spv`\n\n" + + if test_metadata.glsl.spirv_opt_args: + readme += f"`spirv-opt shader{shader_extension}.spv -o temp.spv --validate-after-all {' '.join(test_metadata.glsl.spirv_opt_args)}`\n\n" + + files_to_list = glsl_files + spv_files + asm_files + files_to_list.sort() + + files_to_show = glsl_files + asm_files + files_to_show.sort() + + readme += "The following shader files are included in the attached archive, some of which are also shown inline below:\n\n" + + for file_to_list in files_to_list: + short_path = file_to_list.relative_to(bug_report_dir).as_posix() + readme += f"* {short_path}\n" + + for file_to_show in files_to_show: + short_path = file_to_show.relative_to(bug_report_dir).as_posix() + file_contents = util.file_read_text(file_to_show) + readme += f"\n{short_path}:\n\n" + readme += f"```\n{file_contents}\n```\n" + + util.file_write_text(output_dir / "README.md", readme) + + return bug_report_dir diff --git a/gfauto/gfauto/fuzz_spirv_test.py b/gfauto/gfauto/fuzz_spirv_test.py new file mode 100644 index 000000000..28373fac8 --- /dev/null +++ b/gfauto/gfauto/fuzz_spirv_test.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SPIR-V fuzzing module. + +Functions for handling spirv-fuzz generated tests. +""" + +import random +from pathlib import Path +from typing import List, Optional + +from gfauto import ( + binaries_util, + fuzz, + fuzz_glsl_test, + result_util, + shader_job_util, + spirv_fuzz_util, + test_util, + util, +) +from gfauto.device_pb2 import Device +from gfauto.gflogging import log +from gfauto.settings_pb2 import Settings +from gfauto.test_pb2 import Test, TestSpirvFuzz + + +def make_test( + base_source_dir: Path, + subtest_dir: Path, + spirv_opt_args: Optional[List[str]], + binary_manager: binaries_util.BinaryManager, +) -> Path: + # Create the subtest by copying the base source. + util.copy_dir(base_source_dir, test_util.get_source_dir(subtest_dir)) + + test = Test(spirv_fuzz=TestSpirvFuzz()) + + # TODO: Handle spirv_opt_args. + + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-dis")]) + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-val")]) + if spirv_opt_args: + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-opt")]) + + # Write the test metadata. + test_util.metadata_write(test, subtest_dir) + + return subtest_dir + + +def run( + test_dir: Path, + binary_manager: binaries_util.BinaryManager, + device: Optional[Device] = None, +) -> str: + + test: Test = test_util.metadata_read(test_dir) + if not device: + device = test.device + + log(f"Running test on device:\n{device.name}") + + result_output_dir = fuzz_glsl_test.run_shader_job( + shader_job=test_util.get_shader_job_path(test_dir, is_variant=True), + output_dir=test_util.get_results_directory( + test_dir, device.name, is_variant=True + ), + test=test, + device=device, + binary_manager=binary_manager, + ) + + return result_util.get_status(result_output_dir) + + +def handle_test( + test_dir: Path, + reports_dir: Path, + active_devices: List[Device], + binary_manager: binaries_util.BinaryManager, + settings: Settings, +) -> bool: + report_paths: List[Path] = [] + issue_found = False + + # Run on all devices. + for device in active_devices: + status = run(test_dir, binary_manager, device) + if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): + issue_found = True + if status == fuzz.STATUS_TOOL_CRASH: + # No need to run further on real devices if the pre-processing step failed. + break + + # For each device that saw a crash, copy the test to reports_dir, adding the signature and device info to the test + # metadata. + for device in active_devices: + report_dir = fuzz_glsl_test.maybe_add_report( + test_dir, reports_dir, device, settings + ) + if report_dir: + report_paths.append(report_dir) + + # TODO: reductions. + + # TODO: summaries. + + return issue_found + + +def fuzz_spirv( + staging_dir: Path, + reports_dir: Path, + active_devices: List[Device], + spirv_fuzz_shaders: List[Path], + settings: Settings, + binary_manager: binaries_util.BinaryManager, +) -> None: + + staging_name = staging_dir.name + template_source_dir = staging_dir / "source_template" + + # Copy in a randomly chosen reference. + reference_spirv_shader_job = shader_job_util.copy( + random.choice(spirv_fuzz_shaders), + template_source_dir / test_util.REFERENCE_DIR / test_util.SHADER_JOB, + language_suffix=shader_job_util.SUFFIXES_SPIRV_FUZZ_INPUT, + ) + + # TODO: Allow using downloaded spirv-fuzz. + + spirv_fuzz_util.run_generate_on_shader_job( + util.tool_on_path("spirv-fuzz"), + reference_spirv_shader_job, + template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, + seed=str(random.getrandbits(spirv_fuzz_util.GENERATE_SEED_BITS)), + ) + + test_dirs = [ + make_test( + template_source_dir, + staging_dir / f"{staging_name}_no_opt_test", + spirv_opt_args=None, + binary_manager=binary_manager, + ) + ] + + for test_dir in test_dirs: + if handle_test(test_dir, reports_dir, active_devices, binary_manager, settings): + # If we generated a report, don't bother trying other optimization combinations. + break diff --git a/gfauto/gfauto/generate_cts_test_template.py b/gfauto/gfauto/generate_cts_test_template.py new file mode 100644 index 000000000..8d40fc296 --- /dev/null +++ b/gfauto/gfauto/generate_cts_test_template.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate a CTS test. + +This module/script is copied next to a specific test in your repository of bugs +to generate an Amber script test suitable for adding to the CTS. +In particular, the Amber script test is suitable for use with |add_amber_tests_to_cts.py|. +""" + +import sys +from pathlib import Path + +from gfauto import artifact_util, tool + + +def main() -> None: + + # Checklist: + # - check output_amber + # - check short_description + # - check comment_text + # - check copyright_year + # - check extra_commands + + artifact_util.recipes_write_built_in() + + tool.glsl_shader_job_crash_to_amber_script_for_google_cts( + input_json=Path() / "reduced_glsl" / "variant" / "shader.json", + output_amber=Path() / "name-of-test-TODO.amber", + work_dir=Path() / "work", + # One sentence, 58 characters max., no period, no line breaks. + short_description="A fragment shader with TODO", + comment_text="""The test passes because TODO""", + copyright_year="2019", + extra_commands=tool.AMBER_COMMAND_PROBE_TOP_LEFT_RED, + test_metadata_path=Path() / "reduced_glsl" / "test.json", + ) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/gfauto_interestingness_test.py b/gfauto/gfauto/gfauto_interestingness_test.py new file mode 100644 index 000000000..cd0bef984 --- /dev/null +++ b/gfauto/gfauto/gfauto_interestingness_test.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Interestingness test for gfauto. + +When running a reducer, this module is used as the interestingness test. +An interestingness test runs a test on a device and returns 0 if the test +still exposes the behavior of interest (e.g. a crash, an incorrectly rendered image). + +See |setup.py| to see how this module is added to the entry_points/console_scripts. +""" + +import argparse +import sys +from pathlib import Path + +from gfauto import ( + artifact_util, + binaries_util, + fuzz, + fuzz_glsl_test, + result_util, + signature_util, + test_util, + util, +) +from gfauto.gflogging import log +from gfauto.util import check, check_file_exists + +# TODO: Maybe add helper method and throw exceptions instead of calling sys.exit. + +# TODO: Could we make the interestingness test the only way of running a shader job (or indeed, any test)? +# We would want to pass the output directory (default is one will be created, as is currently the case), the test_json, +# no crash signature nor device (we can get that from the test_json), although perhaps these could be overridden? +# A device (or test?) could then even specify a custom interestingness command, although the default one would probably +# be the same for all devices and it would look at the device info in the test_json? + +# TODO: Support more than just GLSL crash tests. + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Interestingness test that runs a shader using Amber, " + "calculates the crash signature based on the result, and returns 0 " + "if the signature matches the expected crash signature." + ) + + parser.add_argument( + "test_json", + help="The .json test metadata file path that describes how to run the test.", + ) + parser.add_argument("shader_job_json", help="The .json shader job file path.") + + parser.add_argument( + "--use_default_binaries", + help="Use the latest binaries, ignoring those defined in the test.json. " + "Implies --fallback_binaries.", + action="store_true", + ) + + parser.add_argument( + "--fallback_binaries", + help="Fallback to the latest binaries if they are not defined in the test.json.", + action="store_true", + ) + + parser.add_argument("--output", help="Output directory.", default=None) + + parsed_args = parser.parse_args(sys.argv[1:]) + + test_json: Path = Path(parsed_args.test_json) + shader_job_json: Path = Path(parsed_args.shader_job_json) + use_default_binaries: bool = parsed_args.use_default_binaries + fallback_binaries: bool = parsed_args.fallback_binaries or use_default_binaries + output: Path = Path( + parsed_args.output + ) if parsed_args.output else shader_job_json.with_suffix("") + + check_file_exists(test_json) + check_file_exists(shader_job_json) + + artifact_util.recipes_write_built_in() + + test = test_util.metadata_read_from_path(test_json) + + check( + test.HasField("glsl") and bool(test.device) and bool(test.crash_signature), + AssertionError( + f"Provided test json {str(test_json)} does not have entries: glsl, device, crash_signature" + ), + ) + + binary_manager = binaries_util.BinaryManager( + binaries_util.DEFAULT_BINARIES if fallback_binaries else [], + util.get_platform(), + binaries_util.BUILT_IN_BINARY_RECIPES_PATH_PREFIX, + ) + + output_dir = fuzz_glsl_test.run_shader_job( + shader_job_json, + output_dir=output, + test=test, + device=test.device, + binary_manager=binary_manager, + use_default_binaries=use_default_binaries, + ) + + log( + f"gfauto_interestingness_test: finished running {str(shader_job_json)} in {str(output_dir)}." + ) + + status = result_util.get_status(output_dir) + if status not in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): + log("Shader run did not crash; not interesting.") + sys.exit(1) + + log_contents = util.file_read_text(result_util.get_log_path(output_dir)) + signature = signature_util.get_signature_from_log_contents(log_contents) + + log(f"Expected signature: {test.crash_signature}") + log(f"Actual signature: {signature}") + + if signature == test.crash_signature: + log("Interesting!") + return + + log("Not interesting") + sys.exit(1) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/gflogging.py b/gfauto/gfauto/gflogging.py new file mode 100644 index 000000000..f1f553fe2 --- /dev/null +++ b/gfauto/gfauto/gflogging.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Logging utility for gfauto. + +Use to log to stdout and/or to a file. Print statements are banned in gfauto. +""" + +from pathlib import Path +from typing import List, TextIO + +from gfauto import util + +_LOG_TO_STDOUT = True +_LOG_TO_STREAM: List[TextIO] = [] + + +def push_stream_for_logging(stream: TextIO) -> None: + _LOG_TO_STREAM.append(stream) + + +def pop_stream_for_logging() -> None: + _LOG_TO_STREAM.pop() + + +def log(message: str) -> None: + if _LOG_TO_STDOUT: + print(message, flush=True) # noqa T001 + for stream in _LOG_TO_STREAM: + stream.write(message) + stream.write("\n") + stream.flush() + + +def log_a_file(log_file: Path) -> None: + log(f"Logging the contents of {str(log_file)}") + try: + log(util.file_read_text(log_file)) + except IOError: + log(f"Failed to read {str(log_file)}") diff --git a/gfauto/gfauto/glsl_generate_util.py b/gfauto/gfauto/glsl_generate_util.py new file mode 100644 index 000000000..9220fdeff --- /dev/null +++ b/gfauto/gfauto/glsl_generate_util.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""glsl-generate utility module. + +Generates GLSL shaders using GraphicsFuzz. +""" + +import pathlib +from pathlib import Path +from typing import List, Optional + +from gfauto import subprocess_util, util + +GENERATE_SEED_BITS = 64 + + +def run_generate( + graphicsfuzz_tool_path: Path, + reference_shader_json: pathlib.Path, + donors_path: pathlib.Path, + output_shader_json: pathlib.Path, + seed: Optional[str] = None, + other_args: Optional[List[str]] = None, +) -> pathlib.Path: + util.file_mkdirs_parent(output_shader_json) + cmd = [ + str(graphicsfuzz_tool_path), + "com.graphicsfuzz.generator.tool.Generate", + str(reference_shader_json), + str(donors_path), + str(output_shader_json), + "--generate-uniform-bindings", + "--max-uniforms", + "10", + ] + + if seed: + cmd.append(f"--seed={seed}") + + if other_args: + cmd.extend(other_args) + + subprocess_util.run(cmd) + + return output_shader_json diff --git a/gfauto/gfauto/glslang_validator_util.py b/gfauto/gfauto/glslang_validator_util.py new file mode 100644 index 000000000..7407f4305 --- /dev/null +++ b/gfauto/gfauto/glslang_validator_util.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""glslangValidator utility module. + +Runs glslangValidator on GLSL shader jobs to get SPIR-V shader jobs. +""" + +import pathlib +from typing import Optional + +from gfauto import binaries_util, shader_job_util, subprocess_util, util + +GLSLANG_DEFAULT_TIME_LIMIT = 120 + + +def run_glslang_glsl_shader_to_spirv_shader( + glsl_shader_path: pathlib.Path, + output_dir_path: pathlib.Path, + glslang_validator_file_path: Optional[pathlib.Path] = None, + time_limit: int = GLSLANG_DEFAULT_TIME_LIMIT, +) -> pathlib.Path: + + if not glslang_validator_file_path: + glslang_validator_file_path = util.tool_on_path( + binaries_util.GLSLANG_VALIDATOR_NAME + ) + + output_spirv_file_path = output_dir_path / (glsl_shader_path.name + ".spv") + + util.file_mkdirs_parent(output_spirv_file_path) + + subprocess_util.run( + util.prepend_catchsegv_if_available( + [ + str(glslang_validator_file_path), + "-V", + "-o", + str(output_spirv_file_path), + str(glsl_shader_path), + ] + ), + timeout=time_limit, + ) + + return output_spirv_file_path + + +def run_glslang_glsl_to_spirv_job( + glsl_shader_job_json_file_path: pathlib.Path, + spirv_shader_job_json_file_path: pathlib.Path, + glslang_validator_file_path: Optional[pathlib.Path] = None, +) -> pathlib.Path: + + if not glslang_validator_file_path: + glslang_validator_file_path = util.tool_on_path( + binaries_util.GLSLANG_VALIDATOR_NAME + ) + + glsl_shader_files = shader_job_util.get_related_files( + glsl_shader_job_json_file_path + ) + + util.copy_file(glsl_shader_job_json_file_path, spirv_shader_job_json_file_path) + + for glsl_shader_file in glsl_shader_files: + run_glslang_glsl_shader_to_spirv_shader( + glsl_shader_file, + spirv_shader_job_json_file_path.parent, + glslang_validator_file_path, + ) + + return spirv_shader_job_json_file_path diff --git a/gfauto/gfauto/host_device_util.py b/gfauto/gfauto/host_device_util.py new file mode 100644 index 000000000..08937aea5 --- /dev/null +++ b/gfauto/gfauto/host_device_util.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Host device utility module. + +Used to run Amber on the host machine. +""" + +import subprocess +from pathlib import Path +from typing import Dict, Optional + +from gfauto import fuzz, gflogging, result_util, subprocess_util, util +from gfauto.gflogging import log + + +def run_amber( + amber_script_file: Path, + output_dir: Path, + dump_image: bool, + dump_buffer: bool, + skip_render: bool = False, + debug_layers: bool = False, + icd: Optional[Path] = None, +) -> Path: + with util.file_open_text( + result_util.get_amber_log_path(output_dir), "w" + ) as log_file: + try: + gflogging.push_stream_for_logging(log_file) + + run_amber_helper( + amber_script_file, + output_dir, + dump_image, + dump_buffer, + skip_render, + debug_layers, + icd, + ) + finally: + gflogging.pop_stream_for_logging() + + return output_dir + + +def run_amber_helper( + amber_script_file: Path, + output_dir: Path, + dump_image: bool, + dump_buffer: bool, + skip_render: bool = False, + debug_layers: bool = False, + icd: Optional[Path] = None, +) -> Path: + + # TODO: Use binary paths. + + image_file = output_dir / fuzz.IMAGE_FILE_NAME + buffer_file = output_dir / fuzz.BUFFER_FILE_NAME + + cmd = [ + str(util.tool_on_path("amber")), + str(amber_script_file), + "--log-graphics-calls-time", + "--disable-spirv-val", + ] + + if not debug_layers: + cmd.append("-d") + + if skip_render: + # -ps tells amber to stop after pipeline creation + cmd.append("-ps") + else: + if dump_image: + cmd.append("-i") + cmd.append(str(image_file)) + if dump_buffer: + cmd.append("-b") + cmd.append(str(buffer_file)) + cmd.append("-B") + cmd.append("0") + + util.prepend_catchsegv_if_available(cmd) + + status = "UNEXPECTED_ERROR" + + result: Optional[subprocess.CompletedProcess] = None + env: Optional[Dict[str, str]] = None + + if icd: + env = {"VK_ICD_FILENAMES": str(icd)} + + try: + result = subprocess_util.run( + cmd, + check_exit_code=False, + timeout=fuzz.AMBER_RUN_TIME_LIMIT, + verbose=True, + env=env, + ) + except subprocess.TimeoutExpired: + status = fuzz.STATUS_TIMEOUT + + if result: + if result.returncode != 0: + status = fuzz.STATUS_CRASH + else: + status = fuzz.STATUS_SUCCESS + + log("\nSTATUS " + status + "\n") + + util.file_write_text(result_util.get_status_path(output_dir), status) + + return output_dir diff --git a/gfauto/gfauto/proto_util.py b/gfauto/gfauto/proto_util.py new file mode 100644 index 000000000..cf352eb73 --- /dev/null +++ b/gfauto/gfauto/proto_util.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Protobuf utility module. + +Used to read/write protobuf data structures from/to JSON text and from/to JSON files. +""" + +from pathlib import Path +from typing import TypeVar + +from google.protobuf import json_format +from google.protobuf.message import Message + +from gfauto import util + +# pylint: disable=invalid-name; Generic type variable names are usually one letter. +M = TypeVar("M", bound=Message) # noqa + + +def json_to_message(json: str, message: M) -> M: + json_format.Parse(json, message, ignore_unknown_fields=True) + return message + + +def message_to_json( + message: Message, including_default_value_fields: bool = True +) -> str: + return json_format.MessageToJson( + message, + including_default_value_fields=including_default_value_fields, + preserving_proto_field_name=True, + sort_keys=True, + ) + + +def message_to_file( + message: Message, + output_file_path: Path, + including_default_value_fields: bool = True, +) -> Path: + message_json = message_to_json(message, including_default_value_fields) + util.file_write_text(output_file_path, message_json) + return output_file_path + + +def file_to_message(input_file_path: Path, message: M) -> M: + message_json = util.file_read_text(input_file_path) + return json_to_message(message_json, message) diff --git a/gfauto/gfauto/recipe.proto b/gfauto/gfauto/recipe.proto new file mode 100644 index 000000000..274c29679 --- /dev/null +++ b/gfauto/gfauto/recipe.proto @@ -0,0 +1,36 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; + + +// An artifact is a directory containing an ArtifactMetadata proto stored in "artifact.json" and/or a Recipe proto +// stored in "recipe.json", plus any other files and directories within. The recipe is executed to produce the files +// that make up the artifact, which must include the "artifact.json" file. +// See artifact.proto. +message Recipe { + + oneof recipe { + RecipeDownloadAndExtractArchiveSet download_and_extract_archive_set = 1; + } +} + +// Downloads and extracts one or more archives and makes the binaries available. +message RecipeDownloadAndExtractArchiveSet { + ArchiveSet archive_set = 1; +} diff --git a/gfauto/gfauto/recipe_download_and_extract_archive_set.py b/gfauto/gfauto/recipe_download_and_extract_archive_set.py new file mode 100644 index 000000000..c25b9547c --- /dev/null +++ b/gfauto/gfauto/recipe_download_and_extract_archive_set.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Download and extract archive set recipe. + +Implements the RecipeDownloadAndExtractArchiveSet recipe. +See recipe.proto. +""" + +import os +import shutil +import stat +import urllib.request +from zipfile import ZipFile, ZipInfo + +from gfauto import artifact_util, util +from gfauto.artifact_pb2 import ArtifactMetadata +from gfauto.common_pb2 import Archive +from gfauto.gflogging import log +from gfauto.recipe_pb2 import RecipeDownloadAndExtractArchiveSet + +ALL_EXECUTABLE_PERMISSION_BITS = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + + +def recipe_download_and_extract_archive_set( + recipe: RecipeDownloadAndExtractArchiveSet, output_artifact_path: str +) -> None: + + for archive in recipe.archive_set.archives: # type: Archive + util.check_field_truthy(archive.url, "url") + util.check_field_truthy(archive.output_file, "output_file") + util.check_field_truthy(archive.output_directory, "output_directory") + + output_file_path = artifact_util.artifact_get_inner_file_path( + archive.output_file, output_artifact_path + ) + + output_directory_path = artifact_util.artifact_get_inner_file_path( + archive.output_directory, output_artifact_path + ) + + log(f"Downloading {archive.url} to {str(output_file_path)}") + urllib.request.urlretrieve(archive.url, str(output_file_path)) + + if output_file_path.name.lower().endswith(".zip"): + with ZipFile(str(output_file_path), "r") as zip_file: + for info in zip_file.infolist(): # type: ZipInfo + extracted_file = zip_file.extract(info, str(output_directory_path)) + # If the file was created on a UNIX-y system: + if info.create_system == 3: + # Shift away first 2 bytes to get permission bits. + zip_file_exec_bits = info.external_attr >> 16 + # Just consider the executable bits. + zip_file_exec_bits = ( + zip_file_exec_bits & ALL_EXECUTABLE_PERMISSION_BITS + ) + current_attribute_bits = os.stat(extracted_file).st_mode + if ( + current_attribute_bits | zip_file_exec_bits + ) != current_attribute_bits: + os.chmod( + extracted_file, + current_attribute_bits | zip_file_exec_bits, + ) + else: + shutil.unpack_archive(str(output_file_path), str(output_directory_path)) + + output_metadata = ArtifactMetadata() + output_metadata.data.extracted_archive_set.archive_set.CopyFrom(recipe.archive_set) + + artifact_util.artifact_write_metadata(output_metadata, output_artifact_path) diff --git a/gfauto/gfauto/recipe_pb2.py b/gfauto/gfauto/recipe_pb2.py new file mode 100644 index 000000000..f962362c1 --- /dev/null +++ b/gfauto/gfauto/recipe_pb2.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/recipe.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/recipe.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x13gfauto/recipe.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"j\n\x06Recipe\x12V\n download_and_extract_archive_set\x18\x01 \x01(\x0b\x32*.gfauto.RecipeDownloadAndExtractArchiveSetH\x00\x42\x08\n\x06recipe\"M\n\"RecipeDownloadAndExtractArchiveSet\x12\'\n\x0b\x61rchive_set\x18\x01 \x01(\x0b\x32\x12.gfauto.ArchiveSetb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,]) + + + + +_RECIPE = _descriptor.Descriptor( + name='Recipe', + full_name='gfauto.Recipe', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='download_and_extract_archive_set', full_name='gfauto.Recipe.download_and_extract_archive_set', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='recipe', full_name='gfauto.Recipe.recipe', + index=0, containing_type=None, fields=[]), + ], + serialized_start=52, + serialized_end=158, +) + + +_RECIPEDOWNLOADANDEXTRACTARCHIVESET = _descriptor.Descriptor( + name='RecipeDownloadAndExtractArchiveSet', + full_name='gfauto.RecipeDownloadAndExtractArchiveSet', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='archive_set', full_name='gfauto.RecipeDownloadAndExtractArchiveSet.archive_set', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=160, + serialized_end=237, +) + +_RECIPE.fields_by_name['download_and_extract_archive_set'].message_type = _RECIPEDOWNLOADANDEXTRACTARCHIVESET +_RECIPE.oneofs_by_name['recipe'].fields.append( + _RECIPE.fields_by_name['download_and_extract_archive_set']) +_RECIPE.fields_by_name['download_and_extract_archive_set'].containing_oneof = _RECIPE.oneofs_by_name['recipe'] +_RECIPEDOWNLOADANDEXTRACTARCHIVESET.fields_by_name['archive_set'].message_type = gfauto_dot_common__pb2._ARCHIVESET +DESCRIPTOR.message_types_by_name['Recipe'] = _RECIPE +DESCRIPTOR.message_types_by_name['RecipeDownloadAndExtractArchiveSet'] = _RECIPEDOWNLOADANDEXTRACTARCHIVESET +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Recipe = _reflection.GeneratedProtocolMessageType('Recipe', (_message.Message,), { + 'DESCRIPTOR' : _RECIPE, + '__module__' : 'gfauto.recipe_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Recipe) + }) +_sym_db.RegisterMessage(Recipe) + +RecipeDownloadAndExtractArchiveSet = _reflection.GeneratedProtocolMessageType('RecipeDownloadAndExtractArchiveSet', (_message.Message,), { + 'DESCRIPTOR' : _RECIPEDOWNLOADANDEXTRACTARCHIVESET, + '__module__' : 'gfauto.recipe_pb2' + # @@protoc_insertion_point(class_scope:gfauto.RecipeDownloadAndExtractArchiveSet) + }) +_sym_db.RegisterMessage(RecipeDownloadAndExtractArchiveSet) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/recipe_pb2.pyi b/gfauto/gfauto/recipe_pb2.pyi new file mode 100644 index 000000000..b4af77653 --- /dev/null +++ b/gfauto/gfauto/recipe_pb2.pyi @@ -0,0 +1,57 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + ArchiveSet as gfauto___common_pb2___ArchiveSet, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Optional as typing___Optional, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Recipe(google___protobuf___message___Message): + + @property + def download_and_extract_archive_set(self) -> RecipeDownloadAndExtractArchiveSet: ... + + def __init__(self, + download_and_extract_archive_set : typing___Optional[RecipeDownloadAndExtractArchiveSet] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Recipe: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",u"recipe"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",u"recipe"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",b"download_and_extract_archive_set",u"recipe",b"recipe"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"download_and_extract_archive_set",b"recipe"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"recipe",b"recipe"]) -> typing_extensions___Literal["download_and_extract_archive_set"]: ... + +class RecipeDownloadAndExtractArchiveSet(google___protobuf___message___Message): + + @property + def archive_set(self) -> gfauto___common_pb2___ArchiveSet: ... + + def __init__(self, + archive_set : typing___Optional[gfauto___common_pb2___ArchiveSet] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> RecipeDownloadAndExtractArchiveSet: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"archive_set"]) -> None: ... diff --git a/gfauto/gfauto/recipe_wrap.py b/gfauto/gfauto/recipe_wrap.py new file mode 100644 index 000000000..0031c15af --- /dev/null +++ b/gfauto/gfauto/recipe_wrap.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""RecipeWrap class.""" + +import attr + +from gfauto import artifact_util +from gfauto.recipe_pb2 import Recipe + + +@attr.dataclass +class RecipeWrap: + """Wraps a Recipe proto with its path for convenience.""" + + path: str + recipe: Recipe + + def write(self) -> str: + return artifact_util.artifact_write_recipe(self.recipe, self.path) diff --git a/gfauto/gfauto/result_util.py b/gfauto/gfauto/result_util.py new file mode 100644 index 000000000..07e2cb5fe --- /dev/null +++ b/gfauto/gfauto/result_util.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Result utility module. + +When we run Amber, we write certain files to indicate the result. +This module contains functions for getting the paths of these files. +""" + +from pathlib import Path + +from gfauto import util + + +def get_status_path(result_output_dir: Path) -> Path: + return result_output_dir / "STATUS" + + +def get_status(result_output_dir: Path) -> str: + status_file = get_status_path(result_output_dir) + return util.file_read_text_or_else(status_file, "UNEXPECTED_ERROR") + + +def get_log_path(result_output_dir: Path) -> Path: + return result_output_dir / "log.txt" + + +def get_amber_log_path(result_dir: Path) -> Path: + return result_dir / "amber_log.txt" diff --git a/gfauto/gfauto/settings.proto b/gfauto/gfauto/settings.proto new file mode 100644 index 000000000..67b2ccb72 --- /dev/null +++ b/gfauto/gfauto/settings.proto @@ -0,0 +1,36 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; +import "gfauto/device.proto"; + + +// A fuzzing session requires a Settings proto stored in "settings.json". +message Settings { + // A list of devices, including the list of active devices names that will actually be used in this session. + DeviceList device_list = 1; + + // A list of custom binaries; this should rarely be set, but can be used, for example, if you want to override the + // default versions of some binaries for all devices. It is probably more common to override binaries for certain + // devices, or to just update the hardcoded set of default binaries (in binaries_util.py) to their newer versions. + repeated Binary custom_binaries = 2; + + // Allow at most this many duplicate crashes per device. + // If 0 (the protobuf default value), a different default value will actually be used, as determined by gfauto. + uint32 maximum_duplicate_crashes = 3; +} diff --git a/gfauto/gfauto/settings_pb2.py b/gfauto/gfauto/settings_pb2.py new file mode 100644 index 000000000..5b7f9c340 --- /dev/null +++ b/gfauto/gfauto/settings_pb2.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/settings.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 +from gfauto import device_pb2 as gfauto_dot_device__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/settings.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x15gfauto/settings.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\x7f\n\x08Settings\x12\'\n\x0b\x64\x65vice_list\x18\x01 \x01(\x0b\x32\x12.gfauto.DeviceList\x12\'\n\x0f\x63ustom_binaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binary\x12!\n\x19maximum_duplicate_crashes\x18\x03 \x01(\rb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,gfauto_dot_device__pb2.DESCRIPTOR,]) + + + + +_SETTINGS = _descriptor.Descriptor( + name='Settings', + full_name='gfauto.Settings', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='device_list', full_name='gfauto.Settings.device_list', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='custom_binaries', full_name='gfauto.Settings.custom_binaries', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='maximum_duplicate_crashes', full_name='gfauto.Settings.maximum_duplicate_crashes', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=75, + serialized_end=202, +) + +_SETTINGS.fields_by_name['device_list'].message_type = gfauto_dot_device__pb2._DEVICELIST +_SETTINGS.fields_by_name['custom_binaries'].message_type = gfauto_dot_common__pb2._BINARY +DESCRIPTOR.message_types_by_name['Settings'] = _SETTINGS +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Settings = _reflection.GeneratedProtocolMessageType('Settings', (_message.Message,), { + 'DESCRIPTOR' : _SETTINGS, + '__module__' : 'gfauto.settings_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Settings) + }) +_sym_db.RegisterMessage(Settings) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/settings_pb2.pyi b/gfauto/gfauto/settings_pb2.pyi new file mode 100644 index 000000000..13922088f --- /dev/null +++ b/gfauto/gfauto/settings_pb2.pyi @@ -0,0 +1,52 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + Binary as gfauto___common_pb2___Binary, +) + +from gfauto.device_pb2 import ( + DeviceList as gfauto___device_pb2___DeviceList, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Settings(google___protobuf___message___Message): + maximum_duplicate_crashes = ... # type: int + + @property + def device_list(self) -> gfauto___device_pb2___DeviceList: ... + + @property + def custom_binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... + + def __init__(self, + device_list : typing___Optional[gfauto___device_pb2___DeviceList] = None, + custom_binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, + maximum_duplicate_crashes : typing___Optional[int] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Settings: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"device_list"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",u"device_list",u"maximum_duplicate_crashes"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"device_list",b"device_list"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"custom_binaries",b"device_list",b"maximum_duplicate_crashes"]) -> None: ... diff --git a/gfauto/gfauto/settings_util.py b/gfauto/gfauto/settings_util.py new file mode 100644 index 000000000..5ce0ee465 --- /dev/null +++ b/gfauto/gfauto/settings_util.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Settings utility module. + +Used to read and write the Settings proto. +See settings.proto. +""" + +from pathlib import Path + +from gfauto import devices_util, proto_util +from gfauto.settings_pb2 import Settings + +DEFAULT_SETTINGS_FILE_PATH = Path("settings.json") + +DEFAULT_SETTINGS = Settings(maximum_duplicate_crashes=3) + + +def read(settings_path: Path) -> Settings: + result = proto_util.file_to_message(settings_path, Settings()) + if not result.maximum_duplicate_crashes: + result.maximum_duplicate_crashes = DEFAULT_SETTINGS.maximum_duplicate_crashes + return result + + +def write(settings: Settings, settings_path: Path) -> Path: + return proto_util.message_to_file(settings, settings_path) + + +def write_default(settings_path: Path) -> Path: + settings = Settings() + settings.CopyFrom(DEFAULT_SETTINGS) + devices_util.get_device_list(settings.device_list) + return write(settings, settings_path) diff --git a/gfauto/gfauto/shader_job_util.py b/gfauto/gfauto/shader_job_util.py new file mode 100644 index 000000000..5acfccbec --- /dev/null +++ b/gfauto/gfauto/shader_job_util.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shader job utility module. + +Used to copy shader jobs and get the related files of a shader job. +""" +import itertools +import pathlib +from pathlib import Path +from typing import List, Optional, Tuple, Union + +from gfauto import util + +EXT_FRAG = ".frag" +EXT_VERT = ".vert" +EXT_COMP = ".comp" + +EXT_ALL = (EXT_FRAG, EXT_VERT, EXT_COMP) + +SUFFIX_GLSL = "" +SUFFIX_SPIRV = ".spv" +SUFFIX_ASM_SPIRV = ".asm" +SUFFIX_FACTS = ".facts" +SUFFIX_TRANSFORMATIONS = ".transformations" +SUFFIX_TRANSFORMATIONS_JSON = ".transformations_json" + +SUFFIX_SPIRV_ORIG = ".spv_orig" + +SUFFIXES_SPIRV_FUZZ_INPUT = (SUFFIX_SPIRV, SUFFIX_FACTS) + +SUFFIXES_SPIRV_FUZZ = ( + SUFFIX_SPIRV, + SUFFIX_SPIRV_ORIG, + SUFFIX_FACTS, + SUFFIX_TRANSFORMATIONS, + SUFFIX_TRANSFORMATIONS_JSON, +) + + +def get_shader_contents( + shader_job_file_path: pathlib.Path, + extension: str, + language_suffix: str = SUFFIX_GLSL, + must_exist: bool = False, +) -> Optional[str]: + shader_file = shader_job_file_path.with_suffix(extension + language_suffix) + if shader_file.exists(): + return util.file_read_text(shader_file) + if must_exist: + raise AssertionError(f"could not read {shader_file}") + + return None + + +def get_related_suffixes_that_exist( + shader_job_file_path: Path, + extensions: Union[Tuple[str, ...], List[str]] = EXT_ALL, + language_suffix: Union[Tuple[str, ...], List[str]] = (SUFFIX_GLSL,), +) -> List[str]: + # Cross product of extensions and suffixes as tuples. + # (".frag", ".comp", ...) x (".facts", ".spv") + suffixes_as_tuples = itertools.product(extensions, language_suffix) + + # Same as above but as str. E.g. ".frag.facts", ".frag.spv" + suffixes = [file_parts[0] + file_parts[1] for file_parts in suffixes_as_tuples] + + # Filter + suffixes_that_exist = [ + s for s in suffixes if shader_job_file_path.with_suffix(s).is_file() + ] + + return suffixes_that_exist + + +def get_related_files( + shader_job_file_path: pathlib.Path, + extensions: Union[Tuple[str, ...], List[str]] = EXT_ALL, + language_suffix: Union[Tuple[str, ...], List[str]] = (SUFFIX_GLSL,), +) -> List[pathlib.Path]: + + suffixes_that_exist = get_related_suffixes_that_exist( + shader_job_file_path, extensions, language_suffix + ) + + # variant_001.frag.spv, ... + paths = [shader_job_file_path.with_suffix(s) for s in suffixes_that_exist] + + return paths + + +def copy( + shader_job_file_path: pathlib.Path, + output_shader_job_file_path: pathlib.Path, + extensions: Union[Tuple[str, ...], List[str]] = EXT_ALL, + language_suffix: Union[Tuple[str, ...], List[str]] = (SUFFIX_GLSL,), +) -> pathlib.Path: + + util.copy_file(shader_job_file_path, output_shader_job_file_path) + + suffixes_that_exist = get_related_suffixes_that_exist( + shader_job_file_path, extensions, language_suffix + ) + + for suffix in suffixes_that_exist: + util.copy_file( + shader_job_file_path.with_suffix(suffix), + output_shader_job_file_path.with_suffix(suffix), + ) + + return output_shader_job_file_path diff --git a/gfauto/gfauto/signature_util.py b/gfauto/gfauto/signature_util.py new file mode 100644 index 000000000..176bf5f05 --- /dev/null +++ b/gfauto/gfauto/signature_util.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bug signature module. + +Used to compute the "signature" of a bug, typically using the error message or the function name from the top of the +stack trace. +""" + +import re +from pathlib import Path +from typing import Match, Optional, Pattern + +from gfauto import subprocess_util, util + +# .* does not match newlines +# (?: ) non-group parentheses + +HEX_LIKE = r"(?:0x)?[0-9a-fA-F]" + +# 06-15 21:17:00.039 7517 7517 F DEBUG : #00 pc 00000000009d9c34 /my/library.so ((anonymous namespace)::Bar::Baz(aaa::MyInstr*, void* (*)(unsigned int))+456) +# Another example of the function signature: /my/library.so (myFunction+372) +# Another example of the function signature: /my/library.so (myFunction(...)+372) + +# Just look for anything that contains "word(" or "word+" from after the (hex-like) PC address. +PATTERN_ANDROID_BACKTRACE_FUNCTION = re.compile( + r"\n.*#00 pc " + HEX_LIKE + r"+ (.*[\w\d_]+[(+].*)" +) + +ANDROID_BACKTRACE_COMMON_TEXT_TO_REMOVE = re.compile( + r"(?:" + r"vendor/" + r"|hw/" + r"|data/local/(?:tmp/)?" + r"|system/(?:lib(?:64)?/)?" + r"|lib(?:64)?/" + r"|apex/" + r"|bionic/" + r"|com.android.runtime(?:/lib(?:64)?)?/" + r"|anonymous namespace" + r"|\(BuildId: " + HEX_LIKE + r"+\)" + r")" +) + +PATTERN_ANDROID_BACKTRACE_CATCHALL = re.compile(r"\n.*#00 pc " + HEX_LIKE + r"+ (.*)") + +# E.g. ERROR: temp/.../variant/shader.frag:549: 'variable indexing fragment shader output array' : not supported with this profile: es +# variable indexing fragment shader output array <-- group 1 +PATTERN_GLSLANG_ERROR = re.compile(r"ERROR: .*?: '(.*?)'") + + +# E.g. +# glslangValidator: ../glslang/MachineIndependent/ParseHelper.cpp:2212: void glslang::TParseContext::nonOpBuiltInCheck(const glslang::TSourceLoc&, const glslang::TFunction&, glslang::TIntermAggregate&): Assertion `PureOperatorBuiltins == false' failed. + +PATTERN_ASSERTION_FAILURE = re.compile(r"\n.*?:\d+: (.*? [Aa]ssert(?:ion)?)") + + +# Only used if "0 pass, 1 fail" is found. +# E.g. /data/local/tmp/graphicsfuzz/test.amber: 256: probe ssbo format does not match buffer format +# probe ssbo format does not match buffer format +PATTERN_AMBER_ERROR = re.compile(r"\n.*?\w: \d+: (.*)") + +# E.g. error: line 0: Module contains unreachable blocks during merge return. Run dead branch elimination before merge return. +# error: line 0: Module contains unreachable blocks during merge return. Run dead branch elimination before merge return. +# Module contains unreachable blocks during merge return. Run dead branch elimination before merge return. +PATTERN_SPIRV_OPT_ERROR: Pattern[str] = re.compile(r"error: line \d+: (.*)") + +# E.g. +# Backtrace: +# /data/git/graphicsfuzz/graphicsfuzz/target/graphicsfuzz/bin/Linux/spirv-opt(_ZN8spvtools3opt21StructuredCFGAnalysis16SwitchMergeBlockEj+0x369)[0x5bd6d9] +PATTERN_CATCHSEGV_STACK_FRAME = re.compile(r"Backtrace:\n.*/([^/(]*\([^)+\[]+)\+") + +# E.g. +# Backtrace: +# /data/git/graphicsfuzz/gfauto/temp/june_20/binaries/swiftshader_vulkan/Linux/libvk_swiftshader.so(+0x1d537d)[0x7f51ebd1237d] +# /data/git/graphicsfuzz/gfauto/temp/june_20/binaries/swiftshader_vulkan/Linux/libvk_swiftshader.so 0x1d537d +# ^ group 1 ^ group 2 +PATTERN_CATCHSEGV_STACK_FRAME_ADDRESS = re.compile( + r"Backtrace:\n(.*)\(\+([x\da-fA-F]+)+\)\[" +) + +PATTERN_SWIFT_SHADER_ABORT = re.compile(r":\d+ ABORT:(.*)") + +PATTERN_SWIFT_SHADER_WARNING = re.compile(r":\d+ WARNING:(.*)") + + +def remove_hex_like(string: str) -> str: + temp = string + # Remove hex like chunks of 4 or more. + temp = re.sub(HEX_LIKE + r"{4,}", "", temp) + return temp + + +def clean_up(string: str) -> str: + temp: str = string + # Remove numbers. + temp = re.sub(r"\d+", "", temp) + # Replace spaces with _. + temp = re.sub(r" ", "_", temp) + # Remove non-word, non-_ characters. + temp = re.sub(r"[^\w_]", "", temp) + # Replace multiple _ with _. + temp = re.sub(r"__+", "_", temp) + # Strip _ + temp = temp.strip("_") + return temp + + +def reduce_length(string: str) -> str: + return string[:50] + + +def basic_match(pattern: Pattern[str], log_contents: str) -> Optional[str]: + match: Optional[Match[str]] = re.search(pattern, log_contents) + if not match: + return None + group = match.group(1) + group = clean_up(group) + group = reduce_length(group) + return group + + +def get_signature_from_log_contents( # pylint: disable=too-many-return-statements, too-many-branches, too-many-statements; + log_contents: str +) -> str: + + # noinspection PyUnusedLocal + match: Optional[Match[str]] + # noinspection PyUnusedLocal + group: Optional[str] + + # glslang error. + group = basic_match(PATTERN_GLSLANG_ERROR, log_contents) + if group: + return group + + # Assertion error pattern, used by glslang. + group = basic_match(PATTERN_ASSERTION_FAILURE, log_contents) + if group: + return group + + # Spirv-opt error. + group = basic_match(PATTERN_SPIRV_OPT_ERROR, log_contents) + if group: + return group + + # ABORT message from SwiftShader. + group = basic_match(PATTERN_SWIFT_SHADER_ABORT, log_contents) + if group: + return group + + # WARNING message from SwiftShader. + group = basic_match(PATTERN_SWIFT_SHADER_WARNING, log_contents) + if group: + return group + + # Amber error. + if "0 pass, 1 fail" in log_contents: + group = basic_match(PATTERN_AMBER_ERROR, log_contents) + if group: + return group + + # Android stack traces. + if "#00 pc" in log_contents: + lines = log_contents.split("\n") + for line in lines: + pc_pos = line.find("#00 pc") + if pc_pos == -1: + continue + line = line[pc_pos:] + + if "/amber_ndk" in line: + return "amber_ndk" + break + + # Check for stack line with libc alloc. + if re.search(r"\n.*#\d+ pc .*libc\.so \(\w?alloc", log_contents): + # Find the first stack frame without libc.so and replace the log_contents with that frame. + # We do this because the error is better identified by this line and because out of memory errors + # often occur at a nondeterministic location within libc. + for line in lines: + if ( + re.search(r" #\d+ pc ", line) + and "libc.so" not in line + and "operator new" not in line + ): + # Replace the stack frame number so it looks like the 0th frame. + line = re.sub(r" #\d+ ", " #00 ", line) + log_contents = f"\n{line}\n" + break + + match = re.search(PATTERN_ANDROID_BACKTRACE_FUNCTION, log_contents) + if match: + group = match.group(1) + # Remove common text. + group = re.sub(ANDROID_BACKTRACE_COMMON_TEXT_TO_REMOVE, "", group) + group = clean_up(group) + group = reduce_length(group) + return group + + # TODO: Maybe more. + + # If we get here, we found #00 pc, but nothing else. + # This regex essentially matches the entire line after the hex-like PC address. + match = re.search(PATTERN_ANDROID_BACKTRACE_CATCHALL, log_contents) + if match: + group = match.group(1) + # Remove common text. + group = re.sub(ANDROID_BACKTRACE_COMMON_TEXT_TO_REMOVE, "", group) + # Remove hex-like chunks. + group = remove_hex_like(group) + group = clean_up(group) + group = reduce_length(group) + return group + + # catchsegv "Backtrace:" with source code info. + group = basic_match(PATTERN_CATCHSEGV_STACK_FRAME, log_contents) + if group: + return group + + # catchsegv "Backtrace:" with addresses. + if "Backtrace:" in log_contents: + result = get_signature_from_catchsegv_frame_address(log_contents) + if result: + return result + + if "Shader compilation failed" in log_contents: + return "compile_error" + + if "Failed to link shaders" in log_contents: + return "link_error" + + if "Calling vkCreateGraphicsPipelines Fail" in log_contents: + return "pipeline_failure" + + # TODO: Check for Amber fence failure. + + if "Resource deadlock would occur" in log_contents: + return "Resource_deadlock_would_occur" + + return "no_signature" + + +def get_signature_from_catchsegv_frame_address(log_contents: str) -> Optional[str]: + match = re.search(PATTERN_CATCHSEGV_STACK_FRAME_ADDRESS, log_contents) + if not match: + return None + module = Path(match.group(1)) + if not module.exists(): + return None + address = match.group(2) + function_signature = get_function_signature_from_address(module, address) + if not function_signature: + return None + function_signature = clean_up(function_signature) + function_signature = reduce_length(function_signature) + return function_signature + + +def get_function_signature_from_address(module: Path, address: str) -> Optional[str]: + try: + address_tool = util.tool_on_path("addr2line") + result = subprocess_util.run( + [str(address_tool), "-e", str(module), address, "-f", "-C"], + check_exit_code=False, + ) + if result.returncode != 0: + return None + stdout: str = result.stdout + lines = stdout.splitlines() + if not lines: + return None + return lines[0] + except util.ToolNotOnPathError: + return None diff --git a/gfauto/gfauto/spirv_dis_util.py b/gfauto/gfauto/spirv_dis_util.py new file mode 100644 index 000000000..77cd7d30b --- /dev/null +++ b/gfauto/gfauto/spirv_dis_util.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""spirv-dis utility module. + +Converts a SPIR-V shader job to a SPIR-V assembly shader job (the shaders are disassembled). +""" + +import pathlib +from typing import Optional + +from gfauto import binaries_util, shader_job_util, subprocess_util, util + + +def run_spirv_dis_on_spirv_shader( + input_spirv_file_path: pathlib.Path, + output_dir_path: pathlib.Path, + spirv_dis_file_path: Optional[pathlib.Path] = None, +) -> pathlib.Path: + if not spirv_dis_file_path: + spirv_dis_file_path = util.tool_on_path(binaries_util.SPIRV_DIS_NAME) + + output_spirv_file_path = output_dir_path / ( + util.remove_end(input_spirv_file_path.name, ".spv") + ".asm" + ) + + util.file_mkdirs_parent(output_spirv_file_path) + + subprocess_util.run( + util.prepend_catchsegv_if_available( + [ + str(spirv_dis_file_path), + str(input_spirv_file_path), + "-o", + str(output_spirv_file_path), + "--raw-id", + ] + ) + ) + + return output_spirv_file_path + + +def run_spirv_shader_job_to_spirv_asm_shader_job( + input_spirv_job_json_file_path: pathlib.Path, + output_spirv_job_json_file_path: pathlib.Path, + spirv_dis_file_path: Optional[pathlib.Path] = None, +) -> pathlib.Path: + + if not spirv_dis_file_path: + spirv_dis_file_path = util.tool_on_path(binaries_util.SPIRV_DIS_NAME) + + shader_files = shader_job_util.get_related_files( + input_spirv_job_json_file_path, language_suffix=[shader_job_util.SUFFIX_SPIRV] + ) + + util.copy_file(input_spirv_job_json_file_path, output_spirv_job_json_file_path) + + for shader_file in shader_files: + run_spirv_dis_on_spirv_shader( + shader_file, output_spirv_job_json_file_path.parent, spirv_dis_file_path + ) + + return output_spirv_job_json_file_path diff --git a/gfauto/gfauto/spirv_fuzz_util.py b/gfauto/gfauto/spirv_fuzz_util.py new file mode 100644 index 000000000..8b9b41d33 --- /dev/null +++ b/gfauto/gfauto/spirv_fuzz_util.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""spirv-fuzz utility module. + +Runs spirv-fuzz to generate a variant SPIR-V shader job. +""" + +from pathlib import Path +from typing import List, Optional + +from gfauto import shader_job_util, subprocess_util, util + +# TODO: Make this 64 bits. + +GENERATE_SEED_BITS = 32 + + +def run_generate( + spirv_fuzz_path: Path, + reference_shader_spv: Path, + output_shader_spv: Path, + seed: Optional[str] = None, + other_args: Optional[List[str]] = None, +) -> Path: + + util.check( + output_shader_spv.suffix == shader_job_util.SUFFIX_SPIRV, + AssertionError(f"Expected {str(output_shader_spv)} to end with .spv"), + ) + + cmd = [ + str(spirv_fuzz_path), + str(reference_shader_spv), + "-o", + str(output_shader_spv), + ] + + if seed: + cmd.append(f"--seed={seed}") + + if other_args: + cmd.extend(other_args) + + subprocess_util.run(cmd) + + # reference.spv -> output.spv_orig + + util.copy_file( + reference_shader_spv, + output_shader_spv.with_suffix(shader_job_util.SUFFIX_SPIRV_ORIG), + ) + + # reference.spv.facts -> output.spv.facts + + source_facts_path = reference_shader_spv.with_suffix(shader_job_util.SUFFIX_FACTS) + dest_facts_path = output_shader_spv.with_suffix(shader_job_util.SUFFIX_FACTS) + + if source_facts_path.exists(): + util.copy_file(source_facts_path, dest_facts_path) + + return output_shader_spv + + +def run_generate_on_shader_job( + spirv_fuzz_path: Path, + reference_shader_json: Path, + output_shader_json: Path, + seed: Optional[str] = None, + other_args: Optional[List[str]] = None, +) -> Path: + + util.copy_file(reference_shader_json, output_shader_json) + + suffixes_that_exist = shader_job_util.get_related_suffixes_that_exist( + reference_shader_json, shader_job_util.EXT_ALL, [shader_job_util.SUFFIX_SPIRV] + ) + + for suffix in suffixes_that_exist: + run_generate( + spirv_fuzz_path, + reference_shader_json.with_suffix(suffix), + output_shader_json.with_suffix(suffix), + seed, + other_args, + ) + + return output_shader_json diff --git a/gfauto/gfauto/spirv_opt_util.py b/gfauto/gfauto/spirv_opt_util.py new file mode 100644 index 000000000..dc5bbfc5f --- /dev/null +++ b/gfauto/gfauto/spirv_opt_util.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""spirv-opt utility module. + +Runs spirv-opt on a SPIR-V shader job to get a new SPIR-V shader job where the shaders have been optimized. +""" + +import pathlib +import random +from typing import List, Optional + +from gfauto import binaries_util, shader_job_util, subprocess_util, util + +SPIRV_OPT_DEFAULT_TIME_LIMIT = 120 + +OPT_OPTIONS: List[str] = [ + "--ccp", + "--combine-access-chains", + "--convert-local-access-chains", + "--copy-propagate-arrays", + "--eliminate-dead-branches", + "--eliminate-dead-code-aggressive", + "--eliminate-dead-inserts", + "--eliminate-local-multi-store", + "--eliminate-local-single-block", + "--eliminate-local-single-store", + "--if-conversion", + "--inline-entry-points-exhaustive", + "--merge-blocks", + "--merge-return", + "--private-to-local", + "--reduce-load-size", + "--redundancy-elimination", + "--scalar-replacement=100", + "--simplify-instructions", + "--vector-dce", +] + + +def random_spirv_opt_args(max_num_args: int = 30) -> List[str]: + result: List[str] = [] + num_args = random.randint(1, max_num_args) + for _ in range(0, num_args): + arg = random.choice(OPT_OPTIONS) + # --merge-return relies on there not being unreachable code, so we always invoke dead branch + # elimination before --merge-return. + if arg in ("--merge-return", "--merge-blocks"): + result.append("--eliminate-dead-branches") + result.append(arg) + return result + + +def run_spirv_opt_on_spirv_shader( + input_spirv_file_path: pathlib.Path, + output_dir_path: pathlib.Path, + spirv_opt_args: List[str], + spirv_opt_file_path: Optional[pathlib.Path] = None, + spirv_opt_no_validate_after_all: bool = False, + time_limit: int = SPIRV_OPT_DEFAULT_TIME_LIMIT, +) -> pathlib.Path: + + if not spirv_opt_file_path: + spirv_opt_file_path = util.tool_on_path(binaries_util.SPIRV_OPT_NAME) + + output_spirv_file_path = output_dir_path / input_spirv_file_path.name + + util.file_mkdirs_parent(output_spirv_file_path) + + cmd = [ + str(spirv_opt_file_path), + str(input_spirv_file_path), + "-o", + str(output_spirv_file_path), + ] + + if not spirv_opt_no_validate_after_all: + cmd.append("--validate-after-all") + + cmd += spirv_opt_args + + cmd = util.prepend_catchsegv_if_available(cmd) + + subprocess_util.run(cmd, timeout=time_limit) + + return output_spirv_file_path + + +def run_spirv_opt_on_spirv_shader_job( + input_spirv_shader_job_json_file_path: pathlib.Path, + output_spirv_shader_job_json_file_path: pathlib.Path, + spirv_opt_args: List[str], + spirv_opt_file_path: Optional[pathlib.Path] = None, + spirv_opt_no_validate_after_all: bool = False, +) -> pathlib.Path: + + if not spirv_opt_file_path: + spirv_opt_file_path = util.tool_on_path(binaries_util.SPIRV_OPT_NAME) + + shader_files = shader_job_util.get_related_files( + input_spirv_shader_job_json_file_path, + language_suffix=[shader_job_util.SUFFIX_SPIRV], + ) + + util.copy_file( + input_spirv_shader_job_json_file_path, output_spirv_shader_job_json_file_path + ) + + for shader_file in shader_files: + run_spirv_opt_on_spirv_shader( + shader_file, + output_spirv_shader_job_json_file_path.parent, + spirv_opt_args, + spirv_opt_file_path, + spirv_opt_no_validate_after_all, + ) + + return output_spirv_shader_job_json_file_path diff --git a/gfauto/gfauto/subprocess_util.py b/gfauto/gfauto/subprocess_util.py new file mode 100644 index 000000000..559d87e25 --- /dev/null +++ b/gfauto/gfauto/subprocess_util.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Subprocess utility module. + +Used to execute a process. In particular, the stdout and stderr are captured, and logged on failure. +The entire process group is killed on timeout, when this feature is available; this can prevent hangs, +especially when using catchsegv. +""" + +import os +import signal +import subprocess +import time +from typing import Dict, List, Optional, Union + +from gfauto.gflogging import log +from gfauto.util import check + +LOG_COMMAND_FAILED_PREFIX = "Command failed: " +LOG_COMMAND_TIMED_OUT_PREFIX = "Command timed out: " + + +def convert_stdout_stderr( + result: Union[ + subprocess.CalledProcessError, + subprocess.CompletedProcess, + subprocess.TimeoutExpired, + ] +) -> None: + + if result.stdout is not None: + result.stdout = result.stdout.decode(encoding="utf-8", errors="ignore") + if result.stderr is not None: + result.stderr = result.stderr.decode(encoding="utf-8", errors="ignore") + + +def log_stdout_stderr_helper(stdout: str, stderr: str) -> None: + + log("STDOUT:") + log(stdout) + log("") + + log("STDERR:") + log(stderr) + log("") + + +def log_stdout_stderr( + result: Union[ + subprocess.CalledProcessError, + subprocess.CompletedProcess, + subprocess.TimeoutExpired, + ], +) -> None: + log_stdout_stderr_helper(result.stdout, result.stderr) + + +def log_returncode_helper(returncode: int) -> None: + log(f"RETURNCODE: {str(returncode)}") + + +def log_returncode( + result: Union[ + subprocess.CalledProcessError, subprocess.CompletedProcess, subprocess.Popen + ], +) -> None: + log_returncode_helper(result.returncode) + + +def posix_kill_group(process: subprocess.Popen) -> None: + os.killpg(process.pid, signal.SIGTERM) + time.sleep(1) + os.killpg(process.pid, signal.SIGKILL) + + +def run_helper( + cmd: List[str], + check_exit_code: bool = True, + timeout: Optional[float] = None, + env: Optional[Dict[str, str]] = None, +) -> subprocess.CompletedProcess: + check( + bool(cmd) and cmd[0] is not None and isinstance(cmd[0], str), + AssertionError("run takes a list of str, not a str"), + ) + + env_child: Optional[Dict[str, str]] = None + if env: + log(f"Extra environment variables are: {env}") + env_child = os.environ.copy() + env_child.update(env) + + with subprocess.Popen( + cmd, + encoding="utf-8", + errors="ignore", + start_new_session=True, + env=env_child, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as process: + try: + stdout, stderr = process.communicate(input=None, timeout=timeout) + except subprocess.TimeoutExpired: + try: + posix_kill_group(process) + except AttributeError: + process.kill() + stdout, stderr = process.communicate() + assert timeout # noqa + raise subprocess.TimeoutExpired(process.args, timeout, stdout, stderr) + except: # noqa + try: + posix_kill_group(process) + except AttributeError: + process.kill() + raise + + exit_code = process.poll() + if check_exit_code and exit_code != 0: + raise subprocess.CalledProcessError(exit_code, process.args, stdout, stderr) + return subprocess.CompletedProcess(process.args, exit_code, stdout, stderr) + + +def run( + cmd: List[str], + check_exit_code: bool = True, + timeout: Optional[float] = None, + verbose: bool = False, + env: Optional[Dict[str, str]] = None, +) -> subprocess.CompletedProcess: + log("Exec" + (" (verbose):" if verbose else ":") + str(cmd)) + try: + result = run_helper(cmd, check_exit_code, timeout, env) + except subprocess.TimeoutExpired as ex: + log(LOG_COMMAND_TIMED_OUT_PREFIX + str(cmd)) + # no return code to log in case of timeout + log_stdout_stderr(ex) + raise ex + except subprocess.CalledProcessError as ex: + log(LOG_COMMAND_FAILED_PREFIX + str(cmd)) + log_returncode(ex) + log_stdout_stderr(ex) + raise ex + + log_returncode(result) + if verbose: + log_stdout_stderr(result) + + return result diff --git a/gfauto/gfauto/test.proto b/gfauto/gfauto/test.proto new file mode 100644 index 000000000..5079496ba --- /dev/null +++ b/gfauto/gfauto/test.proto @@ -0,0 +1,42 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; +import "gfauto/device.proto"; + +// A test directory (test_dir) contains a Test proto in "test.json", plus the reference and variant shader jobs. +// A Test proto contains all the information needed to execute a test on a specific device, plus the crash signature +// for detecting if the result is interesting (i.e. the bug reproduces). +message Test { + oneof test { + TestGlsl glsl = 1; + TestSpirvFuzz spirv_fuzz = 5; + } + string crash_signature = 2; + Device device = 3; + repeated Binary binaries = 4; +} + +message TestGlsl { + repeated string spirv_opt_args = 1; +} + +// Spirv-fuzz generated spirv test. +message TestSpirvFuzz { + +} diff --git a/gfauto/gfauto/test_create_readme.py b/gfauto/gfauto/test_create_readme.py new file mode 100644 index 000000000..5dacf1f9a --- /dev/null +++ b/gfauto/gfauto/test_create_readme.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create readme module. + +A script that creates a README and bug_report directory for a test and its result_dir. +""" + +import argparse +import itertools +import sys +from pathlib import Path + +from gfauto import artifact_util, binaries_util, fuzz_glsl_test, test_util +from gfauto.binaries_util import BinaryManager +from gfauto.common_pb2 import Binary +from gfauto.gflogging import log +from gfauto.util import check, check_dir_exists + + +def update_test_json(test_json: Path) -> Path: + test = test_util.metadata_read_from_path(test_json) + + for test_binary in itertools.chain( + test.binaries, test.device.binaries + ): # type: Binary + for default_binary in binaries_util.DEFAULT_BINARIES: + if ( + test_binary.name == default_binary.name + and test_binary.version != default_binary.version + ): + log( + f"Updating version: {test_binary.version} -> {default_binary.version}" + ) + test_binary.version = default_binary.version + break + + return test_util.metadata_write_to_path(test, test_json) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="A script that creates a README and bug_report directory for a test and its result_dir." + ) + + parser.add_argument( + "source_dir", help="Source directory containing test.json and shaders." + ) + + parser.add_argument( + "result_dir", + help="Path to the result_dir of a test containing e.g. the intermediate shader files, log.txt, etc.", + ) + + parser.add_argument( + "--output_dir", + help="Output directory where the README and bug_report directory will be written.", + default=".", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + source_dir = Path(parsed_args.source_dir) + result_dir = Path(parsed_args.result_dir) + output_dir = Path(parsed_args.output_dir) + + check_dir_exists(source_dir) + check_dir_exists(result_dir) + + artifact_util.recipes_write_built_in() + + binary_manager = BinaryManager(binaries_util.DEFAULT_BINARIES) + + test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) + + check(test.HasField("glsl"), AssertionError("Only glsl tests currently supported")) + + check( + test.device.HasField("preprocess"), + AssertionError("Only preprocess device tests currently supported"), + ) + + fuzz_glsl_test.tool_crash_summary_bug_report_dir( + source_dir, result_dir, output_dir, binary_manager + ) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/test_pb2.py b/gfauto/gfauto/test_pb2.py new file mode 100644 index 000000000..4e3093d80 --- /dev/null +++ b/gfauto/gfauto/test_pb2.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/test.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 +from gfauto import device_pb2 as gfauto_dot_device__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/test.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x11gfauto/test.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\xb8\x01\n\x04Test\x12 \n\x04glsl\x18\x01 \x01(\x0b\x32\x10.gfauto.TestGlslH\x00\x12+\n\nspirv_fuzz\x18\x05 \x01(\x0b\x32\x15.gfauto.TestSpirvFuzzH\x00\x12\x17\n\x0f\x63rash_signature\x18\x02 \x01(\t\x12\x1e\n\x06\x64\x65vice\x18\x03 \x01(\x0b\x32\x0e.gfauto.Device\x12 \n\x08\x62inaries\x18\x04 \x03(\x0b\x32\x0e.gfauto.BinaryB\x06\n\x04test\"\"\n\x08TestGlsl\x12\x16\n\x0espirv_opt_args\x18\x01 \x03(\t\"\x0f\n\rTestSpirvFuzzb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,gfauto_dot_device__pb2.DESCRIPTOR,]) + + + + +_TEST = _descriptor.Descriptor( + name='Test', + full_name='gfauto.Test', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='glsl', full_name='gfauto.Test.glsl', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='spirv_fuzz', full_name='gfauto.Test.spirv_fuzz', index=1, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='crash_signature', full_name='gfauto.Test.crash_signature', index=2, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='device', full_name='gfauto.Test.device', index=3, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='binaries', full_name='gfauto.Test.binaries', index=4, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='test', full_name='gfauto.Test.test', + index=0, containing_type=None, fields=[]), + ], + serialized_start=72, + serialized_end=256, +) + + +_TESTGLSL = _descriptor.Descriptor( + name='TestGlsl', + full_name='gfauto.TestGlsl', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='spirv_opt_args', full_name='gfauto.TestGlsl.spirv_opt_args', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=258, + serialized_end=292, +) + + +_TESTSPIRVFUZZ = _descriptor.Descriptor( + name='TestSpirvFuzz', + full_name='gfauto.TestSpirvFuzz', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=294, + serialized_end=309, +) + +_TEST.fields_by_name['glsl'].message_type = _TESTGLSL +_TEST.fields_by_name['spirv_fuzz'].message_type = _TESTSPIRVFUZZ +_TEST.fields_by_name['device'].message_type = gfauto_dot_device__pb2._DEVICE +_TEST.fields_by_name['binaries'].message_type = gfauto_dot_common__pb2._BINARY +_TEST.oneofs_by_name['test'].fields.append( + _TEST.fields_by_name['glsl']) +_TEST.fields_by_name['glsl'].containing_oneof = _TEST.oneofs_by_name['test'] +_TEST.oneofs_by_name['test'].fields.append( + _TEST.fields_by_name['spirv_fuzz']) +_TEST.fields_by_name['spirv_fuzz'].containing_oneof = _TEST.oneofs_by_name['test'] +DESCRIPTOR.message_types_by_name['Test'] = _TEST +DESCRIPTOR.message_types_by_name['TestGlsl'] = _TESTGLSL +DESCRIPTOR.message_types_by_name['TestSpirvFuzz'] = _TESTSPIRVFUZZ +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Test = _reflection.GeneratedProtocolMessageType('Test', (_message.Message,), { + 'DESCRIPTOR' : _TEST, + '__module__' : 'gfauto.test_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Test) + }) +_sym_db.RegisterMessage(Test) + +TestGlsl = _reflection.GeneratedProtocolMessageType('TestGlsl', (_message.Message,), { + 'DESCRIPTOR' : _TESTGLSL, + '__module__' : 'gfauto.test_pb2' + # @@protoc_insertion_point(class_scope:gfauto.TestGlsl) + }) +_sym_db.RegisterMessage(TestGlsl) + +TestSpirvFuzz = _reflection.GeneratedProtocolMessageType('TestSpirvFuzz', (_message.Message,), { + 'DESCRIPTOR' : _TESTSPIRVFUZZ, + '__module__' : 'gfauto.test_pb2' + # @@protoc_insertion_point(class_scope:gfauto.TestSpirvFuzz) + }) +_sym_db.RegisterMessage(TestSpirvFuzz) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/test_pb2.pyi b/gfauto/gfauto/test_pb2.pyi new file mode 100644 index 000000000..9af63bb2c --- /dev/null +++ b/gfauto/gfauto/test_pb2.pyi @@ -0,0 +1,87 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + Binary as gfauto___common_pb2___Binary, +) + +from gfauto.device_pb2 import ( + Device as gfauto___device_pb2___Device, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Test(google___protobuf___message___Message): + crash_signature = ... # type: typing___Text + + @property + def glsl(self) -> TestGlsl: ... + + @property + def spirv_fuzz(self) -> TestSpirvFuzz: ... + + @property + def device(self) -> gfauto___device_pb2___Device: ... + + @property + def binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... + + def __init__(self, + glsl : typing___Optional[TestGlsl] = None, + spirv_fuzz : typing___Optional[TestSpirvFuzz] = None, + crash_signature : typing___Optional[typing___Text] = None, + device : typing___Optional[gfauto___device_pb2___Device] = None, + binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Test: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"device",u"glsl",u"spirv_fuzz",u"test"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"binaries",u"crash_signature",u"device",u"glsl",u"spirv_fuzz",u"test"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"device",b"device",u"glsl",b"glsl",u"spirv_fuzz",b"spirv_fuzz",u"test",b"test"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[b"binaries",b"crash_signature",b"device",b"glsl",b"spirv_fuzz",b"test"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"test",b"test"]) -> typing_extensions___Literal["glsl","spirv_fuzz"]: ... + +class TestGlsl(google___protobuf___message___Message): + spirv_opt_args = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + spirv_opt_args : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> TestGlsl: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"spirv_opt_args"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[b"spirv_opt_args"]) -> None: ... + +class TestSpirvFuzz(google___protobuf___message___Message): + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> TestSpirvFuzz: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... diff --git a/gfauto/gfauto/test_update_binaries.py b/gfauto/gfauto/test_update_binaries.py new file mode 100644 index 000000000..5a3049e0e --- /dev/null +++ b/gfauto/gfauto/test_update_binaries.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Update binaries module. + +A script that updates the binaries in a test.json file. +""" + +import argparse +import itertools +import sys +from pathlib import Path +from typing import List + +from gfauto import binaries_util, test_util +from gfauto.common_pb2 import Binary +from gfauto.gflogging import log +from gfauto.util import check_file_exists + + +def update_test_json(test_json: Path) -> Path: + test = test_util.metadata_read_from_path(test_json) + + for test_binary in itertools.chain( + test.binaries, test.device.binaries + ): # type: Binary + for default_binary in binaries_util.DEFAULT_BINARIES: + if ( + test_binary.name == default_binary.name + and test_binary.version != default_binary.version + ): + log( + f"Updating version: {test_binary.version} -> {default_binary.version}" + ) + test_binary.version = default_binary.version + break + + return test_util.metadata_write_to_path(test, test_json) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="A script that updates the binaries in a test.json file." + ) + + parser.add_argument( + "test_json", help="Paths to one or more test.json files.", nargs="*" + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + test_jsons: List[Path] = [Path(json_path) for json_path in parsed_args.test_json] + + for test_json in test_jsons: + check_file_exists(test_json) + + for test_json in test_jsons: + update_test_json(test_json) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/test_util.py b/gfauto/gfauto/test_util.py new file mode 100644 index 000000000..497ec3d67 --- /dev/null +++ b/gfauto/gfauto/test_util.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test utility module. + +A test directory contains a Test proto stored in "source/test.json", the reference and variant shader jobs, and various +other files, including results. +This module is used to read Test proto files and get various paths that exist in test directories. +""" + +from pathlib import Path + +from gfauto import proto_util, util +from gfauto.test_pb2 import Test + +TEST_METADATA = "test.json" +REFERENCE_DIR = "reference" +VARIANT_DIR = "variant" +SHADER_JOB = "shader.json" +SHADER_JOB_RESULT = "shader.info.json" + + +def get_source_dir(test_dir: Path) -> Path: + return test_dir / "source" + + +def get_metadata_path(test_dir: Path) -> Path: + return get_source_dir(test_dir) / TEST_METADATA + + +def metadata_write(metadata: Test, test_dir: Path) -> Path: + metadata_write_to_path(metadata, get_metadata_path(test_dir)) + return test_dir + + +def metadata_read(test_dir: Path) -> Test: + return metadata_read_from_path(get_metadata_path(test_dir)) + + +def metadata_read_from_path(test_metadata_path: Path) -> Test: + text = util.file_read_text(test_metadata_path) + result = Test() + proto_util.json_to_message(text, result) + return result + + +def metadata_write_to_path(metadata: Test, test_metadata_path: Path) -> Path: + text = proto_util.message_to_json(metadata) + util.file_write_text(test_metadata_path, text) + return test_metadata_path + + +def get_shader_job_path(test_dir: Path, is_variant: bool = True) -> Path: + return ( + test_dir + / "source" + / (VARIANT_DIR if is_variant else REFERENCE_DIR) + / SHADER_JOB + ) + + +def get_device_directory(test_dir: Path, device_name: str) -> Path: + return test_dir / "results" / device_name + + +def get_results_directory( + test_dir: Path, device_name: str, is_variant: bool = True +) -> Path: + return get_device_directory(test_dir, device_name) / ( + VARIANT_DIR if is_variant else REFERENCE_DIR + ) + + +def get_reduced_test_dir(test_dir: Path, device_name: str, reduction_name: str) -> Path: + return get_device_directory(test_dir, device_name) / "reductions" / reduction_name + + +def get_reduction_work_directory( + reduced_test_dir: Path, is_variant: bool = True +) -> Path: + return ( + reduced_test_dir + / "reduction_work" + / (VARIANT_DIR if is_variant else REFERENCE_DIR) + ) diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py new file mode 100644 index 000000000..161a05540 --- /dev/null +++ b/gfauto/gfauto/tool.py @@ -0,0 +1,279 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tool module. + +Used to convert shader jobs to Amber script tests that are suitable for adding to the VK-GL-CTS project. +""" + +from pathlib import Path +from typing import List, Optional + +from gfauto import ( + amber_converter, + artifact_util, + binaries_util, + glslang_validator_util, + shader_job_util, + spirv_dis_util, + spirv_opt_util, + subprocess_util, + test_util, + util, +) +from gfauto.util import check + +AMBER_COMMAND_PROBE_TOP_LEFT_RED = "probe rgba (0, 0) (1, 0, 0, 1)\n" + +AMBER_COMMAND_PROBE_TOP_LEFT_WHITE = "probe rgba (0, 0) (1, 1, 1, 1)\n" + + +def get_copyright_header_google(year: str) -> str: + return f"""Copyright {year} Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +def get_binary_paths_using_artifact_system( + artifact_path: str +) -> binaries_util.BinaryManager: + + # Deprecated. + + artifact_util.recipes_write_built_in() + artifact_util.artifact_execute_recipe_if_needed(artifact_path) + artifact_metadata = artifact_util.artifact_read_metadata(artifact_path) + + return binaries_util.BinaryManager( + list(artifact_metadata.data.extracted_archive_set.archive_set.binaries) + ) + + +def amberfy( + input_json: Path, + output_amber: Path, + amberfy_settings: amber_converter.AmberfySettings, + input_glsl_source_json_path: Optional[Path] = None, +) -> Path: + return amber_converter.run_spirv_asm_shader_job_to_amber_script( + input_json, output_amber, amberfy_settings, input_glsl_source_json_path + ) + + +def spirv_dis_shader_job( + input_json: Path, output_json: Path, binary_paths: binaries_util.BinaryGetter +) -> Path: + return spirv_dis_util.run_spirv_shader_job_to_spirv_asm_shader_job( + input_json, + output_json, + binary_paths.get_binary_path_by_name(binaries_util.SPIRV_DIS_NAME).path, + ) + + +def spirv_opt_shader_job( + input_json: Path, + spirv_opt_args: List[str], + output_json: Path, + binary_paths: binaries_util.BinaryGetter, +) -> Path: + spirv_opt_binary = binary_paths.get_binary_path_by_name( + binaries_util.SPIRV_OPT_NAME + ) + return spirv_opt_util.run_spirv_opt_on_spirv_shader_job( + input_json, + output_json, + spirv_opt_args, + spirv_opt_binary.path, + binaries_util.SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG + in spirv_opt_binary.binary.tags, + ) + + +def glslang_glsl_shader_job_to_spirv( + input_json: Path, output_json: Path, binary_paths: binaries_util.BinaryGetter +) -> Path: + return glslang_validator_util.run_glslang_glsl_to_spirv_job( + input_json, + output_json, + binary_paths.get_binary_path_by_name(binaries_util.GLSLANG_VALIDATOR_NAME).path, + ) + + +def run_spirv_val_on_shader(shader_path: Path, spirv_val_path: Path) -> None: + subprocess_util.run( + util.prepend_catchsegv_if_available([str(spirv_val_path), str(shader_path)]) + ) + + +def validate_spirv_shader_job_helper(input_json: Path, spirv_val_path: Path) -> None: + shader_paths = shader_job_util.get_related_files( + input_json, shader_job_util.EXT_ALL, [shader_job_util.SUFFIX_SPIRV] + ) + for shader_path in shader_paths: + run_spirv_val_on_shader(shader_path, spirv_val_path) + + +def validate_spirv_shader_job( + input_json: Path, binary_paths: binaries_util.BinaryGetter +) -> None: + validate_spirv_shader_job_helper( + input_json, + binary_paths.get_binary_path_by_name(binaries_util.SPIRV_VAL_NAME).path, + ) + + +def glsl_shader_job_to_amber_script( + input_json: Path, + output_amber: Path, + work_dir: Path, + binary_paths: binaries_util.BinaryGetter, + amberfy_settings: amber_converter.AmberfySettings, + spirv_opt_args: Optional[List[str]] = None, +) -> Path: + + result = input_json + + # If GLSL: + if shader_job_util.get_related_suffixes_that_exist(input_json): + + result = shader_job_util.copy(result, work_dir / "0_glsl" / result.name) + + result = glslang_glsl_shader_job_to_spirv( + result, work_dir / "1_spirv" / result.name, binary_paths + ) + + # If SPIR-V: + elif shader_job_util.get_related_suffixes_that_exist( + input_json, language_suffix=[shader_job_util.SUFFIX_SPIRV] + ): + result = shader_job_util.copy( + result, + work_dir / "1_spirv" / result.name, + # Copy all spirv-fuzz related files too: + language_suffix=shader_job_util.SUFFIXES_SPIRV_FUZZ, + ) + else: + raise AssertionError(f"Unrecognized shader job type: {str(input_json)}") + + result_spirv = result + + result = spirv_dis_shader_job( + result, work_dir / "1_spirv_asm" / result.name, binary_paths + ) + + validate_spirv_shader_job(result_spirv, binary_paths) + + if spirv_opt_args: + result = result_spirv + result = spirv_opt_shader_job( + result, spirv_opt_args, work_dir / "2_spirv_opt" / result.name, binary_paths + ) + result_spirv = result + result = spirv_dis_shader_job( + result, work_dir / "2_spirv_opt_asm" / result.name, binary_paths + ) + + validate_spirv_shader_job(result_spirv, binary_paths) + + result = amberfy(result, output_amber, amberfy_settings, input_json) + + return result + + +def glsl_shader_job_crash_to_amber_script_for_google_cts( + input_json: Path, + output_amber: Path, + work_dir: Path, + short_description: str, + comment_text: str, + copyright_year: str, + extra_commands: str, + binary_paths: Optional[binaries_util.BinaryGetter] = None, + spirv_opt_args: Optional[List[str]] = None, + spirv_opt_hash: Optional[str] = None, + test_metadata_path: Optional[Path] = None, +) -> Path: + """ + Converts a GLSL shader job to an Amber script suitable for adding to the CTS. + + :param input_json: + :param output_amber: + :param work_dir: + :param short_description: One sentence, 58 characters max., no period, no line breaks. + :param comment_text: Why the test should pass. Can have line breaks. Ideally make sure lines are not too long. + :param copyright_year: + :param extra_commands: + :param binary_paths: + :param spirv_opt_args: + :param spirv_opt_hash: + :param test_metadata_path: + :return: + """ + if not binary_paths: + check( + bool(test_metadata_path), + AssertionError("Must have test_metadata_path or binary_paths"), + ) + assert test_metadata_path # noqa + binary_paths = binaries_util.BinaryManager( + binaries_util.BinaryManager.get_binary_list_from_test_metadata( + test_metadata_path + ) + ) + + if not spirv_opt_args: + check( + bool(test_metadata_path), + AssertionError("Must have test_metadata_path or binary_paths"), + ) + assert test_metadata_path # noqa + spirv_opt_args = list( + test_util.metadata_read_from_path(test_metadata_path).glsl.spirv_opt_args + ) + + if spirv_opt_args and not spirv_opt_hash: + spirv_opt_hash = binary_paths.get_binary_path_by_name( + binaries_util.SPIRV_OPT_NAME + ).binary.version + + return glsl_shader_job_to_amber_script( + input_json, + output_amber, + work_dir, + binary_paths, + amber_converter.AmberfySettings( + copyright_header_text=get_copyright_header_google(copyright_year), + add_graphics_fuzz_comment=True, + short_description=short_description, + comment_text=comment_text, + use_default_fence_timeout=True, + extra_commands=extra_commands, + spirv_opt_args=spirv_opt_args, + spirv_opt_hash=spirv_opt_hash, + ), + spirv_opt_args=spirv_opt_args, + ) diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py new file mode 100644 index 000000000..69bf6a7ed --- /dev/null +++ b/gfauto/gfauto/util.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""General utility module. + +Used for accessing files, file system operations like creating directories, copying, moving, etc., getting the full path +of a tool on the PATH, removing the beginning and/or end of string, and custom assert functions like check, +check_check_field_truthy, check_file_exists, etc. +""" + +import os +import pathlib +import platform +import shutil +from contextlib import contextmanager +from pathlib import Path +from typing import Any, BinaryIO, Iterator, List, TextIO, cast + +from gfauto import gflogging + +MIN_SIGNED_INT_32 = -pow(2, 31) +MAX_SIGNED_INT_32 = pow(2, 31) - 1 + +# Note: Could use the built-in |file.open| and |file.write_text|, etc. + + +def file_open_binary(file: pathlib.Path, mode: str) -> BinaryIO: # noqa VNE002 + check("b" in mode, AssertionError(f"|mode|(=={mode}) should contain 'b'")) + # Type hint (no runtime check). + result = cast(BinaryIO, open(str(file), mode)) + return result + + +def file_open_text(file: pathlib.Path, mode: str) -> TextIO: # noqa VNE002 + check("b" not in mode, AssertionError(f"|mode|(=={mode}) should not contain 'b'")) + if "w" in mode: + file_mkdirs_parent(file) + # Type hint (no runtime check). + result = cast(TextIO, open(str(file), mode, encoding="utf-8", errors="ignore")) + return result + + +def file_read_text_or_else(text_file: pathlib.Path, or_else: str) -> str: + try: + return file_read_text(text_file) + except IOError: + return or_else + + +def file_read_text(file: pathlib.Path) -> str: # noqa VNE002 + with file_open_text(file, "r") as f: + return f.read() + + +def file_read_lines(file: pathlib.Path) -> List[str]: # noqa VNE002 + with file_open_text(file, "r") as f: + return f.readlines() + + +def file_write_text(file: pathlib.Path, text: str) -> int: # noqa VNE002 + file_mkdirs_parent(file) + with file_open_text(file, "w") as f: + return f.write(text) + + +def mkdirs_p(path: pathlib.Path) -> Path: # noqa VNE002 + path.mkdir(parents=True, exist_ok=True) + return path + + +def file_mkdirs_parent(file: pathlib.Path) -> None: # noqa VNE002 + mkdirs_p(file.parent) + + +class ToolNotOnPathError(Exception): + pass + + +def tool_on_path(tool: str) -> pathlib.Path: # noqa VNE002 + result = shutil.which(tool) + if result is None: + raise ToolNotOnPathError( + "Could not find {} on PATH. Please add to PATH.".format(tool) + ) + return pathlib.Path(result) + + +def prepend_catchsegv_if_available(cmd: List[str]) -> List[str]: + try: + cmd.insert(0, str(tool_on_path("catchsegv"))) + except ToolNotOnPathError: + pass + + return cmd + + +def copy_file( + source_file_path: pathlib.Path, dest_file_path: pathlib.Path +) -> pathlib.Path: + file_mkdirs_parent(dest_file_path) + shutil.copy(str(source_file_path), str(dest_file_path)) + return dest_file_path + + +def copy_dir( + source_dir_path: pathlib.Path, dest_dir_path: pathlib.Path +) -> pathlib.Path: + file_mkdirs_parent(dest_dir_path) + shutil.copytree(source_dir_path, dest_dir_path) + return dest_dir_path + + +def move_file(source_path: Path, dest_path: Path) -> Path: + check_file_exists(source_path) + check( + not dest_path.is_dir(), + AssertionError( + f"Tried to move {str(source_path)} to a directory {str(dest_path)}" + ), + ) + file_mkdirs_parent(dest_path) + gflogging.log(f"Move file {str(source_path)} to {str(dest_path)}") + source_path.replace(dest_path) + return dest_path + + +def move_dir( + source_dir_path: pathlib.Path, dest_dir_path: pathlib.Path +) -> pathlib.Path: + file_mkdirs_parent(dest_dir_path) + shutil.move(source_dir_path, dest_dir_path) + return dest_dir_path + + +def make_directory_symlink(new_symlink_file_path: Path, existing_dir: Path) -> Path: + check(existing_dir.is_dir(), AssertionError(f"Not a directory: {existing_dir}")) + file_mkdirs_parent(new_symlink_file_path) + symlink_contents = os.path.relpath( + str(existing_dir), start=str(new_symlink_file_path.parent) + ) + new_symlink_file_path.symlink_to(symlink_contents, target_is_directory=True) + return new_symlink_file_path + + +def remove_start(string: str, start: str) -> str: + check( + string.startswith(start), AssertionError("|string| does not start with |start|") + ) + + return string[len(start) :] + + +def remove_end(str_in: str, str_end: str) -> str: + check( + str_in.endswith(str_end), + AssertionError(f"|str_in|(=={str_in}) should end with |str_end|(=={str_end})"), + ) + return str_in[: -len(str_end)] + + +def norm_path(path: pathlib.Path) -> pathlib.Path: # noqa VNE002 + return pathlib.Path(os.path.normpath(str(path))) + + +@contextmanager +def pushd(path: pathlib.Path) -> Iterator[None]: # noqa VNE002 + current_dir = pathlib.Path().resolve() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(str(current_dir)) + + +def check(condition: bool, exception: Exception) -> None: + if not condition: + raise exception + + +def check_field_truthy(field: Any, field_name: str) -> None: + if not field: + raise ValueError(f"{field_name}(={str(field)}) must be filled") + + +def check_file_exists(path: Path) -> None: + check(path.is_file(), FileNotFoundError(f"Could not find file {str(path)}")) + + +def check_dir_exists(path: Path) -> None: + check(path.is_dir(), FileNotFoundError(f"Could not find directory {str(path)}")) + + +def get_platform() -> str: + host = platform.system() + if host in ("Linux", "Windows"): + return host + if host == "Darwin": + return "Mac" + raise AssertionError("Unsupported platform: {}".format(host)) diff --git a/gfauto/mypy.ini b/gfauto/mypy.ini new file mode 100644 index 000000000..f71b7730b --- /dev/null +++ b/gfauto/mypy.ini @@ -0,0 +1,16 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[mypy] +python_version = 3.6 diff --git a/gfauto/pylintrc b/gfauto/pylintrc new file mode 100644 index 000000000..7f81be881 --- /dev/null +++ b/gfauto/pylintrc @@ -0,0 +1,519 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns=.*_pb2.* + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=C0330, + C0111, + W0511, # Allow to-dos. + E1101, # Missing member; mypy (type checking) should cover this. + R0913, # Too many arguments; could re-enable this. + C0301, # Line too long; we use an auto formatter. + R0903, # too-few-public-methods; good, but I want to store data in classes in case I later add methods. + ungrouped-imports, # We use an imports formatter. + cyclic-import, # We allow cyclic imports, but we should import modules, not functions and variables. + duplicate-code, # Unfortunately, disabling this on a case-by-case basis is broken: https://github.com/PyCQA/pylint/issues/214 + + + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package.. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, while `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[STRING] + +# This flag controls whether the implicit-str-concat-in-sequence should +# generate a warning on implicit string concatenation in sequences defined over +# several lines. +check-str-concat-over-line-jumps=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=yes # This helps. + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _, + f # Good for file handles. + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement. +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/gfauto/pyproject.toml b/gfauto/pyproject.toml new file mode 100644 index 000000000..bcc226527 --- /dev/null +++ b/gfauto/pyproject.toml @@ -0,0 +1,29 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ['wheel', 'setuptools'] +build-backend = 'setuptools.build_meta' + +[tool.black] +line-length = 88 +target-version = ['py36', 'py37', 'py38'] + +# Black currently matches against full paths, so we only match one level deep +# to ensure only files in gfauto (and not in the current directory) are +# matched. +# https://github.com/python/black/issues/712 + +include = '.*[.]py$' +exclude = '__pycache__|_pb2[.]py' diff --git a/gfauto/run_protoc.sh b/gfauto/run_protoc.sh new file mode 100755 index 000000000..15b642696 --- /dev/null +++ b/gfauto/run_protoc.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -u + +python -m grpc.tools.protoc --python_out=. --proto_path=. --mypy_out=. gfauto/*.proto + +# protoc gfauto/artifact.proto gfauto/recipe.proto --python_out=. --plugin=protoc-gen-mypy=github/mypy-protobuf/python/protoc-gen-mypy --mypy_out=. --proto_path=. diff --git a/gfauto/setup.cfg b/gfauto/setup.cfg new file mode 100644 index 000000000..a26d25184 --- /dev/null +++ b/gfauto/setup.cfg @@ -0,0 +1,26 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[metadata] +description-file = README.md + +# This affects pep8.py, which in turn affects PyCharm/IntelliJ PEP8 warnings. +[pep8] +ignore = E203,W503,E501 + +# pycodestyle is the new name and version of pep8.py. I'm not sure if these are +# actually used by flake8 or PyCharm/IntelliJ, but they should be kept in sync +# with [pep8] above and the config in .flake8. +[pycodestyle] +ignore = E203,W503,E501 diff --git a/gfauto/setup.py b/gfauto/setup.py new file mode 100644 index 000000000..04bf6b02c --- /dev/null +++ b/gfauto/setup.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + from setuptools import setup, Extension +except Exception: + from distutils.core import setup, Extension + + +setup( + name="gfauto", + version=0.9, + description="GraphicsFuzz auto.", + keywords="GraphicsFuzz fuzzing GLSL SPIRV", + author="The GraphicsFuzz Project Authors", + author_email="android-graphics-tools-team@google.com", + url="https://github.com/google/graphicsfuzz", + license="Apache License 2.0", + packages=["gfauto"], + python_requires=">=3.6", + install_requires=["protobuf"], + package_data={"gfauto": ["*.proto", "*.pyi"]}, + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved::Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3 :: Only", + ], + entry_points={"console_scripts": [ + "gfauto_fuzz = gfauto.fuzz:main", + "gfauto_interestingness_test = gfauto.gfauto_interestingness_test:main", + "gfauto_write_device_file = gfauto.devices_util:write_device_file", + "add_amber_tests_to_cts = gfauto.add_amber_tests_to_cts:main", + "gfauto_test_update_binaries = gfauto.test_update_binaries:main", + "gfauto_test_create_readme = gfauto.test_create_readme:main", + ]}, +) diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic new file mode 100644 index 000000000..b08b455dd --- /dev/null +++ b/gfauto/whitelist.dic @@ -0,0 +1,166 @@ + +Interm +Builtins +Loc +8spvtools3opt21 +Analysis16 +0x369 +0x5bd6d9 +00000000009d9c34 +0x1d537d +0x7f51ebd1237d +0x1d537d + + +'00000000009d9c34' +'variable +array' +inux +tructured +nalysis16 +erge +false' +creen +ull + + +amberscript +arg +argparse +argv +asm +bak +basename +catchsegv +chdir +cmd +contextlib +contextmanager +copyfile +cpp +dest +dirname +expandtabs +func +gflogging +gl +glsl +glslang +isdir +isfile +isort +mkdir +mkdirs +nargs +noqa +normpath +Oneof +pathlib +pb2 +Popen +posix +proto +python3 +readline +RETURNCODE +returncode +rstrip +shader +shaders +shutil +spirv +spitext +splitext +spv +stderr +stdout +subn +subprocess +usr +util +validator +vk +runtime +readlines +pylint +num +uuid +uuid4 +truthy +graphicsfuzz +randint +amberfy +zipfile +urlretrieve +infolist +attr +chmod +amberfy +amberscriptify +ssbo +SSBO +subtest +interestingness +vulkancts +mustpass +dumpsys +logcat +Vulkan +gfauto +protobuf +SPIR +paulthomson +msan +spvtools +swiftshader +libvk +RELWITHDEBINFO +vkscript +subpath +myuniform +ivec +subdata +fbsize +passthrough +pushd +reinstalls +icd +Getter +prepend +splitlines +adb +namespace +Baz +const +tmp +Backtrace +noinspection +Preprocess +preprocessor +dirs +png +txt +rglob +copytree +symlink +relpath +issubset +exe +repo +environ +subdirectory +killpg +iterdir +unreduced +Pseudocode +ssert +nondeterministic +keyevent +libc +alloc +0th +getrandbits +randbits +jsons +spvt + From 0b31b5ed81608e434ae88a5bfbe3a6ab13e02681 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 14 Aug 2019 11:06:29 -0500 Subject: [PATCH 100/180] Add function to make opaque zero vec3 from cross product of equal vec3s (#682) Fixes #636. --- .../fuzzer/OpaqueExpressionGenerator.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 5db3f07ce..3c6031e40 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -38,6 +38,7 @@ import com.graphicsfuzz.common.util.OpenGlConstants; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.SideEffectChecker; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; import com.graphicsfuzz.generator.transformation.ExpressionIdentity; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.util.Constants; @@ -121,6 +122,7 @@ List waysToMakeZero() { opaqueZeroFactories.add(this::opaqueZeroLogarithm); opaqueZeroFactories.add(this::opaqueZeroTan); opaqueZeroFactories.add(this::opaqueZeroVectorLength); + opaqueZeroFactories.add(this::opaqueZeroVectorCross); return opaqueZeroFactories; } @@ -571,6 +573,55 @@ private Optional opaqueZeroVectorLength(BasicType type, boolean constConte fuzzer))); } + /** + * Function to create an opaque zero (specifically an opaque zero vec3) by taking the cross + * product of two equivalent vec3s. This opaque function relies on the fact that if u == v, where + * u and v are vec3, then cross(u, v) = (0, 0, 0). These equivalent vec3s are produced by + * fuzzing random float values and applying different identity functions to them per vector. + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque zero vec3 + * made from the cross product of two equal vec3s. + */ + private Optional opaqueZeroVectorCross(BasicType type, boolean constContext, + final int depth, + Fuzzer fuzzer, boolean isZero) { + assert isZero; + if (type != BasicType.VEC3) { + // Cross product only works on vec3 and can only produce vec3 as a return type. + return Optional.empty(); + } + // We only want literals, so we need to make our own LiteralFuzzer rather than use the supplied + // Fuzzer object. + final LiteralFuzzer litFuzzer = new LiteralFuzzer(generator); + // cross(vec3(firstVec3ConstructorArgs), vec3(secondVec3ConstructorArgs)) + final List firstVec3ConstructorArgs = new ArrayList(); + final List secondVec3ConstructorArgs = new ArrayList(); + for (int i = 0; i < type.getNumElements(); i++) { + final Optional maybeFuzzedFloatLiteral = litFuzzer.fuzz(type.getElementType()); + // Something went horribly wrong if we got an Optional.empty() - we are guaranteed to obtain a + // fuzzed expression, since the element type of a vec3 is a float, and it is always possible + // to fuzz a float literal. + assert maybeFuzzedFloatLiteral.isPresent(); + final Expr fuzzedFloatLiteral = maybeFuzzedFloatLiteral.get(); + // We apply different identities to the literals per vector - the two vectors are + // semantically the same vector (and so cross() should still return the zero vector). + firstVec3ConstructorArgs.add( + applyIdentityFunction( + fuzzedFloatLiteral, type.getElementType(), constContext, depth, fuzzer)); + secondVec3ConstructorArgs.add( + applyIdentityFunction( + fuzzedFloatLiteral.clone(), type.getElementType(), constContext, depth, fuzzer)); + } + return Optional.of( + new FunctionCallExpr("cross", + new TypeConstructorExpr(type.toString(), firstVec3ConstructorArgs), + new TypeConstructorExpr(type.toString(), secondVec3ConstructorArgs))); + } + private Optional opaqueOneExponential(BasicType type, boolean constContext, final int depth, Fuzzer fuzzer, boolean isZero) { // represent 1 as the exponential function of opaqueZero, e.g. exp(0.0) From 9c316e29f7a7d8d03a01159a32ac3ae5a7b54d67 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Wed, 14 Aug 2019 11:07:26 -0500 Subject: [PATCH 101/180] Add new versions of GraphicsFuzz shaders that use Integer Functions (#605) Implements #543. This PR adds modified versions of the default GraphicsFuzz shaders to the 310es directory that use GLES features not found in GLES 1.00, including do/while loops, while loops and Integer Functions. --- .../310es/bubblesort_flag_intfunctions.frag | 70 +++++++++ .../310es/bubblesort_flag_intfunctions.json | 16 ++ .../310es/colorgrid_modulo_intfunctions.frag | 65 ++++++++ .../310es/colorgrid_modulo_intfunctions.json | 9 ++ .../310es/mandelbrot_blurry_intfunctions.frag | 82 ++++++++++ .../310es/mandelbrot_blurry_intfunctions.json | 9 ++ .../310es/prefix_sum_intfunctions.frag | 84 ++++++++++ .../310es/prefix_sum_intfunctions.json | 9 ++ .../samples/310es/squares_intfunctions.frag | 144 ++++++++++++++++++ .../samples/310es/squares_intfunctions.json | 22 +++ 10 files changed, 510 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.frag create mode 100644 shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.json create mode 100644 shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.frag create mode 100644 shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.json create mode 100644 shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.frag create mode 100644 shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.json create mode 100644 shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.frag create mode 100644 shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.json create mode 100644 shaders/src/main/glsl/samples/310es/squares_intfunctions.frag create mode 100644 shaders/src/main/glsl/samples/310es/squares_intfunctions.json diff --git a/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.frag b/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.frag new file mode 100644 index 000000000..b6af4ef77 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.frag @@ -0,0 +1,70 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +bool checkSwap(float a, float b) +{ + return gl_FragCoord.y < resolution.y / 2.0 ? a > b : a < b; +} +void main() +{ + int msb10 = 1024; + int msb9 = 512; + uint uselessOutVariable; + float data[10]; + for(int i = bitfieldReverse(int(injectionSwitch.x)); i < findMSB(msb10); i++) + { + data[i] = float(usubBorrow(uint(10), uint(i), uselessOutVariable)) * injectionSwitch.y; + } + int i = bitfieldExtract(int(injectionSwitch.x), bitCount(0), int(injectionSwitch.x)); + do + { + for(int j = bitfieldExtract(int(injectionSwitch.x), 0, 0); j < findLSB(msb10); j++) + { + if(uint(j) < uaddCarry(uint(i), 1u, uselessOutVariable)) + { + continue; + } + bool doSwap = checkSwap(data[i], data[j]); + if(doSwap) + { + float temp = data[i]; + data[i] = data[j]; + data[j] = temp; + } + } + i++; + } while(i < findMSB(msb9)); + if(gl_FragCoord.x < resolution.x / 2.0) + { + _GLF_color = vec4(data[findMSB(1)] / 10.0, data[findLSB(32)] / 10.0, data[findMSB(msb9)] / 10.0, 1.0); + } + else + { + _GLF_color = vec4(data[findLSB(32)] / 10.0, data[findMSB(msb9)] / 10.0, data[findMSB(1)] / 10.0, 1.0); + } +} + diff --git a/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.json b/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.frag b/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.frag new file mode 100644 index 000000000..e16d678ff --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.frag @@ -0,0 +1,65 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +int msb8 = 256; + +float nb_mod(float limit, float ref) { + int msb8 = 256; + float s = float(bitfieldExtract(int(resolution.x), 0, 0)); + int i = bitCount(msb8); + while(i < bitfieldInsert(800, i, 0, 0)) { + if (mod(float(i), ref) <= 0.01) { + s += 0.2; + } + if (float(i) >= limit) { + return s; + } + i++; + } + return s; +} + +void main() +{ + int msb8 = 256; + vec4 c = vec4(bitfieldExtract(0, 0, 0), 0.0, 0.0, bitCount(msb8)); + float ref = floor(resolution.x / float(findLSB(msb8))); + + c.x = nb_mod(gl_FragCoord.x, ref); + c.y = nb_mod(gl_FragCoord.y, ref); + c.z = c.x + c.y; + int i = bitfieldReverse(bitfieldExtract(0, 0, 0)); + do { + if (c[i] >= 1.0) { + c[i] = c[i] * c[i]; + } + i++; + } while (i < findMSB(findLSB(msb8))); + c.x = mod(c.x, 1.0); + c.y = mod(c.y, 1.0); + c.z = mod(c.z, 1.0); + _GLF_color = c; +} + diff --git a/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.json b/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.json new file mode 100644 index 000000000..269b946d9 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.frag b/shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.frag new file mode 100644 index 000000000..87e3a039e --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.frag @@ -0,0 +1,82 @@ +#version 310 es + +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +vec3 pickColor(int i) { + return vec3(float(i) / 50.0, float(i) / 120.0, float(i) / 140.0); +} + +vec3 mand(float xCoord, float yCoord) { + float height = resolution.y; + float width = resolution.x; + + float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float x = 0.0, y = 0.0; + if (0.0 > resolution.x) + { + x = 1.0; + y = 1.0; + } + int iteration = bitfieldReverse(int(x)); + int k = bitfieldExtract(int(y), bitCount(int(x)), int(y)); + int iterationCap = 1000; + do { + if (x*x+y*y > 4.0) { + break; + } + float x_new = x*x - y*y + c_re; + y = 2.0*x*y + c_im; + x = x_new; + iteration++; + k++; + } while(k < bitfieldInsert(iterationCap + (257.0 > resolution.y ? 1 : 0), 0, 0, 0)); + if (0.0 > resolution.y) + { + iterationCap += 1; + } + if (iteration < bitfieldInsert(iterationCap, 0, 0, 0)) { + return pickColor(iteration); + } else { + return vec3(0.0); + } +} + +void main() { + int msb16 = 65536; + uint uselessOutVariable; + vec3 data[16]; + for (int i = 0; i < findMSB(16); i++) { + for (int j = 0; j < findLSB(16); j++) { + data[uaddCarry(uint(4*j), uint(i), uselessOutVariable)] = mand(gl_FragCoord.x + float(i - bitCount(1)), gl_FragCoord.y + float(j - bitCount(1))); + } + } + vec3 sum = vec3(0.0); + for (int i = bitfieldReverse(0); i < findMSB(msb16); i++) { + sum += data[i]; + } + sum /= vec3(16.0); + _GLF_color = vec4(sum, 1.0); +} + diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.json b/shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.json new file mode 100644 index 000000000..28c3c3e61 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.frag b/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.frag new file mode 100644 index 000000000..0eeca69ef --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.frag @@ -0,0 +1,84 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +void main(void) { + int msb9 = 512; + int msb10 = 1024; + int msb14 = 16384; + int msb15 = 32768; + int msb19 = 524288; + int msb20 = 1048576; + int msb24 = 16777216; + int msb25 = 33554432; + int msb29 = 536870912; + int msb30 = 1073741824; + uint uselessOutVariable; + float A[50]; + int i = bitfieldExtract(0, 0, 0); + do { + if (i >= int(resolution.x)) { + break; + } + if (findLSB(16) * (i/findMSB(16)) == i) { + A[i/findLSB(16)] = float(i); + } + i++; + } while(i < bitfieldInsert(200, 0, 0, 0)); + i = findLSB(0); + do { + if (i < int(gl_FragCoord.x)) { + break; + } + if (i > findMSB(0)) { + A[i] += A[int(usubBorrow(uint(i), 1u, uselessOutVariable))]; + } + i++; + } while(i < bitfieldInsert(50, 0, 0, 0)); + if (int(gl_FragCoord.x) < findLSB(msb20)) { + _GLF_color = vec4(A[bitfieldReverse(0)]/resolution.x, A[findMSB(16)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(40, 0, 0, 0)) { + _GLF_color = vec4(A[findLSB(32)]/resolution.x, A[findMSB(msb9)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(60, 0, 0, 0)) { + _GLF_color = vec4(A[findMSB(msb10)]/resolution.x, A[findLSB(msb14)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(80, 0, 0, 0)) { + _GLF_color = vec4(A[findLSB(msb15)]/resolution.x, A[findMSB(msb19)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(100, 0, 0, 0)) { + _GLF_color = vec4(A[findMSB(msb20)]/resolution.x, A[findLSB(msb24)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(120, 0, 0, 0)) { + _GLF_color = vec4(A[findLSB(msb25)]/resolution.x, A[findMSB(msb29)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(140, 0, 0, 0)) { + _GLF_color = vec4(A[findMSB(msb30)]/resolution.x, A[bitfieldInsert(34, 0, 0, 0)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(160, 0, 0, 0)) { + _GLF_color = vec4(A[bitfieldInsert(35, 0, 0, 0)]/resolution.x, A[bitfieldInsert(39, 0, 0, 0)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(180, 0, 0, 0)) { + _GLF_color = vec4(A[bitfieldInsert(40, 0, 0, 0)]/resolution.x, A[bitfieldInsert(44, 0, 0, 0)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(180, 0, 0, 0)) { + _GLF_color = vec4(A[bitfieldInsert(45, 0, 0, 0)]/resolution.x, A[bitfieldInsert(49, 0, 0, 0)]/resolution.y, 1.0, 1.0); + } else { + discard; + } + +} + diff --git a/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.json b/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.json new file mode 100644 index 000000000..000b854cf --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/squares_intfunctions.frag b/shaders/src/main/glsl/samples/310es/squares_intfunctions.frag new file mode 100644 index 000000000..2f92a875d --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/squares_intfunctions.frag @@ -0,0 +1,144 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform float time; + +uniform vec2 resolution; + +float h_r; +float s_g; +float b_b; + +void doConvert() +{ + int msb8 = 256; + vec3 temp; + temp = b_b * (float(bitCount(msb8)) - s_g) + (b_b - b_b * (float(bitCount(msb8)) - s_g)) * clamp(abs(abs(6.0 * (h_r - vec3(bitfieldReverse(0), bitCount(msb8), 2) / 3.0)) - 3.0) - float(bitCount(msb8)), float(bitfieldExtract(int(resolution.x), 0, 0)), float(bitCount(int(resolution.x)))); + h_r = temp.x; + s_g = temp.y; + b_b = temp.z; +} + +vec3 computeColor(float c, vec2 position) +{ + int msb8 = 256; + h_r = fract(c); + s_g = float(bitCount(msb8)); + b_b = (0.5 + (sin(time) * 0.5 + 0.5)); + doConvert(); + s_g *= float(bitCount(msb8)) / position.y; + h_r *= float(bitCount(msb8)) / position.x; + if (abs(position.y - position.x) < 0.5) { + b_b = clamp(float(bitfieldExtract(int(resolution.x), 0, 0)), float(bitCount(msb8)), b_b * float(bitCount(msb8)) * 3.0); + } + return vec3(h_r, s_g, b_b); +} + +vec3 defaultColor() { + return vec3(float(bitfieldExtract(int(resolution.y), 0, 0))); +} + +vec3 drawShape(vec2 pos, vec2 square, vec3 setting) +{ + int msb8 = 256; + bool c1 = pos.x - setting.x < square.x; + if (!c1) { + return defaultColor(); + } + bool c2 = pos.x + setting.x > square.x; + if (!c2) { + return defaultColor(); + } + bool c3 = pos.y - setting.x < square.y; + if (!c3) { + return defaultColor(); + } + bool c4 = pos.y + setting.x > square.y; + if (!c4) { + return defaultColor(); + } + + bool c5 = pos.x - (setting.x - setting.y) < square.x; + if (!c5) { + return computeColor(setting.z / (5.0 * float(findMSB(msb8))), pos); + } + bool c6 = pos.x + (setting.x - setting.y) > square.x; + if (!c6) { + return computeColor(setting.z / (5.0 * float(findMSB(msb8))), pos); + } + bool c7 = pos.y - (setting.x - setting.y) < square.y; + if (!c7) { + return computeColor(setting.z / (5.0 * float(findMSB(msb8))), pos); + } + bool c8 = pos.y + (setting.x - setting.y) > square.y; + if (!c8) { + return computeColor(setting.z / (5.0 * float(findMSB(msb8))), pos); + } + return defaultColor(); +} + +vec3 computePoint(mat2 rotationMatrix) +{ + int msb8 = 256; + vec2 aspect; + aspect = resolution.xy / min(resolution.x, resolution.y); + vec2 position; + position = (gl_FragCoord.xy / resolution.xy) * aspect; + vec2 center; + center = vec2(0.5) * aspect; + position *= rotationMatrix; + center *= rotationMatrix; + vec3 result = vec3(0.0); + for(int i = bitfieldInsert(35, 0, 0, bitfieldExtract(int(resolution.x), 0, 0)); i >= bitfieldReverse(bitfieldExtract(int(resolution.x), 0, 0)); i --) + { + vec3 d; + d = drawShape(position, center + vec2(sin(float(i) / (float(bitCount(msb8)) * 10.0) + time) / 4.0 * float(bitCount(msb8)), float(bitfieldExtract(int(resolution.x), 0, 0))), vec3(0.01 + sin(float(i) / 100.0 * float(bitCount(msb8))), 0.01, float(i))); + if(length(d) <= float(bitfieldExtract(int(resolution.x), 0, 0))) { + continue; + } + result = vec3(d); + } + return result; +} + +void main() { + int msb8 = 256; + float angle; + angle = sin(time) * 0.1; + mat2 rotationMatrix; + rotationMatrix = mat2(sin(angle), - cos(angle), cos(angle), sin(angle)); + vec3 point1; + point1 = computePoint(rotationMatrix); + mat2 rotationMatrix2; + rotationMatrix2 = rotationMatrix * rotationMatrix; + vec3 point2; + point2 = computePoint(rotationMatrix2); + mat2 rotationMatrix3; + rotationMatrix3 = rotationMatrix * rotationMatrix * rotationMatrix; + vec3 point3; + point3 = computePoint(rotationMatrix3); + vec3 mixed; + mixed = mix(point1, point2, vec3(0.3)); + mixed = mix(mixed, point3, vec3(0.3)); + _GLF_color = vec4(mixed, float(bitCount(msb8))); +} + diff --git a/shaders/src/main/glsl/samples/310es/squares_intfunctions.json b/shaders/src/main/glsl/samples/310es/squares_intfunctions.json new file mode 100644 index 000000000..14446b79a --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/squares_intfunctions.json @@ -0,0 +1,22 @@ +{ + "time": { + "func": "glUniform1f", + "args": [ + 0.0 + ] + }, + "mouse": { + "func": "glUniform2f", + "args": [ + 0.0, + 0.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From ff86f6c9698246b57cbcc9dcf82de09d3bfc57d4 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Thu, 15 Aug 2019 17:16:49 +0700 Subject: [PATCH 102/180] Expression Generator: find numbers for addition based on known facts (#681) Adds a new case for choosing numbers for the addition based on known facts held by the fact manager. Also refactors a util method that previously provides a pair sum of two numbers that guarantees to compute the expected value. Fixes #663. --- .../ExpressionGenerator.java | 116 +++++++++--------- .../knownvaluegeneration/FactManager.java | 16 +++ 2 files changed, 74 insertions(+), 58 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java index 5b486edb2..255779d5a 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java @@ -52,8 +52,6 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; public class ExpressionGenerator { @@ -290,6 +288,16 @@ private Expr generateForLoopValue(Value value, return new VariableIdentifierExpr(varName); } + // A utility method performing the subtraction of the interface Number. + private Number subtractNumbers(Number firstOperand, Number secondOperand) { + if (firstOperand instanceof Float) { + assert secondOperand instanceof Float; + return firstOperand.floatValue() - secondOperand.floatValue(); + } + assert firstOperand instanceof Integer && secondOperand instanceof Integer; + return firstOperand.intValue() - secondOperand.intValue(); + } + /** * This method generates the expression that performs the addition of two values which result is * equal to the given value. @@ -315,15 +323,53 @@ private Expr generateAdditionValue(Value value, if (value.getType() != BasicType.INT && value.getType() != BasicType.FLOAT) { return null; } + // If the given value is unknown, we are free to choose any arbitrary numbers for addition. + if (value.valueIsUnknown()) { + return new BinaryExpr( + generateExpr(factManager, currentFunction, stmtToInsertBefore, + new NumericValue((BasicType) value.getType(), Optional.empty())), + generateExpr(factManager, currentFunction, stmtToInsertBefore, + new NumericValue((BasicType) value.getType(), Optional.empty())), + BinOp.ADD); + } + + Number summandA; + // Given the expected type, we have to retrieve all values from the fact manager and filter only + // the facts that are known to compute particular values. + final List knownValues = factManager.getValuesFromType(value.getType()) + .stream().filter(item -> !item.valueIsUnknown()).collect(Collectors.toList()); + boolean genSummandsFromKnownValues = generator.nextBoolean(); + // If we are able to find any known values with the correct type, we then consider choosing + // summands based on such values. Otherwise, we will have to pick a random number to make an + // addition expression. + if (!knownValues.isEmpty() && genSummandsFromKnownValues) { + summandA = ((NumericValue) knownValues + .get(generator.nextInt(knownValues.size()))) + .getValue().get(); + } else { + if (value.getType() == BasicType.FLOAT) { + // TODO(https://github.com/google/graphicsfuzz/issues/688): range of numbers [-10, 10] is + // temporarily used here, we have to change how the summand is generated. + summandA = (float) generator.nextInt(21) - 10; + } else { + summandA = generator.nextInt(INT_MAX); + } + } + + final Number expected = ((NumericValue) value).getValue().get(); + // To get the second summand, we subtract original value by the the first summand. + final Number summandB = subtractNumbers(expected, summandA); + // Randomly decide whether summandA or summandB should be the first summand. + final boolean summandAFirst = generator.nextBoolean(); + final Number firstSummand = summandAFirst ? summandA : summandB; + final Number secondSummand = summandAFirst ? summandB : summandA; - final Pair, Optional> pair = getPairSum(value); return new BinaryExpr( generateExpr(factManager, currentFunction, stmtToInsertBefore, - new NumericValue((BasicType) value.getType(), pair.getLeft())), + new NumericValue((BasicType) value.getType(), Optional.of(firstSummand))), generateExpr(factManager, currentFunction, stmtToInsertBefore, - new NumericValue((BasicType) value.getType(), pair.getRight())), - BinOp.ADD - ); + new NumericValue((BasicType) value.getType(), Optional.of(secondSummand))), + BinOp.ADD); } /** @@ -429,13 +475,14 @@ private String genFunctionName(Value value) { + freshId(); } - private String genParamName(Type type, boolean unknownValue) { + private String genParamName(Value value) { // Provides name for a function arguments based on the given value. // For example: // _GLF_UNKNOWN_PARAM_vec4_id_1: a parameter of unknown value of vec4 type. // _GLF_PARAM_int_id_62: a parameter of integer type. - return (unknownValue ? Constants.GLF_UNKNOWN_PARAM : Constants.GLF_PARAM) - + "_" + type.toString() + return (value.valueIsUnknown() ? Constants.GLF_UNKNOWN_PARAM : Constants.GLF_PARAM) + + "_" + value.getType().toString() + + parseNameFromValue(value) + freshId(); } @@ -580,7 +627,7 @@ private Expr generateFunctionFact(Value value, // when calling this function the fact manager will generate any arbitrary value that // matches the parameter type. final Value paramValue = fuzzValue(paramType); - final String paramName = genParamName(paramType, paramValue.valueIsUnknown()); + final String paramName = genParamName(paramValue); argumentValues.add(paramValue); final ParameterDecl parameterDecl = new ParameterDecl( @@ -675,51 +722,4 @@ private Value fuzzValue(Type type) { throw new RuntimeException("Not implemented yet!"); } - /** - * A utility method that returns a pair of two numbers whose sum is equal to the given value. - * - * @param value the original value that will be split into two numbers. - * @return if value is unknown, returns a pair of empty value. Otherwise find and return two - * numbers that will add up to the given value. - */ - public Pair, Optional> getPairSum(Value value) { - assert value instanceof NumericValue; - final NumericValue numericValue = (NumericValue) value; - - if (numericValue.valueIsUnknown()) { - return new ImmutablePair<>(Optional.empty(), Optional.empty()); - } - - // Following the equation a = (a/b) + (a - a/b), we first need to pick a random integer b - // in range 1-10. We then derive the left number by dividing the original number a with the - // random number b. We then generate the right number by subtracting the original value a - // with the left number obtained from the previous step. - // - // For example, if a number 7 is an input and we pick a random integer 3. The left number - // is (7/3) which equals to 2. Next we subtract the original value with the left number - // to find the right number: 7 - 2 = 5. We finally have two numbers 2 and 5 that can add - // up to 7. - - // TODO(https://github.com/google/graphicsfuzz/issues/663): we can derive a number based on the - // known facts hold by FactManager here. - - if (numericValue.getType() == BasicType.FLOAT) { - final float a = numericValue.getValue().get().floatValue(); - final float b = Math.max(1, generator.nextInt(10)); - final float left = a / b; - final float right = a - left; - return new ImmutablePair<>(Optional.of(left), Optional.of(right)); - } - - if (numericValue.getType() == BasicType.INT) { - final int a = numericValue.getValue().get().intValue(); - final int b = Math.max(1, generator.nextInt(10)); - final int left = a / b; - final int right = a - left; - return new ImmutablePair<>(Optional.of(left), Optional.of(right)); - } - - throw new RuntimeException("Should be unreachable"); - } - } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java index 58cc4f712..e0565a12c 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java @@ -16,11 +16,14 @@ package com.graphicsfuzz.generator.knownvaluegeneration; +import com.graphicsfuzz.common.ast.type.Type; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class FactManager { @@ -35,6 +38,19 @@ public FactManager(FactManager prototype) { functionFacts = new HashMap>(); } + /** + * Given an expected type, this method gets available function and variable facts hold by + * this fact manager. + * + * @param type type of the facts we are going to search for. + * @return a list of values that matches the given type. + */ + public List getValuesFromType(Type type) { + return Stream.concat(variableFacts.keySet().stream(), functionFacts.keySet().stream()) + .filter(value -> value.getType() == type) + .collect(Collectors.toList()); + } + /** * Creates a new fact manager where this fact manager is used as a prototype. We basically call * this method only when generating a new function where the return fact manager is non global. From b4e77f9af2d0956656b19a2ed8ae427d49de70db Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 15 Aug 2019 16:32:48 +0100 Subject: [PATCH 103/180] gfauto: use AmberScript instead of VkScript (#689) --- gfauto/.flake8 | 2 + gfauto/gfauto/amber_converter.py | 730 ++++++++++++++++++---------- gfauto/gfauto/test_create_readme.py | 23 - gfauto/gfauto/tool.py | 11 +- gfauto/whitelist.dic | 9 +- 5 files changed, 486 insertions(+), 289 deletions(-) diff --git a/gfauto/.flake8 b/gfauto/.flake8 index fb740b865..bb9bcdbae 100644 --- a/gfauto/.flake8 +++ b/gfauto/.flake8 @@ -53,6 +53,8 @@ ignore = # Allow unindexed string format parameters P101 + P102 + P103 # For flake8-quotes: diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py index fa66ff76e..9ec2ff895 100644 --- a/gfauto/gfauto/amber_converter.py +++ b/gfauto/gfauto/amber_converter.py @@ -19,10 +19,15 @@ Converts a SPIR-V assembly shader job (all shaders are already disassembled) to an Amber script file. """ +import itertools import json import pathlib from copy import copy -from typing import List, Optional +from enum import Enum +from pathlib import Path +from typing import Dict, List, Optional + +import attr from gfauto import shader_job_util, util from gfauto.gflogging import log @@ -31,28 +36,17 @@ AMBER_FENCE_TIMEOUT_MS = 60000 +@attr.dataclass class AmberfySettings: # pylint: disable=too-many-instance-attributes - def __init__( - self, - copyright_header_text: Optional[str] = None, - add_generated_comment: bool = False, - add_graphics_fuzz_comment: bool = False, - short_description: Optional[str] = None, - comment_text: Optional[str] = None, - use_default_fence_timeout: bool = False, - extra_commands: Optional[str] = None, - spirv_opt_args: Optional[List[str]] = None, - spirv_opt_hash: Optional[str] = None, - ): - self.copyright_header_text = copyright_header_text - self.add_generated_comment = add_generated_comment - self.add_graphics_fuzz_comment = add_graphics_fuzz_comment - self.short_description = short_description - self.comment_text = comment_text - self.use_default_fence_timeout = use_default_fence_timeout - self.extra_commands = extra_commands - self.spirv_opt_args = spirv_opt_args - self.spirv_opt_hash = spirv_opt_hash + copyright_header_text: Optional[str] = None + add_generated_comment: bool = False + add_graphics_fuzz_comment: bool = False + short_description: Optional[str] = None + comment_text: Optional[str] = None + use_default_fence_timeout: bool = False + extra_commands: Optional[str] = None + spirv_opt_args: Optional[List[str]] = None + spirv_opt_hash: Optional[str] = None def copy(self: "AmberfySettings") -> "AmberfySettings": # A shallow copy is adequate. @@ -86,9 +80,147 @@ def get_text_as_comment(text: str) -> str: return "\n".join(lines) -def uniform_json_to_amberscript(uniform_json_contents: str) -> str: +def amberscript_comp_buff_def(comp_json: str, make_empty_buffer: bool = False) -> str: + """ + Returns a string (template) containing AmberScript commands for defining the initial in/out buffer. + + Only the "$compute" key is read. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + + "$compute": { + "num_groups": [12, 13, 14]; + "buffer": { + "binding": 123, + "fields": + [ + { "type": "int", "data": [ 0 ] }, + { "type": "int", "data": [ 1, 2 ] }, + ] + } + } + + } + + becomes: + + BUFFER {} DATA_TYPE int DATA + 0 1 2 + END + + Or, if |make_empty_buffer| is True: + + BUFFER {} DATA_TYPE int SIZE 3 0 + + + :param comp_json: The shader job JSON as a string. + :param make_empty_buffer: If true, an "empty" buffer is created that is of the same size and type as the normal + in/out buffer; the empty buffer can be used to store the contents of the in/out buffer via the Amber COPY command. + The only difference is the "empty" buffer is initially filled with just one value, which avoids redundantly + listing hundreds of values that will just be overwritten, and makes it clear(er) for those reading the AmberScript + file that the initial state of the buffer is unused. + """ + ssbo_types = { + "int": "int32", + "ivec2": "vec2", + "ivec3": "vec3", + "ivec4": "vec4", + "uint": "uint32", + "float": "float", + "vec2": "vec2", + "vec3": "vec3", + "vec4": "vec4", + } + + comp = json.loads(comp_json) + + check( + "$compute" in comp.keys(), + AssertionError("Cannot find '$compute' key in JSON file"), + ) + + compute_info = comp["$compute"] + + check( + len(compute_info["buffer"]["fields"]) > 0, + AssertionError("Compute shader test with empty SSBO"), + ) + + field_types_set = {field["type"] for field in compute_info["buffer"]["fields"]} + + check(len(field_types_set) == 1, AssertionError("All field types must be the same")) + + ssbo_type = compute_info["buffer"]["fields"][0]["type"] + if ssbo_type not in ssbo_types.keys(): + raise ValueError(f"Unsupported SSBO datum type: {ssbo_type}") + ssbo_type_amber = ssbo_types[ssbo_type] + + # E.g. [[0, 0], [5], [1, 2, 3]] + field_data = [field["data"] for field in compute_info["buffer"]["fields"]] + + # E.g. [0, 0, 5, 1, 2, 3] + # |*| unpacks the list so each element is passed as an argument. + # |chain| takes a list of iterables and concatenates them. + field_data_flattened = itertools.chain(*field_data) + + # E.g. ["0", "0", "5", "1", "2", "3"] + field_data_flattened_str = [str(field) for field in field_data_flattened] + + result = "" + if make_empty_buffer: + # We just use the first value to initialize every element of the "empty" buffer. + result += f"BUFFER {{}} DATA_TYPE {ssbo_type_amber} SIZE {len(field_data_flattened_str)} {field_data_flattened_str[0]}\n" + else: + result += f"BUFFER {{}} DATA_TYPE {ssbo_type_amber} DATA\n" + result += f" {' '.join(field_data_flattened_str)}\n" + result += "END\n" + + return result + + +def amberscript_comp_num_groups_def(json_contents: str) -> str: + shader_job_info = json.loads(json_contents) + num_groups = shader_job_info["$compute"]["num_groups"] + num_groups_str = [str(dimension) for dimension in num_groups] + return " ".join(num_groups_str) + + +def amberscript_uniform_buffer_bind(uniform_json: str, prefix: str) -> str: + """ + Returns AmberScript commands for uniform binding. + + Skips the special '$compute' key, if present. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + "$compute": { ... will be ignored ... } + } + + becomes: + + BIND BUFFER {prefix}_myuniform AS uniform DESCRIPTOR_SET 0 BINDING 3 """ - Returns the string representing VkScript version of uniform declarations. + result = "" + uniforms = json.loads(uniform_json) + for name, entry in uniforms.items(): + if name == "$compute": + continue + result += f" BIND BUFFER {prefix}_{name} AS uniform DESCRIPTOR_SET 0 BINDING {entry['binding']}\n" + return result + + +def amberscript_uniform_buffer_def(uniform_json_contents: str, prefix: str) -> str: + """ + Returns the string representing AmberScript version of uniform definitions. Skips the special '$compute' key, if present. @@ -103,47 +235,58 @@ def uniform_json_to_amberscript(uniform_json_contents: str) -> str: becomes: + # uniforms for {prefix} + # myuniform - uniform ubo 0:3 float 0 42.0 + BUFFER {prefix}_myuniform DATA_TYPE float DATA + 42.0 + END + + :param uniform_json_contents: + :param prefix: E.g. "reference" or "variant". The buffer names will include this prefix to avoid name + clashes. """ - uniform_types = { - "glUniform1i": "int", - "glUniform2i": "ivec2", - "glUniform3i": "ivec3", - "glUniform4i": "ivec4", + uniform_types: Dict[str, str] = { + "glUniform1i": "int32", + "glUniform2i": "vec2", + "glUniform3i": "vec3", + "glUniform4i": "vec4", "glUniform1f": "float", - "glUniform2f": "vec2", - "glUniform3f": "vec3", - "glUniform4f": "vec4", + "glUniform2f": "vec2", + "glUniform3f": "vec3", + "glUniform4f": "vec4", + "glUniformMatrix2fv": "mat2x2", + "glUniformMatrix2x3fv": "mat2x3", + "glUniformMatrix2x4fv": "mat2x4", + "glUniformMatrix3x2fv": "mat3x2", + "glUniformMatrix3fv": "mat3x3", + "glUniformMatrix3x4fv": "mat3x4", + "glUniformMatrix4x2fv": "mat4x2", + "glUniformMatrix4x3fv": "mat4x3", + "glUniformMatrix4fv": "mat4x4", } - descriptor_set = 0 # always 0 in our tests - offset = 0 # We never have uniform offset in our tests + result = f"# uniforms for {prefix}\n" - result = "" - uniform_json = json.loads(uniform_json_contents) - for name, entry in uniform_json.items(): + result += "\n" + + uniforms = json.loads(uniform_json_contents) + for name, entry in uniforms.items(): if name == "$compute": continue func = entry["func"] - binding = entry["binding"] - - check( - func in uniform_types.keys(), - AssertionError(f"unknown uniform type for function{func}"), - ) - + if func not in uniform_types.keys(): + raise ValueError("Error: unknown uniform type for function: " + func) uniform_type = uniform_types[func] - result += "# " + name + "\n" - result += f"uniform ubo {descriptor_set}:{binding}" - result += " " + uniform_type - result += f" {offset}" + result += f"# {name}\n" + result += f"BUFFER {prefix}_{name} DATA_TYPE {uniform_type} DATA\n" for arg in entry["args"]: result += f" {arg}" result += "\n" + result += "END\n" return result @@ -154,148 +297,267 @@ def translate_type_for_amber(type_name: str) -> str: return type_name -def comp_json_to_amberscript(shader_json_contents: str) -> str: - """ - Returns the string representing VkScript version of compute shader setup. +def is_compute_job(input_asm_spirv_job_json_path: pathlib.Path) -> bool: + comp_files = shader_job_util.get_related_files( + input_asm_spirv_job_json_path, + [shader_job_util.EXT_COMP], + [shader_job_util.SUFFIX_ASM_SPIRV], + ) + check( + len(comp_files) <= 1, + AssertionError(f"Expected 1 or 0 compute shader files: {comp_files}"), + ) + return len(comp_files) == 1 - The compute shader setup is found under the special "$compute" key in the JSON. - { - "my_uniform_name": { ... ignored by this function ... }, +class ShaderType(Enum): + FRAGMENT = "fragment" + VERTEX = "vertex" + COMPUTE = "compute" - "$compute": { - "num_groups": [12, 13, 14]; - "buffer": { - "binding": 123, - "fields": [ - { - "type": "int", - "data": [42, 43, 44, 45] - } - ] - } - } - } +@attr.dataclass +class Shader: + shader_type: ShaderType + shader_spirv_asm: Optional[str] + shader_source: Optional[str] + processing_info: str # E.g. "optimized with spirv-opt -O" - becomes: - offset - | - ssbo 123 subdata int 0 42 43 44 45 - compute 12 13 14 +@attr.dataclass +class ShaderJob: + name_prefix: str # Can be used to create unique ssbo buffer names; uniform names are already unique. + uniform_definitions: str # E.g. BUFFER reference_resolution DATA_TYPE vec2 DATA 256.0 256.0 END ... + uniform_bindings: str # E.g. BIND BUFFER reference_resolution AS uniform DESCRIPTOR_SET 0 BINDING 2 ... - """ - shader_json = json.loads(shader_json_contents) - check( - "$compute" in shader_json.keys(), - AssertionError('Cannot find "$compute" key in JSON file'), - ) - compute_json = shader_json["$compute"] - - result = "## SSBO\n" - - binding = compute_json["buffer"]["binding"] - offset = 0 - for field_info in compute_json["buffer"]["fields"]: - result += ( - "ssbo " - + str(binding) - + " subdata " - + translate_type_for_amber(field_info["type"]) - + " " - + str(offset) + +@attr.dataclass +class ComputeShaderJob(ShaderJob): + + compute_shader: Shader + + # String containing AmberScript command(s) for defining a buffer containing the initial data for the input/output + # buffer that will be used by the compute shader. + # This string is a template (use with .format()) where the template argument is the name of buffer. + # E.g. BUFFER {} DATA_TYPE vec4 DATA 0.0 0.0 END + initial_buffer_definition_template: str + + # Same as above, but defines an empty buffer with the same size and type as the initial buffer. + # E.g. BUFFER {name} DATA_TYPE vec4 SIZE 2 0.0 + empty_buffer_definition_template: str + + # String specifying the number of groups to run when calling the Amber RUN command. E.g. "7 3 4". + num_groups_def: str + + +@attr.dataclass +class GraphicsShaderJob(ShaderJob): + vertex_shader: Shader + fragment_shader: Shader + + +@attr.dataclass +class ShaderJobFile: + name_prefix: str # Uniform names will be prefixed with this name to ensure they are unique. E.g. "reference". + asm_spirv_shader_job_json: Path + glsl_source_json: Optional[Path] + processing_info: str # E.g. "optimized with spirv-opt -O" + + def to_shader_job(self) -> ShaderJob: + json_contents = util.file_read_text(self.asm_spirv_shader_job_json) + + if is_compute_job(self.asm_spirv_shader_job_json): + glsl_comp_contents = None + if self.glsl_source_json: + glsl_comp_contents = shader_job_util.get_shader_contents( + self.glsl_source_json, shader_job_util.EXT_COMP + ) + comp_asm_contents = shader_job_util.get_shader_contents( + self.asm_spirv_shader_job_json, + shader_job_util.EXT_COMP, + shader_job_util.SUFFIX_ASM_SPIRV, + must_exist=True, + ) + + # Guaranteed + assert comp_asm_contents # noqa + + return ComputeShaderJob( + self.name_prefix, + amberscript_uniform_buffer_def(json_contents, self.name_prefix), + amberscript_uniform_buffer_bind(json_contents, self.name_prefix), + Shader( + ShaderType.COMPUTE, + comp_asm_contents, + glsl_comp_contents, + self.processing_info, + ), + amberscript_comp_buff_def(json_contents), + amberscript_comp_buff_def(json_contents, make_empty_buffer=True), + amberscript_comp_num_groups_def(json_contents), + ) + + # Get GLSL contents + glsl_vert_contents = None + glsl_frag_contents = None + if self.glsl_source_json: + glsl_vert_contents = shader_job_util.get_shader_contents( + self.glsl_source_json, shader_job_util.EXT_VERT + ) + glsl_frag_contents = shader_job_util.get_shader_contents( + self.glsl_source_json, shader_job_util.EXT_FRAG + ) + + # Get spirv asm contents + vert_contents = shader_job_util.get_shader_contents( + self.asm_spirv_shader_job_json, + shader_job_util.EXT_VERT, + shader_job_util.SUFFIX_ASM_SPIRV, ) - for datum in field_info["data"]: - result += " " + str(datum) - offset += 4 - result += "\n" - result += "\n" - result += "compute" - result += " " + str(compute_json["num_groups"][0]) - result += " " + str(compute_json["num_groups"][1]) - result += " " + str(compute_json["num_groups"][2]) - result += "\n" + frag_contents = shader_job_util.get_shader_contents( + self.asm_spirv_shader_job_json, + shader_job_util.EXT_FRAG, + shader_job_util.SUFFIX_ASM_SPIRV, + must_exist=True, + ) - return result + return GraphicsShaderJob( + self.name_prefix, + amberscript_uniform_buffer_def(json_contents, self.name_prefix), + amberscript_uniform_buffer_bind(json_contents, self.name_prefix), + Shader( + ShaderType.VERTEX, + vert_contents, + glsl_vert_contents, + self.processing_info, + ), + Shader( + ShaderType.FRAGMENT, + frag_contents, + glsl_frag_contents, + self.processing_info, + ), + ) -def amberscriptify_image( # pylint: disable=too-many-branches,too-many-locals - vert_asm_contents: Optional[str], - frag_asm_contents: str, - shader_job_json_contents: str, - amberfy_settings: AmberfySettings, - vert_glsl_contents: Optional[str] = None, - frag_glsl_contents: Optional[str] = None, - add_red_pixel_probe: bool = False, -) -> str: - """Generates Amberscript representation of an image test.""" - result = "" +@attr.dataclass +class ShaderJobBasedAmberTest: + reference: Optional[ShaderJob] + variant: ShaderJob + + +@attr.dataclass +class ShaderJobFileBasedAmberTest: + reference_asm_spirv_job: Optional[ShaderJobFile] + variant_asm_spirv_job: ShaderJobFile + + def to_shader_job_based(self) -> ShaderJobBasedAmberTest: + return ShaderJobBasedAmberTest( + self.reference_asm_spirv_job.to_shader_job() + if self.reference_asm_spirv_job + else None, + self.variant_asm_spirv_job.to_shader_job(), + ) + + +def get_amber_script_header(amberfy_settings: AmberfySettings) -> str: + result = "#!amber\n" if amberfy_settings.copyright_header_text: - result += get_text_as_comment(amberfy_settings.copyright_header_text) + "\n\n" + result += "\n" + result += get_text_as_comment(amberfy_settings.copyright_header_text) + result += "\n\n" if amberfy_settings.add_generated_comment: - result = "# Generated.\n\n" + result += "\n# Generated.\n\n" if amberfy_settings.add_graphics_fuzz_comment: - result += "# A test for a bug found by GraphicsFuzz.\n\n" + result += "\n# A test for a bug found by GraphicsFuzz.\n" if amberfy_settings.short_description: - result += f"# Short description: {amberfy_settings.short_description}\n\n" + result += f"\n# Short description: {amberfy_settings.short_description}\n" if amberfy_settings.comment_text: - result += get_text_as_comment(amberfy_settings.comment_text) + "\n\n" + result += f"\n{get_text_as_comment(amberfy_settings.comment_text)}\n" if amberfy_settings.spirv_opt_args: + result += "\n" result += get_spirv_opt_args_comment( amberfy_settings.spirv_opt_args, amberfy_settings.spirv_opt_hash ) + result += "\n" - if vert_glsl_contents or frag_glsl_contents: - result += "# Derived from the following GLSL.\n\n" + if not amberfy_settings.use_default_fence_timeout: + result += f"\nSET ENGINE_DATA fence_timeout_ms {AMBER_FENCE_TIMEOUT_MS}\n" - if vert_glsl_contents: - result += "# Vertex shader GLSL:\n" - result += get_text_as_comment(vert_glsl_contents) - result += "\n\n" + return result - if frag_glsl_contents: - result += "# Fragment shader GLSL:\n" - result += get_text_as_comment(frag_glsl_contents) - result += "\n\n" - result += "[require]\n" - result += "fbsize 256 256\n" +def get_amber_script_shader_def(shader: Shader, name: str) -> str: + result = "" + if shader.shader_source: + result += f"\n# {name} is derived from the following GLSL:\n" + result += get_text_as_comment(shader.shader_source) + if shader.shader_spirv_asm: + result += f"\nSHADER {str(shader.shader_type.value)} {name} SPIRV-ASM\n" + result += shader.shader_spirv_asm + result += "END\n" + else: + result += f"\nSHADER {str(shader.shader_type.value)} {name} PASSTHROUGH\n" - if not amberfy_settings.use_default_fence_timeout: - result += "fence_timeout " + str(AMBER_FENCE_TIMEOUT_MS) + "\n" + return result + + +# noinspection DuplicatedCode +def graphics_shader_job_amber_test_to_amber_script( + shader_job_amber_test: ShaderJobBasedAmberTest, amberfy_settings: AmberfySettings +) -> str: + # TODO: handle reference, if present. + + # Guaranteed, and needed for type checker. + assert isinstance(shader_job_amber_test.variant, GraphicsShaderJob) # noqa + + result = get_amber_script_header(amberfy_settings) + + variant = shader_job_amber_test.variant + + variant_vertex_shader_name = f"{variant.name_prefix}_vertex_shader" + variant_fragment_shader_name = f"{variant.name_prefix}_fragment_shader" + + # Define shaders. + + result += get_amber_script_shader_def( + variant.vertex_shader, variant_vertex_shader_name + ) + + result += get_amber_script_shader_def( + variant.fragment_shader, variant_fragment_shader_name + ) + + # Define uniforms for variant shader job. result += "\n" + result += variant.uniform_definitions - if vert_asm_contents: - result += "[vertex shader spirv]\n" - result += vert_asm_contents - else: - result += "[vertex shader passthrough]" - result += "\n\n" + # Currently, this buffer must be called "framebuffer" to be able to dump it from Amber. + result += "\nBUFFER framebuffer FORMAT B8G8R8A8_UNORM\n" - result += "[fragment shader spirv]\n" - result += frag_asm_contents - result += "\n\n" + # Create a pipeline that uses the variant shaders and writes the output to "framebuffer". - result += "[test]\n" + result += "\nPIPELINE graphics gfz_pipeline\n" + result += f" ATTACH {variant_vertex_shader_name}\n" + result += f" ATTACH {variant_fragment_shader_name}\n" + result += " FRAMEBUFFER_SIZE 256 256\n" + result += " BIND BUFFER framebuffer AS color LOCATION 0\n" + result += variant.uniform_bindings + result += "END\n" + result += "CLEAR_COLOR gfz_pipeline 0 0 0 255\n" - uniforms_text = uniform_json_to_amberscript(shader_job_json_contents) - if uniforms_text: - result += "## Uniforms\n" - result += uniforms_text - result += "\n" - result += "draw rect -1 -1 2 2\n" + # Run the pipeline. - if add_red_pixel_probe: - result += "probe rgba (0, 0) (1, 0, 0, 1)\n" + result += "\nCLEAR gfz_pipeline\n" + result += "RUN gfz_pipeline DRAW_RECT POS 0 0 SIZE 256 256\n" if amberfy_settings.extra_commands: result += amberfy_settings.extra_commands @@ -303,69 +565,52 @@ def amberscriptify_image( # pylint: disable=too-many-branches,too-many-locals return result -def is_compute_job(input_asm_spirv_job_json_path: pathlib.Path) -> bool: - comp_files = shader_job_util.get_related_files( - input_asm_spirv_job_json_path, - [shader_job_util.EXT_COMP], - [shader_job_util.SUFFIX_ASM_SPIRV], - ) - check( - len(comp_files) <= 1, - AssertionError(f"Expected 1 or 0 compute shader files: {comp_files}"), - ) - return len(comp_files) == 1 +# noinspection DuplicatedCode +def compute_shader_job_amber_test_to_amber_script( + shader_job_amber_test: ShaderJobBasedAmberTest, amberfy_settings: AmberfySettings +) -> str: + # TODO: handle reference, if present. + # Guaranteed, and needed for type checker. + assert isinstance(shader_job_amber_test.variant, ComputeShaderJob) # noqa -def amberscriptify_comp( - comp_asm_contents: str, - shader_job_json_contents: str, - amberfy_settings: AmberfySettings, - comp_glsl_contents: Optional[str], -) -> str: - result = "" + result = get_amber_script_header(amberfy_settings) - if amberfy_settings.copyright_header_text: - result += get_text_as_comment(amberfy_settings.copyright_header_text) + "\n\n" + variant = shader_job_amber_test.variant - if amberfy_settings.add_generated_comment: - result = "# Generated.\n\n" + variant_compute_shader_name = f"{variant.name_prefix}_compute_shader" + variant_ssbo_name = f"{variant.name_prefix}_ssbo" - if amberfy_settings.add_graphics_fuzz_comment: - result += "# A test for a bug found by GraphicsFuzz.\n\n" + # Define shaders. - if amberfy_settings.short_description: - result += f"# Short description: {amberfy_settings.short_description}\n\n" + result += get_amber_script_shader_def( + variant.compute_shader, variant_compute_shader_name + ) - if amberfy_settings.comment_text: - result += get_text_as_comment(amberfy_settings.comment_text) + "\n\n" + # Define uniforms for variant shader job. - if amberfy_settings.spirv_opt_args: - result += get_spirv_opt_args_comment( - amberfy_settings.spirv_opt_args, amberfy_settings.spirv_opt_hash - ) + result += "\n" + result += variant.uniform_definitions - if comp_glsl_contents: - result += "# Derived from the following GLSL.\n\n" - result += "# Compute shader GLSL:\n" - result += get_text_as_comment(comp_glsl_contents) - result += "\n\n" + # Define in/out buffer for variant shader job. + # Note that |initial_buffer_definition_template| is a string template that takes the buffer name as an argument. - if not amberfy_settings.use_default_fence_timeout: - result += "[require]\n" - result += "fence_timeout " + str(AMBER_FENCE_TIMEOUT_MS) + "\n\n" + result += "\n" + result += variant.initial_buffer_definition_template.format(variant_ssbo_name) - result += "[compute shader spirv]\n" - result += comp_asm_contents - result += "\n\n" + # Create a pipeline that uses the variant compute shader and binds |variant_ssbo_name|. + + result += "\nPIPELINE compute gfz_pipeline\n" + result += f" ATTACH {variant_compute_shader_name}\n" + result += ( + f" BIND BUFFER {variant_ssbo_name} AS storage DESCRIPTOR_SET 0 BINDING 0\n" + ) + result += variant.uniform_bindings + result += "END\n" - result += "[test]\n" - uniforms_text = uniform_json_to_amberscript(shader_job_json_contents) - if uniforms_text: - result += "## Uniforms\n" - result += uniforms_text + # Run the pipeline. - # This also includes the "compute" command that runs the shader. - result += comp_json_to_amberscript(shader_job_json_contents) + result += f"\nRUN gfz_pipeline {variant.num_groups_def}\n" if amberfy_settings.extra_commands: result += amberfy_settings.extra_commands @@ -373,77 +618,36 @@ def amberscriptify_comp( return result -def run_spirv_asm_shader_job_to_amber_script( # pylint: disable=too-many-locals - input_asm_spirv_job_json_path: pathlib.Path, - output_amber_script_file_path: pathlib.Path, +def spirv_asm_shader_job_to_amber_script( + shader_job_file_amber_test: ShaderJobFileBasedAmberTest, + output_amber_script_file_path: Path, amberfy_settings: AmberfySettings, - input_glsl_source_json_path: Optional[pathlib.Path] = None, - add_red_pixel_probe: bool = False, -) -> pathlib.Path: +) -> Path: log( - f"Amberfy: {str(input_asm_spirv_job_json_path)} to {str(output_amber_script_file_path)}" + f"Amberfy: {str(shader_job_file_amber_test.variant_asm_spirv_job.asm_spirv_shader_job_json)} " + + ( + f"with reference {str(shader_job_file_amber_test.reference_asm_spirv_job.asm_spirv_shader_job_json)} " + if shader_job_file_amber_test.reference_asm_spirv_job + else "" + ) + + f"to {str(output_amber_script_file_path)}" ) - # Get the JSON contents. - json_contents = util.file_read_text(input_asm_spirv_job_json_path) + shader_job_amber_test = shader_job_file_amber_test.to_shader_job_based() - if is_compute_job(input_asm_spirv_job_json_path): # pylint:disable=no-else-raise - glsl_comp_contents = None - if input_glsl_source_json_path: - glsl_comp_contents = shader_job_util.get_shader_contents( - input_glsl_source_json_path, shader_job_util.EXT_COMP - ) - comp_asm_contents = shader_job_util.get_shader_contents( - input_asm_spirv_job_json_path, - shader_job_util.EXT_COMP, - shader_job_util.SUFFIX_ASM_SPIRV, - must_exist=True, + if isinstance(shader_job_amber_test.variant, GraphicsShaderJob): + result = graphics_shader_job_amber_test_to_amber_script( + shader_job_amber_test, amberfy_settings ) - assert comp_asm_contents # noqa - - result = amberscriptify_comp( - comp_asm_contents, json_contents, amberfy_settings, glsl_comp_contents + elif isinstance(shader_job_amber_test.variant, ComputeShaderJob): + result = compute_shader_job_amber_test_to_amber_script( + shader_job_amber_test, amberfy_settings ) - else: - # Get GLSL contents - glsl_vert_contents = None - glsl_frag_contents = None - if input_glsl_source_json_path: - glsl_vert_contents = shader_job_util.get_shader_contents( - input_glsl_source_json_path, shader_job_util.EXT_VERT - ) - glsl_frag_contents = shader_job_util.get_shader_contents( - input_glsl_source_json_path, shader_job_util.EXT_FRAG - ) - - # Get spirv asm contents - vert_contents = shader_job_util.get_shader_contents( - input_asm_spirv_job_json_path, - shader_job_util.EXT_VERT, - shader_job_util.SUFFIX_ASM_SPIRV, - ) - - frag_contents = shader_job_util.get_shader_contents( - input_asm_spirv_job_json_path, - shader_job_util.EXT_FRAG, - shader_job_util.SUFFIX_ASM_SPIRV, - must_exist=True, - ) - - # Guaranteed. - assert frag_contents # noqa - - result = amberscriptify_image( - vert_contents, - frag_contents, - json_contents, - amberfy_settings, - glsl_vert_contents, - glsl_frag_contents, - add_red_pixel_probe, + raise AssertionError( + f"Unknown shader job type: {shader_job_amber_test.variant}" ) util.file_write_text(output_amber_script_file_path, result) diff --git a/gfauto/gfauto/test_create_readme.py b/gfauto/gfauto/test_create_readme.py index 5dacf1f9a..bd4f9b4f7 100644 --- a/gfauto/gfauto/test_create_readme.py +++ b/gfauto/gfauto/test_create_readme.py @@ -20,37 +20,14 @@ """ import argparse -import itertools import sys from pathlib import Path from gfauto import artifact_util, binaries_util, fuzz_glsl_test, test_util from gfauto.binaries_util import BinaryManager -from gfauto.common_pb2 import Binary -from gfauto.gflogging import log from gfauto.util import check, check_dir_exists -def update_test_json(test_json: Path) -> Path: - test = test_util.metadata_read_from_path(test_json) - - for test_binary in itertools.chain( - test.binaries, test.device.binaries - ): # type: Binary - for default_binary in binaries_util.DEFAULT_BINARIES: - if ( - test_binary.name == default_binary.name - and test_binary.version != default_binary.version - ): - log( - f"Updating version: {test_binary.version} -> {default_binary.version}" - ) - test_binary.version = default_binary.version - break - - return test_util.metadata_write_to_path(test, test_json) - - def main() -> None: parser = argparse.ArgumentParser( description="A script that creates a README and bug_report directory for a test and its result_dir." diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index 161a05540..1ab132e71 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -79,8 +79,15 @@ def amberfy( amberfy_settings: amber_converter.AmberfySettings, input_glsl_source_json_path: Optional[Path] = None, ) -> Path: - return amber_converter.run_spirv_asm_shader_job_to_amber_script( - input_json, output_amber, amberfy_settings, input_glsl_source_json_path + + shader_job_file_amber_test = amber_converter.ShaderJobFileBasedAmberTest( + reference_asm_spirv_job=None, + variant_asm_spirv_job=amber_converter.ShaderJobFile( + "variant", input_json, input_glsl_source_json_path, "" + ), + ) + return amber_converter.spirv_asm_shader_job_to_amber_script( + shader_job_file_amber_test, output_amber, amberfy_settings ) diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index b08b455dd..d3965608e 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -163,4 +163,11 @@ getrandbits randbits jsons spvt - +enum +Enum +P102 +iterables +unindexed +vec2 +vec4 +framebuffer From fe795ba93e238e84589553f0a6febab093f9bbcf Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Fri, 16 Aug 2019 14:30:48 +0100 Subject: [PATCH 104/180] gfauto: fix issue in tool (#694) Don't require test metadata if you don't provide spirv_opt_args. Also add temp directory that is ignored. --- gfauto/gfauto/tool.py | 7 +++---- gfauto/temp/.gitignore | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 gfauto/temp/.gitignore diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index 1ab132e71..008ea1cdc 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -252,12 +252,11 @@ def glsl_shader_job_crash_to_amber_script_for_google_cts( ) ) - if not spirv_opt_args: + if test_metadata_path: check( - bool(test_metadata_path), - AssertionError("Must have test_metadata_path or binary_paths"), + bool(not spirv_opt_args), + AssertionError("Cannot have spirv_opt_args AND test_metadata_path"), ) - assert test_metadata_path # noqa spirv_opt_args = list( test_util.metadata_read_from_path(test_metadata_path).glsl.spirv_opt_args ) diff --git a/gfauto/temp/.gitignore b/gfauto/temp/.gitignore new file mode 100644 index 000000000..563d85214 --- /dev/null +++ b/gfauto/temp/.gitignore @@ -0,0 +1,3 @@ + +* +!/.gitignore From d3bea6371acc69c84acfd0abc8549acfa63eb588 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Fri, 16 Aug 2019 11:09:57 -0500 Subject: [PATCH 105/180] Fix opaque dot function inverted isZero indices (#695) Fixes #690. --- .../generator/fuzzer/OpaqueExpressionGenerator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 3c6031e40..4d2e0625d 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -327,12 +327,12 @@ private Optional opaqueZeroOrOneDot(BasicType type, boolean constContext, } } else { // The two vectors will be exactly the same - all 0 except for one value, which will be 1. - final int nonZeroIndex = generator.nextInt(vectorWidth); + final int oneIndex = generator.nextInt(vectorWidth); for (int i = 0; i < vectorWidth; i++) { firstVectorArgs.add( - makeOpaqueZeroOrOne(i == nonZeroIndex, type, constContext, depth, fuzzer)); + makeOpaqueZeroOrOne(i != oneIndex, type, constContext, depth, fuzzer)); secondVectorArgs.add( - makeOpaqueZeroOrOne(i == nonZeroIndex, type, constContext, depth, fuzzer)); + makeOpaqueZeroOrOne(i != oneIndex, type, constContext, depth, fuzzer)); } } assert firstVectorArgs.size() == vectorWidth && secondVectorArgs.size() == vectorWidth; From bd9ee784af63c2481eb7db2ecf8d89218e2ed6fa Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Sat, 17 Aug 2019 05:36:54 +0700 Subject: [PATCH 106/180] Expression Generator: introduce uniforms (#693) Fixes #671. --- .../ExpressionGenerator.java | 70 +++++++++++++++++-- .../tool/KnownValueShaderGenerator.java | 28 +++++++- .../java/com/graphicsfuzz/util/Constants.java | 1 + 3 files changed, 92 insertions(+), 7 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java index 255779d5a..db244c0a7 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java @@ -39,7 +39,9 @@ import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.QualifiedType; import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; @@ -56,7 +58,7 @@ public class ExpressionGenerator { private static final int MAX_FUNCTION_PARAMS = 5; - private static final int NUM_WAYS_TO_GENERATE_EXPR = 7; + private static final int NUM_WAYS_TO_GENERATE_EXPR = 8; private static final int MAX_DEPTH = 5; // Theses boundaries are taken from the LiteralFuzzer, we may want to consider // modifying these values. @@ -152,6 +154,9 @@ public Expr generateExpr(FactManager factManager, case 6: result = generateForLoopValue(value, factManager, functionDefinition, stmtToInsertBefore); break; + case 7: + result = generateUniformValue(value); + break; default: throw new RuntimeException("Should be unreachable as switch cases cover all cases"); } @@ -163,6 +168,61 @@ public Expr generateExpr(FactManager factManager, return result; } + + private Expr generateUniformValue(Value value) { + if (!(value instanceof NumericValue) && !(value instanceof CompositeValue)) { + return null; + } + // TODO(https://github.com/google/graphicsfuzz/issues/692): we would be able to support + // matrix uniform once Amber has solved the matrix-related issues. + if (BasicType.allMatrixTypes().contains(value.getType())) { + return null; + } + final String uniformName = Constants.GLF_UNIFORM + "_" + + value.getType().toString() + + parseNameFromValue(value) + + freshId(); + + // We must assign a random value if the given value is empty since we are not allowed to + // inject unknown uniform variables. + if (value.valueIsUnknown()) { + value = fuzzValue(value.getType(), true); + } + + final List uniformValues = new ArrayList<>(); + if (value instanceof CompositeValue) { + CompositeValue compositeValue = (CompositeValue) value; + for (Value element : compositeValue.getValueList().get()) { + // It's also possible that an element of the given composite value is unknown, thus we have + // to initialize the element if it is empty. + final NumericValue elementValue = element.valueIsUnknown() + ? (NumericValue) fuzzValue(element.getType(), true) + : (NumericValue) element; + uniformValues.add(elementValue.getValue().get()); + } + } else { + assert value instanceof NumericValue; + uniformValues.add(((NumericValue) value).getValue().get()); + } + + pipelineInfo.addUniform(uniformName, (BasicType) value.getType(), Optional.empty(), + uniformValues); + // Declare a new variable declaration for the uniform we generated. + final VariableDeclInfo variableDeclInfo = new VariableDeclInfo(uniformName, null, null); + final VariablesDeclaration variablesDecl = new VariablesDeclaration( + new QualifiedType(value.getType(), Arrays.asList(TypeQualifier.UNIFORM)), variableDeclInfo + ); + translationUnit.addDeclaration(variablesDecl); + + // As uniform variables are available at the global scope, we create a new fact given a + // generated uniform and thus keep it at the global scope manager. + final VariableDeclFact variableDeclFact = new VariableDeclFact(variablesDecl, + variableDeclInfo, value); + globalFactManager.addVariableFact(value, variableDeclFact); + + return new VariableIdentifierExpr(uniformName); + } + /** * Assigns value to the newly-generated variable using a for loop. To do so, we first pick a * random number (divisor) and add this number to the variable for each iteration. Then, we @@ -626,7 +686,7 @@ private Expr generateFunctionFact(Value value, // If the fact manager is generating an unknown parameter(value is Optional.empty), // when calling this function the fact manager will generate any arbitrary value that // matches the parameter type. - final Value paramValue = fuzzValue(paramType); + final Value paramValue = fuzzValue(paramType, false); final String paramName = genParamName(paramValue); argumentValues.add(paramValue); @@ -679,10 +739,10 @@ private List getAvailableTypes() { BasicType.VEC3, BasicType.VEC4); } - private Value fuzzValue(Type type) { + private Value fuzzValue(Type type, boolean forceGenerateKnownValue) { // An Unknown value variable is the variable whose value could be anything. The fact manager // could generate any arbitrary value but with the correct type. - final boolean isUnknown = generator.nextBoolean(); + final boolean isUnknown = generator.nextBoolean() && !forceGenerateKnownValue; if (type == BasicType.BOOL) { return new BooleanValue( isUnknown ? Optional.empty() : Optional.of(generator.nextBoolean())); @@ -712,7 +772,7 @@ private Value fuzzValue(Type type) { if (BasicType.allBasicTypes().contains(type)) { final List values = new ArrayList<>(); for (int i = 0; i < ((BasicType) type).getNumElements(); i++) { - values.add(fuzzValue(((BasicType) type).getElementType())); + values.add(fuzzValue(((BasicType) type).getElementType(), forceGenerateKnownValue)); } return new CompositeValue(type, Optional.of(values)); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java index 30f4c827a..e6cf78c05 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java @@ -42,12 +42,14 @@ import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.PipelineInfo; +import com.graphicsfuzz.common.util.PruneUniforms; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.generator.knownvaluegeneration.ExpressionGenerator; import com.graphicsfuzz.generator.knownvaluegeneration.FactManager; import com.graphicsfuzz.generator.knownvaluegeneration.NumericValue; import com.graphicsfuzz.util.ArgsUtil; +import com.graphicsfuzz.util.Constants; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -56,6 +58,7 @@ import java.util.Optional; import java.util.stream.Stream; import net.sourceforge.argparse4j.ArgumentParsers; +import net.sourceforge.argparse4j.impl.Arguments; import net.sourceforge.argparse4j.inf.ArgumentParser; import net.sourceforge.argparse4j.inf.ArgumentParserException; import net.sourceforge.argparse4j.inf.Namespace; @@ -100,6 +103,17 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .help("Seed (unsigned 64 bit long integer) for the random number generator.") .type(String.class); + parser.addArgument("--max-uniforms") + .help("Ensure that generated shaders have no more than the given number of uniforms; " + + "required for Vulkan compatibility.") + .setDefault(0) + .type(Integer.class); + + parser.addArgument("--generate-uniform-bindings") + .help("Put all uniforms in uniform blocks and generate bindings; required for Vulkan " + + "compatibility.") + .action(Arguments.storeTrue()); + return parser.parseArgs(args); } @@ -109,6 +123,8 @@ public static void mainHelper(String[] args) throws ArgumentParserException, IOE final float gFloat = ns.getFloat("g"); final float bFloat = ns.getFloat("b"); final float aFloat = ns.getFloat("a"); + final int maxUniforms = ns.getInt("max_uniforms"); + final boolean generateUniformBindings = ns.getBoolean("generate_uniform_bindings"); final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final String version = ns.getString("version"); final File shaderJobFile = ns.get("output"); @@ -152,8 +168,16 @@ public static void mainHelper(String[] args) throws ArgumentParserException, IOE new ExprStmt(new BinaryExpr(new VariableIdentifierExpr("_GLF_color"), new TypeConstructorExpr("vec4", rgbaExprs), BinOp.ASSIGN))); - final ShaderJob newShaderJob = new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu); - fileOps.writeShaderJobFile(newShaderJob, shaderJobFile); + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), pipelineInfo, tu); + + if (maxUniforms > 0) { + PruneUniforms.prune(shaderJob, maxUniforms, Arrays.asList(Constants.GLF_UNIFORM)); + } + if (generateUniformBindings) { + shaderJob.makeUniformBindings(); + } + + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); LOGGER.info("Generation complete."); } diff --git a/util/src/main/java/com/graphicsfuzz/util/Constants.java b/util/src/main/java/com/graphicsfuzz/util/Constants.java index 97e323f54..31c76384f 100755 --- a/util/src/main/java/com/graphicsfuzz/util/Constants.java +++ b/util/src/main/java/com/graphicsfuzz/util/Constants.java @@ -51,6 +51,7 @@ private Constants() { public static final String GLF_PARAM = "_GLF_PARAM"; public static final String GLF_UNKNOWN_PARAM = "GLF_UNKNOWN_PARAM"; public static final String GLF_INIT_GLOBALS = "_GLF_init_globals"; + public static final String GLF_UNIFORM = "_GLF_UNIFORM"; public static final String INJECTED_LOOP_COUNTER = "_injected_loop_counter"; From 8ffcbd39d7e0c7cf6ca75d64604a209ef86a4084 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 20 Aug 2019 08:55:51 +0100 Subject: [PATCH 107/180] Add shaders that use all matrix types (#685) Adds two synthetic shaders that uses matrices to compute a greyscale grid pattern. Derived from more complex shaders that use matrix uniforms but that cannot currently be handled by Amber due to the following Amber issue: https://github.com/google/amber/issues/616 The more complex shaders will eventually be added in addition, but these simpler shaders allow better coverage of matrix types for now. --- .../samples/310es/matrices_many_loops.frag | 99 ++++++++++++ .../samples/310es/matrices_many_loops.json | 14 ++ .../samples/310es/matrices_smart_loops.frag | 150 ++++++++++++++++++ .../samples/310es/matrices_smart_loops.json | 14 ++ 4 files changed, 277 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/matrices_many_loops.frag create mode 100644 shaders/src/main/glsl/samples/310es/matrices_many_loops.json create mode 100644 shaders/src/main/glsl/samples/310es/matrices_smart_loops.frag create mode 100644 shaders/src/main/glsl/samples/310es/matrices_smart_loops.json diff --git a/shaders/src/main/glsl/samples/310es/matrices_many_loops.frag b/shaders/src/main/glsl/samples/310es/matrices_many_loops.frag new file mode 100644 index 000000000..8dfda73f2 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/matrices_many_loops.frag @@ -0,0 +1,99 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform float one; + +uniform vec2 resolution; + +#define POPULATE_MAT(MAT, COLS, ROWS) \ + for (int c = 0; c < COLS; c++) { \ + for (int r = 0; r < ROWS; r++) { \ + MAT[c][r] = one; \ + } \ + } + +// The following macro populates sums[INDEX] to be the sum of all the +// entries in MAT (which must be a COLS x ROWS matrix) divided by +// 16x.0. + +#define POPULATE_SUMS(MAT, COLS, ROWS, INDEX) \ + sums[INDEX] = 0.0; \ + for (int c = 0; c < COLS; c++) { \ + for (int r = 0; r < ROWS; r++) { \ + sums[INDEX] += MAT[c][r]; \ + } \ + } \ + sums[INDEX] /= 16.0; + +mat2x2 m22; + +mat2x3 m23; + +mat2x4 m24; + +mat3x2 m32; + +mat3x3 m33; + +mat3x4 m34; + +mat4x2 m42; + +mat4x3 m43; + +mat4x4 m44; + +void main() +{ + + POPULATE_MAT(m22, 2, 2); + POPULATE_MAT(m23, 2, 3); + POPULATE_MAT(m24, 2, 4); + POPULATE_MAT(m32, 3, 2); + POPULATE_MAT(m33, 3, 3); + POPULATE_MAT(m34, 3, 4); + POPULATE_MAT(m42, 4, 2); + POPULATE_MAT(m43, 4, 3); + POPULATE_MAT(m44, 4, 4); + + float sums[9]; + + POPULATE_SUMS(m22, 2, 2, 0); + POPULATE_SUMS(m23, 2, 3, 1); + POPULATE_SUMS(m24, 2, 4, 2); + POPULATE_SUMS(m32, 3, 2, 3); + POPULATE_SUMS(m33, 3, 3, 4); + POPULATE_SUMS(m34, 3, 4, 5); + POPULATE_SUMS(m42, 4, 2, 6); + POPULATE_SUMS(m43, 4, 3, 7); + POPULATE_SUMS(m44, 4, 4, 8); + + int region_x = int(gl_FragCoord.x / (resolution.x / 3.0)); + int region_y = int(gl_FragCoord.y / (resolution.x / 3.0)); + int overall_region = region_y * 3 + region_x; + + if (overall_region > 0 && overall_region < 9) { + _GLF_color = vec4(vec3(sums[overall_region]), 1.0); + } else { + _GLF_color = vec4(vec3(0.0), 1.0); + } +} diff --git a/shaders/src/main/glsl/samples/310es/matrices_many_loops.json b/shaders/src/main/glsl/samples/310es/matrices_many_loops.json new file mode 100644 index 000000000..0ad587e72 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/matrices_many_loops.json @@ -0,0 +1,14 @@ +{ + "one": { + "func": "glUniform1f", + "args": [ + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/matrices_smart_loops.frag b/shaders/src/main/glsl/samples/310es/matrices_smart_loops.frag new file mode 100644 index 000000000..cf6dcbb91 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/matrices_smart_loops.frag @@ -0,0 +1,150 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform float one; + +uniform vec2 resolution; + +mat2x2 m22; + +mat2x3 m23; + +mat2x4 m24; + +mat3x2 m32; + +mat3x3 m33; + +mat3x4 m34; + +mat4x2 m42; + +mat4x3 m43; + +mat4x4 m44; + +void main() +{ + uint matrix_number = 0u; + for(int cols = 2; cols <= 4; cols++) + { + for(int rows = 2; rows <= 4; rows++) + { + for(int c = 0; c < cols; c++) + { + for(int r = 0; r < rows; r++) + { + switch(matrix_number) + { + case 0u: + m22[c][r] = one; + break; + case 1u: + m23[c][r] = one; + break; + case 2u: + m24[c][r] = one; + break; + case 3u: + m32[c][r] = one; + break; + case 4u: + m33[c][r] = one; + break; + case 5u: + m34[c][r] = one; + break; + case 6u: + m42[c][r] = one; + break; + case 7u: + m43[c][r] = one; + break; + case 8u: + m44[c][r] = one; + break; + } + } + } + matrix_number = matrix_number + 1u; + } + } + + float sums[9]; + + int sum_index = 0; + for(int cols = 2; cols <= 4; cols++) + { + for(int rows = 2; rows <= 4; rows++) + { + sums[sum_index] = 0.0; + for(int c = 0; c < cols; c++) + { + for(int r = 0; r < rows; r++) + { + switch(sum_index) + { + case 0: + sums[sum_index] += m22[c][r]; + break; + case 1: + sums[sum_index] += m23[c][r]; + break; + case 2: + sums[sum_index] += m24[c][r]; + break; + case 3: + sums[sum_index] += m32[c][r]; + break; + case 4: + sums[sum_index] += m33[c][r]; + break; + case 5: + sums[sum_index] += m34[c][r]; + break; + case 6: + sums[sum_index] += m42[c][r]; + break; + case 7: + sums[sum_index] += m43[c][r]; + break; + case 8: + sums[sum_index] += m44[c][r]; + break; + } + } + } + sums[sum_index] /= 16.0; + sum_index ++; + } + } + + int region_x = int(gl_FragCoord.x / (resolution.x / 3.0)); + int region_y = int(gl_FragCoord.y / (resolution.x / 3.0)); + int overall_region = region_y * 3 + region_x; + + if (overall_region > 0 && overall_region < 9) { + _GLF_color = vec4(vec3(sums[overall_region]), 1.0); + } else { + _GLF_color = vec4(vec3(0.0), 1.0); + } +} diff --git a/shaders/src/main/glsl/samples/310es/matrices_smart_loops.json b/shaders/src/main/glsl/samples/310es/matrices_smart_loops.json new file mode 100644 index 000000000..0ad587e72 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/matrices_smart_loops.json @@ -0,0 +1,14 @@ +{ + "one": { + "func": "glUniform1f", + "args": [ + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, 256.0 + ] + } +} From 2e0c664b7cd291016ae3d080e1d711d45b1fcf6e Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 20 Aug 2019 08:58:52 +0100 Subject: [PATCH 108/180] Add 300 es versions of 310 es shaders (#699) Excluded integer function shaders (as 300 es does not support integer functions), but rewrote bits of some other shaders to back-port them to 300 es. --- .../glsl/samples/300es/binarysearch_bw.frag | 164 ++++++++++++++++ .../glsl/samples/300es/binarysearch_bw.json | 16 ++ .../glsl/samples/300es/binarysearch_tree.frag | 176 ++++++++++++++++++ .../glsl/samples/300es/binarysearch_tree.json | 16 ++ .../300es/binarysearch_tree_simple.frag | 173 +++++++++++++++++ .../300es/binarysearch_tree_simple.json | 16 ++ .../samples/300es/householder_lattice.frag | 119 ++++++++++++ .../samples/300es/householder_lattice.json | 11 ++ .../glsl/samples/300es/mergesort_mosaic.frag | 169 +++++++++++++++++ .../glsl/samples/300es/mergesort_mosaic.json | 16 ++ .../samples/300es/prefix_sum_checkers.frag | 95 ++++++++++ .../samples/300es/prefix_sum_checkers.json | 16 ++ .../glsl/samples/300es/quicksort_palette.frag | 131 +++++++++++++ .../glsl/samples/300es/quicksort_palette.json | 16 ++ .../samples/300es/selection_sort_struct.frag | 75 ++++++++ .../samples/300es/selection_sort_struct.json | 16 ++ .../samples/300es/trigonometric_strip.frag | 74 ++++++++ .../samples/300es/trigonometric_strip.json | 16 ++ 18 files changed, 1315 insertions(+) create mode 100644 shaders/src/main/glsl/samples/300es/binarysearch_bw.frag create mode 100644 shaders/src/main/glsl/samples/300es/binarysearch_bw.json create mode 100644 shaders/src/main/glsl/samples/300es/binarysearch_tree.frag create mode 100644 shaders/src/main/glsl/samples/300es/binarysearch_tree.json create mode 100644 shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.frag create mode 100644 shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.json create mode 100644 shaders/src/main/glsl/samples/300es/householder_lattice.frag create mode 100644 shaders/src/main/glsl/samples/300es/householder_lattice.json create mode 100644 shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag create mode 100644 shaders/src/main/glsl/samples/300es/mergesort_mosaic.json create mode 100644 shaders/src/main/glsl/samples/300es/prefix_sum_checkers.frag create mode 100644 shaders/src/main/glsl/samples/300es/prefix_sum_checkers.json create mode 100644 shaders/src/main/glsl/samples/300es/quicksort_palette.frag create mode 100644 shaders/src/main/glsl/samples/300es/quicksort_palette.json create mode 100644 shaders/src/main/glsl/samples/300es/selection_sort_struct.frag create mode 100644 shaders/src/main/glsl/samples/300es/selection_sort_struct.json create mode 100644 shaders/src/main/glsl/samples/300es/trigonometric_strip.frag create mode 100644 shaders/src/main/glsl/samples/300es/trigonometric_strip.json diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_bw.frag b/shaders/src/main/glsl/samples/300es/binarysearch_bw.frag new file mode 100644 index 000000000..1cef54511 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/binarysearch_bw.frag @@ -0,0 +1,164 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct BinarySearchObject{ + int prime_numbers[10]; +}; + +int FINDMSB(int a) { + int result = -1; + int temp = a; + for (int i = 0; i < 31; i++) { + bool bitIsZero = (temp % 2) == 0; + if (a < 0 && bitIsZero) { + result = i; + } else if (a > 0 && !bitIsZero) { + result = i; + } + temp = temp >> 1; + } + return result; +} + +#define LDEXP(significand, exponent) ((significand)*pow(2.0, exponent)) + +vec2 brick(vec2 uv) { + int a = 4; + do { + uv.y -= step(injectionSwitch.y, uv.x); + uv.x -= fract(tanh(uv.x)) / LDEXP(injectionSwitch.y, float(FINDMSB(a))); + a--; + } while (a > int(injectionSwitch.x)); + int b = 3; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a); + uv.x *= (isnan(uv.y) ? cosh(gl_FragCoord.y) : tanh(gl_FragCoord.x)); + b--; + } while (b > int(injectionSwitch.x)); + int c = 2; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a) + float(b); + uv.x += LDEXP(injectionSwitch.y, isinf(uv.y + uv.x) ? float(FINDMSB(b)) : float(FINDMSB(a))); + c--; + } while (c > int(injectionSwitch.x)); + int d = 1; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a) + float(b) + float(c); + d--; + } while (d > int(injectionSwitch.x)); + return fract(uv); +} + +float patternize(vec2 uv) { + vec2 size = vec2(0.45); + vec2 st = smoothstep(size, size, uv); + switch (int(mod(gl_FragCoord.y, 5.0))) { + case 0: + return mix(pow(st.x, injectionSwitch.y), st.x, size.y); + break; + case 1: + return mix(pow(uv.y, injectionSwitch.y), st.y, size.x); + break; + case 2: + discard; + break; + case 3: + return mix(pow(uv.y, injectionSwitch.y), uv.y, size.y); + break; + case 4: + return mix(pow(st.y, injectionSwitch.y), st.x, size.x); + break; + } +} + +int binarySearch(BinarySearchObject obj, int x) { + int l = 0, r = 9; + while (l <= r) { + int m = (l + r) / 2; + if (obj.prime_numbers[m] == x) { + return m; + } + + if (obj.prime_numbers[m] < x) { + l = m + 1; + } else { + r = m - 1; + } + } + // If an element is not present in the array we return -1. + return -1; +} + +void main() { + BinarySearchObject obj; + // Initialize first 10 prime numbers to the array. + for (int i = 0; i < 10; i++) { + if (i == 0) { + obj.prime_numbers[i] = 2; + } else if (i == 1) { + obj.prime_numbers[i] = 3; + } else if (i == 2) { + obj.prime_numbers[i] = 5; + } else if (i == 3) { + obj.prime_numbers[i] = 7; + } else if (i == 4) { + obj.prime_numbers[i] = 11; + } else if (i == 5) { + obj.prime_numbers[i] = 13; + } else if (i == 6) { + obj.prime_numbers[i] = 17; + } else if (i == 7) { + obj.prime_numbers[i] = 19; + } else if (i == 8) { + obj.prime_numbers[i] = 23; + } else if (i == 9) { + obj.prime_numbers[i] = 29; + } + } + + vec2 uv = (gl_FragCoord.xy / resolution.x) * vec2(resolution.x / resolution.y, 1.0); + vec2 b = brick(uv * 7.0); + vec3 color = vec3(patternize(b)); + + if (gl_FragCoord.y < resolution.y / 1.1) { + // We are going to search the item in array by giving the value of the item index 4 and 0 in an array. + if (binarySearch(obj, obj.prime_numbers[4]) != -(int(resolution.y)) && binarySearch(obj, obj.prime_numbers[0]) >= -(int(resolution.x))) { + color.yz -= dot(float(binarySearch(obj, obj.prime_numbers[4])), float(binarySearch(obj, obj.prime_numbers[0]))); + } else { + discard; + } + } else { + // The following condition is true as there is no value 1 in the array. + if (binarySearch(obj, 1) == -1) { + discard; + } else { + color.yz += color.yz; + } + } + + _GLF_color = vec4(color, injectionSwitch.y); + +} diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_bw.json b/shaders/src/main/glsl/samples/300es/binarysearch_bw.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/binarysearch_bw.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag b/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag new file mode 100644 index 000000000..ad843ed32 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag @@ -0,0 +1,176 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct BST{ + int data; + int leftIndex; + int rightIndex; +}; + +BST tree[10]; + +void makeTreeNode(inout BST tree, int data) +{ + tree.data = data; + tree.leftIndex = -1; + tree.rightIndex = -1; +} + +void insert(int treeIndex, int data) +{ + int baseIndex = 0; + while (baseIndex <= treeIndex) { + // If new value is smaller thatn the current node, we known that we will have + // add this element in the left side. + if (data <= tree[baseIndex].data) { + // If a left subtree of the current node is empty, the new node is added as + // a left subtree of the current node. + if (tree[baseIndex].leftIndex == -1) { + tree[baseIndex].leftIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].leftIndex; + continue; + } + } else { + // If a right subtree of the current node is empty, the new node is added as + // a right subtree of the current node. + if (tree[baseIndex].rightIndex == -1) { + tree[baseIndex].rightIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].rightIndex; + continue; + } + } + } +} + +// Return element data if the given target exists in a tree. Otherwise, we simply return -1. +int search(int target){ + BST currentNode; + int index = 0; + while (index != -1) { + currentNode = tree[index]; + if (currentNode.data == target) { + return target; + } + index = target > currentNode.data ? currentNode.rightIndex : currentNode.leftIndex; + } + return -1; +} + +vec3 hueColor(float angle) { + float nodeData = float(search(15)); + vec3 color; + color = clamp(fract(angle * vec3(1.0, 5.0, nodeData)), 0.0, 1.0); + color.x *= cosh(isnan(float(search(30))) ? injectionSwitch.x : injectionSwitch.y); + return color; +} + +float makeFrame(float v) { + v *= 6.5; + if (v < 1.5) { + return float(search(100)); + } + if (v < 4.0) { + return injectionSwitch.x; + } + if (v < float(search(6))) { + return 1.0; + } + return 10.0 + float(search(30)); +} + +/* +* This shader implements binary search tree using an array data structure. The elements of +* tree are kept in the array that contains a list of BST object holding indices of left and +* right subtree in the array. +* +* - Tree representation of the number used in this shader: +* 9 +* / \ +* 5 12 +* / \ \ +* 2 7 15 +* / \ / \ +* 6 8 13 17 +* +* - Array representation: +* [9, 5, 12, 15, 7, 8, 2, 6, 17, 13] +* +*/ + +void main() { + int treeIndex = int(injectionSwitch.x); + // Initialize root node. + makeTreeNode(tree[int(injectionSwitch.x)], 9); + // Each time we insert a new node into the tree, we increment one. + treeIndex++; + + insert(treeIndex, 5); + treeIndex++; + insert(treeIndex, 12); + treeIndex++; + insert(treeIndex, 15); + treeIndex++; + insert(treeIndex, 7); + treeIndex++; + insert(treeIndex, 8); + treeIndex++; + insert(treeIndex, 2); + treeIndex++; + insert(treeIndex, 6); + treeIndex++; + insert(treeIndex, 17); + treeIndex++; + insert(treeIndex, 13); + + vec2 z = (gl_FragCoord.yx / resolution); + float x = makeFrame(z.x); + float y = makeFrame(z.y); + + int sum = -100; + for (int target = 0; target < 20; target ++) { + int result = search(target); + if (result > 0) { + sum += result; + } else { + switch (result) { + case -1: + sum += int(injectionSwitch.y); + break; + case 0: + return; + } + } + } + float a = sinh(x + y * float(sum)); + _GLF_color = vec4(hueColor(a), 1.); + +} diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_tree.json b/shaders/src/main/glsl/samples/300es/binarysearch_tree.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/binarysearch_tree.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.frag b/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.frag new file mode 100644 index 000000000..25e67e16f --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.frag @@ -0,0 +1,173 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct BST{ + int data; + int leftIndex; + int rightIndex; +}; + +BST tree[10]; + +void makeTreeNode(inout BST tree, int data) +{ + tree.data = data; + tree.leftIndex = -1; + tree.rightIndex = -1; +} + +void insert(int treeIndex, int data) +{ + int baseIndex = 0; + while (baseIndex <= treeIndex) { + // If new value is smaller thatn the current node, we known that we will have + // add this element in the left side. + if (data <= tree[baseIndex].data) { + // If a left subtree of the current node is empty, the new node is added as + // a left subtree of the current node. + if (tree[baseIndex].leftIndex == -1) { + tree[baseIndex].leftIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].leftIndex; + continue; + } + } else { + // If a right subtree of the current node is empty, the new node is added as + // a right subtree of the current node. + if (tree[baseIndex].rightIndex == -1) { + tree[baseIndex].rightIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].rightIndex; + continue; + } + } + } +} + +// Return element data if the given target exists in a tree. Otherwise, we simply return -1. +int search(int target){ + BST currentNode; + int index = 0; + while (index != -1) { + currentNode = tree[index]; + if (currentNode.data == target) { + return target; + } + index = target > currentNode.data ? currentNode.rightIndex : currentNode.leftIndex; + } + return -1; +} + +vec3 hueColor(float angle) { + float nodeData = float(search(15)); + return (30.0 + angle * vec3(1.0, 5.0, nodeData)) / 50.0; +} + +float makeFrame(float v) { + v *= 6.5; + if (v < 1.5) { + return float(search(100)); + } + if (v < 4.0) { + return injectionSwitch.x; + } + if (v < float(search(6))) { + return 1.0; + } + return 10.0 + float(search(30)); +} + +/* +* This shader implements binary search tree using an array data structure. The elements of +* tree are kept in the array that contains a list of BST object holding indices of left and +* right subtree in the array. +* +* - Tree representation of the number used in this shader: +* 9 +* / \ +* 5 12 +* / \ \ +* 2 7 15 +* / \ / \ +* 6 8 13 17 +* +* - Array representation: +* [9, 5, 12, 15, 7, 8, 2, 6, 17, 13] +* +*/ + +void main() { + int treeIndex = int(injectionSwitch.x); + // Initialize root node. + makeTreeNode(tree[int(injectionSwitch.x)], 9); + // Each time we insert a new node into the tree, we increment one. + treeIndex++; + + insert(treeIndex, 5); + treeIndex++; + insert(treeIndex, 12); + treeIndex++; + insert(treeIndex, 15); + treeIndex++; + insert(treeIndex, 7); + treeIndex++; + insert(treeIndex, 8); + treeIndex++; + insert(treeIndex, 2); + treeIndex++; + insert(treeIndex, 6); + treeIndex++; + insert(treeIndex, 17); + treeIndex++; + insert(treeIndex, 13); + + vec2 z = (gl_FragCoord.yx / resolution); + float x = makeFrame(z.x); + float y = makeFrame(z.y); + + int sum = -100; + for (int target = 0; target < 20; target ++) { + int result = search(target); + if (result > 0) { + sum += result; + } else { + switch (result) { + case -1: + sum += int(injectionSwitch.y); + break; + case 0: + return; + } + } + } + float a = x + y * float(sum); + _GLF_color = vec4(hueColor(a), 1.); + +} diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.json b/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/householder_lattice.frag b/shaders/src/main/glsl/samples/300es/householder_lattice.frag new file mode 100644 index 000000000..e061b06c1 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/householder_lattice.frag @@ -0,0 +1,119 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +int MATRIX_N = 4; + +uniform mat4 matrix_a_uni; + +void main() +{ + // We need to modify A in place, so we have to copy the uniform + mat4 matrix_a = mat4(matrix_a_uni); + vec4 matrix_b = gl_FragCoord.wxyz; + + vec4 matrix_u = vec4(0.0); + + float magnitudeX = 0.0; + float alpha1 = 0.0; + float alpha2 = 0.0; + float alpha3 = 0.0; + float beta = 0.0; + // QR factorization + for(int k = 0; k < MATRIX_N - 1; k++) + { + // define vector x as the column A(k) only with entries k - n + // creating our U vector at the same time we create X magnitude. + for(int x = MATRIX_N - 1; x >= k; x--) // start from the bottom and go up. + { + magnitudeX += pow(matrix_a[x][k], 2.0); + matrix_u[x] = matrix_a[x][k]; + } + magnitudeX = sqrt(magnitudeX); + // entry k is the top of our U vector for this iteration + // define U as x - sign(x1) (magnitude X) (elementary matrix 1) + matrix_u[k] -= (sign(matrix_u[k]) * magnitudeX); + + // alpha1 = U(T) * U gives us the dot product + for(int u = MATRIX_N - 1; u >= k; u--) + { + alpha1 += pow(matrix_u[u], 2.0); + } + + // alpha2 = 2.0 / (U(T) * U) = 2.0 / alpha1 + alpha2 = 2.0 / alpha1; + + for(int j = k; j < MATRIX_N; j++) + { + // evaluate alpha3 = U(T) * a(j) updated k - 1 times + for(int a = MATRIX_N - 1; a >= k; a--) + { + alpha3 += matrix_u[a] * matrix_a[a][j]; + } + + beta = alpha2 * alpha3; + + // evaluate a(j) updated k times = a(j) updated k - 1 times - (beta * U) + for(int a = MATRIX_N - 1; a >= k; a--) + { + matrix_a[a][j] = matrix_a[a][j] - (beta * matrix_u[a]); + } + alpha3 = 0.0; + beta = 0.0; + } + + // evaluate alpha3 = U(T) * a(j) updated k - 1 times + for(int b = MATRIX_N - 1; b >= k; b--) + { + alpha3 += matrix_u[b] * matrix_b[b]; + } + + // evaluate beta = alpha2 * alpha3 + beta = alpha2 * alpha3; + + // evaluate b(j) updated k times = b(j) updated k - 1 times - (beta * U) + for(int b = MATRIX_N - 1; b >= k; b--) + { + matrix_b[b] = matrix_b[b] - (beta * matrix_u[b]); + } + + magnitudeX = 0.0; + alpha1 = 0.0; + alpha2 = 0.0; + alpha3 = 0.0; + beta = 0.0; + } + + // back substitution algorithm + for(int i = (MATRIX_N - 1); i >= 0; i--) + { + for(int j = (MATRIX_N - 1); j >= (i + 1); j--) + { + matrix_b[i] -= (matrix_a[i][j] * matrix_b[j]); + } + + // final operation, using the diagonal; matrix_b becomes the solution matrix x + matrix_b[i] /= matrix_a[i][i]; + } + _GLF_color = tan(matrix_b); + _GLF_color.a = 1.0; +} + diff --git a/shaders/src/main/glsl/samples/300es/householder_lattice.json b/shaders/src/main/glsl/samples/300es/householder_lattice.json new file mode 100644 index 000000000..51d6c7a30 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/householder_lattice.json @@ -0,0 +1,11 @@ +{ + "matrix_a_uni": { + "func": "glUniformMatrix4fv", + "args": [ + 1.0, 5.0, 3.0, 7.0, + 9.0, 6.0, 7.0, 8.0, + 1.0, 2.0, 3.0, 4.0, + 5.0, 6.0, 7.0, 8.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag b/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag new file mode 100644 index 000000000..d215742eb --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag @@ -0,0 +1,169 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +// Size of an array. +const int N = 10; +// An array and its temperary array whose elements will be sorted. +int data[10], temp[10]; + +// Merge two sorted subarrays data[from ... mid] and data[mid+1 ... to]. +void merge(int from, int mid, int to) { + int k = from, i = from, j = mid +1; + + while (i <= mid && j <= to) { + if (data[i] < data[j]) { + temp[k++] = data[i++]; + } else { + temp[k++] = data[j++]; + } + } + // Copy remaining elements. + while (i < N && i <= mid) { + temp[k++] = data[i++]; + } + // Copy back to the original array. + for (int i = from; i <= to; i++) { + data[i] = temp[i]; + } +} + +// Sort array using the iterative approach. +void mergeSort() { + int low = 0; + int high = N - 1; + + // Devide the array into blocks of size m. + // m = [1, 2, 4 ,8, 16...]. + for (int m = 1; m <= high; m = 2 * m) { + + // For m = 1, i = [0, 2, 4, 6, 8]. + // For m = 2, i = [0, 4, 8]. + // For m = 4, i = [0, 8]. + for (int i = low; i< high; i += 2 * m) { + int from = i; + int mid = i + m - 1; + int to = min (i + 2 * m - 1, high); + merge(from, mid, to); + } + } +} + +#define LDEXP(significand, exponent) ((significand)*pow(2.0, exponent)) + +void main() { + int i = int(injectionSwitch.x); + do { + switch(i) { + case 0: + data[i] = 4; + break; + case 1: + data[i] = 3 ; + break; + case 2: + data[i] = 2 ; + break; + case 3: + data[i] = 1 ; + break; + case 4: + data[i] = 0 ; + break; + case 5: + data[i] = -1; + break; + case 6: + data[i] = -2; + break; + case 7: + data[i] = -3; + break; + case 8: + data[i] = -4; + break; + case 9: + data[i] = -5; + break; + } + i++; + } while (i < 10); + + for (int j = 0; j < 10; j++) { + temp[j] = data[j]; + } + mergeSort(); + + mat3 pos = mat3( + vec3(4, 0, int(injectionSwitch.y)) * LDEXP(0.5, 2.0), + vec3(0, -5, int(injectionSwitch.y)) * LDEXP(0.2, 5.0), + vec3(1, 8, int(injectionSwitch.y)) * LDEXP(injectionSwitch.y, 0.0) + ); + vec3 vecCoor = roundEven(pos * vec3(gl_FragCoord.xx / resolution.yx, 1)); + vec2 color; + + do { + if (int(gl_FragCoord[1]) < 30) { + color = fract(sin(vecCoor.yx - trunc(float(data[0])))); + color[0] = dFdy(gl_FragCoord.y); + break; + } else if (int(gl_FragCoord[1]) < 60) { + color = fract(sin(vecCoor.yx - trunc(float(data[1])))); + color[1] *= atan(faceforward(injectionSwitch, color.xx, vecCoor.yx).y); + break; + } else if (int(gl_FragCoord[1]) < 90) { + color = fract(sin(vecCoor.yx - trunc(float(data[2])))); + color.x += atanh(color.x) * cosh(injectionSwitch.y) * smoothstep(color, injectionSwitch, gl_FragCoord.yy).x; + break; + } else if (int(gl_FragCoord[1]) < 120) { + color = fract(acosh(vecCoor.yx - trunc(float(data[3])))); + color.x += (isnan(gl_FragCoord.x) ? log2(gl_FragCoord.x) : log2(gl_FragCoord.y)); + break; + } else if (int(gl_FragCoord[1]) < 150) { + discard; + } else if (int(gl_FragCoord[1]) < 180) { + color = fract(sin(vecCoor.yx - trunc(float(data[4])))); + color[1] += asinh(gl_FragCoord.y * LDEXP(color.y, float(-i))); + break; + } else if (int(gl_FragCoord[1]) < 210) { + color = fract(sin(vecCoor.yx - trunc(float(data[5])))); + color.y -= tanh(color.x) / cosh(color.y); + break; + } else if (int(gl_FragCoord[1]) < 240) { + color = fract(asinh(vecCoor.yx - trunc(float(data[6])))); + color.y -= isnan(float(i)) ? tanh(gl_FragCoord.x): atanh(gl_FragCoord.y); + break; + } else if (int(gl_FragCoord[1]) < 270) { + color = fract(sin(vecCoor.yx - trunc(float(data[7])))); + color.y *= mix(normalize(vecCoor), normalize(vec3(color, degrees(color.x))), injectionSwitch.x).y; + break; + } else { + discard; + } + } while(0 == int(injectionSwitch.x)); + + _GLF_color = vec4(vec3(color, trunc(injectionSwitch.y)), injectionSwitch.y); + +} diff --git a/shaders/src/main/glsl/samples/300es/mergesort_mosaic.json b/shaders/src/main/glsl/samples/300es/mergesort_mosaic.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/mergesort_mosaic.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.frag b/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.frag new file mode 100644 index 000000000..36043ed57 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.frag @@ -0,0 +1,95 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +vec2 pattern(in vec2 x) { + vec2 n = floor(x); + vec3 m = vec3(1.0); + + for (int j = -1; j <= int(injectionSwitch.y); j++) { + for (int i = -1; i <= int(injectionSwitch.y); i++) { + vec2 g = vec2(float(j), float(i)); + vec2 o = mix(n, g, 0.2); + if (injectionSwitch.x < (m.x)) { + int k = 1; + while (k >= 0) { + o = o + o; + k--; + } + m = vec3(injectionSwitch.x, cos(o)); + } + } + } + return vec2(m.x, m.y - m.z); +} + +void main() { + vec2 uv = gl_FragCoord.xy / resolution.y; + float A[50]; + for (int i = 0; i < 200; i++) { + if (i >= int(resolution.x)) { + break; + } + if ((4 * (i/4)) == i) { + A[i/4] = float(i); + } + } + for (int i = 0; i < 50; i++) { + if (i < int(gl_FragCoord.x)) { + break; + } + if (i > 0) { + A[i] += A[i - 1]; + } + } + + vec2 c = pattern((15.0 + tan(0.2)) * uv); + vec3 col; + if (int(gl_FragCoord.y) < 20) { + col = .5 + cos(c.y + vec3(resolution.x, A[4]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 40) { + col = .5 + cos(c.y + vec3(resolution.x, A[9]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 60) { + col = .5 + cos(c.y + vec3(resolution.x, A[14]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 80) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 100) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 120) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 140) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 160) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 180) { + col = .5 + cos(c.y + vec3(resolution.x, A[44]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 200) { + col = .5 + cos(c.y + vec3(resolution.x, A[49]/resolution.x + 50.0, 22.0)); + } else { + discard; + } + + _GLF_color = vec4(col, 1.0); +} diff --git a/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.json b/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/quicksort_palette.frag b/shaders/src/main/glsl/samples/300es/quicksort_palette.frag new file mode 100644 index 000000000..c41018986 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/quicksort_palette.frag @@ -0,0 +1,131 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct QuicksortObject{ + int numbers[10]; +}; + +QuicksortObject obj; + + void swap(int i, int j) { + int temp = obj.numbers[i]; + obj.numbers[i] = obj.numbers[j]; + obj.numbers[j] = temp; + } + + // Since "partition" is the preserved word, we add prefix to this function name to prevent an error. + int performPartition(int l, int h) { + // The rightmost element is chosen as a pivot. + int pivot = obj.numbers[h]; + int i = (l - 1); + + for (int j = l; j <= h - 1; j++) { + if (obj.numbers[j] <= pivot) { + i++; + swap(i, j); + } + } + swap(i + 1, h); + return (i + 1); + } + + void quicksort() { + int l = 0, h = 9; + int stack[10]; + int top = -1; + + stack[++top] = l; + stack[++top] = h; + + while (top >= 0) { + h = stack[top--]; + l = stack[top--]; + + int p = performPartition(l, h); + if (p - 1 > l) { + stack[++top] = l; + stack[++top] = p - 1; + } + if (p + 1 < h) { + stack[++top] = p + 1; + stack[++top] = h; + } + } + } + +vec3 palette(vec3 a, vec3 b, vec3 c, vec3 d) { + return fract(mix(d * a, a * c, b + d - c)); +} + +float randomize(vec2 co) +{ + return float(floor(fract(sin(dot(co.xy ,vec2(12.5, 3.))) * 4250.) + 0.5)); +} + +bool puzzlelize(vec2 pos) +{ + return randomize(pos.xy) < .12; +} + +void main() { + // Initialize decreasing values to an array starting from 10. + for (int i = int(injectionSwitch.x); i < 10; i ++) { + obj.numbers[i] = (10 - i) * int(injectionSwitch.y); + } + quicksort(); + vec2 grid = vec2(20, 20); + vec2 uv = gl_FragCoord.xy / resolution; + vec3 color = palette(vec3(float(obj.numbers[4]) * 0.1), vec3(0.9, float(obj.numbers[8]) * 0.1, 0.8), trunc(vec3(injectionSwitch.y)), vec3(injectionSwitch.x, 0.3, 0.7)); + if (uv.x > (1.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color += palette(vec3(0.5, float(obj.numbers[8]) * 0.1, 0.2), vec3(0.5), trunc(vec3(injectionSwitch.y)), vec3(float(obj.numbers[4]) * 0.1, injectionSwitch.x, 0.6)); + count++; + } while (count != obj.numbers[int(injectionSwitch.x)]); + grid = vec2(count + obj.numbers[8], count + obj.numbers[6]); + } + if (uv.x > (2.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color -= palette(vec3(float(obj.numbers[4]) * 0.1), trunc(vec3(0.1)), vec3(float(obj.numbers[int(injectionSwitch.y)]) * 0.1), vec3(injectionSwitch.x, float(obj.numbers[2]) * 0.1 , float(obj.numbers[8]) * 0.1)); + count++; + } while (count != obj.numbers[1]); + grid += vec2(count + int(injectionSwitch.y), count + int(injectionSwitch.x)); + } + if(uv.x > (3.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color -= palette(vec3(float(obj.numbers[int(injectionSwitch.y)]) * 0.1), vec3(0.8), trunc(vec3(0.1)), vec3(injectionSwitch.x, 0.6, float(obj.numbers[int(injectionSwitch.x)]) * 0.1)); + count++; + } while (count != obj.numbers[2]); + grid += vec2(count + obj.numbers[3], count + obj.numbers[3]); + } + + vec2 position = vec2(gl_FragCoord.x, resolution.x - gl_FragCoord.y); + position = floor(position / grid); + _GLF_color = vec4(color, injectionSwitch.y) + vec4(!puzzlelize(position)); + +} diff --git a/shaders/src/main/glsl/samples/300es/quicksort_palette.json b/shaders/src/main/glsl/samples/300es/quicksort_palette.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/quicksort_palette.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag b/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag new file mode 100644 index 000000000..ad151bb49 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag @@ -0,0 +1,75 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct Obj { + float odd_numbers[10]; + float even_numbers[10]; +}; + +void main() { + Obj obj; + + // Initialize first 10 odd numbers to the array. + int odd_index = 0; + float odd_number = 1.0; + while (odd_index <= 9) { + obj.odd_numbers[odd_index] = odd_number; + odd_number += 2.0; + odd_index++; + } + // Similarly, initialize even numbers and iterate backward. + int even_index = 9; + float even_number = 0.0; + while (even_index >= 0) { + obj.even_numbers[even_index] = even_number; + even_number += 2.; + even_index--; + } + + // Perform the selection sort. + for (int i = 0; i < 9; i++) { + int index = i; + for (int j = i + 1; j < 10; j++) { + if (obj.even_numbers[j] < obj.even_numbers[index]) { + index = j; + } + } + float smaller_number = obj.even_numbers[index]; + obj.even_numbers[index] = obj.even_numbers[i]; + obj.even_numbers[i] = smaller_number; + } + + vec2 uv = gl_FragCoord.xy/resolution.xy; + vec3 col = tan(pow(uv.xxx, uv.yyy) + + vec3( + obj.odd_numbers[int(floor(gl_FragCoord.x/1000.0))], + obj.even_numbers[int(floor(gl_FragCoord.y/1000.0))], + sinh(uv.x) + )); + + _GLF_color.rgb = col; + _GLF_color.a = 1.0; +} diff --git a/shaders/src/main/glsl/samples/300es/selection_sort_struct.json b/shaders/src/main/glsl/samples/300es/selection_sort_struct.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/selection_sort_struct.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/trigonometric_strip.frag b/shaders/src/main/glsl/samples/300es/trigonometric_strip.frag new file mode 100644 index 000000000..53d7e47f9 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/trigonometric_strip.frag @@ -0,0 +1,74 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +float compute_derivative_x(float v){ + return dFdx(v) * injectionSwitch.y; +} + +float compute_derivative_y(float v){ + return dFdy(v) * injectionSwitch.y; +} + +float compute_stripe(float v) { + return smoothstep(-.9, 1., (v)/ ((injectionSwitch.y > injectionSwitch.x) ? compute_derivative_x(v): compute_derivative_y(v))); +} + +void main() { + vec2 uv = gl_FragCoord.xy / resolution.x; + vec3 col = vec3(0, 0, 0); + + bool c1 = uv.y < 0.25; + if (c1) { + float stripe = compute_stripe(cos((uv.x + uv.y ) * 20.0 )); + col = mix(vec3(uv.x, 0, 0.75), vec3(.8, .7, uv.x), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c2 = uv.y < 0.5; + if (!c1 && c2) { + float stripe = compute_stripe(tan((uv.x + uv.y) * 20.0 )); + col = mix(vec3(0.5, uv.x, 0.1), vec3(.4, 0, .5), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c3 = uv.y < 0.75; + if (!c1 && !c2 && c3) { + float stripe = compute_stripe(cos((uv.x + uv.y) * 20.0 )); + col = mix(vec3(.7, sinh(uv.x), uv.x ), vec3(.3, .5, uv.x), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c4 = uv.y >= 0.75; + if (!c1 && !c2 && !c3 && c4) { + float stripe = compute_stripe(tan((uv.x + uv.y) * 20.0 )); + col = mix(vec3(uv.x, .8, 0), vec3(1, uv.y, 0), stripe); + _GLF_color = vec4(col, 1.0); + return; + } +} diff --git a/shaders/src/main/glsl/samples/300es/trigonometric_strip.json b/shaders/src/main/glsl/samples/300es/trigonometric_strip.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/trigonometric_strip.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From 3b1cd7928dfe5f837dfaca3841c85eb463808a45 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 20 Aug 2019 10:43:49 +0100 Subject: [PATCH 109/180] gfauto: fix signature_util plus tests (#702) Also, add pytest tests and related changes. --- gfauto/.flake8 | 4 +- gfauto/.idea/runConfigurations/pytest.xml | 18 + gfauto/.idea/scopes/python_files.xml | 2 +- gfauto/Pipfile | 3 + gfauto/Pipfile.lock | 416 +++++++++++++--------- gfauto/README.md | 2 +- gfauto/check_all.sh | 5 +- gfauto/fix_all.sh | 4 +- gfauto/gfauto/signature_util.py | 6 +- gfauto/gfautotests/__init__.py | 15 + gfauto/gfautotests/test_signature_util.py | 44 +++ gfauto/pyproject.toml | 6 +- gfauto/run_protoc.sh | 4 + gfauto/whitelist.dic | 1 + 14 files changed, 350 insertions(+), 180 deletions(-) create mode 100644 gfauto/.idea/runConfigurations/pytest.xml create mode 100644 gfauto/gfautotests/__init__.py create mode 100644 gfauto/gfautotests/test_signature_util.py diff --git a/gfauto/.flake8 b/gfauto/.flake8 index bb9bcdbae..cb61af254 100644 --- a/gfauto/.flake8 +++ b/gfauto/.flake8 @@ -15,7 +15,7 @@ # This is not a TOML file; no quoting. [flake8] -filename = */gfauto/*.py +filename = */gfauto/*.py, */gfautotests/*.py exclude = *_pb2.py* max-line-length = 88 ignore = @@ -61,5 +61,5 @@ ignore = inline-quotes = " # For flake8-spellcheck. Default is whitelist.txt but using whitelist.dic means -# we can add the file to PyCharm/IntelliJ (albeit, globally, not per project). +# we can add the file to PyCharm/IntelliJ. whitelist=whitelist.dic diff --git a/gfauto/.idea/runConfigurations/pytest.xml b/gfauto/.idea/runConfigurations/pytest.xml new file mode 100644 index 000000000..e4a9cac18 --- /dev/null +++ b/gfauto/.idea/runConfigurations/pytest.xml @@ -0,0 +1,18 @@ + + + + + \ No newline at end of file diff --git a/gfauto/.idea/scopes/python_files.xml b/gfauto/.idea/scopes/python_files.xml index a2c0629a3..44ba6e31b 100644 --- a/gfauto/.idea/scopes/python_files.xml +++ b/gfauto/.idea/scopes/python_files.xml @@ -1,3 +1,3 @@ - + \ No newline at end of file diff --git a/gfauto/Pipfile b/gfauto/Pipfile index bf68d9ce4..70071fa91 100644 --- a/gfauto/Pipfile +++ b/gfauto/Pipfile @@ -43,6 +43,9 @@ grpcio-tools = "*" # Type checking. mypy = "*" +# Testing. +pytest = "*" + # PyLint linter. pylint = "*" diff --git a/gfauto/Pipfile.lock b/gfauto/Pipfile.lock index 3fc9134ba..4d718af10 100644 --- a/gfauto/Pipfile.lock +++ b/gfauto/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a4ba469d9d6dc05205708ab9462b64a7d9436c5d1146fdf821e4a39a69b75993" + "sha256": "3aa0659fa85d176cb557e1d584e9a4fbd383af08b47a975ef92b6e6df14264a8" }, "pipfile-spec": 6, "requires": {}, @@ -20,26 +20,24 @@ }, "protobuf": { "hashes": [ - "sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8", - "sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538", - "sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e", - "sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a", - "sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6", - "sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0", - "sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc", - "sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47", - "sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01", - "sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115", - "sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277", - "sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c", - "sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea", - "sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87", - "sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7", - "sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126", - "sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a", - "sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832" - ], - "version": "==3.8.0" + "sha256:00a1b0b352dc7c809749526d1688a64b62ea400c5b05416f93cfb1b11a036295", + "sha256:01acbca2d2c8c3f7f235f1842440adbe01bbc379fa1cbdd80753801432b3fae9", + "sha256:0a795bca65987b62d6b8a2d934aa317fd1a4d06a6dd4df36312f5b0ade44a8d9", + "sha256:0ec035114213b6d6e7713987a759d762dd94e9f82284515b3b7331f34bfaec7f", + "sha256:31b18e1434b4907cb0113e7a372cd4d92c047ce7ba0fa7ea66a404d6388ed2c1", + "sha256:32a3abf79b0bef073c70656e86d5bd68a28a1fbb138429912c4fc07b9d426b07", + "sha256:55f85b7808766e5e3f526818f5e2aeb5ba2edcc45bcccede46a3ccc19b569cb0", + "sha256:64ab9bc971989cbdd648c102a96253fdf0202b0c38f15bd34759a8707bdd5f64", + "sha256:64cf847e843a465b6c1ba90fb6c7f7844d54dbe9eb731e86a60981d03f5b2e6e", + "sha256:917c8662b585470e8fd42f052661fc66d59fccaae450a60044307dcbf82a3335", + "sha256:afed9003d7f2be2c3df20f64220c30faec441073731511728a2cb4cab4cd46a6", + "sha256:bf8e05d638b585d1752c5a84247134a0350d3a8b73d3632489a014a9f6f1e758", + "sha256:d831b047bd69becaf64019a47179eb22118a50dd008340655266a906c69c6417", + "sha256:de2760583ed28749ff885789c1cbc6c9c06d6de92fc825740ab99deb2f25ea4d", + "sha256:eabc4cf1bc19689af8022ba52fd668564a8d96e0d08f3b4732d26a64255216a4", + "sha256:fcff6086c86fb1628d94ea455c7b9de898afc50378042927a59df8065a79a549" + ], + "version": "==3.9.1" }, "six": { "hashes": [ @@ -64,6 +62,13 @@ ], "version": "==2.2.5" }, + "atomicwrites": { + "hashes": [ + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + ], + "version": "==1.3.0" + }, "attrs": { "hashes": [ "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", @@ -100,6 +105,13 @@ ], "version": "==7.0" }, + "ddt": { + "hashes": [ + "sha256:474546b4020ce8a2f9550ba8899c28aa2c284c7bbf175bddede98be949d1ca7c", + "sha256:d13e6af8f36238e89d00f4ebccf2bda4f6d1878be560a6600689e42077e164e3" + ], + "version": "==1.2.1" + }, "decorator": { "hashes": [ "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", @@ -116,11 +128,11 @@ }, "flake8": { "hashes": [ - "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", - "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" + "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", + "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" ], "index": "pypi", - "version": "==3.7.7" + "version": "==3.7.8" }, "flake8-bandit": { "hashes": [ @@ -146,19 +158,19 @@ }, "flake8-broken-line": { "hashes": [ - "sha256:3eb823f48b4ec67735ebbe8e282319826c8e8be7dfc185c9e259228084c21de2", - "sha256:84147d38a1562d011a8de0bb1de299a715f7dea1a7332bd6946db04a1e4c3ddd" + "sha256:30378a3749911e453d0a9e03204156cbbd35bcc03fb89f12e6a5206e5baf3537", + "sha256:7721725dce3aeee1df371a252822f1fcecfaf2766dcf5bac54ee1b3f779ee9d1" ], "index": "pypi", - "version": "==0.1.0" + "version": "==0.1.1" }, "flake8-bugbear": { "hashes": [ - "sha256:5070774b668be92c4312e5ca82748ddf4ecaa7a24ff062662681bb745c7896eb", - "sha256:fef9c9826d14ec23187ae1edeb3c6513c4e46bf0e70d86bac38f7d9aabae113d" + "sha256:d8c466ea79d5020cb20bf9f11cf349026e09517a42264f313d3f6fddb83e0571", + "sha256:ded4d282778969b5ab5530ceba7aa1a9f1b86fa7618fc96a19a1d512331640f8" ], "index": "pypi", - "version": "==19.3.0" + "version": "==19.8.0" }, "flake8-builtins": { "hashes": [ @@ -186,11 +198,11 @@ }, "flake8-comprehensions": { "hashes": [ - "sha256:35f826956e87f230415cde9c3b8b454e785736cf5ff0be551c441b41b937f699", - "sha256:f0b61d983d608790abf3664830d68efd3412265c2d10f6a4ba1a353274dbeb64" + "sha256:7b174ded3d7e73edf587e942458b6c1a7c3456d512d9c435deae367236b9562c", + "sha256:e36fc12bd3833e0b34fe0639b7a817d32c86238987f532078c57eafdc7a8a219" ], "index": "pypi", - "version": "==2.1.0" + "version": "==2.2.0" }, "flake8-debugger": { "hashes": [ @@ -201,11 +213,11 @@ }, "flake8-docstrings": { "hashes": [ - "sha256:4e0ce1476b64e6291520e5570cf12b05016dd4e8ae454b8a8a9a48bc5f84e1cd", - "sha256:8436396b5ecad51a122a2c99ba26e5b4e623bf6e913b0fea0cb6c2c4050f91eb" + "sha256:3ad372b641f4c8e70c7465f067aed4ff8bf1e9347fce14f9eb71ed816db36257", + "sha256:d8d72ccd5807c1ab9ff1466cb9bece0c4d94b8669e9bc4f472abc80dbc5d399e" ], "index": "pypi", - "version": "==1.3.0" + "version": "==1.3.1" }, "flake8-isort": { "hashes": [ @@ -268,10 +280,10 @@ }, "flake8-quotes": { "hashes": [ - "sha256:10c9af6b472d4302a8e721c5260856c3f985c5c082b04841aefd2f808ac02038" + "sha256:5dbaf668887873f28346fb87943d6da2e4b9f77ce9f2169cff21764a0a4934ed" ], "index": "pypi", - "version": "==2.0.1" + "version": "==2.1.0" }, "flake8-spellcheck": { "hashes": [ @@ -313,92 +325,101 @@ }, "gitpython": { "hashes": [ - "sha256:563221e5a44369c6b79172f455584c9ebbb122a13368cc82cb4b5addff788f82", - "sha256:8237dc5bfd6f1366abeee5624111b9d6879393d84745a507de0fda86043b65a8" + "sha256:259a8b6d6a4a118738c4a65fa990f8c8c91525bb43970aed2868952ebb86ceb8", + "sha256:73aa7b59e58dd3435121421c33c284e5ef51bc7b2f4373e1a1e4cc06e9c928ec" ], - "version": "==2.1.11" + "version": "==3.0.1" }, "grpcio": { "hashes": [ - "sha256:03b78b4e7dcdfe3e257bb528cc93923f9cbbab6d5babf15a60d21e9a4a70b1a2", - "sha256:1ce0ccfbdfe84387dbcbf44adb4ae16ec7ae70e166ffab478993eb1ea1cba3ce", - "sha256:22e167a9406d73dd19ffe8ed6a485f17e6eac82505be8c108897f15e68badcbb", - "sha256:31d0aeca8d8ee2301c62c5c340e0889d653b1280d68f9fa203982cb6337b050e", - "sha256:44c7f99ca17ebbcc96fc54ed00b454d8313f1eac28c563098d8b901025aff941", - "sha256:5471444f53f9db6a1f1f11f5dbc173228881df8446380b6b98f90afb8fd8348e", - "sha256:561bca3b1bde6d6564306eb05848fd155136e9c3a25d2961129b1e2edba22fce", - "sha256:5bf58e1d2c2f55365c06e8cb5abe067b88ca2e5550fb62009c41df4b54505acf", - "sha256:6b7163d1e85d76b0815df63fcc310daec02b44532bb433f743142d4febcb181f", - "sha256:766d79cddad95f5f6020037fe60ea8b98578afdf0c59d5a60c106c1bdd886303", - "sha256:770b7372d5ca68308ff66d7baee53369fa5ce985f84bcb6aa1948c1f2f7b02f2", - "sha256:7ab178da777fc0f55b6aef5a755f99726e8e4b75e3903954df07b27059b54fcf", - "sha256:8078305e77c2f6649d36b24d8778096413e474d9d7892c6f92cfb589c9d71b2e", - "sha256:85600b63a386d860eeaa955e9335e18dd0d7e5477e9214825abf2c2884488369", - "sha256:857d9b939ae128be1c0c792eb885c7ff6a386b9dea899ac4b06f4d90a31f9d87", - "sha256:87a41630c90c179fa5c593400f30a467c498972c702f348d41e19dafeb1d319e", - "sha256:8805d486c6128cc0fcc8ecf16c4095d99a8693a541ef851429ab334e028a4a97", - "sha256:8d71b7a89c306a41ccc7741fc9409b14f5b86727455c2a1c0c7cfcb0f784e1f2", - "sha256:9e1b80bd65f8f160880cb4dad7f55697f6d37b2d7f251fc0c2128e811928f369", - "sha256:9e290c84a145ae2411ee0ec9913c41cd7500e2e7485fe93632434d84ef4fda67", - "sha256:9ec9f88b5bc94bd99372f27cdd53af1c92ba06717380b127733b953cfb181174", - "sha256:a0a02a8b4ba6deadf706d5f849539b3685b72b186a3c9ef5d43e8972ed60fb6f", - "sha256:a4059c59519f5940e01a071f74ae2a60ea8f6185b03d22a09d40c7959a36b16b", - "sha256:a6e028c2a6da2ebfa2365a5b32531d311fbfec0e3600fc27e901b64f0ff7e54e", - "sha256:adcdebf9f8463df4120c427cf6c9aed39258bccd03ed37b6939e7a145d64d6e0", - "sha256:bdec982610259d07156a58f80b8c3e69be7751a9208bc577b059c5193d087fad", - "sha256:cefc4d4251ffb73feb303d4b7e9d6c367cb60f2db16d259ea28b114045f965aa", - "sha256:d4145c8aa6afbac10ad27e408f7ce15992fe89ba5d0b4abca31c0c2729864c03", - "sha256:da76dc5ad719ee99de5ea28a5629ff92172cbb4a70d8a6ae3a5b7a53c7382ce1", - "sha256:dde2452c08ef8b6426ccab6b5b6de9f06d836d9937d6870e68153cbf8cb49348", - "sha256:e3d88091d2539a4868750914a6fe7b9ec50e42b913851fc1b77423b5bd918530", - "sha256:f9c67cfe6278499d7f83559dc6322a8bbb108e307817a3d7acbfea807b3603cc" - ], - "version": "==1.22.0" + "sha256:1303578092f1f6e4bfbc354c04ac422856c393723d3ffa032fff0f7cb5cfd693", + "sha256:229c6b313cd82bec8f979b059d87f03cc1a48939b543fe170b5a9c5cf6a6bc69", + "sha256:3cd3d99a8b5568d0d186f9520c16121a0f2a4bcad8e2b9884b76fb88a85a7774", + "sha256:41cfb222db358227521f9638a6fbc397f310042a4db5539a19dea01547c621cd", + "sha256:43330501660f636fd6547d1e196e395cd1e2c2ae57d62219d6184a668ffebda0", + "sha256:45d7a2bd8b4f25a013296683f4140d636cdbb507d94a382ea5029a21e76b1648", + "sha256:47dc935658a13b25108823dabd010194ddea9610357c5c1ef1ad7b3f5157ebee", + "sha256:480aa7e2b56238badce0b9413a96d5b4c90c3bfbd79eba5a0501e92328d9669e", + "sha256:4a0934c8b0f97e1d8c18e76c45afc0d02d33ab03125258179f2ac6c7a13f3626", + "sha256:5624dab19e950f99e560400c59d87b685809e4cfcb2c724103f1ab14c06071f7", + "sha256:60515b1405bb3dadc55e6ca99429072dad3e736afcf5048db5452df5572231ff", + "sha256:610f97ebae742a57d336a69b09a9c7d7de1f62aa54aaa8adc635b38f55ba4382", + "sha256:64ea189b2b0859d1f7b411a09185028744d494ef09029630200cc892e366f169", + "sha256:686090c6c1e09e4f49585b8508d0a31d58bc3895e4049ea55b197d1381e9f70f", + "sha256:7745c365195bb0605e3d47b480a2a4d1baa8a41a5fd0a20de5fa48900e2c886a", + "sha256:79491e0d2b77a1c438116bf9e5f9e2e04e78b78524615e2ce453eff62db59a09", + "sha256:825177dd4c601c487836b7d6b4ba268db59787157911c623ba59a7c03c8d3adc", + "sha256:8a060e1f72fb94eee8a035ed29f1201ce903ad14cbe27bda56b4a22a8abda045", + "sha256:90168cc6353e2766e47b650c963f21cfff294654b10b3a14c67e26a4e3683634", + "sha256:94b7742734bceeff6d8db5edb31ac844cb68fc7f13617eca859ff1b78bb20ba1", + "sha256:962aebf2dd01bbb2cdb64580e61760f1afc470781f9ecd5fe8f3d8dcd8cf4556", + "sha256:9c8d9eacdce840b72eee7924c752c31b675f8aec74790e08cff184a4ea8aa9c1", + "sha256:af5b929debc336f6bab9b0da6915f9ee5e41444012aed6a79a3c7e80d7662fdf", + "sha256:b9cdb87fc77e9a3eabdc42a512368538d648fa0760ad30cf97788076985c790a", + "sha256:c5e6380b90b389454669dc67d0a39fb4dc166416e01308fcddd694236b8329ef", + "sha256:d60c90fe2bfbee735397bf75a2f2c4e70c5deab51cd40c6e4fa98fae018c8db6", + "sha256:d8582c8b1b1063249da1588854251d8a91df1e210a328aeb0ece39da2b2b763b", + "sha256:ddbf86ba3aa0ad8fed2867910d2913ee237d55920b55f1d619049b3399f04efc", + "sha256:e46bc0664c5c8a0545857aa7a096289f8db148e7f9cca2d0b760113e8994bddc", + "sha256:f6437f70ec7fed0ca3a0eef1146591bb754b418bb6c6b21db74f0333d624e135", + "sha256:f71693c3396530c6b00773b029ea85e59272557e9bd6077195a6593e4229892a", + "sha256:f79f7455f8fbd43e8e9d61914ecf7f48ba1c8e271801996fef8d6a8f3cc9f39f" + ], + "version": "==1.23.0" }, "grpcio-tools": { "hashes": [ - "sha256:0d8e85d7e62ea4e04ff75eace98dd0b53b22bd57dc1efbed0f8398db884e1cf6", - "sha256:160e6fd9c44a3f6c76a8c2f84014e367d41a3cae27c451406b616f9a08ba2b1f", - "sha256:337554658b4a4c0a6e1ee75acd6681ec45007e1d6d1975e885f92b665881e33a", - "sha256:36390281f31861b8b685f7e8366ff8abbcdcedf3dda7b73133be56db71d3283b", - "sha256:3dd362bcd1db5247760b7f8b8157abd53e47210a17df9de994f064819ba64c04", - "sha256:4381f3722f3c5d02626796f9db6dd0dcb75c13245ea5655b54880ab7bd239a73", - "sha256:4b5a5fe3e949bd03e068c05070f3cdc6a01b68323efb667199d3c67a30ef515c", - "sha256:4f43d03e3e1f5f7bb14b8bba35a607e35ea4589fed778b8321c97810bc0a76ac", - "sha256:67d1a8d71b2572250124ebdebbfac7248563bb97ab139916fb265705d1b0dbe2", - "sha256:6e6fab6e8e92aadb70b4167e5eadd843e8c850d97213fdf4a39f6e4d3596e6f0", - "sha256:75fe16b642564e47c65cf6d23362079cdbbbc5c544b59c7bc1a3a7c8865d73e3", - "sha256:78bc91e38fe6a6c80de084014e22365c788811773829f28621878a803dd6af48", - "sha256:8dde6f660e8390380e987b0661b2ced31bafa2d3dde9845bd918a007d998e7a8", - "sha256:928de5a03fa2664d20433d3023e512fa1f8bc0588ff3eceee33c2d93b82e0110", - "sha256:9a4348b0309c444e5b7bfd3a2cf38bdb354c56e5f4b966eb88b249aff46b0a4c", - "sha256:9af68a58dc1a339e5735425a66b0a7146114b6fbc860d1f8b74b277d151f7bf1", - "sha256:a3447b842ab2964a834776be0a38b93d1977887d43cb69a2ce9941388923d8a9", - "sha256:a83b82b39e32a3110e45a11f0df9aceacefd201c25da79a66edda60d37a9f2e6", - "sha256:b05de8f16752b50851cd1caa3e63a49b35f8adb3eee0cec8991b231fac0c158d", - "sha256:b659d2306dd79818b2fae0543377e199bcf656404032e24f2af53a0e5338b5c9", - "sha256:b688dd1d83fdb46ab84d14a0711a0033984add9717619513f21e00f1b005f986", - "sha256:bbe9c13774ecaf99beb39d05c3ee659771c903add0c5e7a761c79387f653f69f", - "sha256:c23f59a04ed9db331922e5f02425958a907a019e157f14a6b95dd0ca1bcf05a6", - "sha256:cf0c35fe4a8b0dc78d9bf181317d9fe9cd21a397429d9c0c01c0178cf20e4442", - "sha256:cfd2320a0b4233ec4b9d3ac8d19df69194548bd32473fdba366cb50930f59dc2", - "sha256:d5c82c63ec24f0de5151e478cabfca23afd39005b6d0ba62dccf81cf19b0ae7f", - "sha256:d8d3dcb3832c209c66c67bd0f62289358b2cb942a68c00ac94c2ebf6870f731e", - "sha256:d98ed0a731d6c4cee39e76ba0bc2225455da097f41fa11ebfa27fe3854b6cc98", - "sha256:dbb2dd5367bfd71a6a779f6b237de015d6e508df129a126fc2a39da8a675457b", - "sha256:f3c135ad51ec95667bb945be107d0176f3048e615283104bf30027fe8bc06a8e", - "sha256:f8d75ead0ef7d060699b6c87b11f3c54e0c4b8e24065cbd262b64d300558a420" - ], - "index": "pypi", - "version": "==1.22.0" + "sha256:056f2a274edda4315e825ac2e3a9536f5415b43aa51669196860c8de6e76d847", + "sha256:0c953251585fdcd422072e4b7f4243fce215f22e21db94ec83c5970e41db6e18", + "sha256:142a73f5769f37bf2e4a8e4a77ef60f7af5f55635f60428322b49c87bd8f9cc0", + "sha256:1b333e2a068d8ef89a01eb23a098d2a789659c3178de79da9bd3d0ffb944cc6d", + "sha256:2124f19cc51d63405a0204ae38ef355732ab0a235975ab41ff6f6f9701905432", + "sha256:24c3a04adfb6c6f1bc4a2f8498d7661ca296ae352b498e538832c22ddde7bf81", + "sha256:3a2054e9640cbdd0ce8a345afb86be52875c5a8f9f5973a5c64791a8002da2dd", + "sha256:3fd15a09eecef83440ac849dcda2ff522f8ee1603ebfcdbb0e9b320ef2012e41", + "sha256:457e7a7dfa0b6bb608a766edba6f20c9d626a790df802016b930ad242fec4470", + "sha256:49ad5661d54ff0d164e4b441ee5e05191187d497380afa16d36d72eb8ef048de", + "sha256:561078e425d21a6720c3c3828385d949e24c0765e2852a46ecc3ad3fca2706e5", + "sha256:5a4f65ab06b32dc34112ed114dee3b698c8463670474334ece5b0b531073804c", + "sha256:8883e0e34676356d219a4cd37d239c3ead655cc550836236b52068e091416fff", + "sha256:8d2b45b1faf81098780e07d6a1c207b328b07e913160b8baa7e2e8d89723e06c", + "sha256:b0ebddb6ecc4c161369f93bb3a74c6120a498d3ddc759b64679709a885dd6d4f", + "sha256:b786ba4842c50de865dd3885b5570690a743e84a327b7213dd440eb0e6b996f8", + "sha256:be8efa010f5a80f1862ead80c3b19b5eb97dc954a0f59a1e2487078576105e03", + "sha256:c29106eaff0e2e708a9a89628dc0134ef145d0d3631f0ef421c09f380c30e354", + "sha256:c3c71236a056ec961b2b8b3b7c0b3b5a826283bc69c4a1c6415d23b70fea8243", + "sha256:cbc35031ec2b29af36947d085a7fbbcd8b79b84d563adf6156103d82565f78db", + "sha256:d47307c22744918e803c1eec7263a14f36aaf34fe496bff9ccbcae67c02b40ae", + "sha256:db088c98e563c1bb070da5984c8df08b45b61e4d9c6d2a8a1ffeed2af89fd1f3", + "sha256:df4dd1cb670062abdacc1fbce41cae4e08a4a212d28dd94fdbbf90615d027f73", + "sha256:e3adcf1499ca08d1e036ff44aedf55ed78653d946f4c4426b6e72ab757cc4dec", + "sha256:e3b3e32e0cda4dc382ec5bed8599dab644e4b3fc66a9ab54eb58248e207880b9", + "sha256:ed524195b35304c670755efa1eca579e5c290a66981f97004a5b2c0d12d6897d", + "sha256:edb42432790b1f8ec9f08faf9326d7e5dfe6e1d8c8fe4db39abc0a49c1c76537", + "sha256:eff1f995e5aa4cc941b6bbc45b5b57842f8f62bbe1a99427281c2c70cf42312c", + "sha256:f2fcdc2669662d77b400f80e20315a3661466e3cb3df1730f8083f9e49465cbc", + "sha256:f52ec9926daf48f41389d39d01570967b99c7dbc12bffc134cc3a3c5b5540ba2", + "sha256:fd007d67fdfbd2a13bf8a8c8ced8353b42a92ca72dbee54e951d8ddbc6ca12bc", + "sha256:ff9045e928dbb7943ea8559bfabebee95a43a830e00bf52c16202d2d805780fb" + ], + "index": "pypi", + "version": "==1.23.0" + }, + "importlib-metadata": { + "hashes": [ + "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8", + "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3" + ], + "markers": "python_version < '3.8'", + "version": "==0.19" }, "ipython": { "hashes": [ - "sha256:11067ab11d98b1e6c7f0993506f7a5f8a91af420f7e82be6575fcb7a6ca372a0", - "sha256:60bc55c2c1d287161191cc2469e73c116d9b634cff25fe214a43cba7cec94c79" + "sha256:1d3a1692921e932751bc1a1f7bb96dc38671eeefdc66ed33ee4cbc57e92a410e", + "sha256:537cd0176ff6abd06ef3e23f2d0c4c2c8a4d9277b7451544c6cbf56d1c79a83d" ], "index": "pypi", - "version": "==7.6.1" + "version": "==7.7.0" }, "ipython-genutils": { "hashes": [ @@ -416,11 +437,11 @@ }, "jedi": { "hashes": [ - "sha256:49ccb782651bb6f7009810d17a3316f8867dde31654c750506970742e18b553d", - "sha256:79d0f6595f3846dffcbe667cc6dc821b96e5baa8add125176c31a3917eb19d58" + "sha256:786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27", + "sha256:ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e" ], "index": "pypi", - "version": "==0.14.0" + "version": "==0.15.1" }, "lazy-object-proxy": { "hashes": [ @@ -453,22 +474,29 @@ "index": "pypi", "version": "==0.6.1" }, + "more-itertools": { + "hashes": [ + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + ], + "version": "==7.2.0" + }, "mypy": { "hashes": [ - "sha256:12d18bd7fc642c5d54b1bb62dde813a7e2ab79b32ee11ff206ac387c68fc2ad4", - "sha256:23e24bc1683a36f39dee67d8ac74ea414654642eee26d420bada95b8ee8c9095", - "sha256:2b38e64c52a8968df4ebcae0ddba4a54eb94d184695dd4e54e14509a9389b78c", - "sha256:3d4f551466a76e278187ec3a5b26cfb50f72f6760b749aa00ac69a6f9c99898d", - "sha256:53d5dacb8d844e50be698830509aa592b093547e7ab90aee63eb23db61109007", - "sha256:56f981d246010ba21cac6b2455eaecfaf68fc8a5663d865b26c8e579c36f751d", - "sha256:8c57f6f59f1e8479d9fc6e1bf034353e54626ed64e32394c613afc493a441dc1", - "sha256:bbed4a593d87476b592d52867ef86da2155ccd0becf0c4c02e6567d842e43368", - "sha256:d6ff850e2ba18b2db7704897c8f2f1384478e3b75ad292ec06196bf7794f3a40", - "sha256:e13b1bb8785d7f785e0b88873f1c21cda58ceba9ce1153b58cbfa24b09a111d5", - "sha256:e2b9ee6f648ce72d6741925a47c88c2391168ef973b6f74f17969450c5b1ffdd" + "sha256:0107bff4f46a289f0e4081d59b77cef1c48ea43da5a0dbf0005d54748b26df2a", + "sha256:07957f5471b3bb768c61f08690c96d8a09be0912185a27a68700f3ede99184e4", + "sha256:10af62f87b6921eac50271e667cc234162a194e742d8e02fc4ddc121e129a5b0", + "sha256:11fd60d2f69f0cefbe53ce551acf5b1cec1a89e7ce2d47b4e95a84eefb2899ae", + "sha256:15e43d3b1546813669bd1a6ec7e6a11d2888db938e0607f7b5eef6b976671339", + "sha256:352c24ba054a89bb9a35dd064ee95ab9b12903b56c72a8d3863d882e2632dc76", + "sha256:437020a39417e85e22ea8edcb709612903a9924209e10b3ec6d8c9f05b79f498", + "sha256:49925f9da7cee47eebf3420d7c0e00ec662ec6abb2780eb0a16260a7ba25f9c4", + "sha256:6724fcd5777aa6cebfa7e644c526888c9d639bd22edd26b2a8038c674a7c34bd", + "sha256:7a17613f7ea374ab64f39f03257f22b5755335b73251d0d253687a69029701ba", + "sha256:cdc1151ced496ca1496272da7fc356580e95f2682be1d32377c22ddebdf73c91" ], "index": "pypi", - "version": "==0.711" + "version": "==0.720" }, "mypy-extensions": { "hashes": [ @@ -479,24 +507,32 @@ }, "mypy-protobuf": { "hashes": [ - "sha256:feff06ebda0462d663761f135840c1d0e2d1470499efd5b4f847f4b60585368c" + "sha256:69f37e166ace422ef8c8266ff73be5384968152e1d7769f3d866d0e923c354ba", + "sha256:a10bb9f24cc2c1f5077a5e64fc5e22b7f7e230a1a399a34cebc57c80a8a532e6" ], "index": "pypi", - "version": "==1.10" + "version": "==1.15" + }, + "packaging": { + "hashes": [ + "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", + "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" + ], + "version": "==19.1" }, "parso": { "hashes": [ - "sha256:5052bb33be034cba784193e74b1cde6ebf29ae8b8c1e4ad94df0c4209bfc4826", - "sha256:db5881df1643bf3e66c097bfd8935cf03eae73f4cb61ae4433c9ea4fb6613446" + "sha256:63854233e1fadb5da97f2744b6b24346d2750b85965e7e399bec1620232797dc", + "sha256:666b0ee4a7a1220f65d367617f2cd3ffddff3e205f3f16a0284df30e774c2a9c" ], - "version": "==0.5.0" + "version": "==0.5.1" }, "pbr": { "hashes": [ - "sha256:36ebd78196e8c9588c972f5571230a059ff83783fabbbbedecc07be263ccd7e6", - "sha256:5a03f59455ad54f01a94c15829b8b70065462b7bd8d5d7e983306b59127fc841" + "sha256:56e52299170b9492513c64be44736d27a512fa7e606f21942160b68ce510b4bc", + "sha256:9b321c204a88d8ab5082699469f52cc94c5da45c51f114113d01b3d993c24cdf" ], - "version": "==5.4.0" + "version": "==5.4.2" }, "pep8-naming": { "hashes": [ @@ -521,6 +557,13 @@ ], "version": "==0.7.5" }, + "pluggy": { + "hashes": [ + "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", + "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" + ], + "version": "==0.12.0" + }, "prompt-toolkit": { "hashes": [ "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", @@ -531,26 +574,24 @@ }, "protobuf": { "hashes": [ - "sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8", - "sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538", - "sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e", - "sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a", - "sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6", - "sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0", - "sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc", - "sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47", - "sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01", - "sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115", - "sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277", - "sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c", - "sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea", - "sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87", - "sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7", - "sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126", - "sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a", - "sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832" - ], - "version": "==3.8.0" + "sha256:00a1b0b352dc7c809749526d1688a64b62ea400c5b05416f93cfb1b11a036295", + "sha256:01acbca2d2c8c3f7f235f1842440adbe01bbc379fa1cbdd80753801432b3fae9", + "sha256:0a795bca65987b62d6b8a2d934aa317fd1a4d06a6dd4df36312f5b0ade44a8d9", + "sha256:0ec035114213b6d6e7713987a759d762dd94e9f82284515b3b7331f34bfaec7f", + "sha256:31b18e1434b4907cb0113e7a372cd4d92c047ce7ba0fa7ea66a404d6388ed2c1", + "sha256:32a3abf79b0bef073c70656e86d5bd68a28a1fbb138429912c4fc07b9d426b07", + "sha256:55f85b7808766e5e3f526818f5e2aeb5ba2edcc45bcccede46a3ccc19b569cb0", + "sha256:64ab9bc971989cbdd648c102a96253fdf0202b0c38f15bd34759a8707bdd5f64", + "sha256:64cf847e843a465b6c1ba90fb6c7f7844d54dbe9eb731e86a60981d03f5b2e6e", + "sha256:917c8662b585470e8fd42f052661fc66d59fccaae450a60044307dcbf82a3335", + "sha256:afed9003d7f2be2c3df20f64220c30faec441073731511728a2cb4cab4cd46a6", + "sha256:bf8e05d638b585d1752c5a84247134a0350d3a8b73d3632489a014a9f6f1e758", + "sha256:d831b047bd69becaf64019a47179eb22118a50dd008340655266a906c69c6417", + "sha256:de2760583ed28749ff885789c1cbc6c9c06d6de92fc825740ab99deb2f25ea4d", + "sha256:eabc4cf1bc19689af8022ba52fd668564a8d96e0d08f3b4732d26a64255216a4", + "sha256:fcff6086c86fb1628d94ea455c7b9de898afc50378042927a59df8065a79a549" + ], + "version": "==3.9.1" }, "ptyprocess": { "hashes": [ @@ -559,6 +600,13 @@ ], "version": "==0.6.0" }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" + }, "pycodestyle": { "hashes": [ "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", @@ -597,21 +645,38 @@ "index": "pypi", "version": "==2.3.1" }, - "pyyaml": { + "pyparsing": { "hashes": [ - "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", - "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", - "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", - "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", - "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", - "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", - "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", - "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", - "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", - "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", - "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" ], - "version": "==5.1.1" + "version": "==2.4.2" + }, + "pytest": { + "hashes": [ + "sha256:3805d095f1ea279b9870c3eeae5dddf8a81b10952c8835cd628cf1875b0ef031", + "sha256:abc562321c2d190dd63c2faadf70b86b7af21a553b61f0df5f5e1270717dc5a3" + ], + "index": "pypi", + "version": "==5.1.0" + }, + "pyyaml": { + "hashes": [ + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + ], + "version": "==5.1.2" }, "six": { "hashes": [ @@ -682,6 +747,14 @@ "markers": "implementation_name == 'cpython'", "version": "==1.4.0" }, + "typing-extensions": { + "hashes": [ + "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", + "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", + "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + ], + "version": "==3.7.4" + }, "wcwidth": { "hashes": [ "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", @@ -694,6 +767,13 @@ "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" ], "version": "==1.11.2" + }, + "zipp": { + "hashes": [ + "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a", + "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec" + ], + "version": "==0.5.2" } } } diff --git a/gfauto/README.md b/gfauto/README.md index 5fd1a18dc..450b11457 100644 --- a/gfauto/README.md +++ b/gfauto/README.md @@ -33,7 +33,7 @@ Install and configure plugins: * Protobuf Support * File Watchers (may already be installed) * The watcher task should already be under version control. -* Mypy: the built-in PyCharm type checking uses Mypy behinds the scenes, but this plugin enhances it by using the latest version and allowing the use of stricter settings, matching the settings used by the `./check_all.sh` script. +* Mypy: the built-in PyCharm type checking uses Mypy behind-the-scenes, but this plugin enhances it by using the latest version and allowing the use of stricter settings, matching the settings used by the `./check_all.sh` script. Add `whitelist.dic` as a custom dictionary (search for "Spelling" in Actions). Do not add words via PyCharm's "Quick Fixes" feature, as the word will only be added to your personal dictionary. Instead, manually add the word to `whitelist.dic`. diff --git a/gfauto/check_all.sh b/gfauto/check_all.sh index 26f6050d9..5133e139f 100755 --- a/gfauto/check_all.sh +++ b/gfauto/check_all.sh @@ -22,7 +22,8 @@ if [ -z ${VIRTUAL_ENV+x} ]; then source .venv/bin/activate fi -mypy --strict gfauto -pylint gfauto +mypy --strict gfauto gfautotests +pylint gfauto gfautotests # Flake checks formatting via black. flake8 . +pytest gfautotests diff --git a/gfauto/fix_all.sh b/gfauto/fix_all.sh index 390539620..681d8d04e 100755 --- a/gfauto/fix_all.sh +++ b/gfauto/fix_all.sh @@ -22,5 +22,5 @@ if [ -z ${VIRTUAL_ENV+x} ]; then source .venv/bin/activate fi -isort -rc gfauto -black gfauto +isort -rc gfauto gfautotests +black gfauto gfautotests diff --git a/gfauto/gfauto/signature_util.py b/gfauto/gfauto/signature_util.py index 176bf5f05..aa718a1c8 100644 --- a/gfauto/gfauto/signature_util.py +++ b/gfauto/gfauto/signature_util.py @@ -20,6 +20,9 @@ stack trace. """ +# Disable spell-checking for this file. +# flake8: noqa: SC100 + import re from pathlib import Path from typing import Match, Optional, Pattern @@ -59,7 +62,8 @@ # E.g. ERROR: temp/.../variant/shader.frag:549: 'variable indexing fragment shader output array' : not supported with this profile: es # variable indexing fragment shader output array <-- group 1 -PATTERN_GLSLANG_ERROR = re.compile(r"ERROR: .*?: '(.*?)'") +# E.g. ERROR: reports/.../part_1_preserve_semantics/reduction_work/variant/shader_reduced_0173/0_glsl/shader_reduced_0173.frag:456: '=' : cannot convert from ' const 3-component vector of bool' to ' temp bool' +PATTERN_GLSLANG_ERROR = re.compile(r"ERROR: .*?:\d+: (.*)") # E.g. diff --git a/gfauto/gfautotests/__init__.py b/gfauto/gfautotests/__init__.py new file mode 100644 index 000000000..f7bad95d3 --- /dev/null +++ b/gfauto/gfautotests/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/gfauto/gfautotests/test_signature_util.py b/gfauto/gfautotests/test_signature_util.py new file mode 100644 index 000000000..351957ae8 --- /dev/null +++ b/gfauto/gfautotests/test_signature_util.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gfauto import signature_util + +# Disable spell-checking for this file. +# flake8: noqa: SC100 + + +def test_glslang_assertion() -> None: + log = """ +glslangValidator: ../glslang/MachineIndependent/ParseHelper.cpp:2212: void glslang::TParseContext::nonOpBuiltInCheck(const glslang::TSourceLoc&, const glslang::TFunction&, glslang::TIntermAggregate&): Assertion `PureOperatorBuiltins == false' failed. +""" + signature = signature_util.get_signature_from_log_contents(log) + assert signature == "void_glslangTParseContextnonOpBuiltInCheckconst_gl" + + +def test_glslang_error_1() -> None: + log = """ +ERROR: temp/.../variant/shader.frag:549: 'variable indexing fragment shader output array' : not supported with this profile: es +""" + signature = signature_util.get_signature_from_log_contents(log) + assert signature == "variable_indexing_fragment_shader_output_array_not" + + +def test_glslang_error_2() -> None: + log = """ +ERROR: reports/.../part_1_preserve_semantics/reduction_work/variant/shader_reduced_0173/0_glsl/shader_reduced_0173.frag:456: '=' : cannot convert from ' const 3-component vector of bool' to ' temp bool' +""" + signature = signature_util.get_signature_from_log_contents(log) + assert signature == "cannot_convert_from_const_component_vector_of_bool" diff --git a/gfauto/pyproject.toml b/gfauto/pyproject.toml index bcc226527..4b2af62fb 100644 --- a/gfauto/pyproject.toml +++ b/gfauto/pyproject.toml @@ -20,9 +20,9 @@ build-backend = 'setuptools.build_meta' line-length = 88 target-version = ['py36', 'py37', 'py38'] -# Black currently matches against full paths, so we only match one level deep -# to ensure only files in gfauto (and not in the current directory) are -# matched. +# Black currently matches against full paths, and can be quite slow at +# filtering files. Thus, we just use the following filters and always +# specify the "gfauto" directory when running. # https://github.com/python/black/issues/712 include = '.*[.]py$' diff --git a/gfauto/run_protoc.sh b/gfauto/run_protoc.sh index 15b642696..c945f0ef3 100755 --- a/gfauto/run_protoc.sh +++ b/gfauto/run_protoc.sh @@ -18,6 +18,10 @@ set -x set -e set -u +if [ -z ${VIRTUAL_ENV+x} ]; then + source .venv/bin/activate +fi + python -m grpc.tools.protoc --python_out=. --proto_path=. --mypy_out=. gfauto/*.proto # protoc gfauto/artifact.proto gfauto/recipe.proto --python_out=. --plugin=protoc-gen-mypy=github/mypy-protobuf/python/protoc-gen-mypy --mypy_out=. --proto_path=. diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index d3965608e..b3aebb496 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -171,3 +171,4 @@ unindexed vec2 vec4 framebuffer +gfautotests From 23cbd13853be60171d558fe2e63eea867d4db469 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 20 Aug 2019 12:47:20 +0100 Subject: [PATCH 110/180] gfauto: use cdb on Windows for getting stack traces (#703) * gfauto: use cdb on Windows for stack traces * gfauto: signature_util: add cdb regex to recognize stack traces --- gfauto/gfauto/fuzz.py | 3 +++ gfauto/gfauto/host_device_util.py | 2 +- gfauto/gfauto/signature_util.py | 25 +++++++++++++++++++++++-- gfauto/gfauto/subprocess_util.py | 14 -------------- gfauto/gfauto/util.py | 28 ++++++++++++++++++++++++++-- gfauto/whitelist.dic | 2 ++ 6 files changed, 55 insertions(+), 19 deletions(-) diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index 2f33f5411..d4d46bd52 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -192,6 +192,9 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches; artifact_util.recipes_write_built_in() + # Log a warning if there is no tool on the PATH for printing stack traces. + util.prepend_catchsegv_if_available([], log_warning=True) + # TODO: make GraphicsFuzz find donors recursively. references = sorted(donors_dir.rglob("*.json")) diff --git a/gfauto/gfauto/host_device_util.py b/gfauto/gfauto/host_device_util.py index 08937aea5..035dff229 100644 --- a/gfauto/gfauto/host_device_util.py +++ b/gfauto/gfauto/host_device_util.py @@ -95,7 +95,7 @@ def run_amber_helper( cmd.append("-B") cmd.append("0") - util.prepend_catchsegv_if_available(cmd) + cmd = util.prepend_catchsegv_if_available(cmd) status = "UNEXPECTED_ERROR" diff --git a/gfauto/gfauto/signature_util.py b/gfauto/gfauto/signature_util.py index aa718a1c8..febe4e503 100644 --- a/gfauto/gfauto/signature_util.py +++ b/gfauto/gfauto/signature_util.py @@ -58,6 +58,10 @@ r")" ) +PATTERN_CDB_CALL_SITE = re.compile( + fr"\n{HEX_LIKE}+`{HEX_LIKE}+ {HEX_LIKE}+`{HEX_LIKE}+ (.*)" +) + PATTERN_ANDROID_BACKTRACE_CATCHALL = re.compile(r"\n.*#00 pc " + HEX_LIKE + r"+ (.*)") # E.g. ERROR: temp/.../variant/shader.frag:549: 'variable indexing fragment shader output array' : not supported with this profile: es @@ -108,10 +112,11 @@ def remove_hex_like(string: str) -> str: return temp -def clean_up(string: str) -> str: +def clean_up(string: str, remove_numbers: bool = True) -> str: temp: str = string # Remove numbers. - temp = re.sub(r"\d+", "", temp) + if remove_numbers: + temp = re.sub(r"\d+", "", temp) # Replace spaces with _. temp = re.sub(r" ", "_", temp) # Remove non-word, non-_ characters. @@ -177,6 +182,22 @@ def get_signature_from_log_contents( # pylint: disable=too-many-return-statemen if group: return group + # Cdb stack trace + cdb_call_site = re.search( + PATTERN_CDB_CALL_SITE, log_contents + ) # type: Optional[Match[str]] + if cdb_call_site: + site = cdb_call_site.group(1) + if "!" in site: + # We probably have symbols, so remove the address and everything after e.g. "+0x111 [file/path @ 123]" + site = re.sub(rf"\+{HEX_LIKE}+.*", "", site) + site = clean_up(site) + else: + # We don't have symbols so we may as well keep offsets around; don't remove numbers. + site = clean_up(site, remove_numbers=False) + site = reduce_length(site) + return site + # Android stack traces. if "#00 pc" in log_contents: lines = log_contents.split("\n") diff --git a/gfauto/gfauto/subprocess_util.py b/gfauto/gfauto/subprocess_util.py index 559d87e25..741da3062 100644 --- a/gfauto/gfauto/subprocess_util.py +++ b/gfauto/gfauto/subprocess_util.py @@ -34,20 +34,6 @@ LOG_COMMAND_TIMED_OUT_PREFIX = "Command timed out: " -def convert_stdout_stderr( - result: Union[ - subprocess.CalledProcessError, - subprocess.CompletedProcess, - subprocess.TimeoutExpired, - ] -) -> None: - - if result.stdout is not None: - result.stdout = result.stdout.decode(encoding="utf-8", errors="ignore") - if result.stderr is not None: - result.stderr = result.stderr.decode(encoding="utf-8", errors="ignore") - - def log_stdout_stderr_helper(stdout: str, stderr: str) -> None: log("STDOUT:") diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py index 69bf6a7ed..205ae07fb 100644 --- a/gfauto/gfauto/util.py +++ b/gfauto/gfauto/util.py @@ -98,12 +98,36 @@ def tool_on_path(tool: str) -> pathlib.Path: # noqa VNE002 return pathlib.Path(result) -def prepend_catchsegv_if_available(cmd: List[str]) -> List[str]: +def prepend_catchsegv_if_available( + cmd: List[str], log_warning: bool = False +) -> List[str]: try: - cmd.insert(0, str(tool_on_path("catchsegv"))) + return [str(tool_on_path("catchsegv"))] + cmd except ToolNotOnPathError: pass + try: + # cdb is the command line version of WinDbg. + return [ + str(tool_on_path("cdb")), + "-g", + "-G", + "-lines", + "-nosqm", + "-o", + "-x", + "-c", + "kp;q", + ] + cmd + except ToolNotOnPathError: + pass + + if log_warning: + gflogging.log( + "WARNING: Could not find catchsegv (Linux) or cdb (Windows) on your PATH; you will not be able to get " + "stack traces from tools or the host driver." + ) + return cmd diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index b3aebb496..e6674e05d 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -172,3 +172,5 @@ vec2 vec4 framebuffer gfautotests +cdb +Dbg From b04faa280f59a5b7cc863fb070284709a63c70c3 Mon Sep 17 00:00:00 2001 From: asuonpaa <34128694+asuonpaa@users.noreply.github.com> Date: Tue, 20 Aug 2019 15:44:31 +0300 Subject: [PATCH 111/180] gfauto: generate amber code for comparing two shader jobs (#705) Modified amber generation script to add two pipelines and comparison of their framebuffers when reference shader job is present. --- gfauto/gfauto/amber_converter.py | 66 +++++++++++++++++-------------- gfauto/gfauto/android_device.py | 2 +- gfauto/gfauto/host_device_util.py | 2 + gfauto/whitelist.dic | 1 + 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py index 9ec2ff895..b644f8a22 100644 --- a/gfauto/gfauto/amber_converter.py +++ b/gfauto/gfauto/amber_converter.py @@ -513,51 +513,59 @@ def get_amber_script_shader_def(shader: Shader, name: str) -> str: def graphics_shader_job_amber_test_to_amber_script( shader_job_amber_test: ShaderJobBasedAmberTest, amberfy_settings: AmberfySettings ) -> str: - # TODO: handle reference, if present. - # Guaranteed, and needed for type checker. assert isinstance(shader_job_amber_test.variant, GraphicsShaderJob) # noqa result = get_amber_script_header(amberfy_settings) - variant = shader_job_amber_test.variant + jobs = [shader_job_amber_test.variant] - variant_vertex_shader_name = f"{variant.name_prefix}_vertex_shader" - variant_fragment_shader_name = f"{variant.name_prefix}_fragment_shader" + if shader_job_amber_test.reference: + assert isinstance(shader_job_amber_test.reference, GraphicsShaderJob) # noqa + jobs.insert(0, shader_job_amber_test.reference) - # Define shaders. + for job in jobs: + prefix = job.name_prefix - result += get_amber_script_shader_def( - variant.vertex_shader, variant_vertex_shader_name - ) + vertex_shader_name = f"{prefix}_vertex_shader" + fragment_shader_name = f"{prefix}_fragment_shader" - result += get_amber_script_shader_def( - variant.fragment_shader, variant_fragment_shader_name - ) + # Define shaders. - # Define uniforms for variant shader job. + result += get_amber_script_shader_def(job.vertex_shader, vertex_shader_name) - result += "\n" - result += variant.uniform_definitions + result += get_amber_script_shader_def(job.fragment_shader, fragment_shader_name) - # Currently, this buffer must be called "framebuffer" to be able to dump it from Amber. - result += "\nBUFFER framebuffer FORMAT B8G8R8A8_UNORM\n" + # Define uniforms for shader job. - # Create a pipeline that uses the variant shaders and writes the output to "framebuffer". + result += "\n" + result += job.uniform_definitions - result += "\nPIPELINE graphics gfz_pipeline\n" - result += f" ATTACH {variant_vertex_shader_name}\n" - result += f" ATTACH {variant_fragment_shader_name}\n" - result += " FRAMEBUFFER_SIZE 256 256\n" - result += " BIND BUFFER framebuffer AS color LOCATION 0\n" - result += variant.uniform_bindings - result += "END\n" - result += "CLEAR_COLOR gfz_pipeline 0 0 0 255\n" + result += f"\nBUFFER {prefix}_framebuffer FORMAT B8G8R8A8_UNORM\n" - # Run the pipeline. + # Create a pipeline. + + result += f"\nPIPELINE graphics {prefix}_pipeline\n" + result += f" ATTACH {vertex_shader_name}\n" + result += f" ATTACH {fragment_shader_name}\n" + result += " FRAMEBUFFER_SIZE 256 256\n" + result += f" BIND BUFFER {prefix}_framebuffer AS color LOCATION 0\n" + result += job.uniform_bindings + result += "END\n" + result += f"CLEAR_COLOR {prefix}_pipeline 0 0 0 255\n" + + # Run the pipeline. + + result += f"\nCLEAR {prefix}_pipeline\n" + result += f"RUN {prefix}_pipeline DRAW_RECT POS 0 0 SIZE 256 256\n" + result += "\n" + + # Add fuzzy compare of framebuffers if there's more than one pipeline - result += "\nCLEAR gfz_pipeline\n" - result += "RUN gfz_pipeline DRAW_RECT POS 0 0 SIZE 256 256\n" + for pipeline_index in range(1, len(jobs)): + prefix_0 = jobs[0].name_prefix + prefix_1 = jobs[pipeline_index].name_prefix + result += f"EXPECT {prefix_0}_framebuffer RMSE_BUFFER {prefix_1}_framebuffer TOLERANCE 7" if amberfy_settings.extra_commands: result += amberfy_settings.extra_commands diff --git a/gfauto/gfauto/android_device.py b/gfauto/gfauto/android_device.py index c0db103e5..1933ec0d6 100644 --- a/gfauto/gfauto/android_device.py +++ b/gfauto/gfauto/android_device.py @@ -233,7 +233,7 @@ def run_amber_on_device_helper( amber_flags += " -ps" else: if dump_image: - amber_flags += f" -i {fuzz.IMAGE_FILE_NAME}" + amber_flags += f" -i {fuzz.IMAGE_FILE_NAME} -I variant_framebuffer" if dump_buffer: amber_flags += f" -b {fuzz.BUFFER_FILE_NAME} -B 0" diff --git a/gfauto/gfauto/host_device_util.py b/gfauto/gfauto/host_device_util.py index 035dff229..9aaef93e0 100644 --- a/gfauto/gfauto/host_device_util.py +++ b/gfauto/gfauto/host_device_util.py @@ -89,6 +89,8 @@ def run_amber_helper( if dump_image: cmd.append("-i") cmd.append(str(image_file)) + cmd.append("-I") + cmd.append("variant_framebuffer") if dump_buffer: cmd.append("-b") cmd.append(str(buffer_file)) diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index e6674e05d..d45a75b97 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -171,6 +171,7 @@ unindexed vec2 vec4 framebuffer +framebuffers gfautotests cdb Dbg From c7293b781d85e1cdb9efc6f4bda6e56e8ead7efe Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 20 Aug 2019 23:37:11 +0100 Subject: [PATCH 112/180] gfauto: fixes for Windows (#707) --- gfauto/README.md | 10 +++++++++- gfauto/gfauto/fuzz.py | 13 ++++++++++--- gfauto/gfauto/fuzz_glsl_test.py | 8 ++++---- gfauto/gfauto/util.py | 34 ++++++++++++++++++++++++++++++++- gfauto/whitelist.dic | 3 +++ 5 files changed, 59 insertions(+), 9 deletions(-) diff --git a/gfauto/README.md b/gfauto/README.md index 450b11457..75e23b7fb 100644 --- a/gfauto/README.md +++ b/gfauto/README.md @@ -13,6 +13,8 @@ GraphicsFuzz auto (this project) provides scripts for running these tools with m > Optional: if you have just done `git pull` to get a more recent version of GraphicsFuzz auto, consider deleting `.venv/` to start from a fresh virtual environment. This is rarely needed. +> On Windows, you can use the Git Bash shell, or adapt the commands (including those inside `dev_shell.sh.template`) to the Windows command prompt. + Execute `./dev_shell.sh.template`. If the default settings don't work, make a copy of the file called `dev_shell.sh` and modify according to the comments before executing. `pip` must be installed for the version of Python you wish to use. The script generates and activates a Python virtual environment (located at `.venv/`) with all dependencies installed. @@ -112,7 +114,13 @@ You can execute scripts in this repository by opening a Terminal in PyCharm. ## Terminal -The `Terminal` tab in PyCharm is useful and will use the project's Python virtual environment. In any other terminal, use `source .venv/bin/activate`. You can alternatively execute the `./dev_shell.sh` script, but this is fairly slow as it checks and reinstalls all dependencies +The `Terminal` tab in PyCharm is useful and will use the project's Python virtual environment. In any other terminal, use: + +* `source .venv/bin/activate` (on Linux) +* `source .venv/Scripts/activate` (on Windows with the Git Bash shell) +* `.venv/Scripts/activate.bat` (on Windows with cmd) + +You can alternatively execute the `./dev_shell.sh` script, but this is fairly slow as it checks and reinstalls all dependencies ## Fuzzing diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index d4d46bd52..2000efa22 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -157,7 +157,7 @@ def main() -> None: gflogging.pop_stream_for_logging() -def main_helper( # pylint: disable=too-many-locals, too-many-branches; +def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements; settings_path: Path, iteration_seed_override: Optional[int] ) -> None: @@ -234,10 +234,17 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches; log(f"Iteration seed: {iteration_seed}") random.seed(iteration_seed) - staging_name = get_random_name() + staging_name = get_random_name()[:8] staging_dir = temp_dir / staging_name - util.mkdirs_p(staging_dir) + try: + util.mkdir_p_new(staging_dir) + except FileExistsError: + if iteration_seed_override: + raise + log(f"Staging directory already exists: {str(staging_dir)}") + log(f"Starting new iteration.") + continue # Pseudocode: # - Create test_dir(s) in staging directory. diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py index 64e7f824d..398ecae40 100644 --- a/gfauto/gfauto/fuzz_glsl_test.py +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -45,7 +45,7 @@ from gfauto.gflogging import log from gfauto.settings_pb2 import Settings from gfauto.test_pb2 import Test, TestGlsl -from gfauto.util import check +from gfauto.util import check, tool_on_path class ReductionFailedError(Exception): @@ -255,14 +255,14 @@ def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: test_dir_reduction_output=test_dir, test_dir_to_reduce=test_dir, preserve_semantics=True, - reduction_name="part_1_preserve_semantics", + reduction_name="1", ) part_2_reduced_test = run_reduction( test_dir_reduction_output=test_dir, test_dir_to_reduce=part_1_reduced_test, preserve_semantics=False, - reduction_name="part_2_change_semantics", + reduction_name="2", ) device_name = test.device.name @@ -506,7 +506,7 @@ def run_glsl_reduce( ) -> Path: cmd = [ - "glsl-reduce", + str(tool_on_path("glsl-reduce")), str(input_shader_job), "--output", str(output_dir), diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py index 205ae07fb..9d7fc8679 100644 --- a/gfauto/gfauto/util.py +++ b/gfauto/gfauto/util.py @@ -76,6 +76,19 @@ def file_write_text(file: pathlib.Path, text: str) -> int: # noqa VNE002 return f.write(text) +def mkdir_p_new(path: Path) -> Path: # noqa VNE002 + """ + Creates a directory (and all parents, if needed), but fails if the directory already exists. + + :param path: the path to create + :return: the created path + """ + + file_mkdirs_parent(path) + path.mkdir() + return path + + def mkdirs_p(path: pathlib.Path) -> Path: # noqa VNE002 path.mkdir(parents=True, exist_ok=True) return path @@ -170,12 +183,31 @@ def move_dir( def make_directory_symlink(new_symlink_file_path: Path, existing_dir: Path) -> Path: + gflogging.log(f"symlink: from {str(new_symlink_file_path)} to {str(existing_dir)}") check(existing_dir.is_dir(), AssertionError(f"Not a directory: {existing_dir}")) file_mkdirs_parent(new_symlink_file_path) + + # symlink_to takes a path relative to the location of the new file (or an absolute path, but we avoid this). symlink_contents = os.path.relpath( str(existing_dir), start=str(new_symlink_file_path.parent) ) - new_symlink_file_path.symlink_to(symlink_contents, target_is_directory=True) + try: + new_symlink_file_path.symlink_to(symlink_contents, target_is_directory=True) + except OSError: + if get_platform() != "Windows": + raise + # Retry using junctions under Windows. + try: + # noinspection PyUnresolvedReferences + import _winapi # pylint: disable=import-error; + + # Unlike symlink_to, CreateJunction takes a path relative to the current directory. + _winapi.CreateJunction(str(existing_dir), str(new_symlink_file_path)) + return new_symlink_file_path + except ModuleNotFoundError: + pass + raise + return new_symlink_file_path diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index d45a75b97..a003eedad 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -175,3 +175,6 @@ framebuffers gfautotests cdb Dbg +mklink +fullmatch +winapi From b5f768e53069c502f8a8c2579bcc368d0712ff70 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 22 Aug 2019 13:39:26 +0100 Subject: [PATCH 113/180] Apply small fix to GLSL grammar (#708) --- ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 b/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 index 517b450df..add4ebe5c 100644 --- a/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 +++ b/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 @@ -825,7 +825,7 @@ SAMPLER1D: 'sampler1D' ; SAMPLER2D: 'sampler2D' ; SAMPLER3D: 'sampler3D' ; SAMPLER2DRECT: 'sampler2DRect' ; - +SAMPLEREXTERNALOES: 'samplerExternalOES' ; SAMPLER1DSHADOW: 'sampler1DShadow' ; SAMPLER2DSHADOW: 'sampler2DShadow' ; SAMPLER2DRECTSHADOW: 'sampler2DRectShadow' ; From bfe5f4410c99b01076cf72f5d95d0bbad1fd2eac Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 22 Aug 2019 14:57:00 +0100 Subject: [PATCH 114/180] Fix type checking of vector and relational operators (#706) These have been fixed to always return 'bool', even when comparing vector types. Tests for the vector relational builtins that allow comparisons between vector types, to yield boolean vector results, have been added. Some incorrect tests have been removed. Fixes #700. --- .../com/graphicsfuzz/common/typing/Typer.java | 62 ++--- .../graphicsfuzz/common/typing/TyperTest.java | 243 +++++++++++++++--- 2 files changed, 237 insertions(+), 68 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java index c1c6a8ac3..9d5f322d1 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java @@ -41,6 +41,7 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.util.OpenGlConstants; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -59,10 +60,6 @@ public class Typer extends ScopeTrackingVisitor { private final Map structDeclarationMap; - public Map> getUserDefinedFunctions() { - return userDefinedFunctions; - } - public Typer(TranslationUnit tu) { this.tu = tu; this.types = new HashMap<>(); @@ -104,27 +101,32 @@ public void visitFunctionDefinition(FunctionDefinition functionDefinition) { public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { super.visitFunctionCallExpr(functionCallExpr); - List candidateBuiltins = + // First, see if there is a builtin with a matching prototype. + final Optional maybeMatchingBuiltinFunctionReturn = + lookForMatchingFunction(functionCallExpr, TyperHelper.getBuiltins(tu.getShadingLanguageVersion(), tu.getShaderKind()) - .get(functionCallExpr.getCallee()); - if (candidateBuiltins != null) { - for (FunctionPrototype prototype : candidateBuiltins) { - if (prototypeMatches(prototype, functionCallExpr)) { - types.put(functionCallExpr, prototype.getReturnType()); - } - } + .get(functionCallExpr.getCallee())); + if (maybeMatchingBuiltinFunctionReturn.isPresent()) { + types.put(functionCallExpr, maybeMatchingBuiltinFunctionReturn.get()); + return; } - Set candidateUserDefined = - userDefinedFunctions.get(functionCallExpr.getCallee()); - if (candidateUserDefined != null) { - for (FunctionPrototype prototype : candidateUserDefined) { + // If there was no relevant builtin, see whether there is a user-defined type. + lookForMatchingFunction(functionCallExpr, + userDefinedFunctions.get(functionCallExpr.getCallee())) + .ifPresent(type -> types.put(functionCallExpr, type)); + } + + private Optional lookForMatchingFunction(FunctionCallExpr functionCallExpr, + Collection candidateFunctions) { + if (candidateFunctions != null) { + for (FunctionPrototype prototype : candidateFunctions) { if (prototypeMatches(prototype, functionCallExpr)) { - types.put(functionCallExpr, prototype.getReturnType()); + return Optional.of(prototype.getReturnType()); } } } - + return Optional.empty(); } /** @@ -339,7 +341,9 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { case LT: case LXOR: case NE: - types.put(binaryExpr, resolveBooleanResultType(lhsType, rhsType)); + // The above all yield 'bool', even if (as is allowed in the case of '==' and '!=' they are + // applied to vector types. + types.put(binaryExpr, BasicType.BOOL); return; case COMMA: // The type of "e1, e2" is the type of "e2". @@ -432,26 +436,6 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { } } - private Type resolveBooleanResultType(Type lhsType, Type rhsType) { - return maybeComputeBooleanVectorType(lhsType) - .orElse(maybeComputeBooleanVectorType(rhsType) - .orElse(BasicType.BOOL)); - } - - private Optional maybeComputeBooleanVectorType(Type lhsType) { - if (lhsType instanceof BasicType) { - final int numElements = ((BasicType) lhsType).getNumElements(); - if (1 < numElements && numElements <= 4) { - return Optional.of(BasicType.makeVectorType(BasicType.BOOL, numElements)); - } - } - return Optional.empty(); - } - - public Set getTypedExpressions() { - return Collections.unmodifiableSet(types.keySet()); - } - public Type lookupType(Expr expr) { return types.get(expr); } diff --git a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java index c0b6984dd..40147618a 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java @@ -250,35 +250,7 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { }; } - - @Test - public void testBooleanVectorType() throws Exception { - final String program = "#version 440\n" - + "void main() { vec3(1.0) > vec3(2.0); }"; - TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu) { - @Override - public void visitBinaryExpr(BinaryExpr binaryExpr) { - super.visitBinaryExpr(binaryExpr); - assertEquals(BasicType.BVEC3, lookupType(binaryExpr)); - } - }; - } - - @Test - public void testBooleanVectorType2() throws Exception { - final String program = "#version 440\n" - + "void main() { vec3(1.0) > 2.0; }"; - TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu) { - @Override - public void visitBinaryExpr(BinaryExpr binaryExpr) { - super.visitBinaryExpr(binaryExpr); - assertEquals(BasicType.BVEC3, lookupType(binaryExpr)); - } - }; - } - + @Test public void testSwizzleTyped() throws Exception { TranslationUnit tu = ParseHelper.parse("void main() { vec2 v; v.xy = v.yx; }"); @@ -500,6 +472,219 @@ public void visitInitializer(Initializer initializer) { }.visit(tu); } + @Test + public void testEqualityAndInequalityVectorsMatrices() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 300 es\n" + + "void main() {\n" + + " float v1a, v1b;\n" + + " vec2 v2a, v2b;\n" + + " vec3 v3a, v3b;\n" + + " vec4 v4a, v4b;\n" + + " int i1a, i1b;\n" + + " ivec2 i2a, i2b;\n" + + " ivec3 i3a, i3b;\n" + + " ivec4 i4a, i4b;\n" + + " uint u1a, u1b;\n" + + " uvec2 u2a, u2b;\n" + + " uvec3 u3a, u3b;\n" + + " uvec4 u4a, u4b;\n" + + " bool b1a, b1b;\n" + + " bvec2 b2a, b2b;\n" + + " bvec3 b3a, b3b;\n" + + " bvec4 b4a, b4b;\n" + + " mat2x2 m22a, m22b;\n" + + " mat2x3 m23a, m23b;\n" + + " mat2x4 m24a, m24b;\n" + + " mat3x2 m32a, m32b;\n" + + " mat3x3 m33a, m33b;\n" + + " mat3x4 m34a, m34b;\n" + + " mat4x2 m42a, m42b;\n" + + " mat4x3 m43a, m43b;\n" + + " mat4x2 m44a, m44b;\n" + + " v1a == v1b;\n" + + " v2a == v2b;\n" + + " v3a == v3b;\n" + + " v4a == v4b;\n" + + " v1a != v1b;\n" + + " v2a != v2b;\n" + + " v3a != v3b;\n" + + " v4a != v4b;\n" + + " u1a == u1b;\n" + + " u2a == u2b;\n" + + " u3a == u3b;\n" + + " u4a == u4b;\n" + + " u1a != u1b;\n" + + " u2a != u2b;\n" + + " u3a != u3b;\n" + + " u4a != u4b;\n" + + " i1a == i1b;\n" + + " i2a == i2b;\n" + + " i3a == i3b;\n" + + " i4a == i4b;\n" + + " i1a != i1b;\n" + + " i2a != i2b;\n" + + " i3a != i3b;\n" + + " i4a != i4b;\n" + + " b1a == b1b;\n" + + " b2a == b2b;\n" + + " b3a == b3b;\n" + + " b4a == b4b;\n" + + " b1a != b1b;\n" + + " b2a != b2b;\n" + + " b3a != b3b;\n" + + " b4a != b4b;\n" + + " m22a == m22b;\n" + + " m22a != m22b;\n" + + " m23a == m23b;\n" + + " m23a != m23b;\n" + + " m24a == m24b;\n" + + " m24a != m24b;\n" + + " m32a == m32b;\n" + + " m32a != m32b;\n" + + " m33a == m33b;\n" + + " m33a != m33b;\n" + + " m34a == m34b;\n" + + " m34a != m34b;\n" + + " m42a == m42b;\n" + + " m42a != m42b;\n" + + " m43a == m43b;\n" + + " m43a != m43b;\n" + + " m44a == m44b;\n" + + " m44a != m44b;\n" + + "}\n"); + new NullCheckTyper(tu) { + @Override + public void visitBinaryExpr(BinaryExpr binaryExpr) { + super.visitBinaryExpr(binaryExpr); + if (binaryExpr.getOp() == BinOp.EQ || binaryExpr.getOp() == BinOp.NE) { + assertSame(lookupType(binaryExpr), BasicType.BOOL); + } + } + }.visit(tu); + } + + @Test + public void testVectorRelational() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 300 es\n" + + "void main() {\n" + + " vec2 v2a, v2b;\n" + + " vec3 v3a, v3b;\n" + + " vec4 v4a, v4b;\n" + + " ivec2 i2a, i2b;\n" + + " ivec3 i3a, i3b;\n" + + " ivec4 i4a, i4b;\n" + + " uvec2 u2a, u2b;\n" + + " uvec3 u3a, u3b;\n" + + " uvec4 u4a, u4b;\n" + + " bvec2 b2a, b2b;\n" + + " bvec3 b3a, b3b;\n" + + " bvec4 b4a, b4b;\n" + + "\n" + + " lessThan(v2a, v2b);\n" + + " lessThan(v3a, v3b);\n" + + " lessThan(v4a, v4b);\n" + + " lessThan(i2a, i2b);\n" + + " lessThan(i3a, i3b);\n" + + " lessThan(i4a, i4b);\n" + + " lessThan(u2a, u2b);\n" + + " lessThan(u3a, u3b);\n" + + " lessThan(u4a, u4b);\n" + + "\n" + + " lessThanEqual(v2a, v2b);\n" + + " lessThanEqual(v3a, v3b);\n" + + " lessThanEqual(v4a, v4b);\n" + + " lessThanEqual(i2a, i2b);\n" + + " lessThanEqual(i3a, i3b);\n" + + " lessThanEqual(i4a, i4b);\n" + + " lessThanEqual(u2a, u2b);\n" + + " lessThanEqual(u3a, u3b);\n" + + " lessThanEqual(u4a, u4b);\n" + + "\n" + + " greaterThan(v2a, v2b);\n" + + " greaterThan(v3a, v3b);\n" + + " greaterThan(v4a, v4b);\n" + + " greaterThan(i2a, i2b);\n" + + " greaterThan(i3a, i3b);\n" + + " greaterThan(i4a, i4b);\n" + + " greaterThan(u2a, u2b);\n" + + " greaterThan(u3a, u3b);\n" + + " greaterThan(u4a, u4b);\n" + + "\n" + + " greaterThanEqual(v2a, v2b);\n" + + " greaterThanEqual(v3a, v3b);\n" + + " greaterThanEqual(v4a, v4b);\n" + + " greaterThanEqual(i2a, i2b);\n" + + " greaterThanEqual(i3a, i3b);\n" + + " greaterThanEqual(i4a, i4b);\n" + + " greaterThanEqual(u2a, u2b);\n" + + " greaterThanEqual(u3a, u3b);\n" + + " greaterThanEqual(u4a, u4b);\n" + + "\n" + + " equal(v2a, v2b);\n" + + " equal(v3a, v3b);\n" + + " equal(v4a, v4b);\n" + + " equal(i2a, i2b);\n" + + " equal(i3a, i3b);\n" + + " equal(i4a, i4b);\n" + + " equal(u2a, u2b);\n" + + " equal(u3a, u3b);\n" + + " equal(u4a, u4b);\n" + + " equal(b2a, b2b);\n" + + " equal(b3a, b3b);\n" + + " equal(b4a, b4b);\n" + + "\n" + + " notEqual(v2a, v2b);\n" + + " notEqual(v3a, v3b);\n" + + " notEqual(v4a, v4b);\n" + + " notEqual(i2a, i2b);\n" + + " notEqual(i3a, i3b);\n" + + " notEqual(i4a, i4b);\n" + + " notEqual(u2a, u2b);\n" + + " notEqual(u3a, u3b);\n" + + " notEqual(u4a, u4b);\n" + + " notEqual(b2a, b2b);\n" + + " notEqual(b3a, b3b);\n" + + " notEqual(b4a, b4b);\n" + + "\n" + + " any(b2a);\n" + + " any(b3a);\n" + + " any(b4a);\n" + + "\n" + + " all(b2a);\n" + + " all(b3a);\n" + + " all(b4a);\n" + + "\n" + + " not(b2a);\n" + + " not(b3a);\n" + + " not(b4a);\n" + + "}\n"); + new NullCheckTyper(tu) { + @Override + public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { + super.visitFunctionCallExpr(functionCallExpr); + + // There must be a first argument type and it must be a vec type. + BasicType firstArgType = + (BasicType) lookupType(functionCallExpr.getArg(0)).getWithoutQualifiers(); + assertTrue(firstArgType.isVector()); + if (functionCallExpr.getNumArgs() > 1) { + // If there is a second argument type it must match the first argument type. + assertEquals(2, functionCallExpr.getNumArgs()); + assertEquals(firstArgType, lookupType(functionCallExpr.getArg(1)).getWithoutQualifiers()); + } + if (functionCallExpr.getCallee().equals("any") || functionCallExpr.getCallee().equals( + "all")) { + // 'any' and 'all' return 'bool' in all cases. + assertSame(BasicType.BOOL, lookupType(functionCallExpr)); + } else { + // If the first argument is a vector of length n, the result type must be 'bvecn'. + assertEquals(BasicType.makeVectorType(BasicType.BOOL, firstArgType.getNumElements()), + lookupType(functionCallExpr)); + } + } + }.visit(tu); + } + private void checkComputeShaderBuiltin(String builtin, String builtinConstant, BasicType baseType, TypeQualifier qualifier) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { From 6e3021d2a43ae3f43d4613696474ab35b8e06710 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 22 Aug 2019 14:59:54 +0100 Subject: [PATCH 115/180] Update some binaries (#713) --- gfauto/gfauto/binaries_util.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py index 6cb296037..25c270a28 100644 --- a/gfauto/gfauto/binaries_util.py +++ b/gfauto/gfauto/binaries_util.py @@ -56,7 +56,7 @@ "Mac_x64_RelWithDebInfo", ] -DEFAULT_SPIRV_TOOLS_VERSION = "9559cdbdf011c487f67f89e2d694bd4a18d5c1e0" +DEFAULT_SPIRV_TOOLS_VERSION = "06407250a169c6a03b3765e86619075af1a8c187" DEFAULT_BINARIES = [ Binary( @@ -71,7 +71,7 @@ Binary( name="swift_shader_icd", tags=["Debug"], - version="fa0175c0988dd542f008257232207a8b87ad6c63", + version="f25a1c68473b868ce61e97fe5b830c0cdd7e8181", ), ] @@ -510,4 +510,12 @@ def get_graphics_fuzz_121() -> List[recipe_wrap.RecipeWrap]: version_hash="fa0175c0988dd542f008257232207a8b87ad6c63", build_version_hash="ea3b929604da6873ace48988b8d4651bbcd2e573", ) + + _get_built_in_swift_shader_version( + version_hash="f25a1c68473b868ce61e97fe5b830c0cdd7e8181", + build_version_hash="ad0a59319c4a3e23db2688c593a1e0459a99340d", + ) + + _get_built_in_spirv_tools_version( + version_hash="06407250a169c6a03b3765e86619075af1a8c187", + build_version_hash="04b2b8e2543b4643c533b20ca1a9d88c72fea370", + ) ) From 0269cb93d92a390ec2df095a8ccc53fd3db4d647 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Fri, 23 Aug 2019 10:56:59 +0100 Subject: [PATCH 116/180] Update SPIRV-Tools (#714) --- assembly-binaries/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assembly-binaries/pom.xml b/assembly-binaries/pom.xml index dcade1a0f..2fc61903b 100644 --- a/assembly-binaries/pom.xml +++ b/assembly-binaries/pom.xml @@ -58,7 +58,7 @@ limitations under the License. - f2170cc791d0eaa5789ec7528862ae00b984b3b8 + 04b2b8e2543b4643c533b20ca1a9d88c72fea370 9d3202492d78aae5ced08ff14679b81c98d71c15 0565d5aee279a4782689297ada0aa2489e24ad3e 1586e566f4949b1957e7c32454cbf27e501ed632 From 50d417081a0947a499c5197fe4a873ef2c690629 Mon Sep 17 00:00:00 2001 From: asuonpaa <34128694+asuonpaa@users.noreply.github.com> Date: Thu, 29 Aug 2019 11:19:46 +0300 Subject: [PATCH 117/180] gfauto: Add AmberScript conversion support for multiple variants (#717) Added support for converting a gfauto source directory containing a wrong image case into an amber script that runs all the variants and compares the results against the reference. --- gfauto/gfauto/amber_converter.py | 34 ++++--- gfauto/gfauto/tool.py | 161 +++++++++++++++++++++++++------ 2 files changed, 151 insertions(+), 44 deletions(-) diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py index b644f8a22..f973e11e7 100644 --- a/gfauto/gfauto/amber_converter.py +++ b/gfauto/gfauto/amber_converter.py @@ -444,20 +444,24 @@ def to_shader_job(self) -> ShaderJob: @attr.dataclass class ShaderJobBasedAmberTest: reference: Optional[ShaderJob] - variant: ShaderJob + variants: List[ShaderJob] @attr.dataclass class ShaderJobFileBasedAmberTest: reference_asm_spirv_job: Optional[ShaderJobFile] - variant_asm_spirv_job: ShaderJobFile + variants_asm_spirv_job: List[ShaderJobFile] def to_shader_job_based(self) -> ShaderJobBasedAmberTest: + variants = [ + variant_asm_spirv_job.to_shader_job() + for variant_asm_spirv_job in self.variants_asm_spirv_job + ] return ShaderJobBasedAmberTest( self.reference_asm_spirv_job.to_shader_job() if self.reference_asm_spirv_job else None, - self.variant_asm_spirv_job.to_shader_job(), + variants, ) @@ -513,18 +517,19 @@ def get_amber_script_shader_def(shader: Shader, name: str) -> str: def graphics_shader_job_amber_test_to_amber_script( shader_job_amber_test: ShaderJobBasedAmberTest, amberfy_settings: AmberfySettings ) -> str: - # Guaranteed, and needed for type checker. - assert isinstance(shader_job_amber_test.variant, GraphicsShaderJob) # noqa result = get_amber_script_header(amberfy_settings) - jobs = [shader_job_amber_test.variant] + jobs = shader_job_amber_test.variants.copy() if shader_job_amber_test.reference: assert isinstance(shader_job_amber_test.reference, GraphicsShaderJob) # noqa jobs.insert(0, shader_job_amber_test.reference) for job in jobs: + # Guaranteed, and needed for type checker. + assert isinstance(job, GraphicsShaderJob) # noqa + prefix = job.name_prefix vertex_shader_name = f"{prefix}_vertex_shader" @@ -566,6 +571,7 @@ def graphics_shader_job_amber_test_to_amber_script( prefix_0 = jobs[0].name_prefix prefix_1 = jobs[pipeline_index].name_prefix result += f"EXPECT {prefix_0}_framebuffer RMSE_BUFFER {prefix_1}_framebuffer TOLERANCE 7" + result += "\n" if amberfy_settings.extra_commands: result += amberfy_settings.extra_commands @@ -579,12 +585,12 @@ def compute_shader_job_amber_test_to_amber_script( ) -> str: # TODO: handle reference, if present. - # Guaranteed, and needed for type checker. - assert isinstance(shader_job_amber_test.variant, ComputeShaderJob) # noqa - result = get_amber_script_header(amberfy_settings) - variant = shader_job_amber_test.variant + variant = shader_job_amber_test.variants[0] + + # Guaranteed, and needed for type checker. + assert isinstance(variant, ComputeShaderJob) # noqa variant_compute_shader_name = f"{variant.name_prefix}_compute_shader" variant_ssbo_name = f"{variant.name_prefix}_ssbo" @@ -633,7 +639,7 @@ def spirv_asm_shader_job_to_amber_script( ) -> Path: log( - f"Amberfy: {str(shader_job_file_amber_test.variant_asm_spirv_job.asm_spirv_shader_job_json)} " + f"Amberfy: {[str(variant.asm_spirv_shader_job_json) for variant in shader_job_file_amber_test.variants_asm_spirv_job]} " + ( f"with reference {str(shader_job_file_amber_test.reference_asm_spirv_job.asm_spirv_shader_job_json)} " if shader_job_file_amber_test.reference_asm_spirv_job @@ -644,18 +650,18 @@ def spirv_asm_shader_job_to_amber_script( shader_job_amber_test = shader_job_file_amber_test.to_shader_job_based() - if isinstance(shader_job_amber_test.variant, GraphicsShaderJob): + if isinstance(shader_job_amber_test.variants[0], GraphicsShaderJob): result = graphics_shader_job_amber_test_to_amber_script( shader_job_amber_test, amberfy_settings ) - elif isinstance(shader_job_amber_test.variant, ComputeShaderJob): + elif isinstance(shader_job_amber_test.variants[0], ComputeShaderJob): result = compute_shader_job_amber_test_to_amber_script( shader_job_amber_test, amberfy_settings ) else: raise AssertionError( - f"Unknown shader job type: {shader_job_amber_test.variant}" + f"Unknown shader job type: {shader_job_amber_test.variants[0]}" ) util.file_write_text(output_amber_script_file_path, result) diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index 008ea1cdc..76406d738 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -20,7 +20,7 @@ """ from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Tuple from gfauto import ( amber_converter, @@ -82,9 +82,11 @@ def amberfy( shader_job_file_amber_test = amber_converter.ShaderJobFileBasedAmberTest( reference_asm_spirv_job=None, - variant_asm_spirv_job=amber_converter.ShaderJobFile( - "variant", input_json, input_glsl_source_json_path, "" - ), + variants_asm_spirv_job=[ + amber_converter.ShaderJobFile( + "variant", input_json, input_glsl_source_json_path, "" + ) + ], ) return amber_converter.spirv_asm_shader_job_to_amber_script( shader_job_file_amber_test, output_amber, amberfy_settings @@ -153,12 +155,10 @@ def validate_spirv_shader_job( ) -def glsl_shader_job_to_amber_script( +def compile_shader_job( input_json: Path, - output_amber: Path, work_dir: Path, binary_paths: binaries_util.BinaryGetter, - amberfy_settings: amber_converter.AmberfySettings, spirv_opt_args: Optional[List[str]] = None, ) -> Path: @@ -206,40 +206,31 @@ def glsl_shader_job_to_amber_script( validate_spirv_shader_job(result_spirv, binary_paths) - result = amberfy(result, output_amber, amberfy_settings, input_json) - return result -def glsl_shader_job_crash_to_amber_script_for_google_cts( +def glsl_shader_job_to_amber_script( input_json: Path, output_amber: Path, work_dir: Path, - short_description: str, - comment_text: str, - copyright_year: str, - extra_commands: str, + binary_paths: binaries_util.BinaryGetter, + amberfy_settings: amber_converter.AmberfySettings, + spirv_opt_args: Optional[List[str]] = None, +) -> Path: + + result = compile_shader_job(input_json, work_dir, binary_paths, spirv_opt_args) + result = amberfy(result, output_amber, amberfy_settings, input_json) + + return result + + +def get_compilation_settings( binary_paths: Optional[binaries_util.BinaryGetter] = None, spirv_opt_args: Optional[List[str]] = None, spirv_opt_hash: Optional[str] = None, test_metadata_path: Optional[Path] = None, -) -> Path: - """ - Converts a GLSL shader job to an Amber script suitable for adding to the CTS. +) -> Tuple[binaries_util.BinaryGetter, Optional[List[str]], Optional[str]]: - :param input_json: - :param output_amber: - :param work_dir: - :param short_description: One sentence, 58 characters max., no period, no line breaks. - :param comment_text: Why the test should pass. Can have line breaks. Ideally make sure lines are not too long. - :param copyright_year: - :param extra_commands: - :param binary_paths: - :param spirv_opt_args: - :param spirv_opt_hash: - :param test_metadata_path: - :return: - """ if not binary_paths: check( bool(test_metadata_path), @@ -266,6 +257,43 @@ def glsl_shader_job_crash_to_amber_script_for_google_cts( binaries_util.SPIRV_OPT_NAME ).binary.version + return (binary_paths, spirv_opt_args, spirv_opt_hash) + + +def glsl_shader_job_crash_to_amber_script_for_google_cts( + input_json: Path, + output_amber: Path, + work_dir: Path, + short_description: str, + comment_text: str, + copyright_year: str, + extra_commands: str, + binary_paths: Optional[binaries_util.BinaryGetter] = None, + spirv_opt_args: Optional[List[str]] = None, + spirv_opt_hash: Optional[str] = None, + test_metadata_path: Optional[Path] = None, +) -> Path: + """ + Converts a GLSL shader job to an Amber script suitable for adding to the CTS. + + :param input_json: + :param output_amber: + :param work_dir: + :param short_description: One sentence, 58 characters max., no period, no line breaks. + :param comment_text: Why the test should pass. Can have line breaks. Ideally make sure lines are not too long. + :param copyright_year: + :param extra_commands: + :param binary_paths: + :param spirv_opt_args: + :param spirv_opt_hash: + :param test_metadata_path: + :return: + """ + # Get compilation settings + (binary_paths, spirv_opt_args, spirv_opt_hash) = get_compilation_settings( + binary_paths, spirv_opt_args, spirv_opt_hash, test_metadata_path + ) + return glsl_shader_job_to_amber_script( input_json, output_amber, @@ -283,3 +311,76 @@ def glsl_shader_job_crash_to_amber_script_for_google_cts( ), spirv_opt_args=spirv_opt_args, ) + + +def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( + source_dir: Path, + output_amber: Path, + work_dir: Path, + short_description: str, + comment_text: str, + copyright_year: str, + binary_paths: Optional[binaries_util.BinaryGetter] = None, + spirv_opt_args: Optional[List[str]] = None, + spirv_opt_hash: Optional[str] = None, +) -> Path: + """ + Converts a GLSL shader job of a wrong image case to an Amber script suitable for adding to the CTS. + + :param source_dir: + :param output_amber: + :param work_dir: + :param short_description: One sentence, 58 characters max., no period, no line breaks. + :param comment_text: Why the test should pass. Can have line breaks. Ideally make sure lines are not too long. + :param copyright_year: + :param binary_paths: + :param spirv_opt_args: + :param spirv_opt_hash: + :return + """ + test_metadata_path = source_dir / test_util.TEST_METADATA + + # Populate shader jobs with reference and all available variants + shader_jobs = [test_util.REFERENCE_DIR] + sorted( + variant.name + for variant in source_dir.glob(test_util.VARIANT_DIR + "*") + if variant.is_dir() + ) + + # Get compilation settings + (binary_paths, spirv_opt_args, spirv_opt_hash) = get_compilation_settings( + binary_paths, spirv_opt_args, spirv_opt_hash, test_metadata_path + ) + + # Compile all shader jobs + shader_job_files = [ + amber_converter.ShaderJobFile( + shader_job, + compile_shader_job( + source_dir / shader_job / test_util.SHADER_JOB, + work_dir / shader_job, + binary_paths, + spirv_opt_args=spirv_opt_args, + ), + source_dir / shader_job / test_util.SHADER_JOB, + "", + ) + for shader_job in shader_jobs + ] + + return amber_converter.spirv_asm_shader_job_to_amber_script( + amber_converter.ShaderJobFileBasedAmberTest( + reference_asm_spirv_job=shader_job_files[0], + variants_asm_spirv_job=shader_job_files[1:], + ), + output_amber, + amber_converter.AmberfySettings( + copyright_header_text=get_copyright_header_google(copyright_year), + add_graphics_fuzz_comment=True, + short_description=short_description, + comment_text=comment_text, + use_default_fence_timeout=True, + spirv_opt_args=spirv_opt_args, + spirv_opt_hash=spirv_opt_hash, + ), + ) From 9a384894ae05286d62c309e9132832300df24d9e Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Mon, 2 Sep 2019 19:21:53 +0100 Subject: [PATCH 118/180] gfauto: fix some issues with parallel running of gfauto (#723) Fix #719 use separate donors and references directories Fix #721 iteration_seed == 0 does not work Fix #722 add option to skip writing default recipes --- gfauto/README.md | 22 +++++++++++++++++++--- gfauto/gfauto/artifact_util.py | 4 ++++ gfauto/gfauto/fuzz.py | 31 ++++++++++++++++++++++--------- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/gfauto/README.md b/gfauto/README.md index 75e23b7fb..91dd13c33 100644 --- a/gfauto/README.md +++ b/gfauto/README.md @@ -124,11 +124,12 @@ You can alternatively execute the `./dev_shell.sh` script, but this is fairly sl ## Fuzzing -To start fuzzing, create and change to a directory outside the `gfauto/` directory. E.g. `/data/temp/gfauto_fuzzing/2019_06_24`. From here, create a `donors/` directory containing GLSL shader jobs as used by GraphicsFuzz. +To start fuzzing, create and change to a directory outside the `gfauto/` directory. E.g. `/data/temp/gfauto_fuzzing/2019_06_24`. From here, create `references/` and `donors/` directories containing GLSL shader jobs as used by GraphicsFuzz. You can get some samples from the GraphicsFuzz project. ```sh -mkdir donors/ +mkdir references/ donors/ +cp /data/graphicsfuzz_zip/samples/310es/* references/ cp /data/graphicsfuzz_zip/samples/310es/* donors/ ``` @@ -145,5 +146,20 @@ The `active_device_names` list should be modified to include only the unique dev * Including multiple, identical devices (with the same GPU and drivers) is not recommended, as it will currently result in redundant testing. * `host_preprocessor` should always be included first, as this virtual device detects failures in tools such as `glslangValidator` and `spirv-opt`, and will ensure such failures will not be logged for real devices. -* You can use `--settings SETTINGS.json` to use a settings file other than `settings.json` (the default). In this way, you can run multiple instances of `gfauto_fuzz` in parallel to test *different* devices (`host_preprocessor` and `swift_shader` can be included in parallel instances). The code that downloads the binaries is not safe to run concurrently at the time of writing; running one instance for a few minutes to download the latest binaries is usually sufficient to then allow additional instances to be started without conflicts. +* You can use `--settings SETTINGS.json` to use a settings file other than `settings.json` (the default). In this way, you can run multiple instances of `gfauto_fuzz` in parallel to test *different* devices. When starting, `gfauto_fuzz` writes the built-in recipes to `//binaries/`, and this code is not safe to run concurrently; run `gfauto_fuzz` at least once to ensure the recipes are written and then use the `--skip_writing_binary_recipes` option to prevent other instances of `gfauto_fuzz` from writing them on start-up. The code that downloads binaries is not safe to run concurrently at the time of writing; running one instance for a few minutes to download the latest binaries is usually sufficient to then allow additional instances to be started without conflicts. +You can generate a space-separated list of seeds as follows: + +```python +import secrets + +" ".join([str(secrets.randbits(256)) for i in range(0, 1000)]) +``` + +Assuming you saved those to `../seeds.txt`, you can run parallel instances of `gfauto_fuzz` using: + +```sh +parallel -j 64 gfauto_fuzz --skip_writing_binary_recipes --iteration_seed -- $(cat ../seeds.txt) +``` + +This is probably only suitable for testing the `host_preprocessor` and `swift_shader` virtual devices; running parallel tests on actual hardware is likely to give unreliable results. diff --git a/gfauto/gfauto/artifact_util.py b/gfauto/gfauto/artifact_util.py index 5d715cd16..f5b9348e1 100644 --- a/gfauto/gfauto/artifact_util.py +++ b/gfauto/gfauto/artifact_util.py @@ -31,6 +31,7 @@ ) from gfauto.artifact_pb2 import ArtifactMetadata from gfauto.common_pb2 import ArchiveSet +from gfauto.gflogging import log from gfauto.recipe_pb2 import Recipe from gfauto.util import check @@ -49,6 +50,9 @@ def __init__(self, path: str, metadata: Optional[ArtifactMetadata] = None): def recipes_write_built_in() -> None: + log( + "Writing built-in binary recipes to //binaries/ (where // is the ROOT directory)" + ) for recipe_wrap in binaries_util.BUILT_IN_BINARY_RECIPES: if not artifact_get_metadata_file_path(recipe_wrap.path).exists(): recipe_wrap.write() diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index 2000efa22..d9ebceef1 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -142,23 +142,33 @@ def main() -> None: help="The seed to use for one fuzzing iteration (useful for reproducing an issue).", ) + parser.add_argument( + "--skip_writing_binary_recipes", + help="Don't write the default binary recipes to ROOT/binaries. " + "This is useful when running multiple, parallel instances of gfauto because otherwise the writes will race.", + action="store_true", + ) + parsed_args = parser.parse_args(sys.argv[1:]) settings_path = Path(parsed_args.settings) - iteration_seed: Optional[int] = int( + iteration_seed: Optional[int] = None if parsed_args.iteration_seed is None else int( parsed_args.iteration_seed - ) if parsed_args.iteration_seed else None + ) + skip_writing_binary_recipes: bool = parsed_args.skip_writing_binary_recipes with util.file_open_text(Path(f"log_{get_random_name()}.txt"), "w") as log_file: gflogging.push_stream_for_logging(log_file) try: - main_helper(settings_path, iteration_seed) + main_helper(settings_path, iteration_seed, skip_writing_binary_recipes) finally: gflogging.pop_stream_for_logging() def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements; - settings_path: Path, iteration_seed_override: Optional[int] + settings_path: Path, + iteration_seed_override: Optional[int], + skip_writing_binary_recipes: bool, ) -> None: try: @@ -177,6 +187,7 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many reports_dir = Path() / "reports" temp_dir = Path() / "temp" + references_dir = Path() / "references" donors_dir = Path() / "donors" spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders" @@ -190,13 +201,14 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many ) util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") - artifact_util.recipes_write_built_in() + if not skip_writing_binary_recipes: + artifact_util.recipes_write_built_in() # Log a warning if there is no tool on the PATH for printing stack traces. util.prepend_catchsegv_if_available([], log_warning=True) # TODO: make GraphicsFuzz find donors recursively. - references = sorted(donors_dir.rglob("*.json")) + references = sorted(references_dir.rglob("*.json")) # Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file. references = [ref for ref in references if shader_job_util.get_related_files(ref)] @@ -226,7 +238,8 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many while True: - if iteration_seed_override: + # We have to use "is not None" because the seed could be 0. + if iteration_seed_override is not None: iteration_seed = iteration_seed_override else: iteration_seed = secrets.randbits(ITERATION_SEED_BITS) @@ -240,7 +253,7 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many try: util.mkdir_p_new(staging_dir) except FileExistsError: - if iteration_seed_override: + if iteration_seed_override is not None: raise log(f"Staging directory already exists: {str(staging_dir)}") log(f"Starting new iteration.") @@ -275,7 +288,7 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many shutil.rmtree(staging_dir) - if iteration_seed_override: + if iteration_seed_override is not None: log("Stopping due to iteration_seed") break From f63c48aade490ef84d5bf18279aa5130d78f5dae Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Mon, 2 Sep 2019 20:10:59 +0100 Subject: [PATCH 119/180] Use '1;' instead of ';' when replacing jump statements (#724) Before this change, the null statement, ';', was used when replacing break, continue, case label and discard statements during live code injection, and void return statements were completely removed. The latter - completely removing a return - caused problems when a return appeared at the end of a switch statement. Since it is questionable whether it is OK for the final case/default label of a switch to be followed by simply a null statement (e.g. glslangValidator gives a warning about this), this change makes it so that the statement '1;' is used in all cases where a null statement was previously used, and is also used as a replacement for a void return. Fixes #718. --- .../util/RemoveDiscardStatements.java | 4 +-- .../util/RemoveImmediateBreakStatements.java | 4 +-- .../util/RemoveImmediateCaseLabels.java | 4 +-- .../RemoveImmediateContinueStatements.java | 4 +-- .../util/RemoveReturnStatements.java | 4 +-- .../generator/util/RemoveStatements.java | 24 +++++++++------- .../DonateLiveCodeTransformationTest.java | 9 +++--- .../injection/RemoveReturnStatementsTest.java | 28 ++++++++++++++++--- 8 files changed, 49 insertions(+), 32 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveDiscardStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveDiscardStatements.java index f96c4ceb1..d709fc2c8 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveDiscardStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveDiscardStatements.java @@ -18,14 +18,12 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.stmt.DiscardStmt; -import com.graphicsfuzz.common.ast.stmt.NullStmt; -import java.util.Optional; public class RemoveDiscardStatements extends RemoveStatements { public RemoveDiscardStatements(IAstNode node) { super(item -> item instanceof DiscardStmt, - item -> Optional.of(new NullStmt()), node); + item -> makeIntConstantExprStmt(), node); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java index 810d181fd..e96438a2b 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java @@ -20,10 +20,8 @@ import com.graphicsfuzz.common.ast.stmt.BreakStmt; import com.graphicsfuzz.common.ast.stmt.DoStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; -import com.graphicsfuzz.common.ast.stmt.NullStmt; import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.stmt.WhileStmt; -import java.util.Optional; /** * This class removes break statements that are not nested inside loop or switch statements. @@ -32,7 +30,7 @@ public class RemoveImmediateBreakStatements extends RemoveStatements { public RemoveImmediateBreakStatements(IAstNode node) { super(item -> item instanceof BreakStmt, - item -> Optional.of(new NullStmt()), node); + item -> makeIntConstantExprStmt(), node); } @Override diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java index 0c320be22..401266e08 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java @@ -18,9 +18,7 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.stmt.CaseLabel; -import com.graphicsfuzz.common.ast.stmt.NullStmt; import com.graphicsfuzz.common.ast.stmt.SwitchStmt; -import java.util.Optional; /** * This class removes case labels (including default) that are not nested inside switch statements. @@ -29,7 +27,7 @@ public class RemoveImmediateCaseLabels extends RemoveStatements { public RemoveImmediateCaseLabels(IAstNode node) { super(item -> item instanceof CaseLabel, - item -> Optional.of(new NullStmt()), node); + item -> makeIntConstantExprStmt(), node); } @Override diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java index 6d2b5d5e5..146f194fe 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java @@ -20,9 +20,7 @@ import com.graphicsfuzz.common.ast.stmt.ContinueStmt; import com.graphicsfuzz.common.ast.stmt.DoStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; -import com.graphicsfuzz.common.ast.stmt.NullStmt; import com.graphicsfuzz.common.ast.stmt.WhileStmt; -import java.util.Optional; /** * This class removes continue statements that are not nested inside loops. @@ -31,7 +29,7 @@ public class RemoveImmediateContinueStatements extends RemoveStatements { public RemoveImmediateContinueStatements(IAstNode node) { super(item -> item instanceof ContinueStmt, - item -> Optional.of(new NullStmt()), node); + item -> makeIntConstantExprStmt(), node); } @Override diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveReturnStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveReturnStatements.java index 1c0378c2e..cea3bdd84 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveReturnStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveReturnStatements.java @@ -26,8 +26,8 @@ public class RemoveReturnStatements extends RemoveStatements { public RemoveReturnStatements(IAstNode node) { super(item -> item instanceof ReturnStmt, item -> ((ReturnStmt) item).hasExpr() - ? Optional.of(new ExprStmt(((ReturnStmt) item).getExpr())) - : Optional.empty(), + ? new ExprStmt(((ReturnStmt) item).getExpr()) + : makeIntConstantExprStmt(), node); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveStatements.java index 5f54f9851..2d42bb1af 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveStatements.java @@ -17,12 +17,13 @@ package com.graphicsfuzz.generator.util; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.DoStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.IfStmt; import com.graphicsfuzz.common.ast.stmt.LoopStmt; -import com.graphicsfuzz.common.ast.stmt.NullStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.stmt.WhileStmt; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; @@ -35,12 +36,12 @@ public abstract class RemoveStatements extends StandardVisitor { private final Predicate shouldRemove; - private final Function> maybeReplaceWith; + private final Function replaceWith; public RemoveStatements(Predicate shouldRemove, - Function> maybeReplaceWith, IAstNode node) { + Function replaceWith, IAstNode node) { this.shouldRemove = shouldRemove; - this.maybeReplaceWith = maybeReplaceWith; + this.replaceWith = replaceWith; visit(node); } @@ -49,10 +50,7 @@ public void visitBlockStmt(BlockStmt stmt) { List newStmts = new ArrayList<>(); for (Stmt s : stmt.getStmts()) { if (shouldRemove.test(s)) { - Optional possibleReplacement = maybeReplaceWith.apply(s); - if (possibleReplacement.isPresent()) { - newStmts.add(possibleReplacement.get()); - } + newStmts.add(getReplacementStmt(s)); continue; } newStmts.add(s); @@ -101,8 +99,14 @@ private void handleLoop(LoopStmt loop) { } private Stmt getReplacementStmt(Stmt stmt) { - Optional possibleReplacement = maybeReplaceWith.apply(stmt); - return possibleReplacement.isPresent() ? possibleReplacement.get() : new NullStmt(); + return replaceWith.apply(stmt); + } + + /** + * Provides a statement of the form "1;", which is what most statements are replaced with. + */ + static Stmt makeIntConstantExprStmt() { + return new ExprStmt(new IntConstantExpr("1")); } } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java index 19a095e12..2cad5ca76 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java @@ -138,7 +138,7 @@ TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), ShadingLanguageVersion.ESSL_310); assertEquals("{\n" + " if(i > 5)\n" - + " ;\n" + + " 1;\n" + "}\n", donated.getText()); } @@ -159,6 +159,7 @@ public void prepareStatementToDonateTopLevelContinueRemoved() throws Exception { final TranslationUnit reference = ParseHelper.parse("#version 100\n" + "void main() {\n" + " for(int i = 0; i < 100; i++) {\n" + + " 1;\n" + " }\n" + "}\n"); @@ -176,7 +177,7 @@ TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), ShadingLanguageVersion.ESSL_100); assertEquals("{\n" + " if(i > 5)\n" - + " ;\n" + + " 1;\n" + "}\n", donated.getText()); } @@ -223,9 +224,9 @@ TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), ShadingLanguageVersion.ESSL_310); assertEquals("{\n" + " {\n" - + " ;\n" + + " 1;\n" + " x ++;\n" - + " ;\n" + + " 1;\n" + " x ++;\n" + " }\n" + "}\n", donated.getText()); diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/RemoveReturnStatementsTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/RemoveReturnStatementsTest.java index 56fe718c7..6830c6343 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/RemoveReturnStatementsTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/RemoveReturnStatementsTest.java @@ -29,7 +29,7 @@ public class RemoveReturnStatementsTest { @Test public void testDo() throws Exception { final String prog = "void main() { do return; while(true); }"; - final String expectedProg = "void main() { do ; while(true); }"; + final String expectedProg = "void main() { do 1; while(true); }"; TranslationUnit tu = ParseHelper.parse(prog); new RemoveReturnStatements(tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), @@ -39,7 +39,7 @@ public void testDo() throws Exception { @Test public void testWhile() throws Exception { final String prog = "void main() { while(true) return; }"; - final String expectedProg = "void main() { while(true) ; }"; + final String expectedProg = "void main() { while(true) 1; }"; TranslationUnit tu = ParseHelper.parse(prog); new RemoveReturnStatements(tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), @@ -49,7 +49,7 @@ public void testWhile() throws Exception { @Test public void testFor() throws Exception { final String prog = "void main() { for(int i = 0; i < 100; i++) return; }"; - final String expectedProg = "void main() { for(int i = 0; i < 100; i++) ; }"; + final String expectedProg = "void main() { for(int i = 0; i < 100; i++) 1; }"; TranslationUnit tu = ParseHelper.parse(prog); new RemoveReturnStatements(tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), @@ -59,7 +59,27 @@ public void testFor() throws Exception { @Test public void testIf() throws Exception { final String prog = "void main() { if(true) return; else return; }"; - final String expectedProg = "void main() { if(true) ; else ; }"; + final String expectedProg = "void main() { if(true) 1; else 1; }"; + TranslationUnit tu = ParseHelper.parse(prog); + new RemoveReturnStatements(tu); + assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), + PrettyPrinterVisitor.prettyPrintAsString(tu)); + } + + @Test + public void testSwitch() throws Exception { + final String prog = "void main() {\n" + + " switch(0) {\n" + + " case 0:\n" + + " return;\n" + + " }\n" + + "}\n"; + final String expectedProg = "void main() {\n" + + " switch(0) {\n" + + " case 0:\n" + + " 1;\n" + + " }\n" + + "}\n"; TranslationUnit tu = ParseHelper.parse(prog); new RemoveReturnStatements(tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), From ff52614929d223b1868b2019159fe775db137816 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 3 Sep 2019 08:40:34 +0100 Subject: [PATCH 120/180] gfauto: add offline compiler support (#725) * gfauto: Add support for offline shader compilers * gfauto: improve tool.py and generate_cts_test_template.py for more concise scripts. * gfauto: Fix shader_compiler_util.py * gfauto: improve signature_util.py --- gfauto/.flake8 | 1 + gfauto/gfauto/artifact_pb2.pyi | 16 ++++- gfauto/gfauto/common_pb2.pyi | 16 ++++- gfauto/gfauto/device.proto | 7 ++ gfauto/gfauto/device_pb2.py | 79 ++++++++++++++++++--- gfauto/gfauto/device_pb2.pyi | 50 +++++++++++-- gfauto/gfauto/devices_util.py | 11 +++ gfauto/gfauto/fuzz_glsl_test.py | 60 +++++++++++----- gfauto/gfauto/generate_cts_test_template.py | 5 +- gfauto/gfauto/recipe_pb2.pyi | 12 +++- gfauto/gfauto/settings_pb2.pyi | 8 ++- gfauto/gfauto/shader_compiler_util.py | 75 +++++++++++++++++++ gfauto/gfauto/signature_util.py | 16 +++++ gfauto/gfauto/test_pb2.pyi | 13 +++- gfauto/gfauto/tool.py | 45 +++++++++--- gfauto/whitelist.dic | 1 + 16 files changed, 359 insertions(+), 56 deletions(-) create mode 100644 gfauto/gfauto/shader_compiler_util.py diff --git a/gfauto/.flake8 b/gfauto/.flake8 index cb61af254..6cde794b4 100644 --- a/gfauto/.flake8 +++ b/gfauto/.flake8 @@ -28,6 +28,7 @@ ignore = D101 D103 D104 + D105 D107 D102 diff --git a/gfauto/gfauto/artifact_pb2.pyi b/gfauto/gfauto/artifact_pb2.pyi index ac7a3528c..373b6e3b1 100644 --- a/gfauto/gfauto/artifact_pb2.pyi +++ b/gfauto/gfauto/artifact_pb2.pyi @@ -4,6 +4,10 @@ from gfauto.common_pb2 import ( ArchiveSet as gfauto___common_pb2___ArchiveSet, ) +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + from google.protobuf.internal.containers import ( RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, ) @@ -24,12 +28,15 @@ from typing_extensions import ( class ArtifactMetadata(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... class Data(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property def extracted_archive_set(self) -> ArtifactMetadataExtractedArchiveSet: ... def __init__(self, + *, extracted_archive_set : typing___Optional[ArtifactMetadataExtractedArchiveSet] = None, ) -> None: ... @classmethod @@ -41,7 +48,7 @@ class ArtifactMetadata(google___protobuf___message___Message): def ClearField(self, field_name: typing_extensions___Literal[u"data",u"extracted_archive_set"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"data",b"data",u"extracted_archive_set",b"extracted_archive_set"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[b"data",b"extracted_archive_set"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"data",b"data",u"extracted_archive_set",b"extracted_archive_set"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"data",b"data"]) -> typing_extensions___Literal["extracted_archive_set"]: ... derived_from = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] @@ -51,6 +58,7 @@ class ArtifactMetadata(google___protobuf___message___Message): def data(self) -> ArtifactMetadata.Data: ... def __init__(self, + *, data : typing___Optional[ArtifactMetadata.Data] = None, derived_from : typing___Optional[typing___Iterable[typing___Text]] = None, comment : typing___Optional[typing___Text] = None, @@ -64,14 +72,16 @@ class ArtifactMetadata(google___protobuf___message___Message): def ClearField(self, field_name: typing_extensions___Literal[u"comment",u"data",u"derived_from"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"data",b"data"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[b"comment",b"data",b"derived_from"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"comment",b"comment",u"data",b"data",u"derived_from",b"derived_from"]) -> None: ... class ArtifactMetadataExtractedArchiveSet(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property def archive_set(self) -> gfauto___common_pb2___ArchiveSet: ... def __init__(self, + *, archive_set : typing___Optional[gfauto___common_pb2___ArchiveSet] = None, ) -> None: ... @classmethod @@ -83,4 +93,4 @@ class ArtifactMetadataExtractedArchiveSet(google___protobuf___message___Message) def ClearField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[b"archive_set"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> None: ... diff --git a/gfauto/gfauto/common_pb2.pyi b/gfauto/gfauto/common_pb2.pyi index 8dd92eaa2..cb7667b84 100644 --- a/gfauto/gfauto/common_pb2.pyi +++ b/gfauto/gfauto/common_pb2.pyi @@ -1,5 +1,9 @@ # @generated by generate_proto_mypy_stubs.py. Do not edit! import sys +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + from google.protobuf.internal.containers import ( RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, @@ -21,12 +25,14 @@ from typing_extensions import ( class Binary(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... name = ... # type: typing___Text tags = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] path = ... # type: typing___Text version = ... # type: typing___Text def __init__(self, + *, name : typing___Optional[typing___Text] = None, tags : typing___Optional[typing___Iterable[typing___Text]] = None, path : typing___Optional[typing___Text] = None, @@ -39,14 +45,16 @@ class Binary(google___protobuf___message___Message): if sys.version_info >= (3,): def ClearField(self, field_name: typing_extensions___Literal[u"name",u"path",u"tags",u"version"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[b"name",b"path",b"tags",b"version"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"path",b"path",u"tags",b"tags",u"version",b"version"]) -> None: ... class Archive(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... url = ... # type: typing___Text output_file = ... # type: typing___Text output_directory = ... # type: typing___Text def __init__(self, + *, url : typing___Optional[typing___Text] = None, output_file : typing___Optional[typing___Text] = None, output_directory : typing___Optional[typing___Text] = None, @@ -58,9 +66,10 @@ class Archive(google___protobuf___message___Message): if sys.version_info >= (3,): def ClearField(self, field_name: typing_extensions___Literal[u"output_directory",u"output_file",u"url"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[b"output_directory",b"output_file",b"url"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"output_directory",b"output_directory",u"output_file",b"output_file",u"url",b"url"]) -> None: ... class ArchiveSet(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property def archives(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Archive]: ... @@ -69,6 +78,7 @@ class ArchiveSet(google___protobuf___message___Message): def binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Binary]: ... def __init__(self, + *, archives : typing___Optional[typing___Iterable[Archive]] = None, binaries : typing___Optional[typing___Iterable[Binary]] = None, ) -> None: ... @@ -79,4 +89,4 @@ class ArchiveSet(google___protobuf___message___Message): if sys.version_info >= (3,): def ClearField(self, field_name: typing_extensions___Literal[u"archives",u"binaries"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[b"archives",b"binaries"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"archives",b"archives",u"binaries",b"binaries"]) -> None: ... diff --git a/gfauto/gfauto/device.proto b/gfauto/gfauto/device.proto index 8bd000e97..a45055619 100644 --- a/gfauto/gfauto/device.proto +++ b/gfauto/gfauto/device.proto @@ -34,6 +34,7 @@ message Device { DeviceSwiftShader swift_shader = 3; DeviceHost host = 4; DeviceAndroid android = 5; + DeviceShaderCompiler shader_compiler = 7; } // A device can specify/override some binaries. E.g. you may want several SwiftShader "devices" with different // versions or configurations of SwiftShader. @@ -58,3 +59,9 @@ message DeviceAndroid { string serial = 1; string model = 2; } + +// A virtual device that just executes an offline shader compiler. +message DeviceShaderCompiler { + string binary = 1; + repeated string args = 2; +} diff --git a/gfauto/gfauto/device_pb2.py b/gfauto/gfauto/device_pb2.py index 9cf837838..f93fab568 100644 --- a/gfauto/gfauto/device_pb2.py +++ b/gfauto/gfauto/device_pb2.py @@ -21,7 +21,7 @@ package='gfauto', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\x13gfauto/device.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"J\n\nDeviceList\x12\x1b\n\x13\x61\x63tive_device_names\x18\x01 \x03(\t\x12\x1f\n\x07\x64\x65vices\x18\x02 \x03(\x0b\x32\x0e.gfauto.Device\"\xf3\x01\n\x06\x44\x65vice\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\npreprocess\x18\x02 \x01(\x0b\x32\x18.gfauto.DevicePreprocessH\x00\x12\x31\n\x0cswift_shader\x18\x03 \x01(\x0b\x32\x19.gfauto.DeviceSwiftShaderH\x00\x12\"\n\x04host\x18\x04 \x01(\x0b\x32\x12.gfauto.DeviceHostH\x00\x12(\n\x07\x61ndroid\x18\x05 \x01(\x0b\x32\x15.gfauto.DeviceAndroidH\x00\x12 \n\x08\x62inaries\x18\x06 \x03(\x0b\x32\x0e.gfauto.BinaryB\x08\n\x06\x64\x65vice\"\x13\n\x11\x44\x65viceSwiftShader\"\x12\n\x10\x44\x65vicePreprocess\"\x0c\n\nDeviceHost\".\n\rDeviceAndroid\x12\x0e\n\x06serial\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\tb\x06proto3') + serialized_pb=_b('\n\x13gfauto/device.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"J\n\nDeviceList\x12\x1b\n\x13\x61\x63tive_device_names\x18\x01 \x03(\t\x12\x1f\n\x07\x64\x65vices\x18\x02 \x03(\x0b\x32\x0e.gfauto.Device\"\xac\x02\n\x06\x44\x65vice\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\npreprocess\x18\x02 \x01(\x0b\x32\x18.gfauto.DevicePreprocessH\x00\x12\x31\n\x0cswift_shader\x18\x03 \x01(\x0b\x32\x19.gfauto.DeviceSwiftShaderH\x00\x12\"\n\x04host\x18\x04 \x01(\x0b\x32\x12.gfauto.DeviceHostH\x00\x12(\n\x07\x61ndroid\x18\x05 \x01(\x0b\x32\x15.gfauto.DeviceAndroidH\x00\x12\x37\n\x0fshader_compiler\x18\x07 \x01(\x0b\x32\x1c.gfauto.DeviceShaderCompilerH\x00\x12 \n\x08\x62inaries\x18\x06 \x03(\x0b\x32\x0e.gfauto.BinaryB\x08\n\x06\x64\x65vice\"\x13\n\x11\x44\x65viceSwiftShader\"\x12\n\x10\x44\x65vicePreprocess\"\x0c\n\nDeviceHost\".\n\rDeviceAndroid\x12\x0e\n\x06serial\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\t\"4\n\x14\x44\x65viceShaderCompiler\x12\x0e\n\x06\x62inary\x18\x01 \x01(\t\x12\x0c\n\x04\x61rgs\x18\x02 \x03(\tb\x06proto3') , dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,]) @@ -109,7 +109,14 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='binaries', full_name='gfauto.Device.binaries', index=5, + name='shader_compiler', full_name='gfauto.Device.shader_compiler', index=5, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='binaries', full_name='gfauto.Device.binaries', index=6, number=6, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -131,7 +138,7 @@ index=0, containing_type=None, fields=[]), ], serialized_start=129, - serialized_end=372, + serialized_end=429, ) @@ -154,8 +161,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=374, - serialized_end=393, + serialized_start=431, + serialized_end=450, ) @@ -178,8 +185,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=395, - serialized_end=413, + serialized_start=452, + serialized_end=470, ) @@ -202,8 +209,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=415, - serialized_end=427, + serialized_start=472, + serialized_end=484, ) @@ -240,8 +247,46 @@ extension_ranges=[], oneofs=[ ], - serialized_start=429, - serialized_end=475, + serialized_start=486, + serialized_end=532, +) + + +_DEVICESHADERCOMPILER = _descriptor.Descriptor( + name='DeviceShaderCompiler', + full_name='gfauto.DeviceShaderCompiler', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='binary', full_name='gfauto.DeviceShaderCompiler.binary', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='args', full_name='gfauto.DeviceShaderCompiler.args', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=534, + serialized_end=586, ) _DEVICELIST.fields_by_name['devices'].message_type = _DEVICE @@ -249,6 +294,7 @@ _DEVICE.fields_by_name['swift_shader'].message_type = _DEVICESWIFTSHADER _DEVICE.fields_by_name['host'].message_type = _DEVICEHOST _DEVICE.fields_by_name['android'].message_type = _DEVICEANDROID +_DEVICE.fields_by_name['shader_compiler'].message_type = _DEVICESHADERCOMPILER _DEVICE.fields_by_name['binaries'].message_type = gfauto_dot_common__pb2._BINARY _DEVICE.oneofs_by_name['device'].fields.append( _DEVICE.fields_by_name['preprocess']) @@ -262,12 +308,16 @@ _DEVICE.oneofs_by_name['device'].fields.append( _DEVICE.fields_by_name['android']) _DEVICE.fields_by_name['android'].containing_oneof = _DEVICE.oneofs_by_name['device'] +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['shader_compiler']) +_DEVICE.fields_by_name['shader_compiler'].containing_oneof = _DEVICE.oneofs_by_name['device'] DESCRIPTOR.message_types_by_name['DeviceList'] = _DEVICELIST DESCRIPTOR.message_types_by_name['Device'] = _DEVICE DESCRIPTOR.message_types_by_name['DeviceSwiftShader'] = _DEVICESWIFTSHADER DESCRIPTOR.message_types_by_name['DevicePreprocess'] = _DEVICEPREPROCESS DESCRIPTOR.message_types_by_name['DeviceHost'] = _DEVICEHOST DESCRIPTOR.message_types_by_name['DeviceAndroid'] = _DEVICEANDROID +DESCRIPTOR.message_types_by_name['DeviceShaderCompiler'] = _DEVICESHADERCOMPILER _sym_db.RegisterFileDescriptor(DESCRIPTOR) DeviceList = _reflection.GeneratedProtocolMessageType('DeviceList', (_message.Message,), { @@ -312,5 +362,12 @@ }) _sym_db.RegisterMessage(DeviceAndroid) +DeviceShaderCompiler = _reflection.GeneratedProtocolMessageType('DeviceShaderCompiler', (_message.Message,), { + 'DESCRIPTOR' : _DEVICESHADERCOMPILER, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceShaderCompiler) + }) +_sym_db.RegisterMessage(DeviceShaderCompiler) + # @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/device_pb2.pyi b/gfauto/gfauto/device_pb2.pyi index 2d4120652..3cb8345e9 100644 --- a/gfauto/gfauto/device_pb2.pyi +++ b/gfauto/gfauto/device_pb2.pyi @@ -4,6 +4,10 @@ from gfauto.common_pb2 import ( Binary as gfauto___common_pb2___Binary, ) +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + from google.protobuf.internal.containers import ( RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, @@ -25,12 +29,14 @@ from typing_extensions import ( class DeviceList(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... active_device_names = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] @property def devices(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Device]: ... def __init__(self, + *, active_device_names : typing___Optional[typing___Iterable[typing___Text]] = None, devices : typing___Optional[typing___Iterable[Device]] = None, ) -> None: ... @@ -41,9 +47,10 @@ class DeviceList(google___protobuf___message___Message): if sys.version_info >= (3,): def ClearField(self, field_name: typing_extensions___Literal[u"active_device_names",u"devices"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[b"active_device_names",b"devices"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"active_device_names",b"active_device_names",u"devices",b"devices"]) -> None: ... class Device(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... name = ... # type: typing___Text @property @@ -58,15 +65,20 @@ class Device(google___protobuf___message___Message): @property def android(self) -> DeviceAndroid: ... + @property + def shader_compiler(self) -> DeviceShaderCompiler: ... + @property def binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... def __init__(self, + *, name : typing___Optional[typing___Text] = None, preprocess : typing___Optional[DevicePreprocess] = None, swift_shader : typing___Optional[DeviceSwiftShader] = None, host : typing___Optional[DeviceHost] = None, android : typing___Optional[DeviceAndroid] = None, + shader_compiler : typing___Optional[DeviceShaderCompiler] = None, binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, ) -> None: ... @classmethod @@ -74,14 +86,15 @@ class Device(google___protobuf___message___Message): def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"android",u"device",u"host",u"preprocess",u"swift_shader"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"android",u"binaries",u"device",u"host",u"name",u"preprocess",u"swift_shader"]) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"android",u"device",u"host",u"preprocess",u"shader_compiler",u"swift_shader"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"android",u"binaries",u"device",u"host",u"name",u"preprocess",u"shader_compiler",u"swift_shader"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"android",b"android",u"device",b"device",u"host",b"host",u"preprocess",b"preprocess",u"swift_shader",b"swift_shader"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[b"android",b"binaries",b"device",b"host",b"name",b"preprocess",b"swift_shader"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions___Literal[u"device",b"device"]) -> typing_extensions___Literal["preprocess","swift_shader","host","android"]: ... + def HasField(self, field_name: typing_extensions___Literal[u"android",b"android",u"device",b"device",u"host",b"host",u"preprocess",b"preprocess",u"shader_compiler",b"shader_compiler",u"swift_shader",b"swift_shader"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"android",b"android",u"binaries",b"binaries",u"device",b"device",u"host",b"host",u"name",b"name",u"preprocess",b"preprocess",u"shader_compiler",b"shader_compiler",u"swift_shader",b"swift_shader"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"device",b"device"]) -> typing_extensions___Literal["preprocess","swift_shader","host","android","shader_compiler"]: ... class DeviceSwiftShader(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... def __init__(self, ) -> None: ... @@ -91,6 +104,7 @@ class DeviceSwiftShader(google___protobuf___message___Message): def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... class DevicePreprocess(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... def __init__(self, ) -> None: ... @@ -100,6 +114,7 @@ class DevicePreprocess(google___protobuf___message___Message): def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... class DeviceHost(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... def __init__(self, ) -> None: ... @@ -109,10 +124,12 @@ class DeviceHost(google___protobuf___message___Message): def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... class DeviceAndroid(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... serial = ... # type: typing___Text model = ... # type: typing___Text def __init__(self, + *, serial : typing___Optional[typing___Text] = None, model : typing___Optional[typing___Text] = None, ) -> None: ... @@ -123,4 +140,23 @@ class DeviceAndroid(google___protobuf___message___Message): if sys.version_info >= (3,): def ClearField(self, field_name: typing_extensions___Literal[u"model",u"serial"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[b"model",b"serial"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"model",b"model",u"serial",b"serial"]) -> None: ... + +class DeviceShaderCompiler(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + binary = ... # type: typing___Text + args = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + *, + binary : typing___Optional[typing___Text] = None, + args : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceShaderCompiler: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"args",u"binary"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"args",b"args",u"binary",b"binary"]) -> None: ... diff --git a/gfauto/gfauto/devices_util.py b/gfauto/gfauto/devices_util.py index b5b4b4536..202962e32 100644 --- a/gfauto/gfauto/devices_util.py +++ b/gfauto/gfauto/devices_util.py @@ -27,6 +27,7 @@ DeviceHost, DeviceList, DevicePreprocess, + DeviceShaderCompiler, DeviceSwiftShader, ) from gfauto.gflogging import log @@ -85,6 +86,16 @@ def get_device_list(device_list: Optional[DeviceList] = None) -> DeviceList: "Android devices will not be added to settings.json" ) + # Offline compiler. + device = Device( + name="offline_compiler_1", + shader_compiler=DeviceShaderCompiler( + binary="amdllpc", args=["-gfxip=9.0.0", "-verify-ir", "-auto-layout-desc"] + ), + ) + device_list.devices.extend([device]) + # Don't add to active devices, since this is mostly just an example. + return device_list diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py index 398ecae40..c54204553 100644 --- a/gfauto/gfauto/fuzz_glsl_test.py +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -33,6 +33,7 @@ glsl_generate_util, host_device_util, result_util, + shader_compiler_util, shader_job_util, signature_util, spirv_opt_util, @@ -324,7 +325,7 @@ def handle_test( return issue_found -def run_shader_job( +def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branches; shader_job: Path, output_dir: Path, test: Test, @@ -348,24 +349,18 @@ def run_shader_job( # TODO: If Amber is going to be used, check if Amber can use Vulkan debug layers now, and if not, pass that # info down via a bool. - try: - - spirv_opt_hash: Optional[str] = None - if test.glsl.spirv_opt_args: - spirv_opt_hash = child_binary_manager.get_binary_by_name( - binaries_util.SPIRV_OPT_NAME - ).version + spirv_opt_hash: Optional[str] = None + if test.glsl.spirv_opt_args: + spirv_opt_hash = child_binary_manager.get_binary_by_name( + binaries_util.SPIRV_OPT_NAME + ).version - amber_script_file = tool.glsl_shader_job_to_amber_script( + try: + spirv_asm_shader_job_path, spirv_shader_job_path = tool.compile_shader_job( shader_job, - output_dir / "test.amber", output_dir, child_binary_manager, - amber_converter.AmberfySettings( - spirv_opt_args=list(test.glsl.spirv_opt_args), - spirv_opt_hash=spirv_opt_hash, - ), - spirv_opt_args=list(test.glsl.spirv_opt_args), + list(test.glsl.spirv_opt_args), ) except subprocess.CalledProcessError: util.file_write_text( @@ -384,7 +379,7 @@ def run_shader_job( ) ) - # Consider device type. + # Device types: |preprocess| and |shader_compiler| don't need an AmberScript file. if device.HasField("preprocess"): # The "preprocess" device type just needs to get this far, so this is a success. @@ -393,6 +388,39 @@ def run_shader_job( ) return output_dir + if device.HasField("shader_compiler"): + try: + shader_compiler_util.run_shader_job( + device.shader_compiler, spirv_shader_job_path, output_dir + ) + # The shader compiler succeeded; this is a success. + util.file_write_text( + result_util.get_status_path(output_dir), fuzz.STATUS_SUCCESS + ) + return output_dir + except subprocess.CalledProcessError: + util.file_write_text( + result_util.get_status_path(output_dir), fuzz.STATUS_CRASH + ) + return output_dir + except subprocess.TimeoutExpired: + util.file_write_text( + result_util.get_status_path(output_dir), fuzz.STATUS_TIMEOUT + ) + return output_dir + + # Other device types need an AmberScript file. + + amber_script_file = tool.amberfy( + spirv_asm_shader_job_path, + output_dir, + amber_converter.AmberfySettings( + spirv_opt_args=list(test.glsl.spirv_opt_args), + spirv_opt_hash=spirv_opt_hash, + ), + shader_job, + ) + if device.HasField("host") or device.HasField("swift_shader"): icd: Optional[Path] = None diff --git a/gfauto/gfauto/generate_cts_test_template.py b/gfauto/gfauto/generate_cts_test_template.py index 8d40fc296..9561f345b 100644 --- a/gfauto/gfauto/generate_cts_test_template.py +++ b/gfauto/gfauto/generate_cts_test_template.py @@ -40,15 +40,14 @@ def main() -> None: artifact_util.recipes_write_built_in() tool.glsl_shader_job_crash_to_amber_script_for_google_cts( - input_json=Path() / "reduced_glsl" / "variant" / "shader.json", + source_dir=Path() / "reduced_glsl_manual", output_amber=Path() / "name-of-test-TODO.amber", work_dir=Path() / "work", # One sentence, 58 characters max., no period, no line breaks. short_description="A fragment shader with TODO", comment_text="""The test passes because TODO""", copyright_year="2019", - extra_commands=tool.AMBER_COMMAND_PROBE_TOP_LEFT_RED, - test_metadata_path=Path() / "reduced_glsl" / "test.json", + extra_commands=tool.AMBER_COMMAND_EXPECT_RED, ) diff --git a/gfauto/gfauto/recipe_pb2.pyi b/gfauto/gfauto/recipe_pb2.pyi index b4af77653..654eee0ad 100644 --- a/gfauto/gfauto/recipe_pb2.pyi +++ b/gfauto/gfauto/recipe_pb2.pyi @@ -4,6 +4,10 @@ from gfauto.common_pb2 import ( ArchiveSet as gfauto___common_pb2___ArchiveSet, ) +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + from google.protobuf.message import ( Message as google___protobuf___message___Message, ) @@ -18,11 +22,13 @@ from typing_extensions import ( class Recipe(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property def download_and_extract_archive_set(self) -> RecipeDownloadAndExtractArchiveSet: ... def __init__(self, + *, download_and_extract_archive_set : typing___Optional[RecipeDownloadAndExtractArchiveSet] = None, ) -> None: ... @classmethod @@ -34,15 +40,17 @@ class Recipe(google___protobuf___message___Message): def ClearField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",u"recipe"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",b"download_and_extract_archive_set",u"recipe",b"recipe"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[b"download_and_extract_archive_set",b"recipe"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",b"download_and_extract_archive_set",u"recipe",b"recipe"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"recipe",b"recipe"]) -> typing_extensions___Literal["download_and_extract_archive_set"]: ... class RecipeDownloadAndExtractArchiveSet(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property def archive_set(self) -> gfauto___common_pb2___ArchiveSet: ... def __init__(self, + *, archive_set : typing___Optional[gfauto___common_pb2___ArchiveSet] = None, ) -> None: ... @classmethod @@ -54,4 +62,4 @@ class RecipeDownloadAndExtractArchiveSet(google___protobuf___message___Message): def ClearField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[b"archive_set"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> None: ... diff --git a/gfauto/gfauto/settings_pb2.pyi b/gfauto/gfauto/settings_pb2.pyi index 13922088f..72b5f1291 100644 --- a/gfauto/gfauto/settings_pb2.pyi +++ b/gfauto/gfauto/settings_pb2.pyi @@ -8,6 +8,10 @@ from gfauto.device_pb2 import ( DeviceList as gfauto___device_pb2___DeviceList, ) +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + from google.protobuf.internal.containers import ( RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, ) @@ -27,6 +31,7 @@ from typing_extensions import ( class Settings(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... maximum_duplicate_crashes = ... # type: int @property @@ -36,6 +41,7 @@ class Settings(google___protobuf___message___Message): def custom_binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... def __init__(self, + *, device_list : typing___Optional[gfauto___device_pb2___DeviceList] = None, custom_binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, maximum_duplicate_crashes : typing___Optional[int] = None, @@ -49,4 +55,4 @@ class Settings(google___protobuf___message___Message): def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",u"device_list",u"maximum_duplicate_crashes"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"device_list",b"device_list"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[b"custom_binaries",b"device_list",b"maximum_duplicate_crashes"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",b"custom_binaries",u"device_list",b"device_list",u"maximum_duplicate_crashes",b"maximum_duplicate_crashes"]) -> None: ... diff --git a/gfauto/gfauto/shader_compiler_util.py b/gfauto/gfauto/shader_compiler_util.py new file mode 100644 index 000000000..9201434ca --- /dev/null +++ b/gfauto/gfauto/shader_compiler_util.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""(Offline) Shader compiler utility module. + +Used to run offline shader compilers on all shaders in a shader job. +""" + +from pathlib import Path +from typing import List + +from gfauto import shader_job_util, subprocess_util, util +from gfauto.device_pb2 import DeviceShaderCompiler +from gfauto.gflogging import log + +DEFAULT_TIMEOUT = 600 + + +def run_shader( + compiler_path: Path, shader_path: Path, args: List[str], output_dir: Path +) -> Path: + output_file_path = output_dir / (shader_path.name + ".out") + cmd = [str(compiler_path)] + cmd += args + cmd += [str(shader_path), "-o", str(output_file_path)] + cmd = util.prepend_catchsegv_if_available(cmd) + subprocess_util.run(cmd, verbose=True, timeout=DEFAULT_TIMEOUT) + return output_file_path + + +def run_shader_job( + shader_compiler_device: DeviceShaderCompiler, + spirv_shader_job_path: Path, + output_dir: Path, +) -> List[Path]: + try: + compiler_path: Path = util.tool_on_path(shader_compiler_device.binary) + except util.ToolNotOnPathError: + # If not found on PATH, then assume it is an absolute path. + compiler_path = Path(shader_compiler_device.binary) + + log(f"Running {str(compiler_path)} on shader job {str(spirv_shader_job_path)}") + + shader_paths = shader_job_util.get_related_files( + spirv_shader_job_path, language_suffix=[shader_job_util.SUFFIX_SPIRV] + ) + + log(f"Running {str(compiler_path)} on shaders: {shader_paths}") + + result = [] + + for shader_path in shader_paths: + result.append( + run_shader( + compiler_path=compiler_path, + shader_path=shader_path, + args=list(shader_compiler_device.args), + output_dir=output_dir, + ) + ) + + return result diff --git a/gfauto/gfauto/signature_util.py b/gfauto/gfauto/signature_util.py index febe4e503..100fe2570 100644 --- a/gfauto/gfauto/signature_util.py +++ b/gfauto/gfauto/signature_util.py @@ -104,6 +104,12 @@ PATTERN_SWIFT_SHADER_WARNING = re.compile(r":\d+ WARNING:(.*)") +PATTERN_CATCH_ALL_ERROR = re.compile(r"\nERROR: (.*)") + +PATTERN_LLVM_FATAL_ERROR = re.compile( + r"LLVM FATAL ERROR:Broken function found, compilation aborted![\s\S]*STDERR:\n(.*)" +) + def remove_hex_like(string: str) -> str: temp = string @@ -151,6 +157,12 @@ def get_signature_from_log_contents( # pylint: disable=too-many-return-statemen # noinspection PyUnusedLocal group: Optional[str] + # LLVM FATAL ERROR. + # [\s\S] matches anything, including newlines. + group = basic_match(PATTERN_LLVM_FATAL_ERROR, log_contents) + if group: + return group + # glslang error. group = basic_match(PATTERN_GLSLANG_ERROR, log_contents) if group: @@ -262,6 +274,10 @@ def get_signature_from_log_contents( # pylint: disable=too-many-return-statemen if result: return result + group = basic_match(PATTERN_CATCH_ALL_ERROR, log_contents) + if group: + return group + if "Shader compilation failed" in log_contents: return "compile_error" diff --git a/gfauto/gfauto/test_pb2.pyi b/gfauto/gfauto/test_pb2.pyi index 9af63bb2c..6338db327 100644 --- a/gfauto/gfauto/test_pb2.pyi +++ b/gfauto/gfauto/test_pb2.pyi @@ -8,6 +8,10 @@ from gfauto.device_pb2 import ( Device as gfauto___device_pb2___Device, ) +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + from google.protobuf.internal.containers import ( RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, @@ -29,6 +33,7 @@ from typing_extensions import ( class Test(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... crash_signature = ... # type: typing___Text @property @@ -44,6 +49,7 @@ class Test(google___protobuf___message___Message): def binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... def __init__(self, + *, glsl : typing___Optional[TestGlsl] = None, spirv_fuzz : typing___Optional[TestSpirvFuzz] = None, crash_signature : typing___Optional[typing___Text] = None, @@ -59,13 +65,15 @@ class Test(google___protobuf___message___Message): def ClearField(self, field_name: typing_extensions___Literal[u"binaries",u"crash_signature",u"device",u"glsl",u"spirv_fuzz",u"test"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"device",b"device",u"glsl",b"glsl",u"spirv_fuzz",b"spirv_fuzz",u"test",b"test"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[b"binaries",b"crash_signature",b"device",b"glsl",b"spirv_fuzz",b"test"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"binaries",b"binaries",u"crash_signature",b"crash_signature",u"device",b"device",u"glsl",b"glsl",u"spirv_fuzz",b"spirv_fuzz",u"test",b"test"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"test",b"test"]) -> typing_extensions___Literal["glsl","spirv_fuzz"]: ... class TestGlsl(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... spirv_opt_args = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] def __init__(self, + *, spirv_opt_args : typing___Optional[typing___Iterable[typing___Text]] = None, ) -> None: ... @classmethod @@ -75,9 +83,10 @@ class TestGlsl(google___protobuf___message___Message): if sys.version_info >= (3,): def ClearField(self, field_name: typing_extensions___Literal[u"spirv_opt_args"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[b"spirv_opt_args"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"spirv_opt_args",b"spirv_opt_args"]) -> None: ... class TestSpirvFuzz(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... def __init__(self, ) -> None: ... diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index 76406d738..f5e688146 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -20,7 +20,9 @@ """ from pathlib import Path -from typing import List, Optional, Tuple +from typing import Iterator, List, Optional, Tuple + +from attr import dataclass from gfauto import ( amber_converter, @@ -40,6 +42,10 @@ AMBER_COMMAND_PROBE_TOP_LEFT_WHITE = "probe rgba (0, 0) (1, 1, 1, 1)\n" +AMBER_COMMAND_EXPECT_RED = ( + "EXPECT variant_framebuffer IDX 0 0 SIZE 256 256 EQ_RGBA 255 0 0 255\n" +) + def get_copyright_header_google(year: str) -> str: return f"""Copyright {year} Google LLC @@ -155,12 +161,21 @@ def validate_spirv_shader_job( ) +@dataclass +class SpirvAndSpirvAsmShaderJob: + spirv_asm_shader_job: Path + spirv_shader_job: Path + + def __iter__(self) -> Iterator[Path]: + return iter((self.spirv_asm_shader_job, self.spirv_shader_job)) + + def compile_shader_job( input_json: Path, work_dir: Path, binary_paths: binaries_util.BinaryGetter, spirv_opt_args: Optional[List[str]] = None, -) -> Path: +) -> SpirvAndSpirvAsmShaderJob: result = input_json @@ -206,7 +221,7 @@ def compile_shader_job( validate_spirv_shader_job(result_spirv, binary_paths) - return result + return SpirvAndSpirvAsmShaderJob(result, result_spirv) def glsl_shader_job_to_amber_script( @@ -218,10 +233,15 @@ def glsl_shader_job_to_amber_script( spirv_opt_args: Optional[List[str]] = None, ) -> Path: - result = compile_shader_job(input_json, work_dir, binary_paths, spirv_opt_args) - result = amberfy(result, output_amber, amberfy_settings, input_json) + spirv_asm_shader_job = compile_shader_job( + input_json, work_dir, binary_paths, spirv_opt_args + ).spirv_asm_shader_job - return result + amber_test_path = amberfy( + spirv_asm_shader_job, output_amber, amberfy_settings, input_json + ) + + return amber_test_path def get_compilation_settings( @@ -261,7 +281,6 @@ def get_compilation_settings( def glsl_shader_job_crash_to_amber_script_for_google_cts( - input_json: Path, output_amber: Path, work_dir: Path, short_description: str, @@ -272,6 +291,8 @@ def glsl_shader_job_crash_to_amber_script_for_google_cts( spirv_opt_args: Optional[List[str]] = None, spirv_opt_hash: Optional[str] = None, test_metadata_path: Optional[Path] = None, + input_json: Optional[Path] = None, + source_dir: Optional[Path] = None, ) -> Path: """ Converts a GLSL shader job to an Amber script suitable for adding to the CTS. @@ -287,8 +308,16 @@ def glsl_shader_job_crash_to_amber_script_for_google_cts( :param spirv_opt_args: :param spirv_opt_hash: :param test_metadata_path: + :param source_dir: :return: """ + if source_dir: + input_json = source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB + test_metadata_path = source_dir / test_util.TEST_METADATA + else: + check(bool(input_json), AssertionError("Need input_json or source_dir")) + assert input_json # noqa + # Get compilation settings (binary_paths, spirv_opt_args, spirv_opt_hash) = get_compilation_settings( binary_paths, spirv_opt_args, spirv_opt_hash, test_metadata_path @@ -361,7 +390,7 @@ def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( work_dir / shader_job, binary_paths, spirv_opt_args=spirv_opt_args, - ), + ).spirv_asm_shader_job, source_dir / shader_job / test_util.SHADER_JOB, "", ) diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index a003eedad..8d03230e6 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -178,3 +178,4 @@ Dbg mklink fullmatch winapi +iter From 14558e2e38b2aab2aeba0a7737946b5ae41e0772 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 3 Sep 2019 20:51:15 +0100 Subject: [PATCH 121/180] gfauto: fix bad path in amberfy call (#726) * gfauto: fix bad path in amberfy call * Fix Maven download for CI --- build/travis/install-maven.sh | 2 +- gfauto/gfauto/fuzz_glsl_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/travis/install-maven.sh b/build/travis/install-maven.sh index 553f1d0fa..01c137827 100755 --- a/build/travis/install-maven.sh +++ b/build/travis/install-maven.sh @@ -20,7 +20,7 @@ set -u MAVEN_VERSION="3.6.0" MAVEN_FILE="apache-maven-${MAVEN_VERSION}-bin.zip" -MAVEN_URL="https://www-us.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${MAVEN_FILE}" +MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${MAVEN_FILE}" echo "Installing maven ${MAVEN_VERSION} to $(pwd)/apache-maven-${MAVEN_VERSION}" diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py index c54204553..4f0a59fc8 100644 --- a/gfauto/gfauto/fuzz_glsl_test.py +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -413,7 +413,7 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc amber_script_file = tool.amberfy( spirv_asm_shader_job_path, - output_dir, + output_dir / "test.amber", amber_converter.AmberfySettings( spirv_opt_args=list(test.glsl.spirv_opt_args), spirv_opt_hash=spirv_opt_hash, From 72f5ea8f695808dd3f86a0b55d320b354d9037f7 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Wed, 4 Sep 2019 06:28:38 +0100 Subject: [PATCH 122/180] Prevent the reducer from trying to simplify case labels (#727) The reducer could throw an exception due to trying to simplify case statement labels, for which child querying and replacement is not supported. Rather than supporting child querying and replacement, this change prevents the reducer from trying to simplify case labels. Case labels already need to be literal expressions - so there is not much to simplify - and changing the values of these literals would be risky; for example it could easily lead to a switch statement with duplicate case labels, which is not valid. Fixes #697. --- .../ReductionOpportunitiesBase.java | 5 ++++- ...ExprToSubExprReductionOpportunitiesTest.java | 17 +++++++++++++++++ ...xprToConstantReductionOpportunitiesTest.java | 16 ++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java index 8ca3c3728..aeb4c722d 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java @@ -94,7 +94,10 @@ public void visitSwitchStmt(SwitchStmt switchStmt) { @Override public void visitExprCaseLabel(ExprCaseLabel exprCaseLabel) { - super.visitExprCaseLabel(exprCaseLabel); + // Do *not* invoke super.visitExprCaseLabel(...), as we do not wish to look for opportunities + // to simplify the literal expression that is used as a case label. Literals are simple enough + // already, and attempting to change the value of a case label literal runs the risk of + // introducing duplicate case labels. injectionTracker.notifySwitchCase(exprCaseLabel); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java index baf5af88d..aa77c1b18 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java @@ -172,4 +172,21 @@ private List getOps(TranslationUnit tu, new RandomWrapper(0), new IdGenerator())); } + @Test + public void testSwitch() throws Exception { + final String program = "#version 310 es\n" + + "void main() {\n" + + " switch(0) {\n" + + " case - 1:\n" + + " 1;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + CompoundExprToSubExprReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(true, ShadingLanguageVersion.ESSL_310, new RandomWrapper(0), + new IdGenerator())); + assertEquals(0, ops.size()); + } + } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java index cc3730258..668820754 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java @@ -82,4 +82,20 @@ public void testSingleLiveVariable() throws Exception { CompareAsts.assertEqualAsts("void main() { int GLF_live3_a; 1; }", tu); } + @Test + public void testSwitch() throws Exception { + final String program = "#version 310 es\n" + + "void main() {\n" + + " switch(0) {\n" + + " case - 1:\n" + + " 1;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(true, ShadingLanguageVersion.ESSL_310, new RandomWrapper(0), + new IdGenerator())); + assertEquals(0, ops.size()); + } + } From e7513d810385e1b0e6ebb92b424edd06e8640ee0 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Mon, 9 Sep 2019 09:16:42 +0100 Subject: [PATCH 123/180] Include GSoC students in list of contributors (#728) Added Abel Briggs and Jiradet Ounjai to the list of contributors in recognition of their contributions to GraphicsFuzz via Google Summer of Code. --- CONTRIBUTORS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index aed80321e..e3828d8a2 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -17,3 +17,7 @@ Imperial College London David MacIver Dragos Martac Paul Thomson + +Independent contributors via Google Summer of Code + Abel Briggs + Jiradet Ounjai From f667ee5fd0a3c51014a9993935f6fb4e55331b2c Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Mon, 9 Sep 2019 03:17:16 -0500 Subject: [PATCH 124/180] Google Summer of Code 2019 Report - Abel Briggs (#687) --- .../gsoc-2019-abelbriggs.md | 367 ++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 docs/summer-of-code-ideas/summer-of-code-reports/gsoc-2019-abelbriggs.md diff --git a/docs/summer-of-code-ideas/summer-of-code-reports/gsoc-2019-abelbriggs.md b/docs/summer-of-code-ideas/summer-of-code-reports/gsoc-2019-abelbriggs.md new file mode 100644 index 000000000..779b97580 --- /dev/null +++ b/docs/summer-of-code-ideas/summer-of-code-reports/gsoc-2019-abelbriggs.md @@ -0,0 +1,367 @@ +# Google Summer of Code 2019 Report - Abel Briggs + +###### Personal Links: +###### [Github](https://github.com/abelbriggs1) - [Linkedin](https://www.linkedin.com/in/abelbriggs/) + +###### Proposal: +###### [Enhancing Metamorphic Testing Tools For Graphics Drivers](https://summerofcode.withgoogle.com/projects/#6749896743321600) + +###### Mentors: +###### Alastair Donaldson, Hugues Evrard + +## Preface + +Before this report begins, I'd like to deliver a short spiel directed towards potential future +viewers (and likely GSoC hopefuls) of this document. + +While Google Summer of Code may look daunting to the prospective student applicant, I implore anyone +who even has a passing interest in the program to consider applying/proposing for an organization +that strikes your fancy (and do not be intimidated when looking at past GSoC projects!). Mentorship +is an invaluable experience to any engineer (especially junior engineers) and not all workplaces +have the budget and/or manpower to support it. The best feature of Google Summer of Code is that +this problem vanishes - you are paired up with mentors that have 'help my student succeed' as a +major goal, and this is a fundamental cornerstone of the program. I had little experience in +proper software collaboration and in working on large/domain-specific codebases before I began +submitting PRs to GraphicsFuzz - the code reviews and mentoring that I was given during the program +were instrumental in making me a better and more confident engineer. + +Thanks to Ally, Hugues and Paul for their continued mentorship and support over +the 2019 summer. + +With that said... + +## Deliverables + +During and throughout the summer of 2019, my mentors and I worked out the kinks in my original +proposal and developed some fairly solid objectives to work towards. This report is intended to +detail what those deliverables are, how they were delivered and what could still use work in the +future. + +These deliverables were: + + - [Enhance GraphicsFuzz's shader generator by making it more aware of OpenGL Shading Language's built-in functions.](#graphicsfuzz-shader-generator---glsl-built-in-support) + - [Enhance GraphicsFuzz's shader generator by adding additional ways to generate opaque values and new identity transformations.](#graphicsfuzz-shader-generator---identities-and-opaque-value-generation) + - [Add a new 'worker' program](#write-worker-script-that-uses-piglit-test-framework-to-render-images) + that takes GraphicsFuzz shader jobs from a server, renders the shader job via Mesa's open-source + OpenGL test framework Piglit, and sends the results back to the server. + - [Add new shaders to GraphicsFuzz's fuzz test set](#add-new-shaders-to-graphicsfuzzs-test-set) - brand new shaders as well as derivatives of the + original test set. + - [Apply GraphicsFuzz to the Mesa open-source graphics driver suite](#apply-graphicsfuzz-to-the-nouveau-open-source-graphics-driver), + specifically the NVIDIA reverse-engineered driver nouveau, the open-source driver that was easiest + for me to run tests on. + +### GraphicsFuzz shader generator - GLSL built-in support + +When GraphicsFuzz generates fuzzed/arbitrary expressions with the assumption that they won't be +executed (e.g. in `(true ? x : y)`, y will never be executed), it is able to generate calls to GLSL +built-in functions(think `abs()`, `sqrt()`, etc.) with arbitrary values as arguments. To do this, however, +GraphicsFuzz needs to know what types to fuzz expressions for, or it will cause type errors. + +The following PRs involve cross-checking built-in functions with the language version they were +introduced in, then adding their function prototypes to GraphicsFuzz. + +[#521](https://github.com/google/graphicsfuzz/pull/521): +Add built-in support for GLSL Integer Functions + +[#528](https://github.com/google/graphicsfuzz/pull/528): +Add built-in function support for GLSL Vector Relational Functions + +[#549](https://github.com/google/graphicsfuzz/pull/549): +Add built-in function support for GLSL 3.20 Fragment Processing Functions + +[#563](https://github.com/google/graphicsfuzz/pull/563): +Refactor exponential builtins into separate function + +Additionally, minor issues arose from adding support for these functions - some of these functions +involve using `out` parameters and modify said parameters during their execution. This would +potentially cause 'side effects' - lvalues being modified when not expected. The following PR +relates to mitigating this issue. + +[#533](https://github.com/google/graphicsfuzz/pull/533): +Check for side effects with lvalue parameters of built-in functions + +#### Leftovers + +The following issues are nits or minor problems related to built-ins that were left unfixed due to +time or knowledge constraints. + +[#550](https://github.com/google/graphicsfuzz/issues/550): +Ensure certain function prototypes can use non-uniform shader input variables + +[#570](https://github.com/google/graphicsfuzz/issues/570): +Be less conservative about when FunctionCallExprTemplates yield expressions that have side effects + +### GraphicsFuzz shader generator - identities and opaque value generation + +Among the various operations that GraphicsFuzz's generator can do to a shader when fuzzing, two of +the most important are identity transformations and opaque value generation. + +An identity transformation is a function that, given an expression `e`, transforms `e -> f`, where `f` is +an expression equivalent for all intents and purposes to `e` (*semantically equivalent*). GraphicsFuzz +is able to use various properties of GLSL data types and built-in functions to craft these identity +transformations, and can recursively apply them to create obscenely complicated expressions. + +For a toy example of an identity: Let `e` be a float of any valid value and let `identity(x) = x * 1.0`. +Then `e` is semantically equivalent to `identity(e)`. + +An opaque value in the context of GraphicsFuzz is an rvalue expression that is guaranteed to have +a certain value, yet the value is not obvious to a compiler. The generator introduces opaque zeroes +or opaque ones whenever required by other GraphicsFuzz transformations. Like identity transformations, +these opaque values exploit invariants and properties of GLSL data types and built-in functions, and +they can be recursively applied to complicate expressions. + +For a toy example of opaque values: Let `e = 0.0`. Notice that `0.0 == sqrt(0.0) == abs(0.0) == +float(0 >> 8) == injectionSwitch.x == abs(sqrt(float(0 >> 8)))`. Then `0.0` can be replaced with +any of these expressions. + +The following PRs directly involve adding new ways to generate opaque values or identity transformations. + +[#509](https://github.com/google/graphicsfuzz/pull/509): +Generate ternary identities for nonscalar types + +[#512](https://github.com/google/graphicsfuzz/pull/512): +Add new transformation for rewriting nonscalars as their constructors + +[#525](https://github.com/google/graphicsfuzz/pull/525): +Fix matrix constructors generating identities for the wrong base type + +[#527](https://github.com/google/graphicsfuzz/pull/527): +Enhance matrix constructor identity to potentially use all elements of matrix + +[#553](https://github.com/google/graphicsfuzz/pull/553): +Add new identities for integer types that use bitwise operations + +[#562](https://github.com/google/graphicsfuzz/pull/562): +Add new ways to generate opaque zero/one from bitwise operations + +[#580](https://github.com/google/graphicsfuzz/pull/580): +Ensure bitwise shift opaque one shifts left/right by same value + +[#641](https://github.com/google/graphicsfuzz/pull/641): +Refactor shift opaque to use identities instead of fuzzed clamped expressions + +[#642](https://github.com/google/graphicsfuzz/pull/642): +Add function to generate opaque zero/one from dot product of two vectors + +[#648](https://github.com/google/graphicsfuzz/pull/648): +Add function to make opaque zero/one from determinant of a matrix + +[#649](https://github.com/google/graphicsfuzz/pull/649): +Added identity to double transpose a matrix + +[#682](https://github.com/google/graphicsfuzz/pull/682): +Add function to make opaque zero vec3 from cross product of equal vec3s + +[#683](https://github.com/google/graphicsfuzz/pull/683): +Add identity to multiply rectangular matrix/vector by identity matrix + +[#695](https://github.com/google/graphicsfuzz/pull/695): +Fix opaque dot function inverted isZero indices + +[#704](https://github.com/google/graphicsfuzz/pull/704): +Add identity to put data in a larger type and pull it back out + +#### Leftovers + +The following issues are nits or minor problems related to opaque values/identity transformations +that were left unfixed due to time or knowledge constraints, or are issues out of the scope of +GraphicsFuzz. + +[#552](https://github.com/google/graphicsfuzz/issues/552): +Rewrite composite identity tries to index into shader output variables in GLES + +[#637](https://github.com/google/graphicsfuzz/issues/637): +Construct identity matrix by multiplying rectangular zero-one matrices + +[#653](https://github.com/google/graphicsfuzz/issues/653): +Be less conservative with matrix functions in constexpr contexts + +Related glslangValidator issue: https://github.com/KhronosGroup/glslang/issues/1865. + +### Write worker script that uses Piglit test framework to render images + +[Mesa](https://mesa.freedesktop.org/) is an open-source graphics driver suite that is able to power +a massive range of graphics hardware across the last twenty years of computing. To help deal with the +variance in hardware and driver implementations, contributors to Mesa wrote a graphics driver +test framework called [Piglit](https://piglit.freedesktop.org/). Among its huge suite of GLSL and +Vulkan tests, Piglit includes a rendering program, `shader_runner`, that can take user-supplied +`.shader_test` files, render the included shader programs, test images for correct color values, + and more. + +For running shaders, GraphicsFuzz uses a client-server model. In GraphicsFuzz's toolset, a webserver +and several 'workers', or clients, are supplied. These workers and the server communicate to each +other via an [Apache Thrift](https://thrift.apache.org/) specification. + +To run a test set, the user starts the webserver and a worker, then selects a test set to run +(via UI or otherwise). For each shader, the webserver sends the shader source code and a JSON file +that details any additional information the shader needs (such as uniform data) to the worker - the +worker then renders the shader in any way it sees fit and returns the rendered image +(if there was one), logs, etc. + +When reporting bugs in Mesa's drivers, it is most convenient for the Mesa developers to have a +`.shader_test` file that they can easily plug into Piglit as a test case. Additionally, the OpenGL ES +GraphicsFuzz worker is rather unwieldy to compile/use for desktop platforms, and so it was decided +that the development of a Piglit/`shader_runner` worker was worth pursuing, along with a script to +systematically convert GraphicsFuzz shader jobs into Piglit `.shader_test` files. + +The following PRs directly involve the Piglit converter script or worker. + +[#577](https://github.com/google/graphicsfuzz/pull/577): +Add script to convert a GraphicsFuzz shader job to a piglit shader test + +[#584](https://github.com/google/graphicsfuzz/pull/584): +Fix piglit converter script not inserting GL ES version + +[#613](https://github.com/google/graphicsfuzz/pull/613): +Fix piglit converter not accepting blank JSON + +[#617](https://github.com/google/graphicsfuzz/pull/617): +Add piglit shader_runner_gles3 worker script + +[#631](https://github.com/google/graphicsfuzz/pull/631): +docs: Add documentation for piglit worker + +[#632](https://github.com/google/graphicsfuzz/pull/632): +Piglit worker improvements + +[#638](https://github.com/google/graphicsfuzz/pull/638): +graphicsfuzz-piglit-converter: support more uniform types + +In addition, a contribution was made to Piglit upstream to allow `shader_runner` to handle a case +when unused uniform data was optimized out of a compiled shader. + +[shader_runner](https://gitlab.freedesktop.org/mesa/piglit/merge_requests/92): +Add command line option to ignore uniform if it isn't found in shader + +#### Leftovers + +The following issues are nits or minor problems related to the Piglit worker or converter script +that were left unfixed due to time or knowledge constraints. + +[#597](https://github.com/google/graphicsfuzz/issues/597): +Support compute shaders in graphicsfuzz_piglit_converter + +### Add new shaders to GraphicsFuzz's test set + +With the exception of compute shaders, GraphicsFuzz had a fairly limited set of five simple test +shaders for mutation. These shaders sufficed for finding simple bugs, but were limited to GLES 1.00 +features and were unable to test the full extent of GLSL even with significant additions +to the shader generator. + +The following PRs directly involve adding new shaders to GraphicsFuzz's test suite. + +[#605](https://github.com/google/graphicsfuzz/pull/605): +Add new versions of GraphicsFuzz shaders that use Integer Functions + +[#647](https://github.com/google/graphicsfuzz/pull/647): +New 310es shader: householder_lattice + +| householder_lattice.frag | +| ------------------------ | +| ![householder_lattice](https://i.imgur.com/ih3ORPH.png) | + +[#698](https://github.com/google/graphicsfuzz/pull/698): +New 310es shader - Muller's method + +| muller.frag | +| ----------- | +| ![muller](https://i.imgur.com/recmnjw.png) | + +### Apply GraphicsFuzz to the nouveau open-source graphics driver + +[nouveau](https://nouveau.freedesktop.org/wiki/) is a reverse-engineered open-source graphics driver +for NVIDIA graphics hardware that is developed under the umbrella of the Mesa driver suite. Because +of limitations on modern NVIDIA cards due to NVIDIA requiring signed firmware to access +key hardware on graphics cards past the [Maxwell](https://nouveau.freedesktop.org/wiki/CodeNames/#nv110familymaxwell) +generation, nouveau receives little use in comparison to other Mesa drivers or NVIDIA's closed-source +binaries, making it a prime target for fuzzing. + +The following are bugs reported to the Mesa bug tracker found via fuzzing. + +[Bug 110953](https://bugs.freedesktop.org/show_bug.cgi?id=110953): +Adding a redundant single-iteration do-while loop causes different image to be rendered + +| Reference | Variant | +| --------- | ------- | +| ![reference](https://i.imgur.com/PBY0tCP.png) | ![variant](https://i.imgur.com/BHbVc3x.png) + +[Bug 111006](https://bugs.freedesktop.org/show_bug.cgi?id=111006): +Adding a uniform-dependent if-statement in shader renders a different image + +| Reference | Variant | +| --------- | ------- | +| ![reference](https://i.imgur.com/0Q8dSRZ.png) | ![variant](https://i.imgur.com/uaZCU8t.png) + +[Bug 111167](https://bugs.freedesktop.org/show_bug.cgi?id=111167): +Dividing zero by a uniform in loop header causes segfault in nv50_ir::NVC0LegalizeSSA::handleDIV + +``` +for (int i = 1; 1 <= (0 / int(injectionSwitch.y)); 1) +{ +} +``` + +This snippet of shader code causes nouveau to crash when trying to handle the division. + +Stacktrace: +``` +#0 std::_Deque_iterator::_Deque_iterator ( + __x=, + this=) at /usr/include/c++/8/bits/stl_deque.h:1401 +#1 std::_Deque_iterator::operator+ (__n=0, this=0xb0) at /usr/include/c++/8/bits/stl_deque.h:230 +#2 std::_Deque_iterator::operator[] (__n=0, this=0xb0) at /usr/include/c++/8/bits/stl_deque.h:247 +#3 std::deque >::operator[] (__n=0, this=0xa0) at /usr/include/c++/8/bits/stl_deque.h:1404 +#4 nv50_ir::Instruction::getSrc (s=0, this=0x0) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir.h:827 +#5 nv50_ir::NVC0LegalizeSSA::handleDIV (this=0x7ffd7753af60, i=0x55d2e1b132a0) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir_lowering_nvc0.cpp:54 +#6 0x00007fc7191cb4b3 in nv50_ir::NVC0LegalizeSSA::visit ( + this=0x7ffd7753af60, bb=) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir_lowering_nvc0.cpp:334 +#7 0x00007fc719111928 in nv50_ir::Pass::doRun (this=0x7ffd7753af60, + func=, ordered=, skipPhi=true) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir_bb.cpp:500 +#8 0x00007fc7191119f4 in nv50_ir::Pass::doRun (this=0x7ffd7753af60, + prog=, ordered=false, skipPhi=true) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir_inlines.h:413 +``` + +#### Leftovers + +[Bug 111006](https://bugs.freedesktop.org/show_bug.cgi?id=111006) involves a critical component of +GraphicsFuzz's fuzzing techniques - preventing a compiler from optimizing uni-directional +branches (like an if-statement that is always true) by using a variable that is unknown until runtime. +While this bug remains unfixed, it is difficult to continue fuzz testing without running into a +large number of potential duplicate issues. This issue prevented continued fuzzing over the second +half of GSoC 2019. + +### Miscellaneous + +While working on GraphicsFuzz for Google Summer of Code 2019, minor problems that were not directly +related to the detailed deliverables cropped up. + +The following PRs involve miscellaneous refactoring or changes to GraphicsFuzz. + +[#507](https://github.com/google/graphicsfuzz/pull/507): +Add additional tests for empty for statements + +[#532](https://github.com/google/graphicsfuzz/pull/532): +Refactor BasicType with documentation and exception improvements + +[#535](https://github.com/google/graphicsfuzz/pull/535): +Remove pointless static numColumns function + +[#537](https://github.com/google/graphicsfuzz/pull/537): +Refactor code to use isType functions when possible + +[#587](https://github.com/google/graphicsfuzz/pull/587): +Refactor common python script functions into one common lib + +[#668](https://github.com/google/graphicsfuzz/pull/668): +Refactor transposedMatrixType() to use makeMatrixType() + +[#670](https://github.com/google/graphicsfuzz/pull/670): +Fix making array accesses inbounds with uint index accesses + + + From 202f252febe5579ef60d512c70f75c311df2d73b Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Mon, 9 Sep 2019 10:17:43 +0200 Subject: [PATCH 125/180] GSoC 2019 summary report - Jiradet Ounjai (#701) --- .../2019-JiradetOunjai.md | 156 ++++++++++++++++++ .../images/jiradet/shader_binarysearch_bw.png | Bin 0 -> 3336 bytes .../jiradet/shader_binarysearch_tree.png | Bin 0 -> 1069 bytes .../jiradet/shader_mergesort_mosaic.png | Bin 0 -> 1671 bytes .../jiradet/shader_prefix_sum_checkers.png | Bin 0 -> 2279 bytes .../jiradet/shader_quicksort_palette.png | Bin 0 -> 2196 bytes .../jiradet/shader_selection_sort_struct.png | Bin 0 -> 11763 bytes .../jiradet/shader_trigonometric_strip.png | Bin 0 -> 19546 bytes .../jiradet/shadergenerator_overview.png | Bin 0 -> 185423 bytes 9 files changed, 156 insertions(+) create mode 100644 docs/summer-of-code-ideas/2019-JiradetOunjai.md create mode 100644 docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_bw.png create mode 100644 docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_tree.png create mode 100644 docs/summer-of-code-ideas/images/jiradet/shader_mergesort_mosaic.png create mode 100644 docs/summer-of-code-ideas/images/jiradet/shader_prefix_sum_checkers.png create mode 100644 docs/summer-of-code-ideas/images/jiradet/shader_quicksort_palette.png create mode 100644 docs/summer-of-code-ideas/images/jiradet/shader_selection_sort_struct.png create mode 100644 docs/summer-of-code-ideas/images/jiradet/shader_trigonometric_strip.png create mode 100644 docs/summer-of-code-ideas/images/jiradet/shadergenerator_overview.png diff --git a/docs/summer-of-code-ideas/2019-JiradetOunjai.md b/docs/summer-of-code-ideas/2019-JiradetOunjai.md new file mode 100644 index 000000000..57f1d8670 --- /dev/null +++ b/docs/summer-of-code-ideas/2019-JiradetOunjai.md @@ -0,0 +1,156 @@ +# Google Summer of Code 2019 Report - Jiradet Ounjai + +The Google Summer of Code program has turned out to be a great learning opportunity for me. I gain fresh knowledge from my fantastic mentors through the discussions and the intense code reviews we had. I would like to thank my mentors for the time and energy you put into helping me. + +I have made this document to summarize the work done for my Google Summer of Code project *Improve Shading Language Support in GraphicsFuzz*. + + +###### Personal Links +###### [Github](https://github.com/jiradeto) - [Linkedin](https://www.linkedin.com/in/jiradeto/) + + +###### Project Link +###### [Improve Shading Language Support in GraphicsFuzz](https://summerofcode.withgoogle.com/projects/#4859963594244096) + + +###### Mentors +###### Alastair Donaldson, Paul Thomson + + +## Deliverables +My project idea focused on enhancing a shading language support in GraphicsFuzz. To accomplish the goal, I have improved the tool in various aspects which could be elaborated as follow: + +- [Add support for GLSL Built-In Functions](#add-support-for-glsl-built-in-functions) +- [Add new ways to generate an opaque expression](#add-new-ways-to-generate-an-opaque-expression) +- [New sample shaders](#new-sample-shaders) +- [Enhancement for Reducer](#enhancement-for-reducer) +- [Use GraphicsFuzz to find bugs in other tools](#use-graphicsfuzz-to-find-bugs-in-other-tools) +- [New shader generator tool](#new-shader-generator-tool) + + +### Add support for GLSL Built-In Functions +GLSL built-in functions are functions that available for use in a shader and GraphicsFuzz donates these built-ins into the unreachable statements so that they will not be actually executed. By providing a support for the GLSL built-ins, we teach GraphicsFuzz how to call functions appropriately by ensuring that the required function arguments are matched with the GLSL specifications. Moreover, we have to check that the built-in function being injected into a shader is compatible with the given shading language version. The following PRs have provided a support for the GLSL built-in functions. + +[#554](https://github.com/google/graphicsfuzz/pull/554): Add support for GLSL built-in functions: Angle and Trigonometric Functions + +[#555](https://github.com/google/graphicsfuzz/pull/555): Add support for GLSL built-in functions: Matrix Functions + +[#581](https://github.com/google/graphicsfuzz/pull/581): Add support for GLSL built-in functions: Geometric Functions + +[#585](https://github.com/google/graphicsfuzz/pull/585): Add support for GLSL built-in functions: Floating-Point Pack and Unpack Functions + +[#592](https://github.com/google/graphicsfuzz/pull/592): Add support for GLSL built-in functions: Common Functions + + +### Add new ways to generate an opaque expression +Fuzzing a shader in GraphicsFuzz involves applying a number of transformations that might inject code fragment to the original shader aiming to generate a variant shader that is much larger than the original. Opaque expression, one of various expressions the generator employs when injecting code fragment, is an expression whose value is unknown by a compiler. Therefore, donating opaque expressions can help preventing the optimization performed by the compiler. In GraphicsFuzz, we have plenty of opaque expressions representing 0 and 1 however by introducing new opaque values we provide a new expression that is likely to trigger a new bug. The following PRs involve adding new opaque expressions to the GraphicsFuzz's generator. + + +[#413](https://github.com/google/graphicsfuzz/pull/413): Represent 1 as length of normalized vector + +[#421](https://github.com/google/graphicsfuzz/pull/421): Enlarge test in OpaqueExpressionGeneratorTest + +[#424](https://github.com/google/graphicsfuzz/pull/424): Represent 1 using cos() function + +[#428](https://github.com/google/graphicsfuzz/pull/428): Represent 1 using exp(0.0) + +[#429](https://github.com/google/graphicsfuzz/pull/429): Represent 0 using log(1.0) + +[#447](https://github.com/google/graphicsfuzz/pull/447): Seperate functions making opaque 1 and 0 + +[#459](https://github.com/google/graphicsfuzz/pull/459): Represent 0 and 1 using abs(opaque) + +[#472](https://github.com/google/graphicsfuzz/pull/472): Represent 0 as length of a zero vector + +[#475](https://github.com/google/graphicsfuzz/pull/475): Rename opaque factories variable + + +### New sample shaders + +In GraphicsFuzz, sample shaders play an important role in exposing bugs in shader compilers as they are the initial set of shaders that would be mutated whose final result are the variant shaders that might expose the potential compiler bugs. + +Having said that, we have only few sample shaders that come with GraphicsFuzz by default, the following PRs thus focus on adding a brand new set of 310 es sample shaders implementing different sorting and searching algorithms. + + +[#602](https://github.com/google/graphicsfuzz/pull/602): New 310es sample shaders - v1 + +| trigonometric_strip.frag | selection_sort_struct.frag | prefix_sum_checkers.frag | +| :---: | :---: | :---: | +| ![trigonometric_strip](./images/jiradet/shader_trigonometric_strip.png) | ![selection_sort_struct](./images/jiradet/shader_selection_sort_struct.png) | ![prefix_sum_checkers](./images/jiradet/shader_prefix_sum_checkers.png) | + +[#643](https://github.com/google/graphicsfuzz/pull/643): New 310es sample shaders - v2 + +| binarysearch_bw.frag | mergesort_mosaic.frag | +| :---: | :---: | +| ![binarysearch_bw](./images/jiradet/shader_binarysearch_bw.png) | ![mergesort_mosaic](./images/jiradet/shader_mergesort_mosaic.png) | + + +[#660](https://github.com/google/graphicsfuzz/pull/660): New 310es sample shaders - quicksort + +| quicksort_palette.frag | +| :---: | +| ![quicksort_palette](./images/jiradet/shader_quicksort_palette.png) | + + +[#676](https://github.com/google/graphicsfuzz/pull/676): New 310es sample shaders - binary search tree + +| binarysearch_tree.frag | +| :---: | +| ![binarysearch_tree](./images/jiradet/shader_binarysearch_tree.png) | + + +### Enhancement for Reducer +To remove code fragment injected by the generator, we use reducer to shrink the variant shader into a very small and simple shader that is still inducing a bug. GraphicsFuzz' reducer essentially finds the interesting code fragment to be removed based on the reducer opportunities. Currently, we have sufficiently large set of reducer opportunities that can significantly reduce a variant shader. However, there were areas need improvement. The following PRs involve adding new reducer opportunities to the GraphicsFuzz's reducer aiming to help reducer reducing GLSL shaders more efficient: + +[#477](https://github.com/google/graphicsfuzz/pull/477): Add tests for Simplify + +[#489](https://github.com/google/graphicsfuzz/pull/489): Remove parentheses after eliminating macros + +[#490](https://github.com/google/graphicsfuzz/pull/490): Reducer opportunity to remove redundant uniform metadata + +[#500](https://github.com/google/graphicsfuzz/pull/500): Add macro wrapper and fix space before comma + +[#508](https://github.com/google/graphicsfuzz/pull/508): Add a test for function call macro parentheses removal + +[#513](https://github.com/google/graphicsfuzz/pull/513): Add class and tests : reduce VariablesDeclaration with expression + +[#514](https://github.com/google/graphicsfuzz/pull/514): Add if_dead opportunities and fix fail tests + +[#529](https://github.com/google/graphicsfuzz/pull/529): Implementation: Reducer Opportunity to replace variable declaration with an expression + +[#538](https://github.com/google/graphicsfuzz/pull/538): Add test for const variable declaration + +[#561](https://github.com/google/graphicsfuzz/pull/561): Reducer: Add class skeleton and tests + +[#576](https://github.com/google/graphicsfuzz/pull/576): Reducer implementation: reduce global variable declarations to expression + +[#622](https://github.com/google/graphicsfuzz/pull/622): Add reducer opportunities + + +### Use GraphicsFuzz to find bugs in other tools +GraphicsFuzz's team has discovered and reported a bunch of bugs lying in many different GPU vendors. In fact, we constantly run a graphics compiler test since we would be delighted if we could help developers detecting uncovering bugs. Over the past few months, I had a chance to use GraphicsFuzz to help finding bugs in [SPIRV Cross](https://github.com/KhronosGroup/SPIRV-Cross). + +SPIRV-Cross is a very convenient tool that helps parsing and converting SPIR-V to other shader languages. Throughout GSoC program, I relied heavily on this tool since I am using [MoltenVK](https://github.com/KhronosGroup/MoltenVK) to run SPIR-V on Mac and MoltenVK internally calls SPIRV-Cross to convert SPIR-V into Apple's Metal Shading Language. + +To see all SPIRV-Cross issues I filed please check [here](https://github.com/KhronosGroup/SPIRV-Cross/issues?utf8=%E2%9C%93&q=+is%3Aissue+author%3Ajiradeto+). + + +### New shader generator tool +![new_shaders](./images/jiradet/shadergenerator_overview.png) + +GraphicsFuzz has now equipped with a new tool called Known Value Shader Generator which generates a shader job from the given RGBA colors. This tool mutates the numeric inputs by applying various transformations which eventually generates the mutated expressions that guarantee to produce the original input values. With the help of this tool, we have a brand new way to generate a variant shader just by simply providing the expected values. + +The following PRs involve implementing a new shader generator tool. + +[#625](https://github.com/google/graphicsfuzz/pull/625): Add a new tool: Known Value Shader Generator + +[#677](https://github.com/google/graphicsfuzz/pull/677): Expression Generator: add global initializer method + +[#681](https://github.com/google/graphicsfuzz/pull/681): Expression Generator: find numbers for addition based on known facts + +[#693](https://github.com/google/graphicsfuzz/pull/693): Expression Generator: introduce uniforms + +#### Future development +Currently, the shader generator tool has a limited number of transformations. The next steps for this tool involve extending transformations set and integrating this tool into the fuzzing chain of GraphicsFuzz. Afterward, it would also be interesting to apply new capabilities of GraphicsFuzz armed with the new tool to find bugs in shader compilers. + + diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_bw.png b/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_bw.png new file mode 100644 index 0000000000000000000000000000000000000000..d305eff7612801f8504621e925096e6a2977470a GIT binary patch literal 3336 zcmbVPdpOkT9)BkrJsD)TP9v8*X_rn}#x2LaT`9uaRL+t^jZ4XGH7FA^$3D4~a3XRY zZO*hRl!Qp*oS16a2$AcElhQDgj0iJx-uE}oOf%>FvCcp5Z=RWXpZELyd_R}>m+IlZ zM`i7%wEzGW=e>^m0l>kJIIu<$J`P9y5(EIg-Pv*1fm682ZlVi$-@2qD%vS#`PKj?6 z)9|h>ftMRnSDoDGp=`Xt%YO$!W!&lOUCnK)`L@ZL-$h=z!4NWsm8`O7+-uK>w6wJF z?!!+!?#@PpUY}18C9A5b_1drM$W#^jr~s9T$;mVv$nT6I&>VyQAXJGp>a)6f!`ri& z5@bOHPgp6;aOW?5g2@L@oIBOt_Eiy`ZVDVcIo-x^t-p8g2%s^lwG?&#mUou=aA;r6 z(>K?9)=wAbx#1nEh@+K0`=9h^_&(3yGHiOsIX5>K4?4Q7%;B2N<#Gw2WSe0!Zf`)) zOLpYk#d6d7>*l-f2M2wjkQtgb&&fGZ74778myt5p7TVd`s;mIa?z*eM_3z2aN{ZlE z$VoD}nwNB_ZO4)(_b;FPH;TpgqZP?l9#-pvHWh7xt4n)(J1S+I!#TlLDZl)~{LaC~ z08Uy(h?_buudnn~+5{6=Ur?tGU38JQw6L@k0Nd>;LrwFA!lwYZh1Cp9J<#+f-gy^M zNBN}JR*fzyE@qRSu>Q+Sp+(OQmTw!iH7(%J%shmf%01CKvA9?Ok?*&$wVi6XvcXhY zl+=(b{cm+4QLxefnmmYlWA>jrIE(SOva<_X0si{>`cnKWDk@fje@n}1DgFZk2uhdM zBjEEqIp~_fR!zC4$?@^ONt+f5i&4{s#l_s;-*iMogtT&<&6_nqN!f2a9&ZK6Pf!0t zTEy(^B^(fXh^jo3EpI#STAy_C@m2#1My zT_pu~b-4Vk%UjXK9zm>El(3g85X5sZ_GwGl2Q_p2G>dQ4^gXFrB+*S>a8TBR2M-|J zWziuBbZu3PzjO2Rb~SapV``D7Ch9Srmg%JxnH3ih;YkJ2pKv zh0Moo|75@9E=v?1B~}mHuxy#nNIiXh8q(>VKd+2U?;+S~a_3#FgtpwgKJoNo_x*9% z17J1egapL%Wy+3^Cn2+VZwykQ>|`3p%t9m)%#YA4e0_b1QnP4iAR)5a;2pM@DpQjs zT>p<*$hKQqSt)Io%j2Pu53$+o9!##*61k%G-lt9Y(XO&mWKVBwY?PiJ7Z-OFo*qM| z(|vYU9}RlRlQ<3nx7CzE#ic{J%Jy!lB^_G)nKQ`q8la1nV}h4KuJn~g7ZdEcpe|~V z2p%@JAQU2nE?b6J{Zl$RIyK?PTIA-Il@shLTBT*~w8WD*ALP>npPvmO_r8V!pTG=W z0fvhLq_yR(v$N*NJ70-KD+AzBa8bVS`uq2#UPq=-S|JCHpB86?3|k4b0C}z#)9Gkh zi0l4(q8DY~7M{x2#gCoNB2pECjA-5&|I#Te#q0R$Hb#jm zfm&n-VY*i$I#e2+z8O_sfCUa@E~0Z3g+iIcOam{GJTeUdi*Mhwi`48VkvuG|D=r7o zFAVdHqT*r`NI!3TENkf!#iLMG)a{c$d?+0C93~E$rKLio+65w}TA7poN5oIfQKWT? zGdsH{1B>ZkiCMrrPsQ!EkVkEFG*OD$^XD6o>rBQpkTA)~>~`ktWQcI8|CNyQ^N*#4 zi|xicflAF7>NuBoR<`YO_GO`xQIb<<`g{kWlq8fvb#VkV=-mm7uZ<(<6QujDkhx$HyUcPOt z_>xSR);HlDLenHK`LeHZSk(D zc=NU=`^WZQzY1$VEL>f8{OJ-5xIoUiif}iLxtpP#Jf0EU)Q?Qkb%Mh=4AAD+3M)`tg0Pm@WpIJxP^s4&ORpGl10M5JJ9V@=^Py9D1TP3vs literal 0 HcmV?d00001 diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_tree.png b/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_tree.png new file mode 100644 index 0000000000000000000000000000000000000000..a81a1dd54df302e001fc9b94562330ed63d62bc5 GIT binary patch literal 1069 zcmeAS@N?(olHy`uVBq!ia0y~yU<5K5893O0R7}x|GzJFdFP<)rAr*{ot}OID#vtN$ zaX)X9t3;PTtf!0GR2Vz=F2|2C+w*nDrp-LKz| ze`yK+^|QwM%RQ^V0tW633O+*wy?OQNkJa9X_jTUuUlzV`*Zz*5zs1t!$CKZ$z0A*8 z%))R8jQrm8&$UuvU}$JmxU&DNlg}Otj)T)dJ`_-6;1FhL;RLBmWNI*AbeJ*7g%4cV z7<^ofqt#J@<>D2qoTFJf=k8Roa0~df+&{u6(trG%NI}i?Hztfgzt9A2xVtO<`j?J+ zOWXX*!apoz&R-Tbd-&Jlp$ZS{ zMaFrH4g@gmJUhQ~eZ$%9X<_Q1cpYH0?QY-l^Itml+RS_YlJif@{pl|`y>Go+zyF@K wk3Gv7K~S)`)e&*q!zqxPUuYVm>vDKvP)yLbZ&z!b@*d%vf>fZi)x;871-*5kJU&_6F zapCKz?H|_2#Y^7%e^YnQy5sU@wf}2u?@8a=e_i*^`s2ILKEBN@;LV^ggTcXou_1wp z;UEh`6DI?UFaw7ggMbfZ65IUzu5FFladYoOciVjye-7l|J=9zEr?ldK%=Uuqa_@QH z$G;LQd;M|x!@uTt4+Yx(Gu>0aJKAE+asKx2<-1xhSN%z<_+Plaz~I-O)M~~gYPkx{ zgEy{K?OC_j{Laeb)7#~LY>BQ2>A%nWUcQ_=eR{9eWnG)~$NAg7+m~~*fBo^;{=e(L zqFpzy?|(1G_xj`2hkw8QFAnLiH}B8achCU`-~WyFz}O9B4wP`a7;ECpIi=Apn0cDr z29X9o(V3p=E0v6@CUXgW>=yk#XJ+W{eU<)qH5nU#Ax|x|;p>*1+l!g!ALicI{(Jkw ze+GB{^M1en@L&0k|MKtR+kel0_%D0MexC2gAO5qsv!DOFy#K%bIu6qq9DoUcy68sL z|80Ba8Q(ZFC;*cYDoSuSFMB0ny2_&C@sHcOu`b+>N4P5(b|3Z5P`oycTdAr1LXzW# ztK2QBE1hq#Ff~q5RQH&=!E>V8*=KD3HJ5pO{9SWv=JwAO-1nDLo=Abo;q0~7cMTZ2 zUy0ZC9AEv6yKa?OUGMSR&;B-NE&So|{Os@i)vfOzf8130-*xMURGWPn_x7(5dw1+{ z_}Sm`t6KMd{NelgU*y&TE4loxe}xA|CUAkFqf-;Gk1QR+5G!D)#v})zO=2sU-N(d)&JY=<^KP>zk7d$hS<8}kMmdG z`+MH~=%MQ$e{B0v_nEUp6>m0Q@NnW>D;GbVjzb2w6xf>`7P_DLv2n-Bu6_=qsk{$P z$X&Rh(|AfQsBle#tqPO#o6~%|j)t!MFLS##{=V0W_shvGMsmJ=tDjXR{;!s2--q4x zHU9&m3x1mJ_;AJEZf(|kdHd|o+iz9e{x2D`<4ZsP(eL7mxzGQ7d^h>+-?zUX9D4A- z`fq6U|1VZMK3uZ5yFM@f^`k$}^8eS|Ylxt3@dhtQ%ywsAzOs9ROzlGJySG;Ecu*KN zd%x)Y`~F*hIv4(TE&Lx__&>Mse|F*jSMQ(4%!wC&ZhmzC|5ZQ!r54!bL(2`Kid~oA kdi;DW!0Lpd;ZHpGK|ZD})48(RC7VGap00i_>zopr04U0SeEHr2Xt;AC5KLP-QrT{-&M261a9`^-CCcy8% z9GjBi26yQ9nc8jJGS^v;%imp*K#Mnn=Z9Y3?oG1~Yc@rpLVY+T4htxh)5CDH@43E| z;Tq@q{+^nv0>rCVz0~--(FXD%f6N= zV#m&NJOvbdf6_s;L!$8Eau>qUX+B{2Ms5M+GHQnrL^ncCLv+1U;S|a0RP_GZx-}9r zDp)V11rB@%*8|{W03dGx;^{g7-4i(P2?IPN03i9rOE#iMaAfeb=P9(a`E0b!uv?s3QUY%Skq>}Rff z?@+sY?7YUtGE19{_kY;1^i7)BQR<$a5r^HCC?n05c*(6fY_FF2QHahgJIYtL-~G59 z&;1bBA2poB@)EIKdl3bdOPNd+e@^ojRxT#z+;VQ*W1Zwy7xSz?o++xc_R-o6xXgM* z!>8+P+xjl7y+QM0%Oj;RUGfV)lw$d)U{NaT*j2T>6x_8V=V4k8!rVI%q90vwr4UX$ zZ{akrG{k+Aayk>0-qmDX>SckHcO(24S}6qln?K3Pb3eYKx_qg#JNg%^*z0%VeoEQZ zp<>_C3jMvS_tgFy)8Ik2s7hanpDu!F($$r}ys$BicsFcDm)b$ZxNUu74JbVRnTFj7 z;wOJ5mpj$&qfbu9%DLP_HAzM6i=WWka&ff5NAcSjn_SA$DiSB}Et-e$)80xNX@c<6;IVs-AV3R$$LX>H{7GZTPfuEH z`Eb5>NK*+0h!noDMc5Y*TUg9HdM!!WCVc7qY)IqNjvBvh^+3rO1wW9_ND@G@1GNn6 zom&4+6D!R1mT--2WLqly!Ms}WM2?GFJ!EM|*WI}Na1zZ-a3<+Y>rNUwFUe`k-R&0k zG*m(^>=f=yo5Ayk- za|#`5#mGXF&zY+%&Nsca+{fxpR#LGI2gyIoHy8nHrja&4jrUI-__I(zz6%k7T!u@B z$k}GEsg}T+jI+2Ny`>wAejzTwvdJOxs}(UUEQUvt*OS!shJMxAolDLnh@s(WNpPad zu$jDFDUThG#f;|)VKvFWq7TdVjNESFKci0Gx*2IxwBqXhvM~5-E`EM@gXlfqvp>;n zjJ5129csj=(1pF1E&G@dBg@mJ!=}$2LOx54lJBc3S29HVv~yyh8e&`7@VZ6|25C&i#wvtprNnS{y(Gf)X3-Tr@DZFK5m=ZJ!WnK`~v1CP7;^^ z25((H)bv$-4?JKS^#8tySJ0z_zC{27efSb|o*M~(L7=ksc9E;7`WPB9;7tx2PA){2gM6=zWwT-=>ql7Gwddhegn;TF{gQ02T zm{4G8me{k|S85t4sY=jrsr;-3l;dZB+>!K~dv<;?bTj{0CTtImL7GLmc%>aY$o_rHKH^Q7Y%T>B}ldrRo=g4$jzn{37+W?!W8%KqHXw{3WlPs zPQjJrk8Y-z4j{mk=Wi=ur_u9j+2Z*j5vf_JM|o5a70)M$Aiv6?OV4>BbX%fuL3B_* z+RF+s(eFlJ2Xq7_R$%z~3Yk|n=H!=h@CS^zXvgxnW|+#$KGK!0Z9wowWlXm%HajE? zYkCu;tG4wB)SaQtLj`f4D%KyPac0RTy{sa9xj`qeeabJQ#hBPySjNTLW*MFS)DFq6>a~Y#_F^wzZqHPWkN0hYSO0^m9&oJ2(Y_TQY zB5z<1_3Xg?`(B=~TX@*w+gLF1(LO=x;-sYdbC;bAeP9PeZyi9qtV=0Ol9bFniWBf~w!GjcyxLpgX!r-IDMhw-WhX9& z2n5cH4uu%zrE{QvocEn??9jSvhwRo8dM40pK0j^5P#tN7Ygi>Md7 z(0jf=Dbq9if#UP0ue{rqk8a-(YAawK$8RaxWKsLtytXwgpmG9o+Ged#@i)Ys_CP(E zO(%EvoKbOpIi80G=idh=k$}Am3DUNKot*#|t`7{O`NVH!mwod!<5VXFO&vbmlAs9W z#$ji2XCWNJiLUA#xtdc=0xpG;yh5|*S)-)gM6{jsxIsiM@UZzL+PmPWErq2<{-m5idgl>6w`49eZ*vr z2MQdtv*Oj{snh)!A{XbP?ESY67wlIb>I>)p=f>A{B#?1tWuS*7+?Z1nq)e%uS^DRN z*VjU!;!W?#dl9&6Fy7kS4@n?E^Xqj&?WbZbS##P7Ep9A@0?kt!W|sUE_0{SJ&9ph^ zr4zwe(7t;jg;n4pfbByPE&ps?W)8IpVdm!av|9hz|KMOvQLrxsFSW`jeR}>?y2Y1nD8PgYp z5NG^|yWN_9n)z90VGGh9t%h9oSQBM2_Wf&u4fY}8TVF4V1j?#;{<3w@5v><+URE7X z#)7P;!nR;VU}B}r7AoQ(nU-*_-jG@(_Yx$u*6S$;iR{&f|1-ZfE~<_mjGtygI;zGq z6hsG6bMV5&aMcJjs4P-YBE_&2l+!w#D{u8DqHw)cw3D{3u!$LceFzr6Tc+n2sO56D z!VOZK^67~OXE05fjA%GZZ7cD9hieo2UGGL|fOJG5{rL2nu`1~(t?nXwl1H>1yBqIR z&^yd}wkm?|h?@uEz#yzdE2)q+p|4ZF5)57e+9ukmpxdS1av|xF;wG&rIdUu?T8`58 zA`nqejQaK~0;Fxp7?zI2x@XaIr1I4S4*sJOcz<_P%&kSA)_%aX&_ItvDK% F`d^a>%jN(8 literal 0 HcmV?d00001 diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_selection_sort_struct.png b/docs/summer-of-code-ideas/images/jiradet/shader_selection_sort_struct.png new file mode 100644 index 0000000000000000000000000000000000000000..34a290284d84489c27fb6e10de1ee02e2904eadc GIT binary patch literal 11763 zcmXYWc{J4D|Nm>oV8(9j5o6!8WXn2Z$&wJtk__3ihRSZnlBHx%ku4MoPz(E$L!Xl$f!2>_srAqard zTyzeBS7ox$CQh}%BY@dmh`)T}2|FFM-!MHf=*7id)5(!i`mFA$(A_(OI{}5=se9)I=Igt^?pP&ZOo?`3SJ3~&04NBMi!c0^`kxD+ADB@; z{Ywh)ntcecfB{Fajz5JR%)={;$gpxa7`|^QDDO6F3#>Jb{-&sQOt4YZh#fi_Q7dFca%3>d zC(yD`0HU+bRY{=oq2l2|*p$yb#?pYg@Sy8Vc%>gJKuEfkS)eH7!6OZO zU6z_DIo6e%p1R2Xy^|6UfPPW)ubyF*MdWyK?B&|#_L^59n5|6ZgE5Rel|2 z;wjtmN9_@G+B^K9h5@r3ONV?}xLEpxbuO7~-EA&BQ6GOh;wuwAVsipidy}O+sIBQN zd?Kz4j4x3FrG*xDuXBUGh$&ZYqTNL!gZlmMhJ~|WSdV3OiS}d6%$jWN+4IeTzI~-Y z_oiG=IukN`?#i+h@X!YxCflN=7A{ycprs4oy~aV%KILa*C0hJa&;zO>y`i5qy|acz zPki&#hLTts6aawlU*moyv^j7M@I1EZsX$bHj@CR2+w(Ul5^vL9ML-&M7&KWAsKA3U zs#dnN$RC43T2pn$+hB{2UmR>!J@)pS|AX@7Z;m%D6fQV42m-D~iJoaXGa;wRBD}MvcIOqzI+ODPc2y%Y?QP}?0mta2 z%TqUncRUP>j+B8Ygtk9fD5{)om#6T@p{mG;Wf~On(4bxA*}NacnCJGEueW28E=ODT zoKj%!i&}TRg;aeEOH9rC^M4(`Np}0}SoK@8X&x_7^=HQZ(V{v>_bm#7vUJXU3Mr3d zqrYy3&&+T5_@6jIrne2Dc;-%zj_j2a_4uNn+m?K`-S1qj(%&>}?5HnQHP#{K97+_ZyW2h zoE@p)cxLy7(Rxt4sv8Vx(fKuSS|}>u1~i!@qi4KvXEzKd*#1(-OuEf0PU$kLAG6<9 zeQpG>#F&dkAxZI$DTPf@+0idLN*Cm2qUzk%J-*m62uo#Hv4O|gR3l-@FZ7bJeg1VE zF=4pIjDH;3mtuEP3&H-33p6vBlhDkDR(+0SL5*N`EjQwtto3v?z; z;ZZj;?+ze1jm$klpG1q zM4Ly%0j+9hfT^O&i|KPK;ppkiMIR<{C;+ukx%X{Ox+>N}f>p4%7(8g>s|=im{+>oY0yQbK?uWInrahI@m$ zDA8={9Xn{v=j?0B{1h(``K!M&zGWg+*P!1h($ta(r`NDw^zEJxNRFaF(K3M z`JT)8|2Z75)hyB2d4HNRKX>=FMPat;tcGDtxMg1EuFZ~N={y|`VEz>SW1y;ZIRz+J z$iJF=MYo=OFq+S}89npGe8Nu#u%E16AKt`?Qp!np-!^T7@NCZYD^&~r-8EhL9&I%h z-b@X?uDbbeZ_<3*8LYDS}4$*5orlweBct;H7m0 z@wxfW=sWYm%D%uGFS}yLAWM7W(u3NmLwaICz<1hQs?EVz z#Y8BU);SgWZ%~}VHkZB|m&ftl2Z~b0wLb>+(2yu_C*{Gm$iC$?MlBixyhhs5k{VQ# zW)Wd1w)ZcK)`@<04`Q}pQi(PPQ{J@E@7v2apjy`r@gUvriE$PV(7neo4`(Ua`D zB#)AhAC+qr$Ty3_cX@TdI?RqGR0sS-{;~8!$|#(a8NGkk>Q2a1j16{PsXdJ<4%$3= zt_!L9II+~^ipRl+h7AAefTQ$jxPi!;gEp<#zYHX$yQN$IO@kfjvDMdA57)l*E|X30 z-jwU$F>p$(5FSBN6O7Zs3kzp{LAb@{8a#8!^S0;jX#{=DqF#TOv;G!NItn6tP;K@f z=(BuTLhO4W4LwRBid^q~(&#<&KEDi_9jGq|vPb_k(85V$DIGNh#;3%)0eeNz7vBNY zkZ|?3@!!h5gJ?pvrBq_&5~rc8Z(7|L(>>yI88BHKPpZ8@y6(rAlJn0bvk18osYSW^i)qp z#u`~V+aZ%Q8ek2xM5lKQFX%eL4s90K&x>Z<*)Wjsc=YV3uP;{8fS|0C#aA@gLzPIaANpVj?EoRSj$tKQB>51(s>_`BZJ=NBlf zAqo2xT7norepJ%D_nk`47}Gr~kjc7@8e#w2T#1$p-PgPYQJdNi4_ADhpY`Eeb#X@y zMz5hwlmSW*oxMH!OzXtqSz@#(-X0ub3g+{nT*VurmY=z%34IT?~Eo0G*jYVI#wrOBR%~Gw`A05iC@&Ud8E`g5QS%` zdf-5L-{a@Y1X-*J|ClZxwSdNiCM<;EJ4PHTESk1jN+PWHpb;-?`%3&mU~L&#_86M> zn0raiHsA%l-pj;LqWUgR_@3scO+$U+t>b1n@h$PjuP9;8|n1F*9ADn%FWDS(M?gm=S~VH&RY(mMQXQ} zIfqzb2q1(#q}>R@$Br^Vuo>A!vues8ihw|0H6KXgfwq`daZqTR5pasUK7DSqm)aS{ zKIhqIIPP{HVZx~Tpj=0iDtoXZ)eb374hAD^ZT;!R}XbW(-bW4ki2S+g%v05dd=$fV6Ix~>o<*| ztmF$C)l;sPkL}3y{yFM4*hV*wu$%dv4Cl~KA)FE0-XDVsnbH=RmIgJ+@K+hN;3rEUCl|ukc=Th z34iJVE_!5msh1w#&6}q!N_=lsAQ#3Q<%iwdLV=zY$S6&6F0_@0q_8AL988qAcia~5|Agids}km znPX8g2KxLubAL%RPMpT`7B|7ha!9f0=cBG67omt(N|c}J$3L^GkG*2iBZ@L$2BVnU zUE7;a2noaIEbx#w{_;Dwo&GWs+bx-1*W3tgx1~3`#EuNW$6oHav@hyZ9qShrihDeB z#k6F?WBJ7|v>>esjGUb%&PXYA1nCUq>+yflQn^2LdcfBR4{?i)>iQmW+l2b7NbK)W z|0}cSbINN$HX`{hjp!S0>8P3K5ucR@p$nr6dfP9NmpmW;yxzI{)=t^I;qRc54zp&j zMjd0+-mk-ie&Mu#h{a`Ub2PF#K2c-HKR8M?>j^}wPJB<{!q2^na%9cdrg_I`GigeZoErg% z?aQ*P5>t<8oC%`kf%ZL9CJ7)w}@>7#~%D4KZR0cChvD1kbV zub<55<&^E^x=nXZrI0a5Qw6`RaL`PoUA zf{&(ogH6FXjNF9vZ4DIo$>sXHk-_t##$CUR6gtGI{$2U5L?dFi48?nQR1LP39P!|E zi+;dgO?B6Jo=C^1eepFp?c>z7ohx|y2M!5c&pD0FP!r1B!Z1dOe#%qX(?DpSM67M} z&t6&F@jrMgVk<%r|sr@$P*E3~R zLz$R4ZxS6Ry{Uidrvi0C_~y-6@T&3Kj1I*f=S6ey5dPlEFe`}RRtj^$aKRH!+M6tpGRW4eT~bbp>U68-$3;A-wpuwDOuo0G zyfBNM|Eg1b><)8wA02&`7quN`G9uw|A=(a}{U&5ERv$c7CJPSz-a8nkzWe9B)|BVN zq9~QiRL=%G2e@J*v&-jbC{Gr24u9(Zc*H^T_8ys{`W4+~2WQD($;+HY-R8;rHt7;% zhHDv_nD6Z^ZqF-&(AhHDdA^m8$9xuHiEB^$_B*ZDEzxtM_qN3!kP7Fo*ItET&rBAX z2K4a^kD=e4J5uZ9GRPK?5!slP?ZAbHZvQ<~(e!^BvUt^LI4FEQKPCk*$5*9Zua)xrO6BFB4Kzwp?AWffR;5kcNN%JBWMB4pzqtfE?NmEqiLh zo)<7oJ{1MvXprnYV)gD|6d^j##GJ^%iij-+rqZZ6LDe6zub@L2*o*u$-cm;i_xmqM zyrHrbJD}`72YaJz{>O23-JiOR>~sxNUOa};2Q0WqJ!&)%tFtQx+&LtQ`rj6Mdq0zt zx#c{pfFV%xaX;krn6^Ub7Z!)nBJDi8+kh|#tqC~kI~S-GgwJPzazq6^itmZ0a7;Y^ ziBNpJc4#tBtDPwTc9q2TCcp_cktk%b4_LCqsHMb;j+&VBX|6?Nkzw*{20#6bbkEcE zNtXu8sb3-%MxaHe@sTMiYpM~*=#3pDI+$^r2Ny>??V9wb#z-%>(nUSSN2|5BS)O6b z8|-44C6k$8wu(q3{DSxAL7Pv06TK(MSb##_{daAd<#SUR6pYIj6RFCXgizwBfqmk*NO*9Hs%zj8azWTIX0EJ z1D*X{?RtEo_Gio_jIyDnpf)a@WHlq54PzD^-XULtL-kR5u;x7Cv2chGlGlfsbicX} zJJ0gp10%zZKL;}PK?+c^Jh_)!IJ^B^doI6jI9*g~x1E7W3C;m`m_+I#-}|Q7kxE>0 zUvYwae-l1(_N>fu5CsbSEA}@ehX5q|-H(rOIkq?31 zeBysy7yL{*Rsd%^osx(FWPE9BdTwe3=c}n^YH#)}M?&6@=-(qPydyZMPPWCh#xlz( zC4J&dag5Zm2Of#-HH~y>RozL~61K-2;Rk)wcF~wZ`o9T*nvG@c)JT{{pUPE z{e+{`LIi9-)4kCqz>dFOdy_yV9@a+A?%$%4kaI%%p#(C=2R4C%e*| z_p$c5IfS#7MqiG)r0Y5U$-uvCJNEFMoe3fVP+VOs<1D(frbp{QZ9WjRWqiEBNu(5_ z4<}v7({z#^$R04Zm7XkrVkTR&ui1~xA}*uxCehsZJ6><9sY>Q=MktLp0RnuL3m65$ z7UYX+{|<;K0Q0G^Mjk|SIJp^0X*5oU_=pFm|GJK2kIUVt$G8AdezbW{x^VZmuoM?* ztpSO4c{iO0o|XH9ciwTwrs)rxi|!QbeMrKy$EIaSR?Hnb8eOZqa_&rOr0k7lm^_gC z2BRMO?Vc&b(upyeqYIl15JG?<1VcTMdz&D_V=SK?l>k_WDp0Ml!7`o9PPB@M=L?aH z`{&}7RDrF=+=jX6L#`S}yKVNp!U^~c;Bg-!Rmm$W8jYovQ?8O!P>IEv30J(&i|hq> zNbb)vs&16oqr@= zmERrBF#9bQxql9>P*!Ix;M>LKJw#V$| zZ!6%#L&=^jS6OX_{9x5?f|yS5HP5;VLe6@VlOc}pj_4uu`zxQ+Wao)vEJgG{7PA&# z_Y!mG+MDzSkCz+M&9BMwBY%wUGrJr8Q0lqI;{*J8)x}XkNg~z@KDDySE_e@im4MBf zpyW35d_~*#!hd=wfc)@(@=iW7T;k2f_Rk7Zog0u?)$TmnV5zoltNgcEH7GDN#ObDgeg~Cl zZp<>2S=3XUgtS~uC!^pC%f-RVgVz0YX0$}NP4 zOs;%cIyv?=JNsSh65zu{eicVCa%vv=uPW@X(4Wd28Nq&oVmwniK^_8K13ZNvibx+m_if)F-)Wa3x3$0MTv|{%H5nB+3ek+OfC7_ zLq7YrqhFSu42Yi4AeP=RsL^TA^foITPo?jVe=2v0GjanLh4_!tyTbgKLaxN{X;-5M zia&FDSHu|%J?QdRovd2y2_S(jC6LzJ6 zZ|Z4D43{l!^Xy_L@9O<57Nw${ysU^9* zB(jH=Lo_))2!Imh=B8hz8PCL^w=9PZ z-MB_`g_m!chjImv@9DwjGdh4V4NDyD{ov#xTGD0WXb772yx7Fx*+1Dka$`pK8F$~J zV|jc9k=Nd1`XRhDwVWiDASG-=oWg4&cUmmbJR~aE=i9Li*~h{B2DlemZ4#z>fX;4E zT)j0=DMA3H!tbv;aA32?FS!mYy|4fHaNEJOUXk!`EVcAhqhpW>h0nxG){L+jjipgu z(y%a)$|9Ao4H-T8@=olMJ9>&OmbN8#_R<*C==xo6)|CzadXK38IlBungs}Z#|G=hQ z=@PFMT?{9&VbqaL@9Kdvaaay)U;DY}g^@^(9k4bvmI};f?o+^IL-3#MwJV#OxA5j# zpBftyCVK6mVjK4M-o8?`iHS-bTt^CQbD3QL$yS>7FxfdCN)#i8&ycKdzXl+LK-%F~ z6ETH92ZF8UW3;F;rXqJ$s46zZnrM`LhKE}GfQTzudJs>ubBNc z7dk;efmHFFqk=gd|Bs);7H;OkcgN&zT9YebpJNk5{B(zjk0VyOhL*%nvOz_Q$Xn}x zS^U%JE|RGpTMWP`FTX+m-W6r zu9N7Y9($m~oq>t72ORpHsUp;UVP9xcX}$z*yZ<|yy*jBXMZ3fOQGx^_-o<@?)C=x6 zd6cvJPL?D}2*y7aP3ua53vop#O+SCH!I$*i!)hXCg%?st70B&n*c*mHQ%6JgufX2a zNmU3iKQ#^~Ok}i!XJhUf{}m(2%;Yi? z1RYIps~mp)uOSR42CAQj$7Ll@Yj-PKa>4_-30ALrpE9iAaKqZ|sI^oenwFgI?T4XU zlo{85SX#q6Ia(5;7w98S}eY?Ach|+&z_->^kL!V=} zjf#bUbjD`+&hDn|Z0vy*jv<_DNs4%9Bi6Q05;kcFwhICyvSYdPlFiz|H^QLTk8WZG>=z$I79*BOJT=doo(Drmerj5OOijX~ z7z=w`&$&!DrgcWUJnEr-l8(k3*Qav|Lb=J}mIBOgo)l?5V)%whh3NC{$j4IboruD* z_8qcp^XRJz&2o`>LDbXVHHo2?G=BF2zB>F|Pz5NKord~O{aHGU(YI9!Rg?vR&*AV# zHlh}05sx>&ISFm@{4zqi1ZjBHoG@ck`_~x`GrcH(h&B@kURJze%e8Nd-KRf5#xx%> zW;FLK1mt_bis+`38NNrGQw-Lc!R@||u_2mp)B)d=qi1JL6bH;TvQVW_&=5Z|zdM$5 zL<xB9&^u@~lKXhk5$7YdK{2s-H}f5yIDkJ+B9nL8bgYQ#bC@MJG` zq5N!ycPUR>*cTeNucjX|xO;woF50})fJ1$B4an|sCJwiLo^rW3)jTBnk=gjs1yC#I z*fqK!Te8gWyri0s?;9Qw>LDuN_i^JonFnmd{+hn(zw9LQvUbtEs#mqT{ge5pM#)!U)-``QA#gX=eT19b0kS66(^(87ZSfQJyiby#%&mJVxL zRtGZ{4dUZnKI|u!2tG|FIL`HqUFE3HJg0`mc@?6}3;qMOg3UnIl8k~d8m*Wpwm~M~ zjEP7MwV02jC5it{Er=I6&>@)nj=jG#G7ly7H!v3`+;GVQxvrYC{Q<9BhFL0cmbR@M zvIQzFShRh1SAU$@m^Y+h=$#D1MxbZ6z(vHm`zhV2`;o7GW8?ESvyblG9z~BcyoQs75U48njjpEG!EZT}B!vZ>0~? zA>u4DtmP2ZPCdjXXv8x~HuKbtqKl9=ab9elX$sf3x<34ds|gkUcSNA{z6BhyI*3o{ z#uvEolk_jKbuWo@5Zs3}5(UR$g0Pb}7qZ?+zY0K8pDB^=tW*}@TF*o=CMWL+w5vLz zk!dUEeKAn5i3m#u(d9M6$0V~vhV6%EP3^$Y1GLz)o2*%b!@$K#iT&ZA2z|ozt1!=7 z7F5wD_VgKZ;w|a=2Qdix{H3eL&3}+?_>7!iM}x7^%;HITh+MCE(EOnRM$zb*MS-S! zE3kQ!px2(~{SHgXonVQ#h-3R~KKNCrR z+_9Zl1ZKzu-0x3nJ&sFj`|sJz_xBfd4&1X$hbj-5`hmd|sKPAQXuWR@ze9b(K)kYW zSOu-6Bb#@kv$K=z+nfJfybLrmFl^kw^ecc`E8+%S2`HuXPQt1V*0*Die5p+WaURYZ zs3{S%c8#-G^O1g#)*beB`kC^ZdJa{|kRtpGj18>t2PTm^PK1WzG8Kq_A=CsBuxo^g zf=Bd~VT?E@F}q{K$#@$B;y;xUDI`W4bPsq_RvvTovSmCAn~`Wl;cex_T;7khSa6d+ zGfcal-Jb6))9J!EREWU|@v_e!EfOg>#%nR16N4g7Qj53kD~~;Y;`Qyh;B(@~B4W_v z5FXOOSBdR+XOhHfiTJfI$|$XezfP)1?LK5TUsD610xIi{0A(-r;1To|$h&D|_7Je( z%@ELZa7_ki>cGdxUXNW=Ba5bXl zTl2^quvZCEG5KXpmyQ(W!D75etLCmqD_(si=t+tuJzs!KauL-NYC0usO-$IoR|j_g z<=@UnkM+zprjbp+EQ7c4o%6l=pxut|XNa$?SH5YXSS%vXbkrs@6I+qrDSufODkagT zCf1mmm=h(kXY$487iDw{oM`F=+i&XME@r)*<02uZF)RFM>lGo0a@#CAv9U`_BNN5y zs3DP%t)PT8{N#U>`1H92L}2@62dOFK12}+sAY-015FgS5p_y`HUx2FD-5?Y#vYt_s?Y~PwVM85KG!CC-(MY2fp&-eb?qEMk3qkQG{r#G33qSN*#bK-m@O+Q zTfNnmMsK^oWGyq?&&x+>_~!O3FP^8{ZkLx8tn^Ff47Y^&PP?Q2?zB~;TTK7ccDM2a zmSW*JQ-=3{Yrd~jY5T?=3cm3f7(6mxlUD`9%9Jz2P(JME zH-Dzl=O9VlU*VUIyc5xs%HZd_V1?wSmCP-di06x7%D5ML%L>U^xnK_aU&Iu{9Y_u`M_$*CRADX`R;6t#ph zomkT>iLjGX(R+{DHeUDSSg*}|8Gv8SOBHJtC!~%ixXZF^?&#oBrj-(PW6ZN*j5g%b z6N?L(esUe*E>E*+>YTdd;6|OX>)Ld?xJGPzK$*@sSt!EXsAzZg!>^XcXw)<6G|~1d zi5ol6z_bOoU1wdS5H$p>kTY=WL^$m!D{yP$l&;T?s~@1U@GK4xIj7!zKms)leFNjfkn{^QRT5knM9+R59d=FrddI2 z75?#cO{kuKTAm+Szg=pFo_Jqq#0CSqZ+@$nCC9-)W_I$a86C(Nr#VB+w zO&x;YM&hKvcg-X1AM<4)JsA3*?EkeqW{M8rW!jn9u@LVr;zhU~Wd2`p8ST7uX6vr+bam%Ds8mu&OE~?0Vkdv_RkE!v2 zwzAy_S3954+9)L+qnVMkvp^f}EUxldPn?J4R%zfLc9fg;Nkyuw1K2MC;+%3&l6DB~ zIVY%u{FX86^n}_`3uf?(+^c>URy!uQp(e-n#VWF{{-LO-SnSf%s{Kuenl~=2|70HC zVuvKxdfYv7!=I{j-jYcqy!;G(C%CmIAB}o&aF13fLE@Z%jL*27Ck*FzU7p)IprDOT z#qoxF5#m2SA?TGv8;KI8soc4kJ-NJ*weGuB6Mr}_-Hhz z+)R|_1;I*I^X|Qo&meWnPxOh{^NWHbTlDlP^j+Lh#rmSt!Q~{GomdZ3OXQU<-VOec z8zU5{>UFJWLp8~WSelfDl&vYPJMkDA63@OIYo~2kcpBQS%m=;rF5nYOG=_f4)obo`m?Om(W<3WF6em`p}q2Duu*U2338{7kHBWyk` z>%%j7A!ZI(CBH|kishx&w;^|cd6j{-8|F8hXwP5BpdqdPB`UrpmNp24zB6lU_KITq zA%{ZqAJd5*%%i=j7bUCaYl_CDuN*x6!q~AW;e)S0VGOR$(R)mhr)g#95X4Ei<^pP69Wmvv5WOo@XInV)g4yeDiYO0+y(58)xUr<@s9S<741 zu9Un5D7D!oIEr-j@2`IgW?5Ws4Li8uM+CwzcyT%++t;zd%U`%H+m;u~{yI)=q5>1E zg&`2UP4E_VM>h+m{C+@&=^>p>k?hMJ5qg!EKK6VpSW!3R)q{sfaK3`l{LTHdnm6h! lz(r$O@Neq?QxD&hgWlf_b?6n*KKtLav4NTXM_s3w{|9pVLE8WT literal 0 HcmV?d00001 diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_trigonometric_strip.png b/docs/summer-of-code-ideas/images/jiradet/shader_trigonometric_strip.png new file mode 100644 index 0000000000000000000000000000000000000000..206094bc9f2c1b4e19bebc68173441dccc0af739 GIT binary patch literal 19546 zcmXV22{=^W`yZ)Pk_sU))(Dx9B`rhNHndWfA(cv+N+o2jrN~m&Oj1VqmUdbRbuC$k zXjO@^%#c)Lt~7JG?)jhb`#+C|(U^1J^S+<$+?l5&Pj^>^Ig95|C=`VaZcf`M6dCwX zhB9j={Na1z&H)N#v(5%5haG2RT0YG_sUpB!152vwoHi|A3Co?2mgwe4>JU z@^;dD%f%{hiA1N2cR|HP_7u+pjk!Ai{AAL^#wb2*#xJ&u$L2le>=e+Sh zrPGg7z7#{1{cXEjg-+1#wYRM++s%&-O;juOV1UhJ^Vgi)>07bMnmAE#)re}lVSDh=M#ZX+_iTD38O!z&gTrZOBBe`2MB+tvLJB1B zHf;X}zh_7~VGMDYyR@X&EuHOPweeP(yCN%fh%|+Hl&x{_}6Sr*dLvv?|+(h-ERK#J-FBO&Z zmQzgv)8vtq&YgQ*`sm$fFN&8YaUa`^-GRC;Yq+RHCaug>08{p`^voZi<~NGss-7H?i;wUw}tWl?ThlIK5# zuNUcUeyD%XzZ8kVXyijyCPo1*b}{4861gRQbr6rJtry3;MOy>&$RO*&cWBy>%gq*v;4@~ zPqYFF@7A0jPY#%q58Nz&^&L%)ry*%Mb89({_iwcM`n0Q~sr|Am3V-gjg+(kU#8Id5 zPy!kIiyPk zt(_ee5lV2K=gbG|-$U4D;;Keu;*WhlFzOA~ygD#%ORU+jQ9a%G{q%~mtUf70{I$s= zd*uCD8c2u32zSW{rDY$Ha*sJSf^{UPR7+U`zJA~$=f-o3*huyh)nuw>664^8e1Foe z@Nc?g|Gy~8fnBerHPEy2XPO}ouiWR!)fH#R_>ozy^rXl3gkn1s4puz~t*5^}Ve)|2 zc~b4O|_nufUNvf!$!NWtkfnf$k*UXokZ1(S0jmvIdv>U zGC0IlL!SN*x^*>Z7nF~nevmDKf|4BMVsbGRefN9uTbX{>qU&%CBlQ(5b@I~q2yd<* zIrP``mtc7yF*+7?gG=W;qL?91ZJ$%=w|OV*6ekQBWitr8S3FjfP;<#Wo3qFf}VkKs}T3A6fV zgD1g!zo318>jC0x#@dpUyk+X&+UnDM$hr68X<%vLK{v^Mg2hlQnP;{TJfhWc@mc|%|nZBA86=7&j z&KO?{X)Atm^-=8Drv1c%dQMPp!(X3`!Ry(eA{B`{PyN80p*h>{^W#+HfmB3ex80Z| z0?}CqMJtFy&lbEUd{WofQ|S*@m&j36G(eXp(FjYCllB9w9dk5dovb&mfQse-M^PAX z?#q$s&f*o2vJ*0r*^~3ih>*JVE69`|c0VxAQF2i-I6A)SB!qfiz@&eXa6Q{9Q8egu zy4vjXcDu89|8vS*WKL+@x-iY^1(k#vVNuJXcs>^K1VaLP?b&w_mD%Hr48=ET?w6h@ zL2{Ipc<3KI4w3oLMP@GAl1`1GexMd`kt9};+l)Rm=4gx=P#=)@B*Nd*AUU$fWt74F zlgcM|H<**hGp?7UXc#GdQ@n`ihsM%?b&&2{Q2K+pi%u`;Pv6!op?wkt5L=FVC8;NY z&GmG%vQH3&dLo1)@_2CM7{p=yvejx}9u!cQK97$CEF;uq-AR06q1rDwmJ<^9s?wV5 zgqPLon(jw(a@$-*LE~NS!47Qt1JW>=`a!sVjw9QOEgIY-F<0$)XLJy?_r9>;s$v2% zJv817F{FXhLDS{}n<3%Em^`Iop7E_4R(%#wwYFL2y=j9?&5)RU65o)#Yufq3QB)XT zp9UAd`6ZA)>5+r9D^@A~pj8PNdW(jjP;eYeDQt8DQFC@)GT3FOQBGKs7fV>kt->2z zyL?ev+L1)R2RF<|1Rn3F&qb#FrqGa^UuM4n3gfOVELOmvjKLd;>TszAc^e36YYVqRtNcUv=(WV(I_2UW(CLb5XCq`LNz10ixs(V&zr$TqEe})1ag}SF_8xOC2dK&_dzroK9GGT|C*?{ zC`zSMUfSf65$&)dar{!{Jtpuxq1Eu%hKupTkQ~!=&gUfu>fz;aIWIjhi zXY;EhDpGyb$k;a`lW2~=j2Sq5L)_m`$wqIkF5m)_(L+Blp4%X#XW~y(nlnhZ27C6Y zY5kml-UeSHx!z&I;9QhbGNi~mk0|J9`Gdj<`OJ2<^h z@f|AnsZEja|Ce$RxWKCxtAjI&G+_Rd?i?tXS$f}gDdTI4cKF`AmV%2XG3dd#sbG39b=q=8A_9@+dj#~93 zv94%#I}283_RrJu8ON7gs;7~FESwB>>mmw-MRaRYV@)LyRkv!OKNZQE5^-SOPs2tq zj~&+LEG9|7NOtmJ?NctVmCM_gdi)e5jl}xlxmQO7sbJ1zWX4Qi((;FY4<=NBDjUcr4R!g17A>C)Z3&*^a=>vOJpRb=dCD-q4IUg~f4EL7S zVv-E(>V>TWk6>ajHw}ic`yWfyK_u+RZTmw3gu>B|W2qNWKM6!gwB*(H`PkgciMx_a zmc!VS)*fBL_UfaTN}c2KO2u-59!S(Jf>CGi;Np`n<5KAdrO3DvHzH_j|E317FZaBb zjeb&7sAafO9m*9ZWYQ(Y?~Bj(xz_uWuFhKX=@w*}mT3D5*2%bL7bu*WHP>~OE>JaJ z5)bQ*-VNXQUkaU5k3457D5LAe5R7L>7Z|a!cJyGI2qdu&NjPB7zMn2qb+N87W4SrE zv#iNoYNYkgbN_}P=g{R7(3o|x>uSjxtYcL2Rh3xKZqTRn2U&@d?eYU7xcslFL(6U)8$ju&R=w`2t_R|wEha|9g(in>NdPRuD^`kmJ zFowH4l-6tGyOd!=wy%^g_jOK18)4j&z+^gcac=m9tHp`mydUYVr{-X=@C5gyX;L8ZC~%GC@{I1a$`wdd-|cS~ zaBe?3_rLu4{G5pGWL;(U+I4?BtEu<25$S?COpIYzxcZ?~p5jaPru#iy)Snhi9GaNK zbvoVWMe7`G-%`(;^&o!J%zpGsB}c?OIhD>os3V8=UxbRW{pN#UVz}#3uUZn_CuARS z)F$bAcbsiKXHIRJd+B-f7lp8hjgR*1L*LcF?C|xk?JvP)$^BMAy8z8zEqZh%T2obLn4`l|EL02xI;&1Y64AN?uhEqb>WTn3WoE+crip5%T>}-<55* zazwDQTJyW9$h8;Tr(D_@JnCYGG^$3w*Aa22zb5J1*K<d@PM_vj-Z{ytt{Z7 zYE!gAxiqY{3+9-U=}_23p>}h>(X75Nu1kAac75TE$#c(%4J(MkyNg~EW06S*H7TrI zQnYt68p9M$@Fct5?GLQQeK@yy@Ft~`dJ<{v)7 zb|t~;l{G^aJ%lZ))rMR8*UNMw%K|baR-;EEL%F1^8cBXr?)6Z)OPDW>_DD~?e!>O%tsna?_U$KT|d{Dz24v& z!$zI?ySX%NFPrTaFqNe`sr`R4edYN~K?Y5wuRSHrrn#xejR>3m;xLHYH|FH3pWIYn zB9&I(-L;Yqsnrd^g!$sSNL&tM>)9Es1uGUpUpSshx7`QZvLiQW$oUI5q#uQj_vxg< z1I`}*(sI#s-&VdP+Q94E}i?gNzuktB;WXH=En2?MrTA<693DlaH+lA6*(x4 z6=3N0uUCv=sJ^kW*Gq_0mFC0s=(j0_8+{(C<tmt#t%y&R9ZbgyN? zmZh|GuKS{o2S4i<)IZr7ZgW2*?~8Nu&p)p_yJt+7xooVh8zOV)rmn88w$UjWnPphq zj;+emO=JGH&z4zFtB%P|-#0KhDZD=S%)t!<+*K2+gV#)Z8K(88cjD|`vf}iqZME<2 zKTYM-j0oJj-|cZferIEgoXFs;P+7%);J+^0#%0F$`Xt6$r<}=-GuGw z46A>tg4ILkQjF~mrOi7rXWp(bqG|Qsbqrgxx2r0RKCCJyZ(nIEVuqnBA5Lly{=o9C zzwvpxcO57Cl396E6DKq2>s7mbC0X==qk~TxRJ8|98aIYvksnXiG#Kj+Zo>DS%Mx!> z7;KjLkUpyN``$0U)x+P)`x%$6(IR@C42#{D58m4ulrdVz7Hgl^F=ORv00n=d&Y7Ns zPpq%=C9G{oS7i`;*!@ny!u0!f)N_5M%gDh>N$rg@e2p{~x1RY_(yt_+E;RP`Q@O}H zATs=(o_N;l-6i9L#2;K4MI+5HzuusA4PN%9AV2i!G6?b{9Ws#z{9=@XZ}cK<4j;qXvI$KC1RQF_eet}GK0ZSm zAKpl+z~aw)NUk}aqMm5pJVPCwxM@lb>td?wF-K7at+}sQGXv=BgT3Bla3J0*30*`t z1{x)Vral-;CsrFEmfoy7A#x!LcwSGurnLQoCS$jj>0A?1se~9RAav?D3S7n}5jC6e zT945(2n~C@)0kAr1mbieMFs5eB*pq5F%y`3k~R)_(I%2@fY3e3MFj+7BN>^As2d?` z>*&k7CQsdMI@Vt#K&^F=1=@7AsXDLv6IX~$ah`;A9Y|HBFQqR*Tl*ULZJ7kK21Dm^ zdDC)GEmNdxyQuT*t%&xEsJ$pr29M4}-9!voF6u0z$_R>OxZpaUH>;WFA}Y=%)C-6Y z4!E2fc{~%<(g%y1c=FlERlZOk=gA0|%kWZrtnD%}W&pN%lH>GCyweVLjgl_a{UU|} zSFuj`+9(ACdXu*tj97Vn%-?lsP3Sr4SPL<6`7Def*UWojy$Fq32U|8j8$7n`FU8-A z3p5ZNQ^d-fe3(wC=p&E&iXTZv^?^z{vB>}|^(1#CBbWI=Qy;CP%Vwh}p93cshlytj zfHxMNLqyx-KCu|0gKM~v1_gwIGWh%VDT%L|t0;{cAKIEAuY7C#Dhx6hDLfE!BIjD>&G8xvu9KHVAMqaayF`dg9s1uHA zo~A!;xrpTT0hw&z;7#t&L@bSvP2Q{&WT_FtXrgb$I%4?@jSOI`5B4?jc4FcLK5seR z8H2TD5DErB*ON>bOyixNX|Rv@lk!BeT@O=r_wk9s1TzjRogv^r8ish1^~uPA$lUyt@P0+hDSCP&2on3GV@A^YfWMTs z#peXfo01MUa=!i)A6x~9Tn>I2Ey_X7jll6b-ko2N5)t!E!R2JSK5k-<3F8E4 zX9^Elr4Njo&>7ia8SGe|4$d|qzZMX$^a0pFA~Bd)RzRiUjLXDY2fQ;DgG}?-NS-es zJhEvlpEbrnMasz4fw*NL?lt3vBs3Xp@MhVv@}xv?$HeghL?s8+G)4QwJlLi(0#@-@ zOV=2Ae+cuU@OcjK5uS_7cm1j60Ou}qY1Bvp)4O+495 zB#n=_W6bk>jt-vZM&hs%V{-S9d9+8%$1)K^D}z40>HleIR>x6n=BP|5aJ{rllP{7` zJj(!~WTSLb5RwV@xRG58@Eek@QL9*iTwf{PJQ+~QS6oJN-LvTP(h-^|xb9DoTy6ZR zKG-z{lk0q7CFP}DHk!zXbOZ|72*n7fHSruV6gFHd8!6{Ajq$-)%qFm}^m5)38*-Ba zUYm?4ngY!dB4kRuctUBg)LZMQ!6|a`+ z{!%cri9Vv+h$t9=(GsFHVCtWd0(?({*EhMF7TL1f>XsW_ZyC5{TD#pPL% zXYWM4-loK_=cq~Nh$`lQ&ld>ibYl2_l7FR-r;XR96UPYa6zXLV4V{Q{Y`GDTfBqQG za2X4%Eg%BmbM*Ejq5tLDHiX}hJ9|8n{y-vILdYqD*`B12GFZ^Wn30LBHAU7!z%w9y zb`fC?c*O>CC=;}$6B-6U+ml?7iL&{^nN6q{hE8v$@5V~A2~I3lN)Z4zjJ_04n|Cpv z_GtZ=m)g!p3APho(GhBzs{p)GMefX>THUV>U64(^1g#(lV^+?>&}DdWET5-`GtLRT z2(rQf&#T2uNc#`lF?LJ%BiK$ayDg-xJsF>jQqt-ESfVmA(-cmCBj3zuNTG{V(}~^5 zh?WuJ)kJs1!0c?Kgin`gqWfZ~0bUV@g;NAjOMEtvU(<;h$>^;3 z4Q^&w@KyM*GP*?sx>D#%;`ofIY_}Cp ztj5H#a3yZBWHPsaco=}IAU${GxeN=&4)TmQSzAJ&y9ieU;GvD%Y$E*%2xL3)H50f< z>(VhoA2@nP*zD#K9;NlIWSpm_nkN7mvHM7x+eH_^)`Yy~ShmhN3 z2s_v%>DR?prxTvURvZyPI&(Cqri(_xED|s3G=(vsH50Kl1*H2^KsCWTH;{J=h)@T-!HqQ32jr6`#*;HUSVk=2qU{9gNi^sq=K3J# ze~qhyDmX=vLM$ytAe7_sW=KmCP7^R?Vf$n~vi-u@uuKyAi13QX!Uy{r1oRt{PN>VK zq<#N(tYPOY_HfkRa>{oJ;z>9c5H>X!v-MnGS|hlliuA#ttno#%y>k<}DifV)1mp^c zNdr)k31+}>L-G~%k*FrRdJd3xBe!OvNqnAW9${c&6Sx(R{hu5cNYbVf+`|$g&4_#` zW#AvBoz-ofwFP#@PEYf-FRp1)M!cjLRizu*T=5)o3zCrB^mi;%U)#XCiP zP#GWv!s9TY0jDarI6ssAXua)H*kWqrf$ZPYF2DfS%$dhU{V<`dpjc4HxiYVbDWc7U zZ|C7?I`IjD2&Uc)Lb!qaR|0|Eq^A$$GZ9@QB&v?Cnu(l~+8fmF)sdpkjnOh?B2GY` z_^5du^01FC)6CJB!vEC;xTYJq<%rZ4UN1C4>lhSRmeM`Aa*R}nSHs;7PH9|&11!zP z@+RtaPrm9RO39?VV~Hxj3MPO}+IX0OG}|psMk@H|B51vgY489cJsK#k#|HnqAu9Tt zL&WPNtEVR1n+Xh>=vvv-RQjPZK2ryWjY5fjNvLWi<)$c68(!#sARaz_Dli4oSUX5z zfYhi5-AKhioR=sN&gKGLQy@Pj`0_Arqh9su$mwoa7|>Tjc94gYQG3h_YPb%r7=w9v z5ic{rVmGq&2yyqJK3LNPU?{QmB>gj?OCXD+g+$!p3$^ivScn)yg)CyZ0W!}BDSFz# zTs;Vf%b3Mwe3FKe;Q7hC+Y%}Uk5#1-b`91RFQ6(yGAg~-#>K9D-fZbmp~<6Cfs<9R zhg&JNc#B&Gzk*~)3BW%D;NjI6{f-1ixBEKy)31{4J{w6KCSy19;h}@XG8pVUo4{Os z5Zio^XmZp?T$|{NvVn6QFo5#j#M_7kL&~i~h!>V-F$dv!fc$uY2y`R;GJ&=cko6~e zSHJ*<&dmXvJjsWd$TU-_Dfq(^Y9PM)*c3yf;lT$Dub+YZb5)JeK+L5LQ?5g{VG}xd zc<_3WD<1{Ajp&kW^riGvER5&#auVt}?;SD4EF#n%#+)p<4WxS}veJmIhP;RFum@v8 zci2`g34I4gK%dSPa43Wn{7#a^yDUA@A=0Oh z13YdwA*x28AsODmL4u{eXEQ?zm_H`cBODB55ZU9K$U%7K^CnrFiOUS1UrUD0M17@E zY{=$hR4p3>O!1#NI>rBvDh!c_N}54U=H-ZGOhE z?qQeu+d>sDT#&?R>n2O$BymQsZ#Ev-oSGXJwsa{sbAMCj+D$oE)RdHzg1i?!HFdde z`F2(Rwe5og*TR1Kb|g8^+v~H+*MHvn*cYmiu`jY;R5~&bRq#gtcJFnhy?P}cJ;tH` z+wf=q?iU|~tHfvXJ*ovrauzRu&moRH+<5k2F^zUwh7+rO0)kR*iQ#TJ>+4d>#awFr90X{n< z8~#+mlk&3fMR9i^>2dQYCLsC&F>ZB=@SNtBEO~di=k=b`3~8~WtMD<_X9*c|vE%W^z+PtZ|p{DRPBIsNg0H}F1GW8uN23qCC@=U=>I2l2SH zw6#5m&uiuuQ$4>hm#~&%HfMZUtJVyYCvFl6m2xqc0V@L-|EqU9h#%S(+-zA{)sJc5 z6KiNlVS~WyD^`nTy~%SX(MzUi;2YVh^TfDHT@K?-T4S_H4}5ai{WOEUSI}Wu7MhSB zq>E%VDstS%SqF>MR1$Q-fix~XnMeq5!cjep(b}yxTe=}60j`|S<~8zKxE>21^#}}; zG@QtAcQZ8bi%ft3n3O&hl+4 zW^6lx@Ao;WP{z!S$J(ZId4CFsIi_F-elN!C_7nIr>cFEz;obL3g4G@=Cz%^<63^rYo*WE;#ejcI#WlQ;KrX`MGis-rnI z0{SLC3O(h1JjQ)KZ|lVEW1Vwf5XgTVf0pcNN_so?;XkK@tGlle zD|eAoa}SU7W$6K9e^%z1>y4o8J>Mq#Eak_biVr#LL=ImG`5OJ{7#@h9+H_juL44vsi`oYbx`?S8q@JCEUQDtK_bfLH&gP|%Sj1r;j;z079NW^$=X#IiHI zLTtt>OVQ#C;ONeBtm-~`9Bu-Bonf%_kUZ(c4I}Ukd%t>#7nq=dEpjm|m&XFspO5BO z=uAG2#~AQt=~oBbLkYhdtprbE@Ro+^@zu4Tbb<95p5FJ@-v2&Lrd(#w)CrpaX4Db5 zvNui}tt6Ce@k17?@Tlfj`t%n>388;nA46K$bk}Nli9PTV4tzAWV!`ABUvil)5Cq^F zn%UrF38Azk8?m9G{9i^e5-F zanT$&Uw;D(bo$|8sgaZ)4A>ueCvwJjg&&IWhS8i7!lwszv5mr|yT7iAf0KixHFA2T zaJ^R&d{DrQ6;IAJ1#Dg9awj%8>r2D26H;amCJ|w4Xh6S>%WIO7t{;+a*qH=nG~^Og z0rJ*fE>hTt=-K$9ifn2Q!=_5WX59O^xrDg<-8z0b!iLs4UAl)cU1V{$csm})Qc}eR zrMrk;q6fhmBtYNyLqYiP1q+YzC$GR=JW|5%d%#^>J4X3Yr0m05Rb7}@Rz1G9jM(l> zz6`)^BK=v%@@^O!BDC-=6=5=^ADU6wZn4r)hOY?*#2<6$u2t)DdjB_IZ`?42xGeCu zuy{d4Md0Rd?u?lDvljZ09`4b2*j!D@xN0&Dw0`%Td<3UFBL|6($2fIO#d19uckG^c zl@U0L>#~&efJ+8qSSf5sHbFCxLlMm!Jqq|Yf5QTLd0pLQ^#_C(EdJ?QGx#J5Rq7}o zwkZNn(P^i99f80fTYU^KN+LSo%`#yDF$j;nLEXT0Wa_d`nnXA+nv$iL`D{R;(Gz~jYd}?cKv)TA$h=>a0zn7!ezg+Y7~s0# zc>vzAuhgOADLe&pIf3G@81= zo)Ihk`kl$qH8eB>E&jd<>tD~xwCxiZ8sSqHxC<{}3f7bm_rFFzBgTy?+H$~r8k)}y zeTJ1*Pp~LCyhf1A5VUQ5FO#k;Dm>rgSM7}#Pat?oT@JX=AYe!L?t`E3=Y7(V4HVj(4jM*_9P4EHPhw39pp2W@y_08o*b9SSDZ9Ob-H}`Q;T<2 zSHwZ=KgK-z`9)29zK|x)B+Ak`Z<^5>9#&+zae4HM3$ige4D^u%V<_gJ-41x092aSK zz@yIKWBaiglS_AzT_dj(mAy!-O5z|c5c`c^+x9BgT#F(dsBERlm5(!YfdZ80@jbPXTSD1k&!$w%uD4!7g`%YjJSC{K-7GYW5I66{B zBs!D4xoqBwpC%i)#f+Ut@ZbxCO`L!Mz1p%$>eUY7&cg4F#lgqh+M7|iZZV~l=HI|g z%K_<)pe6~^a39~Z)S39OrUvQw5Z>O<#!%ZQ(9esPE=O6; zA^|$v6wHwm$O$^G|61omu7kJtmNDKibt`Iq>q_Czs=LfoUNtEH2ZwS2}%~iBu0G zUqaTsX)0a~k7XP-qpim}VY#l94C_h=Xzr0|T-1vXx_)3937XR3jL(E{-9`F9O{#hDWI2?nz9OQN*Vgld4I))owAf8rXrK^q56Q1M{JgwTo zw`PvvUNBB>dtDH2x4iA>kWik@3s8)Kvx!!!G*N@BgI_)@K?Hu0=o0v6aL(w_=o`wn z^O5~W{@<+oxQK=~=>au+VF~eYUNf!ZaQq{iXJPmDHXFE{uYhn{D-{fS92ZP4rM)sV zL4;ob>TpsO?&5BW6L}sY>SPA;0y0GPL3tco3gS{2C4&Tv-^cOQPNY{#7H__%<6rsY z<)9A6L?vk~*;I`Q&p}!gLvk7ub4}UALN;Rr4qivA^@#)NIbu zem5z6U!N#$^!Q%I^~gM`iUZH9L_($_WwLTF{M-j0m4$Cj`jSbyV81RnbC4l5YV-wx z-G5P?ShP+qGGXtIFK@ZU!fEib3|TfXFCjv8LC7sG6H0{kV{F1#nxA~gW5mY#TxGzDGs(p&^|pAv(WAzaaWGvymrf9Xd|xp#BfaFy z0}f?JU>oh#CO&c@y5rQ{L+sVIg>UxR4xgn-2)=`sWVvzZy7=`K||8$#kfV0 zH{&snbJodQ0}Z9Vor7ku8T7e1@N**wcMBdiz2}wcTxp60#%Wy6F+F-h1`1=E25ysW z2EaiR#TECPLllZvMMnx%WUALM7}9ob{dDKul`lcX+SJt(qB;cwutMnbMzjdgHM!tYgBCmDnS;ru3IjuqOAEcEYlq?=((}9|E5dDs)4<=h|>5L`)*6E ze4OLxpMPv!H#^{Sb)&|hAeYh0+!P0j&lKDAOuiizweS}3c()wf(aQk!)om$Z_b}smUFciWQPqX&TSuwbiB@4~&53cA%-lEet@AJ~Q+bzY6;-R| zI!o|?qDX4IV@@+Wd?hcKG?&pL`rn1j8ec2y%>{4abHJhJVq;!E{B6!zgniz2@9Ux7DN>@(;1tPAMZL>+2OZl zG`vO}zUt7b(E4r6luESX%z%fZzx+bIqVxr6!CivBT!yOKW=nQf$k8hyq?aT7d&NA# zo0Y(c|65ZWbg-vJagX@IFHI^nyY-iw8Ez=TJ!z}c$Cv#be}K#uwOoy*AKel+tUqp`dr2?A!_ptX*5_-PQ4oU|a07l=Q;zk)DpNl8%L~yq zXI~!pyW1wAB{y)FJWHkJjix*y>t*zK8T282I*s*BqhmmbjN zAalPo#e(VM1~(Y9A9HFrKYFUD_1x6;?zZoXo8$v#n%zs?N9G@9@#~aY#};e z`ZqFZ7W#G;|DoR4(-z@M`;viO-+5Oryi`l;Qs}a?innAbO$i{@p?fthE+f2WV#V1&KEY@=%{+Fok~ z=JG@Jt|B)R<uRbB=b8qWE;xjW(d+;}wnxj6{Ze|ir$ zJQFdZ*if%|u~G+CN#_fyf%&>`oyQTS!)FFcd&0xUAJ&F<$n56q5Vdd=ylDp2j0B5i zjXqv=mZ%)h|0+9#{C3Y}fX+Q(HwcYx_ki$X+P_d;!p<_IbnTm_ZKEDolJY{>nqq^i zAn-ir;OEYaQsaQPsU7c?42t!ui$BOjgb)M2)>jv7dAesLDq_H+`svZT*1-w*a!IuB zXva0{5!dT=#8_DUw2>XjEnCAfTE1=S6^nX=Nr{n~5ozfy_Xar1t?@wp$anJ}Hg~QD zlZhD}`O|@GS?TH#7UNMLBs8yfsB{s}&vjq4@7G2maAE*$(U7Sg$o`}*>DZCkO5I|C zv{y0wzEW5c!}05nDax$dxTW5G9<&jlxcHNJ&_Os5K>@3n9=n#&t z;q{5-t9I&Gc(m^1T7sKuT11z~^Xc)eidijJcPgS;AB2aFe1CkVJGLld0CDPtQ;4SY z^Uha_F0X;Widr@K`QruOSfQ|XPQJ8FOC)JQhi>Z%D_kZCov6u5$v+EaR$NRRr7NlO zgUZDg>KX(Us#f_buhM$Ka2hl|aw9xxWVpqp+d6aiC!)9>ijca7eAo=z-CEx}gU(Pj z-A0DLpC9z=wF#~d8B!=IPw(k?IjwEN@8YVbEj%;W;vH9z?YCB0LIO~0V;K#z2aG+E zZAU8SCB}mF^@1vDYlHLJ4HhBH^Yk1!#OrTfN~PR}13O1Ok|4f5F3ULHsBtVgm~7a4 zW&}-(s?dl)ANEMc)G$=KnCEAB77=oKa_OoiFAS?0$+vP7DlZ$B4YZ{ip~Q|)fEq9ufo;=Zy0PJGI@#eI+^MK>7>-Vy!jrME9hO<_$=2w9O)dh_!&^ z%%{t6RB-wG_gcPc?(ZqJTtt|!87+!%w%&CGawtGQWjcCEiMP@DWh{LT@04x(Gc;V` zLEjZ_96etS310JYP1TsaK^w8XAF^iEGHe>AMc-h7OeZgCxuatDr0?+)8sTxlEP3*v zJ1QLM@|*36cztd!etR1qJr=IHchv~B_Wfo#g8QyGe`n`_oZi>tOC++CuJe^y-xMPP zPs-1@+TFGvO?IOmxgnO0$g%sgwxNlf>fb%n{$IA_-iY8ZD zNP1&Ds;6~W2K%$N$vni`zx$b!79kt>k)19+rrt)#$_%UMOH_Ni|2TI4*p`n^$dG>? zUt^ibM61S(|Jy3%NIz_S-TM~%?)QZX1H}s8R?^j)^s1?MEJvvHW^>f)(gCxW*IHMo ziaGWxL7aZZLmBs+tH?}j&-_-f5Zm?X#FLLi{K5#xi^R(Pp9Wge15sP31AdgB*!F=I z3#4dzK5lu_`STLPtJIPK!v(F10~^zvT7=J-tPrk&Rglczt&*%WBc%D#cXA`bjRO<%6oTY!gjK|`c+q&*!-ta8!;&pqPUPSp+tM}U!A%VOirtcG?-25 zd0)+KoDc`*Umb*(Y|S(qsX-uE0EXNxqc;hp!@{SD9t=b zR>wS7n$5aCaU9|8$hCA#SVSDJWckVbd@?8Oz!hE`JtLTGQOo!Q%kUZL&pV!>c0B%N zVd7lEy(pIV*@##+lXYF*#$h+4|C{H z*n!4&NXw354b#ci4R9mqiUc{-eV1vsaO+K6W~a_)n}`!gp^wjXsbQnb~^$g z8KD|;6^QSykf^yXJzb@D!q<9mFV%_1VXI3orK~wu*((<@fX*!W zr&|q{7YHv^;uA>&a)e*z%K^p{PB5uvzcafzmZ#XhwsG|{UKKTNtd^C>(}{ffb{hKD z!;E4DsPD@L7#DQ&b`Q8teEks)#Yrin^hp8~C+gK^n5A|2(SC>C&Q;$tVmLB>D)AOO ze+48(3MHfiW@xV^*5QM7;_VtG?BHrnJ+#~w&F{fI;WrsokMMWg>!(jkPQ86e8OO^; z7;zW>&BOlK)|-c1s^Pqi2N{mKS?S_Dc+ybZ(qD~I#Q$bt3BLz(!JevbdDDS~Jx?|w zObmyk%#8=_hcb@vqaT-66IL=41I!F}3*=!epp37|P@U69&Kd|W;%Kn0KQ)UXW02!^ z9fa^dXpT1VA>@WepRKJ;*ymbm?XY;(A0e-(C0EB$o^ona|)#VnX2#P+LGC@SF8PJ94QEf>XHMUKx@B(1+36zU3J zz=1%=vz^qk)3z|!&i^CpG}_;{I;#iiztdRQs8P-iCOI;n@4Jm4vOl79ng9N{3@Cmb z?wD3`+OQfaS;(7{Cu$uE`xWjVBWHU_YP~unS<6(WyIhxR^ z7I{8z=I8R{2mbxB7^(1`$y%*+ehFjXe0A~Y}livjd{MW-4X!YyJ3Y2#n%Gy)2B`nPl8D9kn$gZ;j!gQ~`!F}*(_SAWgG<4C52cRc%WFGjz zcCSfb0Xr$w{EJ2mN5{uDI-}dd{@KnYP7mlx`HH%`$2HnaY{~CvGDn7wJi^0;+IjSf zLnFgw9|@#+pqjT(<99T_D>5A(m{zTk2ze$JDwBV{zyxEtdyQmJYhlEai1=`<1@SXh zB#YBh{ji*IZM#sMIR?x21P%`1(@plev`m z1_8e>H(4TPEnv()YcPJz;h(<*%Lvu}ywfm)SI+ETQ=p#2N)5grtlr-)uk z`M(aX#Hpzxj3*=n5s(3aNJT>t0V^nl3WbSgS_o(vU_2114B;xEF$A;}UXdh3OH%^i(2gCW zAP>V0qoDu$x?QE{wE3bN*vOLh!7%Kfb5Y?LceG?Kyek9V^3rw#d)EUHWjc+C8rNoT zo3k~htj2r+Lk(9{SL#MWS>KOL9Oc}8Zx{4uoOwynloE_?BGEEL_eVl5Z9yQf&nNOd ziLxYk!px@m@&_Z@F=0CphVC4{3Rz@HxEAjrSyLu_Op96j3={$`G&IA!)pzhmG=kUi zlcDcN)}+9e7h=*M2Asj@hFJ>W2W{b0%E5kRsBY`vWGxlGOH!(YHpO!0lP*{ulvzIH zNrI!Y?>eK3uk=HG^&Lmo%BunUN*sY2b81wN-9?6DKf{w5?Kc@x!3H}#T9ZMtA1`2# zdpWP4Hwm-IuEe!cl0ZLowJo2JU16H?=axh6kdzN5K~>JHaVzta`>xQ=2w<=#v%W}U zZA@gMw<%o-XcUr4d0U|8P2|qF(lQBA1sbTYpNXgcrNiwJx0%UZF+t)x4mh8FLL*gv z)A>2re4__J&42?hjWnKY0k#;KcPu^eGa*Y~0gbnM8~84J%!Y#+I2X)Hyl{Ia8b6G; zRJ-7XhjnstIkZgg?}ey0zVIK>>>)vG{or`n%2L$kjI+$KwkMgX`l2XhCrKEt^BTHPVGNjs)-9!Ae3r(<@gV?Fod0|v` z;rPg>+|@I}PKkqcE|fT+w`4nM@v}~4NU|;@j0(BZZ8dfbytT}ubfPgVk`md71H#TJp z4yzss${|NlSA|nA7TFt)lr#S)c%Z^}sP9M;`EB)mW`$utD%D;i^+!m8<7BT&5w-OV zn`z=d2@Z3BB2OV|7I1doAr`MuwNq7IM6u!A)GFZkxL=_`8~dI&!7elKRZ%VxMzKDV z*Q)0gW64kXokdsX7Q4k*PMspFCoNzo-v*ztgF<@yr zF7Or~eMrv?P~zKOAiP6MA|0HO9jiz|Zgv{pd}t$UOeUL%s7RxPu z(e@&%Wj9bV!$1|5Y|F~CByKV|i825Bu~>P)bEZ!K*)vWuW!m>G07@SDm7ER09Kt&q z5Ov3r-=YVY%LyyLrzVj^s$5}8p8NM#f$GqSR?H)RVU<7AdN;aJ(*Y06Btb5Lfo9gfU%QYhodcF4$H zhwORyy^TH}e&4R&=f1ArzxRDzcg{K9ukqZE=Qv*9)l@!z_UhRaCr+GKRZ-A6ae@SX z;>4+Y#HWEzq%*uifFCEFb(C+N$ivW00dGhgRY1-sPEfP%{eMzb=lXBpfh6mDhAxI0 zcO)zv?06qMa(HOT>tW{zJbmJXl!pZH($3Q50h5Ov#NJuLLz;R22?^l!-p72*O#6?x z*h(`SYTRYI?eN5sNtjoNm!DbYEE5xx)RRY65;_V>hs%L?(#(%tTpT6%_}tyydEEth z9iD*s1jNO~`S@@0-Mq;IJi+7aY47sDgU8;Psb?eS{2BStfhQk4{LIA~Bu~$(`;L&(+$vE1%J7$Z)N|tKG$GdK zULC2iUB815)cxTGs*q07re{`w})n#Z(X?f zzn|UTPmY3I%$emM?7aUpn&~7IA%Euv#XsCV03GlBzpnfrvQCo_!_Pm|dHqjmCB*zZ z^^ceWoWQRlHv}P%RQspTGdYek{6iW~K;;Ptz|Z3DG5=5L9FhtZT>Gaq5}%_a7o+`} zM)?orPfQFiJM$lo{*M>tZvxO*CbsDR@z{R^C^yt3{ZDBmct%KwIW3T-@SoNH;{^eD z?w`^~PIH~fQ6x~6^`F@UF*%^(WdEx8|BJo3=IKe+kq7 zR#g5HrvHCeRM1kSK9jc@)yy^;48}JoLlO@>m!TY#UFsLf)IzxbHi2oGI@{6E2NfVC zhKG8(g@hh+bG$}F2L3Asx`dd@KjK^B_&qL)67P5Xa(+q7#p3ubb~l(Dks9n>mdDzy za%weJa|8#T!hC?pz5<7I*}|2!Y+2IC9&7PHSb%1&olrdQi(|KlAp{3+q zpM(oA#We!(OkfVz@kN(`>$pDXAUSr)5JkBj2)Ez^x&tdm--S*UV51F8$^07_5B>K#~YJI zoJ(Dsqv&yQYbUeRo=> z0-YXiO{xciP{et;Zq{Qo=L7?N*K+>p=f^IFR0BxJ9!-or`1I2~XlIR8YbF|--e9AA z3`4-K0eW$z0V99xQ(9(#gljnxtOvIhxjXa_U_l8Qds;IfkL0Ln3>6U(Vt8S9&m8lt z1ylqzRz~mV;D?eJBR)q0Q`{v0$EN1+9J@cH7`VR@=}hvm`*)cD_wQGARXMm7qyTNU zyJD8f=ktrSACy837F#AS;HUt{QbvG~89jE3Q;q&JIwvR(p%y}Ll zXpm@ZAY04bKgP3v(IEmrtw>ca9VIXTZAkRIxlTf`?oh>Zt`~F_fR!CmK!)dDO%G zm=H6*eOkchs2)R{0mj+MeC9poEG?+XMQE31X=o4iNZma3iv#{C?B8+|l(b&=DO#G+!#NPB&;HVLsLje~+{kX{F#{Q-~Yy?2;^pCDy zPUmwqaQ;2?*=6#_dwc&XQ9?|Y(LMiT8x|x17n;e+;ycPCazj19F7z1A90&+XaTem% z!%YkrwFe8$@c|nbU^)GZ^T$^Ar~sl2?dTLeps|30z=%R5cm@atf!GfAql6RZ!~#ym zOLh8qjMBeEE@Bi>cle}v8i5hjz9vNsTzYwY!b=fInnRCj$4LTOfSJhml@fKUIeT_RE_?q4F#Z!0}8Okfxd^bm?bcx7a?VYA_@mt>yAbj zXe~9cdUnHoZPhvEuL$*lXl)c-FMqh!e7*PZoOYCsz_OUT8 zsHWUdPHMsN*c8jc1aTAh>ZLre7NcwYbp;?XVz^yx|IlIdJfw;L9>9zbQC>%R$yAU6 zh$2Yn!lAOWP|?`-aDM{S(5sz>`j{SFItdjNB{=qxaSX87a>4q}0ec}(blexK&(|E0 zM4~Tr4=;3Z8rmE%s?Yas9RKkDtp*lkuouZ_Y|lLN2+I+TLq7x%EYIq8h1UU#4o)*J z0uEG{Mp(kzg(g zF(Y3Y`a}G4XbibjfPvok2Oo8|Xk);!HFvsP;XYvP8zPyd_?9$2|Bc4PT0_QE?0J3% zV)%+M4**&C?eC-eOGl<58A=KU197UvIt4D1>&ioF*M~j5>7RRC!F%TM+#5i9bqS>4q z{4EaqGN+?kYnJ{YcsdLz$h&|kkS-)9 z92KtxV289bO*EGeHZ6FIb_0OU3JE+{sPUK!kTXBGM}8cU#jyw#2!seFp(`&Boy|GQ z9!-FJb6`fbJ@2?jheQA_nfkNjF(g|DgjbO|kO14k_05H$sO?77oxN1A{KlCJbSkgN z80S_eoNf<67{JYON99o<2HYoDusbj6C_VoNx8Z0+NUZ>|4?zBi=#ZU>0ceo&gJ(w< z1HKRPU*CtWJPYYA05!crV=D)u>yF`^S|9~63h{J{K5A)(+JI3gUOk-t1QZi8Q2=^F z3J+bb6XZT>J>(p~)$NaEble4iVH0Ros~;$b^NoW3FIECP1Yl<`=c^(|B~GmXh*6%c z*Yc=aF~1KSegCV>!Grv6P(>g*&;fV`uPG#R^k{A*Acco5sO%$3XfD48?_7pW4$g_Q zfGPqyaES!oybx+Ld5ppnfC>}EJ&HZ_Kl`kNRtFS-%)T66lAnwq7;ps}T)f zB=%`fVv=il@c%L$O0*Y~(W>s>_s=oM5K;3$sM}c`MJ7aZe;qY{a~R+d?UqQ~JdfV! z17N*+HIj#P^AY$jT=tKs-t5eG`y%Z)Bq!ms?pTU40pb#Jrm6f82OU@%gbbhsClH?Z z^uM<`N$`L4NAWzY_8!2<-!Fx(A7v!@B;brB&e9&eOivPUn?7e#6RQ3D^k4{r=|qA( zsl@s)?xnWJy!$plO!F>X`P1{4#Qb-By1o4?xQgWi(6>ltdkRx426o&G9v|pt$X!6R z&m$kI9GryyND#~*LP`fX7zc_&Aa$r7=D|Q#iS!_V`$-W9-xa}Ew-MV`MTc>*%b8ab z@ltbS*Uxwx+xdG}=MPNo$;G^am>6ak8}CAYz)V~brb^Z+gx)`>sLenJrtpxu$56YZr4@b9o?&Jzu28_;Yw zZqrnmVQb;ze92Eh)_r%FWMvRCatM;Bv8weZr5g>`0@mlhk|1lS$Yz&o$up`YkFUlZj2Q4$6+2I($f=JU8p}JuF7Q3 z3p-NJn;x)L5U?&0-M+HJc(_7NE>dw^&4_6gL9Kq+3k5rL|iFE)Y`l4gT~5)RQuh7#p^I@!??D($tX7h2*gzY+iagot$m8 z%_R50h~8jhn8U&Or}oY#H|Rb%4E>VWXR|$7(`>7^yS)1HhCl-StpRee8QY%YrLV%h zY{*V21pI|?^BThNJNGTC1;6p8cozx(e)m-$l7vKYMuE4hmX;|uekGoOT0`C9zYr(m z`knF!TUN)=*tz}+UgD(M8{k{u_JI9BYF-6gsY_6D7bWMp`5ngv7Pu{w@v)R?k)#e6Vc5Cy)Id0dLtaT z+)sMIL3bvv!yl!Avd?9lFRARx-9k(&fGRgFvT+G4jrdOdjyKAFS4ul`aN>tZ$$(4W zj-NDGdT`kfXR+y9;I`;U>2c|sl9}<#HCxLZ+`@O@?z8u5#&%jXtEkOW z$o0p^3icJ=(--WkayMI}Y5qX(ns*1>_Sr_zf%~V;f})Xs=E8)vZp2^Sf;)%Y_-K3@BS=XDog%p;QzBww~fQ@*VoTk z7TW3lm4kV9GE3{r;`-B>X8Ha}z3#g{D;`lReVygmRcn1+UU->|(p1gepM_>K%Ud5U zouo`%QPtf=9{Kpr)smdKweW7-2Si(TQYtdst2HmC-f!1%w%c!Otb{wP-ZtF=(U6*h z%&(vEL*;Y8^hBq>W+StFJBp@6@LqUa=k(fEpx463iL8;JqFysdw*ksDsuSOnls71g zY+LYyfmTu>&TUvTJ>!b{Dd;&-w zu0OoFMA1QG=4MAvb7GW0@mT+NwHIP$#UCyR8|$5Q?zBo<$rU$~jQMHp9wufx!0B7- z4Hq>nM_^O{<1}=7lh? zj(E(apJSh2RlAz%BXkxfP+F3zv;0Gdr@QAuTCPF8iH+jePi2eNfUkZ#H>?V=ZkJYu zJ8?)_T#1mkz@kUA{OH`G-xOES{dpjW=f7F6BJ*m%VY?jiBX zy+vV?JFGLmt>hr64&4N>67MOkKLrtW;O$EU&@3s{i9EzJ1sntm;*0L6t^VaKE#Diy zJ4?x0$@nI4dw2CtiH;ip^A_Vh&ZjE&&n?v{p1iu?+O_d%u6(AA3rK#JJ2GX2>7_gz zzp{u35nR4ZE>`g=;;J+ET?5hJ=k6w0?iwp)Ex1~}+kDG|NH)#I2(oI~ss)3cOcygv zzgQ;Ks`0lH-v`wKIqJ=a$;MX=Vgcw)QRva7q<>`qv9mLo27 zt@alp-^FyV&3Oz=Mz4slH*9snaK?fW*JCRIP>`*|r+ee$F_aoZ&$5UvidR!@H$8_!DRFOLUjib-yarP$}B zKQv0`3aP^zwA(IK#guxF<)Tc&C)U27?%#27D*jZE6Z2fbZPL|9Wad}AT_)GsTIvHo zjb;m7!waJhI4m1RTe+2MR>`EXx4HtuOXKSP$SZZ5_*>@|k4=8ND;Ot=`k*{RKlLlT z{2IQ+VSBKBO=kUp<#WlFM2%ON$K*0nJ{e z^q!jZVRYpN9Dyy21d1r}%=(k}0cgHHoKcH$YqGvA8IMg)U#p(4no>90)B%~Z%B+R8 zpl86tg(|l;-q=W5G%b`Ub}`AYR8D2U+60p~nwH*3^8t1~ z_=QuF)x%^xem%3*+h-P~CD$^R)GHbn9CrN&@3icG&&*6z7w>tJSErfqP4yzjbiG#Z zB1!4hz$q#>)wEt@_1bhtv)HA4uW2hj|NM!M`0f)wT+GU(2z07s)M7o)IpOU@HS+D+ zUnezc?&uH9Y-$x=%pa4FR!`u*zw?4svt8u1uG<8+uTtuEL!V`-G(%Y`2ggdqX$_Eb zFVbmB%mCBquJ^jlp1=8NOf3Npv6MNAlG9Ob75TLdwb(XOIO>)s<{%pJ8s&q-Kam`60cudwg&hb>^2j0_Oxy{jQ_mL~Ty(QXy^ZE|9T(P$Sz)`FC zHM`6+0k`QpOi)qdled$zEZOMFD<#hhsqd2%Y5h1CwIsYdF#8R~Br}z~)4mLp6hx!z zA1g6>Ytd;K%whHXKp|!{KsYc-YGzoXK;88^2=wX{05ZQMZ1<;dwU?PzRHL!I&BcFe z9uRiudWa-}rP6Jl1>8(GXKuMEtk#Z52@QL!nc`(koDd(J+s0r^f8Q972tTl`3b@Jg`f=UNp8fmQe-d1BUjktM;K)yBUlktvZL zCN$4&8u*sBzS7KS`i56;_1(sl#BADTc`rD(%-UJ~I_rD>qhxm5BkbtF*@@oaVP6B| zh@CMbKX&Qi!th3x@AAdE?CqbO+1sJ`E%nKqyeG)w;(I$R?{;3Hsmc1L_S4KTz&VaT zE;<)|;2Amgg0342G2A2prhk|LL2*pvpVLrV6)1K00Z;Jeq_S*+%=Ry`Ob5Ru*qGq_ zI>frmz}TIWFIj%|((oyf{M?I=WrM=f1s*cG74tS)#xPi2P z3E#L0r09T?Zg(Q>VCGGVCJ_$ zp5JFqHqwMDBxLc&ljN$!MN!V-%b8=T&HDM6vqbGYY^!{alfp4KDhh;bYO`hIarT!= z6VIWNQ(H%~;nxP4!hM=TrLHu`ulx^#7fD6k(_=MS7 zhW*ljAqYr?Eoqex72D~o&$Mo`n%VAVcg5#ge_AsjA@8pajRul@eWcT$o_Ikb(Zg%e zT0X-Ur!*YAbT_{#%e<}L>YvYuWY-)oxfk=yYK2pxpWw`byuk*$%#_WvtJ171MUtY{`0p0>fL5FC0ETreg*#9TD#Zri+t6eq=xNF zf?vx%<0Ho8beaDs)blbKyX#^?*1e{ubvb%ic$wg|Yp+XL#=Hq?L0$9uCw8OmFDt%_ zlVfwah26sXj56gJBd_P!-4c?lE}fTE%n~1wdgk>hM3Bs;+$&u#JkQIK$6l9S1LwBu zW^y;FMIamU(ssLTy~5p($;27A7Q-eP=2t4EFX(3-Qg5R)$-Q$5@aw0OnD?T{|0xC% z!%POg0!p8h-1SH4Juiv5PT%wN8Lt>v$y9bE=&v}UmLhqsQJg1S8}qB)c*X+J>2@Dl5bCeGgnRG-R3G%w(t74)0)N( zZ202`9^!`>XcEdM@8lOI3+pFK;KxdmrXd{_&d%w!S0?XI=efFdRm{6iIw6AoRHk1l2r|$?V3S+8>-9JV1Gt|8t7v~9%S^s*e@$V?e_XN z8@fQ1dgGu;{t%bMzoaGgY~W{DTZ$rFA{$;pH0KYP3Y z(tZA8noIxo($#P|pins?v=kYENt8BtYDFOxj0X8g|Avy?;q&~i-i9k3OvqHxu-`wi5LGsQh ztJ#kxNJCSNx=ii3D12?fp}MCWzEFfUm^M#ebiLw=do-d|lp&atzIcDStI=ekY6{Yw z*FU9)T0~`J?tImXHY09fs1h}C8doS^Y~X5a`&3wiO_KI*DBNTu;BuSVcZl4TKxJf(o$t_}={Vy@3wSq#>&JMdc~jJd+Bw zcib`tH~KuI5u1}u<)7p|OVhBAGbL-R3Tr^;;hhE8toF=?TjJd&?j~Lvjb=$n3nAb% z+x)7or&Otls;6dvQY6r7aUg~AUW0$JT6iWw0AA~NHzR@fd(TG{k0i`2>ZDdQAR_#C zu)B(zYSfUYaYxqGuMTKIOwc~!0?Lm~Kp>bAl?3|4;4Jn7RA=3>Z-53fCIWvPg$q`u|T)#2aCbRX5U+I3c;hud_8 z(+;ee%=sBX3`>-8E`N0OAw%-^D0?PYdImJrW#phy7*Mzi?8WMA3M9}wLfMO1kA(g9Y=x#)NhD8hBg!p z%5kKYt(-5LNGhJ#d^Lf_Vq7F-&ukHi?i!Y_%~ecnJq=Ad0o5kq9y6OYTMtG>`HgRs zmT1>i&EeucJT?g+Lj$R(Nwu=CBWW&ew zT9|#Poqb2v`r}O3%DdC|h6yb-m0g5i-ZH-hILxPCFmB|NXE2&$cR^A*f=S*cr@&hb z%Yl?)lF^OH{!8s`t(A-MdwsJmbtVFEDQYx*3T*1=y+O;789t`GF|ZXrcDcqQ$7=hG z93f`-^3GzMRnfptx{3c&DOay6YRbQmp8>9F%{^fC(Jd@uEWo9XcERca%uxx>aE5iH9C&=z? z2ECnkp=j)q->`CfVXAW1ubgtgpUbh3LlJ#*>)=NBquE&h)T2Via17u)7~+#wi!-BtV4-mc3?2^m|SuP7V4G4Q5)b(c9iUFYZDe-XY~Qt?-d6sTNNWY$yfHHfZkkqT8P_ zZkszxR$B|Fa0B0g73py!z#rJeC)=Uf0b;mxF{7AwZ!ZBD_coUHDbn6I-$}g_K!LV%73SoL#YBy2jA#Ooq{O51&peq5s+0BLkvmZJ60|(uWlLSVb zzsYuglTG=~dz)>JneB|Z3>I#+h1V>|J^4Jke`oN?wI2M zgl8+-{&4g^?3>w@cXl@&n?>Jt;PC$zqinPD$S=*x^cQ3?{O2kGd(q9RsK(`$(pM>; zmUXkc)=cs1w^@=}Di^t~mMHhx2pWB5)YL_B+*uYM@o=fI_&EPsq+K#ye*2`MRwwnH z?#3;XQZ55)qm>)WA|vKaMtDnbTh>gKxbA>-a(>nQHF1NvRB3!uX|F$>aT3hlqhAT- zgeoXC5uNew9)0?BTm|(B)n8&{XHg{IpFQ7NgbkZ7tZGZ_c4!s8VVB^r;^CLCnm^E* zrzx&K)8*V9Wan8~Ya3sN%^&Q8<=$%?OfcH6LE5Q0`!+SNXJ;A>0M{XV zVK~+wiU?Q*24nWdLSX+L3*iq3+$XK9?aFRTh+w~4pWW1IpTppKq{OMio?rm_<=L;` z_(HGFWdH4!_Wa$={7f>Rp;HSBx+Ij*1=ytiW(%R%1m(xPIO7}1EQ&p|_5Q>4?Wqha zA9y`3C1=-x?+C;72e&TYmG#EOVh4<&aZBq#?8UXr;{Lz4{ND_$4XmYag$ zx80}F36f9RxEx}ZqJ9{t7K}cn44CqAxMffl+xAh~O(wYjo^9QeC@B#fvv7karQ*Yt zmEq){)%_3xLA$Xh(~mE$IIoO3br#kYet>&7(+hQ9^44&=qCa4dX^pC%TQqO@-Uxlt zFyN`yhj~<{ZEWEgZQG=w{w3o>#|`!tme&j8yU`=fm?!Bs19Y62-*g$@maHt`&J4O3 z6>7RwiIsJb41D*vu7A1T?FX{)jK!>pK_R1wsIxAZBS?Wb+4^7%1EcUPfwsntetXP8y$m0Nj+TiNZ}TlU`eTXtR{ zGPJNZ5YUK;mFer(Dam52A0B)qrY_#w6}*_?#yejCOSu_lOpQ9H+_-7u%^@9KWSm(x zux+}3m|YKYF|Oij0MD4e2k;BE*QSZozOKco1K_4PUIdD0c5@W*Ss52#9j5u z6CA{R*IqYZC{@%;jSEbLNzSR1{(PcdzB!%Eu4NvMralV{5nQX*bqJBe zLyb>?0f_9IHZHvpk#NQKdmq(7jwdOZ_-%FaBaBNrOCszOp8brn~pu58D|5vx3zuG+5vZuac+OPQg(X_Fvy zEr+;r6wfBsQOrXT=Odqfzv|5sXsg8(M0`RwL7}+NLOuAw*KBDsM0}sHMk6%O9xjFo@)QbG8#C9Hu2Eu5hzf@N`l=+|g}H4V z3yay{eYND+7q(?QWs8^8N8M z?UF#d#$|dmt5`DMhWc}%$4$+g$JEyfuPOwQV@DWw*OzmWJ@w0hn!8#L_jKnw(DN3X z6Fj|&_^H?**y3;FIML+fOiH@o--g>^yA?ofYOF9J;z3VIadwk?2*)%~WTD`f^qqDA zV7W)}IMx~5IPI42+icseB|DPY2#0ky!<51I_vRUZ(#!b0cRLTH1sz3JRGKIH=>UlQ z-Jq9HmsyJPb@W?C>r&F&{k-!U9oHLde|*4x%kA|Ney($yGd=vZgGQ}R{>I_K%68Vs0db>ydV)H`c! z)UbRqtnc$ro~`VDFqOF>V&e4~cePXPCU3Z`I|E*8)#$7~Z=#HR*$^-ZnZ5@n0W)=d zfnb?3-co74{pv#cI*fw=+=xU1X>!5dBv7!#QXXtSscy;#lIn3xA{smHC3!e~EoYue zr1pkGmTdu%jE8vDa+yZR>~6Hg7HIoMSrL9`e!SN!Uy#}q2-lR|{vr!c+RGVzw^pO? z-uJ1yWTq`!&FrY(JBD0HppxJ^RKK(7u$f)UOMUpkCHP3rd#tLcpDKLOF6E7 zfOBHwFOIOck=pMRx(G33VS^JCtkR4MKLkaksa;ggQe5$6c5a)y6EzmpzExu9X|`f6 zgrRB((9idAB22$?r+;EEcf!noluzY4cE z+Atn)tBoA4=pigmOC@W+OS{I@doth0Byhz~K-59&LefUcaS@I=@ zY?xvIHvxEWoH@~Ncg=^pW6gc*+WSfq zgQ95(oJfjxWz6~V4E#=ln$eE0mx_s;jVDPi)&g!LR?=4-59*n>cYE=@ttU7{&p{aB zcX=$f$8u)pef(Tx;#ziofpn@oZ^C!okC?o{3VN@Jscw@2(}8I{oP+7z_tZl3&2&h% zmAfcki2Io2;vgGHt#lyb6uAE^Sb2#>7^TOMW&HwSEw)#l@rzWpnJpJV?w>aQ?8S{!?(j^0!Q}MIli!7Gu;#@BUM}CeX+~TB+b0062rC zw)&QPXUK-jg)F6Z*Lw;rSXmilzplxSYLAg&b;T98x05#uS0fT&FIv-eL}s32ZkunB zhRm8}r*Aht0R752jhWjkm<|Fm-EmLh0>=H(iU9iIRZBY@4(GoKuq77;q`kQacdI|> z#|yyT-3vkJAN8_33(MZ9HSej!c@Hq?>chb#Uu#ajM?c8IYHt`&Fna?9`2@dJtSl_A zYPFa0X0|+nm`-tsd%B~ua8tfyg|4ES*X$jt2HWpGSYliT31IaQ8aj^Guo9FF<~d}=w`H`xV!9ISQC%N5`8(?bztzLU47cpN;{H`~Lvt$Pcdm3u3| z4wpr;xuO;2IG;Xh@tkvRYfs);c3J5(Fe4WOG|2{EF42`?92l7cVk#mPSX&@&z6M2z z$rrz_w@H#71oEfCgjQujjAo*QUYIiZLag6ZY=aFLsD>u~z$So|o&obcVteyFA!!}FBLK6uDpstpeXy6|x#o_D_Vb|?+TGX4Q#i-0QuKdcIz|xOs&pQ$|E- zNQVF{@YZV(F6qsiDDIs>0w1flO_K2aD0N>jOm@yGTz#)h+>S_E$)UXSGmh&A7xb{f z^GK6unrtvyU#w#IbNDwdIPT7ZO7rFIWh)|#Lj3r$+tu2Fe(A<`ellvHIDxVSDotvoQeYt=N|;bJQ61hML0D*)BZyvufe<1Va$%JdG1wQ_;J)`^YV&= zk2KD-H!;R-qTYJWcv_z;xByf+=iZ5LQ{UZCZ@-wl{?%m&4W$$T8sy0Z*fGsI>f-z* zR16j7VVq|uf>Nf*VG80`Y(>~I%%_YpS7)$qOVOicZFe|t>! z-0ECE%nj%V@0?=Z=?dDPg@QTbDrk7XghR%FOA z_Q>?ynDo4#)@JkWh-kZL^-k#GmA1CLbf8=-#UN-Z?6?}RCCgMbOU0)y`}>sa-CGI= zeSid!4QJN(DUI^qnIv~saHQ>+Vfax~aP{H9>Dw?2x)_!7B9fBAyJrMd+ ziBzYU^tY$yGTy74jYTB5_q!T^qRhK>fWqEYrr_utOtNHhUfFoU^2gmBy(v{KFP)(1 z=3xpL#q}Wb`?=nnBdu0TM7->d)TyDX{k~E2;ewe@hGrob#e%6?n44(UzU@y)q+G80 z@KbOuqB(g&Q92ckO~Gu7RL{+wvETW1RrXOhqij+by=2TBP>fzInU~*9+m<&b0JnH} zNw_HtL1+grID1%&XCw`Ojxe3!DM*T#6$m+OC3b5J^h-6a(^M=1>+r&!t^w=2C zTHX@#P(|PyXVoiT>M~pw?MqQyh0qAQ+Rx8QvYDhtW)u0BweO&wk9#QxhHZ zaV}r!GHTZayg|ILeeQFFgEAm>oLDxgpN8#OINs#~oZjvf*lzOrz_4rUH?8woO&+qz zv@MY2UAgbF{m%0x+TwaUL@Cszw2`m+)Lic(E{N!D4pzu)y%0pEnx1nT_o&$9N_HOi zET49@*pzlR4sICVD7H{6wkH=99gfYT7uf7I^GybM8~&=T_$JKzpc1i}$Qz9J&}gb} zs>vAmIG8DER`SHzv4ky~p7zJeS-oD;uR z1jz--Js=?lF#QGW$>l>JVF~~VxGsmI2*y+3#-9LspKzEWegW(H{MqzbFwkrnHyMO5 zgvTdGA&Q-|T(D$t>E8)wG7L}>XD~bMFdbjG)$_<4MSd^5B?@H$_@W>}3xduM}1qe0fMA5@gU3 zTDq7U#eKQh)l1L$1uughGUeGe--CATsiJOezVB4&{ao`=4@RGkImLX_GfEd%hG)mD zdg02mXiMhlomdyN`aHc~Si;kj$AX;3w^6;`%N>T%KMV(WS<8~gWYn~sb6fp!T=&QO z>zL+?>(xM0f}4R28d@t}1{b`?${RcTaAOP!+DZnv>27kB z=*h_Jk6kYf+$ENzE-J+ng$%LYWJxv6V7fwck@(v4?=w`jCp#SVDvj|;uXg>y4S300 zTYd#Sy-Y!P?i*c%_GuytOX{2akI*h}=hE7Dy}AcxI&6zr$d^+_%lg z&n5bP^hQ-7R|LSFmrC+ZUSyX3{jDY=rmsEs)h*p%r4pHCY|;-wpLaQoL`=n3^q#V4 zb3}c>T|aON?l#P|2tz2HT10EHQ>SQp=A%BPKt|e7BwW4Me~F zOCyHjB(of0;0cNks)Q;c*#i$)f8}`9zfAh`F%yBpy9(j}y654FsW#RTlJh`PUa=|x zDsp)(Bq9_P`OUl~Q1xonS?wIQJ8?{AI}Nd8sf_p!^s#{>LeiA_g^vyUS6)4-3P+Dq zVlIdcoe&>FpBjw^Cq*dx?FJ^!jxT*z0blJrw{-Tapp>-q!%^b0*`mhL(MYWrr;)lV zs)cLb&D1kQCn+)0mT5B|l1(Tur|l7`6b_DQ;~;jQny`*@kEqiV6(}#za4Oln_8gJF zl0`WB7R^f{MXe#3S{uiaP|AMdOeoQsNAQ)@i&rk5q4IkFlUGZD_D4qw)76VNz?+^* z?;%9vL1nL)yM-qr$Zmc{!5!?$?j`>);S%uu8Nd#Sm)wLHyeA~%QHcH?f3BB!vnxT* zX5i+#1w!4sXZce!)jOGbRf)CvvkV9SW%RvI6MK?aoz%_713bxL&dJZ-Y+3KvQuLl{K0Ps; zCpr^RlISMa-o?HiQoNH-W-H&$X6-EB{_VOO6YQ!PPiX^3B&CS=2Upaamw6^O9le2W zw}X3AiFMZB+EZG!Z3gUors({-U2jPfcmC3;HN%XL>%YvA3|Ujp5uvuQbm()zFQKA) zIwQx!B3A_B3FqBX+EbuCO%b7B!se9MM?XhQGcbB7AATG_Ob(r68X~;*1}as9E$KGp zQjg^oX1=S7A}=|IEHHSXqSy27s;pn7&sesZKn-^2^D|3yz{#g9!k(67Cqrz@lx=Q- zckyenTH8zPvnrit{fk^1GiJLtvJLW)o=?4GKT;uida*C{SrXsd*VDU4Eny^EQWEVD z%_7)Sp$y>Dfco4NyysF}L2ORp3XIv=X6G1{Y9_#7b|>qYr5s^o9BXOXx9=do#jA-K zWdozlDS9_99+ zo6x3S7Uyg<^FCC= zU6F3XJT;7!HM+-n=gqr_ZiV7BuQ-Qm6Uci~omQsKMDbFQt*TLFs5z~h)E{*E-lxAy ztY-hX@lrb{=svO87nYbQ7F_nsRQu+0@#yMG8566idx|&WUA{_o$deJ>MN1b`{ak46 zEPsTFj~g$^`H?TR<#|DY)s{)1^TTt5V5p;8#!iW9B&35Fe16xElSMf!ih@Py8-Os0 z{9iw1JV(As;~!BIyysu!2#Z2be7C|f$a-o0Q4R)*^)HM&OzJV*s^wuCIjyP6?4E3! z41jPp{mxCE+*2SxQ})&(FF}QVo%8MrrsjD&lh;lR-ZY`}R!F44RF@BBds1M?e+L+> zlA}!>%AN~0!A`Bp%7ux~(ZmZ`zulhF@_$t7AnohC3WmSlz7;ZL$e z8$L&5?Q%EU?zBQ5wd*0o8&r8aqcuP9MYW_-j>S)MOerwE0WW$=Wzdj1yF%|>%u@Hf zD$mAFVmpKlh}4xk*>)@U8_U?Iq>}rK<2tHjRZI&5dXZ=O3cw3GmsPvH%7@~W=@unA zwOIS_w-%)peNt1V7o|%|a<=n+J@fP2bN1U(LkVwYo?Lw_@ws{XTmFn;+KPBaoT&A~ zGNXbB2hlFJ@n?y}Sz-7lkiw1JggZ%TKP+#{my4;glf*V7Y+gLzFd^46E{mTQ=Fn(U zW!8)1edY)!N1;P6KX_Ekm1k4Xx%b(LzQufW4|>!q2pF@pk4rTT!Co-n_;qfSksLG5@e9%{ zyQAH7N2wvG)x@iPg$LqK+UbqRgk;N*fA|_j_zSY?LzkrrbVHi1uD%64l9y7zBZL=& z?`qu#nJiMbrj{qb^xi(Sbl1ro`!VdIALF5D!-=?&U_pu|2L46NnVWzt9Vga{k-ONXSw zhJ~-q=!yl59D|8V40c3XQH6Kp^qgoX-*@d2&ROW4e|sIS z_l4ufnIk#@xk7LqZg<76Bbs@iSL~Syv2%&NQqfi`G45@%CIWCt!ug)C8bm8lxEW)jnXZ#X^@ic?nX*dN{|Lc zBvrZ_luiNZu5WG5InVR{`2KQn!QSg$bIvix9An;t4#ug}wU2mFz(Fy+XC^juO}x1~ zZ%YSFw7QYcp-i%*u?GM(32aCZfh@jzgp5yeHvB*7_{hP1o}wPphkbSkF+fFA$dC%g ze^OC74t&yJxr2POYeYyLJ59kuo%?T^DyM{J!VSDh7Sl+^A1e3TdbVLA7{Y`1UF7v_ zemLc`k{tI;ySCf2d@%O=;<~Nz<@yttuyj)t`WX9fl>+!WQMUGUES~)e0e#oYW$qTGLDHTL&rp-QGd?qVk`y(;4Ti!)5q>d7T znK)RioW*{ByLfNU5#+Wr)w}YeEg6h{DQ?{%eKw^mlH;fd9uW^yc=<0gPnkTezl zZ$sjza!9-)_SD(R!lG}BEX{c6vftP|JXjlK@O^D8=x1^N_b^X`VTt^SEx}N&puBX_;#8HmV^ED)Jf%6T?Ab}nd8>L}mc>iFJlK^JpyhxuKzrBOOb z#j09vqHaL1m6C4krOV%F|7cad-jTbO%zUqKJ;FWAOWmtu=8f5U*5XZZ;%wub4a@Rj zCZEWmSD7t+_9?-LUGX{IR+1C{8{v+`O!AaT;mz-@OxdP1a%laMof&jtcL9MhZL&WU zS36LtQh?B9uzVbeFMTr^m>o#r->?Qdpr>8(bY!kG{#JvSI zctWHmlcif5ctiXSGtA*sZ$1y?2y43o-UL$*5rH(W2Whs#!Z*6Y+K8^r|Kj{eSX6MY zVcJ+RXv$=W<^`F&?M1zv6_@DDucDB8L|hTIhhn_(&dhwK+LSL!=vD{w z@>n``lL>|(6E^nNiFM+h29Ko$Y)8zhMUyy>KV%-Ff!Ze~EG}L!m;lug;2lV2BoP>d2)+Tax zRMXbxck89wV3X=|f3@E~_?+B~g?}Vx|hnZEaN98hHh-CwF6Q!2#{K@s6zgq~4B_F^=>Vk{h>tm@(0Y#=K z$lT-l^tsUPVH${)*)&@D+j1#5%u=qmZlgb5W2QN}xYSh`eVY@z4B?R$&^L?b5E`RI z*9~o9b!tv2L_xph-J+#9riqTi(Pq+A=a)Bu&lURyA}s$+i`oG}2Uk{v#~HYsvhV%x zJgpate5cdVez8ubOX?pWXa|2O{La5;ynQFdeG&U=o26$>VZ#E0S9roRP#kkD+QM)m z_gV6bjSu;8LSK1JZ7gbt%*D8D4-TZ7PJ)&QbEQIlTEy-*{U_lHmw7OeEN zRVsyd7hn1c%4qpt4c!lZu4138GEN@I^lt@4@{tnJL;2>82FN7LX%&_f9v&~qaS6ys zX>9tI4!|sHaIl<80g_n%kk07K|9kl*148OFdL~pa(+GynX3XN(>gn;O^>|Bh#Fj7u zP-~pmUwEF)Lf5B^e?McXxT{_CyY2;5eNBq$Dgld> zn|?O=Q*NTEDoY?7Iy4oM$X|{T$c_#rY}E9V2e~RqR^YmuY>+nS-PjviM3nFNQ2J2X zgaEbD=LrD>6 z0g}>oZFpFm>EojRq7hNn(#M9XyHbR|`MEW|6NQR3;zseT&E#LrF6bFL60~G#lE)=) zsn$x1+MGE1Lb#=5Fe@c0qXcw!YWSK+eZ$ZVrp)b#8?pRLXV!Jf*IyZ{ThMA1la9x~ z&?6cMZFwFoSx3}OFk_-tDWj}EuRy_?XT;JjQhPtiQ$%=zKxys&TR%DIld*&GaVZaZ z>D1XEE=RDq7552yUl@@xlAq4Eb&V#2{ta3i6OtqKzB`IMfh)&bm1hl-^Ji(l^qkl+ zbongR6Jj?8jYbCAy0V8!y~~T6D3q*lNZ$Qe`eJ$fyVRVxE2L4BHI1x-G+r)Nwl+{*Yy^cdO6Q3Er|DA#HZ0Z?2pT0+kVn zd%Qp_IyHHLEM9AkFmI{p$p77!Kre{N_SgqWfC=R?L+W zvcb3!?o-YAlYL>>6pSS~ewo=$o#@V6`QeO3jaL*;&wO0)sF&?r)CyHJ=>lZPF-tf; z>sXEB4_!S4xR*#+1N27sXwn39nR&J^oYl$mT0<0HW=<5WNybJ7Cj@D{(xchprpbAf zFZQfmMJ*!qg`M?`WG5->WoY^j0E}qjMUEnT?yncqH{W{?8R4|kB{&9|55*=RP%fbE zGub%l-b-ritBV;r$auLV;g}Cn62wJ6Ua`DDvMXG#FaLuqjX2spr#RnmW{TWwCBSQe z&V~>PjQCrexjxl;YO7O%Z-1BSmQ*G#xf{bPNX)JKg*sqbWj zBeS3CFANrvrDp}$js1}3U>spZ8bynE{#rvqLPiJ2q}XZ z?XH^enIss&!TQ-JP$`331_hhUyqofDXIixhp{D5viz1pG`|@rNUHl`aZ0sijoTT1( zLpSou15Z`kZjSoh>o*GH#h%%d&)bR3-n<+m^&n(MoW&}ti}m`&h-TFlj-4)rBPtDB zTi3hby`MAigLPTa_?igTSrhp;E+C@0DhQCb#j^EOR;b7(3FrH$7cE;ZRL_$^KjO;YZ2sDTrq+a42P`PA0I{}^^nH0vIR8q?u%4PKUz;ykM)5{ zMr)Owah4b;^g~`Nrb=Vh`2Qda%J~lAeblcJ;feXBWapAcJ30NCR!#~-WJ=Ukgjjy^ zVWLabJ0A!gnOfi@tsmd>}AIZSSz~9QI59t@Ci8n|VX|D#~diL@Hh@j2TLuP18v#54D}Dz)u*W^xozA za^HH5?Xihf?>NS6o5fFdh~-6{bqNJ{hTDFQ_i_L#MhD1K9tCK2tpyGEzq(H90+?tQ zox0CDbqfiHT)A?DD*@>tO<)izlA%Us#TaUy-072W>is5~`WN-9^|w`b4xK7*W7&LH z6tRqmE|7t4@ouWA)k3v;F);qnYD9i%f!=qond<2)A!3#05#<``1%l#Nk~~jU2-LNY zpGx9k7$T?|Jl&TH{B8J5GbJ7C)RVtiL;Y5oUSW+?nVaIHa2kE541t7@F{+@!_FWKe z7(xoem;%Ds@-vO1$3GMeOAd5nytS-oYu?e1X$cV#F_b>GXSVMi?=X`4sJ9<_r-Wh9 zbUn<-08VC>iEbI|q51c{VODtH6STn5ew<3oH~!~TAy2`NSVEQf`~H8F0tmO9{FhX2 z`Y#0LU4}&oaRT=bg>ex;f7NpnF{t@bs}~H8X`kQS$m%xwWZ2nY%Gq>=H!9nwp$cN( zz)O!KdSbgglIFo1iW68tn*}S3Sv^#JV{p8`d;rL@@dKJh+aJ6wKwC7hB?)M_{8eDu z68y;kIxLOhgV*gadr;GHBChz2)2w6LtvOf&NkoC}miqaiwjFa#)&DB_7x+xhz(vz{ zJS?8exkLC~nSrbjn^vtGjbz!55)BHK7Z){h8NYI)&z?N=F@Y77QW(_um{Vh}`ad)> zfeEgag?AD7{%04X-v7%r;Dq2n>736U z_tPvCQ(cRnz)wq%$=M3c4itRYKFp-9R(>y{8z;Cy@?|Lq(;S%a>2u!vamNIIuNj#z zv^FGwqeSITIhuyq=6JO_Sg>Ies z_XvFYoK#A!DQWOnS6_h!eJq-RXc{1et{H_!fr7tJo|Khsl4`Gpa!_@J<2I%DjLK25 z0T*M6edFv;A5^ZvUTgh7!gHS6AJW&B!>Q6>OvQVT6?E^K4)*Fb(5pf`(E^;@Q9%B( z390iVTduIkIWcaNm5qC1Z9Ihc4NsipWPfq7l|(X(_&@0Ol{oBF?@31g?c}Jbz>_~Z z?9q!0sjG-3xn=klyg=l@2j7WM_?CL|Xc@}|)0~UxeIOxxEvcA%+u59Lt;gYO-J6lu zbk*>0x88|^A&nPxqbDBKnuN{uOt%F>0<9L3X;T)*bsx?wR{v91E+$bPOaLOBN`Fl|2q7`qo3 zoUQG(qkd_j8(RmXy%I){707xnPP`lwQfEWOef2O+*dhY|#toAZ{=FxW2k>k?kUXKNI>DU3NBzP&I?ik(t23WK_rAfwiqAV31v@EO z%rhz5r>_@*N~Bzm3ewHU697I6DG7bE4kdyPa}7j@yvZ!2u2Q-ySix5D<3Lwp=HMjh za)nM+O%U(Ce{drMterMy;$8fI zU=90-AOc_kNykb@`{!bvrUuZUIbA>XU`Ycq32>5uuEhkqzKD9_}PI-XoRc#PL)GyTkD0k?qcnNJd~efzUvonr`}H3n#WKq*)5g< z5>or+3UM1V>Jul3vyfg67U@@ASQqzP8R`#m&{^l|Cd*>mn6O?j$@4JpR)9APEH{o)!f5KMHaJ}2C(him@|85SNtI7iU`IgC6k zSPep~c=ia)Juy&2*&pszpEMl4JuDM|@?H~&ODQ88BnUMk|{oQ1y6X-YhFT`*B#p_GWY*PwHNP*=CEW`nG3B6*ajMugr zG|N~5g4+Xy*$s`2WN)zcN#ota-i~y&dFxR6EJ<8mXZhm_W$^~!qCqd}pVBz5qRi$M z@w7CFjFAeU!YD-_U@}hzjwPNk-Ad&FjMMOtIt+LPp2=XybVyw!`tokJHB;XlZAje{ zU+I^7WN8|gCKSx3c%7-`J>yaoj~G&#WAloQ4V1aaxCEKEQfrAZoX?qa2u2I}3tt3t zXUbubvqOh-SdKRalex@(lFXARZN3O;{*EgNQdfN;e;2OgTyOi&aOybBc_tp^9qxR^bVOO0%vrBb7-pR2ttblA z7f)V|C`obGFFzfR{qR04sv-SvMVeaxBX7vpEbQ&IzeIzXsj}&X#Ck}HOG%UZ{0kbQ zJ;y@EGA_&k(tUBH{XFl-xesUbq4|Ir?ENIU%Fck~hosc+H9^$QCOJV4O25m=s`3^@ z$_Qln74TMFhBrS!V_F&tkWOl5_>EpqIPwyMqe>zDW6GQj=gNz~u+&X+bxh2_T?AAO z`%5ISc3Vm34oPQV;nh+CR8@6TndX|D9 zS7Xj5`XGosz4$^G%VnkU($n$ zy#IX7{*GZ!4)dz|3p%-Hy0z5@vC1yJGDb6N+DH-e>c2A|tii-Z0NR>Of)3rI&TioD z)P;+9`;gb&0RzqY)l6>qfADC61$Ga;*ZxI}e=yWXgkBy+mZ!1G zj-G4goGqB``kzyp~fQW6Z>geCZA=)N{n^-RZo{w$19|xP?iN0EKSj!j*I5dE`Mv9Pq#7fCesoBzQ zy;WPBnHjcQiXQO0-eWr69BxX;GNHmH;iO&+byz!~&EzsG7P;D@vuRT2bN1zZJ`mPu z=&CU4{27^I4^JM#1yyYe{fl&8$(p*0Mr$WaMW7udsW&)gBY?LsQQj}>Mt-HU;pE;v zz8SiWw2zqhw0_Mizj8*ll)K6B_nwy~5`(K|sPo7dX`c;oA<1BZo~5HZYl#E=P(9Kd zr#lBbvp6mA$%tun`ZG@^%hrIqqI-UjH+s@GW%k6-2>35sC7UM~so#U`p z2NBS>-OPz6;gSr)p?HRfijx9d)8zP%koX>0F94BlVG!xoOd9upcMLDk2NcErZaP~* zz;^JzK1qj7OQqq+j~rVL!$c0+5uJe;qaUr9jj^hI9j3fM%+0-?x=#f8fc0F{?`0+VKL>ZVbu@`T$>k0Yj->-_hB` z%u?aCC8@Qse)d#=yjl?jD@On{@47}eh=8d9Zu`?Y(qUT;j^;jA>9$kO1o)5G&le%_ zFl2mT?*W)3>~;Xx?;yC=qyqudJy=x5NV*+g4M(=4iK1#b8O{7rpt<^lOf?u3rw|V5 znJ>hsh5=#0dTYXNQenm;xJ26QBHN@D`ovGlC0`%OV=|eiw{*G|w;QmNYXpXjn{Wi9 z6aMV~`_*49(u`*f2zo>CO`7{9`8#I^NM8tezw!;*Gf=dCD=DPCY0j8FJee}`M=7uH zMTyZP{ON)q6yE5X?sf62FCpSmm5VMP-XAg1JKwrpa_7o^3X1c7S?q^?SRZeVpR)RF zR`2PLytH4!g#DAA7^(&g`q-1~l+Fx)P5H@4$JYuq3bN>huzcl$pU%Fr7#%AJl_*m{ zhn9e$I|v?`5HKCrNZ|fY&4Tq<7S1zUy#K3N&9bnXg)9DV&Dx`b)0kD{_-5|S2wN*8 z0w&h(cBEc^uh~6{6)bzg7p+PTI`%U?Pbm=IWFdHCwYEiUg8dR~o4Nk2u!+ylIEt(o z>qWu7w7MurCQ=BU8!?+6Jti@G$_K@?7gA=yCaPsM_<>@s(@L9^Xks&B90AyyfRkKO(B0#z7!iptwI5zpnzzk^HwHw1~Z!->YKDJ4fFMj2+G2 zZ_X3FsS$R8yZYRo2=@kyYCzbLd@uNkFX+p9UHrXqOaLmUk4t-B#T*Zx}>(2zK|Fqy%R(x$thvH~t2bhUmxZkUHhUpY#1BZkt2xiJ7*I zSam89O!}6eYv;~*qVG-zfRx%CNQ;tI@ReXrZ3<>bi2CyDe{M)+GJ`{mxKqBy8 znrea_VaB_Y%L=qo>GmycU>xdhSDe@j;T%`K#Ie4i5%Ei{NLL>^2;cPkkYw~7%0aVa z&cmQrgbt++4>+h>RT1vn6gwBhT;jAID>8D98k``5ozGlE+9DNGc3>7<=QIEnOw%{# ze6Y3<-cwMS{=4u`x>yM!H#z?stIH;^DCh9yfDj#pkScc4qm`rMMRRb#dU)$BUlnJkVpwOsZ+XHkOWy!e@g6Fp-Ue z{49;JyduGntk80W%ur4w8Z;1~>yyK;7Mg)N167^1oh@(SIG)@%1*c;W-M&;2{4wOY zU>Z)5;=$j-O5{VGJ*d{10gm#lYlJ9>_)4vuX`%4Fw}VQ5R^hmd=vJq5arRyT2l6CE z^HhEwgvRpgQ!RrpR@T+;@P<@+Wdz7S2Zq!YB-A9pXut#-SW+A@8P)u!d1wbM7VA7A z_$9k|s&8N{0Q7I|#n)!>qKt2Uy$Qd$*%VKIyXZSDgh8Nd)+QTm8HVjC&DXUMlQ_&9 ze}5p(PB4Ukqvz_~aiTG`&K!hqh##1XqF3cU1d)wu(im}HjD(Z9k*NJ6fh z6n=otM`Y{~UAE@2)oF|)iGKwPwa>CBLg%#d^9A#6uC|jSMWXegX`);X>L>|HuR_`Q zq>rDP`{FKSzRR@e3w<3kK#$fawl7A9%D3I$T_3@Qzm9)zVq2~kz7ox1*^^%q74g_F zekS7TBt5!N$`(*BR>^q=3g`68N%!GREZ2=kJ2o(0*~Z;!r^#$v=20|1(IC(>=zjBR z@a2zvQnI*3ZXP`K0Ug>9MnZH8V>*U3ex@&X;ffo`##4Tq#S}+_^6+-f7UswbeB^sH zOGT?DhHznv80}OXQYWk2WE)Z^K|aOPse7rW@gJqCCagYbc1b}0>FWa>U<2sqkn-2~ z1ViN>PFrLk!2yDve*u4RH`Df0qW9k~atl7kgW&$(GTx}npj&$A87V-ib4jPMI^a*l zzpAC&72Y*V*EI{-ht!e$MokQbU?ChLT-%X+%z3+@%ieb5EO@n5$o3hryQ=wT)f`wA zfsYQI6w?sh%pHb5AUc3|ga@1MKq-_pQK9^AFBH%#ujC@ZudC)saeIVB!lGE|$-``X*QwdvEC}oK$ zfBL+F5y)Xc_?0Yn?i15;MPqjpyLY+~#CKVVaA;08rY^KflX%-jRZL;(CmJ*Z3DX~+ zkS;oZ@qz{g%DCUW^U1lFA)TZbYg6zVS4h8hDLuw1O<@TXfh|Fxir_yt$jFRx;$MrIoBGrf~;}7D&5<0vpUh7RXs;{Rmh2pwa$MYzT2-mk( zRMs=d+q-3ewS@IS3jhM*9paBt;_tD#wG>|#+!f-Mk_)+(Qd9~;hn|p%`}w4+PYWF< zyUjjSNf6N-K@hJ)Ruutq+i31T+^PcOcdHg-6BG2;`)zHZ)i+>;-!tL8`efhbAGrL6 z-+;qwCLhiC6TY4c*dl%eXfCbqmf^BG5F{HU_#5YCUWV)(k3Q?O=rDJNrX(jVy5N2X zkXO{EVfTBwYk25?Q;44b)@_0k1p&o8lOS0387V;NKkDA8}M=ihv)e#%3A=jiy2A_6FA0J(LcicUoRibN2}JpSEZ;Iw)I zvXk?lvJ$-_o|v|H>wM9TNC+8t++$V~j&tbHAw#&)oe$|oNkB%_eYDFnOg4(SpT;|E zYBn#cAFQXZ7wf@qH^V7-G9nf0+Bom<75!-Bsar>y{|Nip^&!$S);@p2@By&ge);7# zZFn?O7_V?VXNl=vS0)Oia{kM5^C+SOu2PB-9>wlzac}~F)7f0F?!u`n$)?@UTz#(>Kr$2ER zN-REOx4llXB6B=vTh$+oD@HAT(Sf_m;cSjAlT83cG1JndXG=#h(9RX_3R>2SU|>?Xf-11M{6480!Pkm(zsc9-Ao0MK=;3Zzbb6O)S5N^Gd(#A{q7@wzlNr{gC6zu|PPr>_p$EGp5s)te0W zkT4yX2$}!*k8BfMn zPHW5_kYyH5ykOt-n-OM=Az8%|1*#`c&6Q`96vPyC46io$ zmbV4vsrR(Exj17>i736ohp%#dM}HV_pPF>Bt(!Msqdx86Xe3u!Kkc_6_#*J4j4W6OgYy-!L8lLwH1}sn^=&YN1`ZF1gW7D`u-(_+8zak2t!C`Q6(vOY- zZ@QF?Au%-t%kSXP3NDUx(36-&mLBwQ)Ci){fVLKm=BWH%2BJntl>?x&&ulJ>NI_sn zxqg-iR9NXeI)n>D@v}q-7T6NrYKCue@YO(?*9BoV;101UH)-&|+sNnO`prGW9jN!Y zEeA+4I8B29V=yFki+Zv@L!x-~1U`m5f_hlwHs)siN1J^A6<#l@#S9PPEMm7k4yDM< z*=Fn6Y1OR9(fPHnn5*VK9GdWjz$j2&mQ&SIz8TpJsAMA3#9EU|_y)1LyBzj&=++Cu z3v1{(-4=BESQM z%7k26zrt6*=r|vhs+N1knS$=U%vc8g<^=ARDYkTd!~I44J?9bWnfNkQ)fEhbUy<%K zZ(G-*?F3n47O|%R9eSAHGAq!c#et!uZ08j*k+U&|1nnW_8f^)Bpc)|>q%_u?a3V_} z0%AS;PFr8A(oG(zFDmsruTrQGnvQWj)%><1f}DkrJ}C`%jgGc+$4|`l!)~hd#RcD{ zo1-*+^WHIrv*6@YL&I7>rlk+F$7lH{ zg0@=8%2(@uCh9hOBe!AFba;mU2z8WT!;a$c_do6Dl z=z(hiKQixyL`xQAykwZt%89xhOhNu+FF+1{B_nz1;J6#q685)j&HeIIpB!Cud;@O`hY|rg)x^=KiaIomO_2_3HpW0T+t<9A%8?VBNa zmBGFsq7%uFW&EQttHJd~X>QMhtg6F#H5e1>LdH(keOO-}MHP);7MKn*s@h7s2wpRI zkvH4kAL<0$RnS^H=+hy z#V8sRbBfvTMdCmJwo?a3MSEK7&Zn`EcYV4t)Jv_bTeujTQ8fo&>^^D7kL%NO$!C^mx=B~rt<#N z`=dj6vp7X{{jEM3Fwq~;X!z=^%W8^6Ke$DnC(m}gz1Rb0)XH`}M^);{fuoJX#0 zjNVOw$pK~sS-kQ6>j^48k?PZP{IW+`KvOLith(AAoBlvH^low_f}v*Uz5tvyD)0rl zarT@}$rvBJAQ%dZ>%_1&uRj*1%eH5nEi!1S(ZSW+I4mp4VI@4bc~yqrC>+*!kLZ*<(Za;Br{={TuGuJhf}o-Qc{f^CCGa~!Hd z)3h2nJMKnVj@F9KsWv5F-J8>P^k_kkbrgDuV4Y)iXVGsXL>%AsO5{(*$CZB4xQT49 zS~&sRR2us1b;RK(4?!6W3(5|O!v24P^2tL`!apdw9>g`Z(Qy2Kb3Ly}VU+V)vGFf+ z@}rrQZj>AFe#tJb%x3@95T27qiH33FcOJ7YRiYN1A?Ur7xcoP_!HeW(;%MoF@Cj$! zMV_Z(2(2Y(ok(cVwu>JeYk}8P)y(1~DYadVK*S*MGV1*gp*Z7yR#i)q3o( z39#j-rYL(Xw&kO%5m$((7>xU?%W%GZVi(6DL*i-4g_Kgx_?u9_ILF-=s0iFXe@vU5 zPf?FqKtAX2yHxs@e2~;^kwUWho#*jh&oUSqA5%KKoBd}g)iWZl26k%La91IQ$S0TG_y#*f{e#jG#hyzwOQS<~V2yVlp{fYYu-I|M zA^GlwqG7OI<(L~<5xX>DFm{DLb^q0qV5}Z%>g-wnp7hUGe9Y+0>Wh|F9eU9HFO0EH z#ryo;)7Fnvs}!>!`ctT%T7=TSNS3iwpTIXyho2?EcD2G3<+F)h*1pBIxZq~n zI4bS5%C)M!jd$WVM28OgMv)R7lJZ#g@J$=M#HS^>is$$tuyZ*iyq^Md2Q^q(T6G)o z`p)U|t037>1wsUb<%)-tWCUzMvwbG_UKA)t@}^+PaZ2vpE!}^cTs3c4RPAvw0G@MI zZXfW`J(LJD;vGp9qTmH!Luk&s1^cqB|kp1=y> z?;=FP#1m6erMt!=7aNF6{CcqxZ2@Eyp7?=R#ZbG3OgtxUJV40tKJ(qgoTAWIU1<(3 zdVUZKE~GS4|B-T9W{`WmCs4g_I92eB~% z|4uho2H0?-x{I2BzTiDiw&Nw!ts*dEX(vySdUMEj`)wCgi_jy+ny%6ows)NMT>Tq*UAKDSY3> zqTtPd@qkNsBgC-_-KE4uaaxX>OJvXO^>p*tV^ZEA(jIf2mYvwYd9c z7gMp>!_3=Gu1pWP<<)&5k!V8|2dR0LRySJq4I#UL0I=rRZ>m5Xhm<>H=QS6#Z61)4 zk&#Hd(@??d8?54@fV~K8F9f<)BAbs@wL5jGsLIfdEr-;AM}1vJKuWwOpbeyM*udip zcw#U`V8Z!=zNxXtvASs-+5|ts#au|2%*W-<^wO6pPW-_2ZbO1|1$odnl7RV%Ce{nG zf9WDGbP%;(=v$OFVZgbEkY@QX-I2ko9`yX}V(sA2c$?PG{&j#hClG?tnL+how$1py z#aNokb0X1sL;|%J?3T)uehyj~X&$EB%b&hp;|u6?cdRP}(O&TIF~aP-4K&SoXiz|? zR{rwr&Ou+sS`B{_RFNo@G7!jw_E-*g+$TN^>ic(t8MUr-GiIX~>a1dso^aPFc+Uc}W`svOq74*nP_wA#} zDawfIo_SPr9-o6;v9B((HJQy_f4ZSpgo#5=9~KQm&suPa)|IA5!Tk0vIKX!$lBtil z^#FMmv?_bQb*zFfcVWHUS$BdcGXxIv8PR8677}rKkC3`FiLPWcC~<(3WsHfMX0wFu zHnoPDTEd_Un`XLVBe|_dGr`cOJrM4`$9~^{-T$bN>R#UO;@*#WzwN%2Z}L;jMoX@{ zq_t~dLR|24Eb>6i>2Ke)mVJ%F`GOzF-zIV?=8wfbhltI;yj2RD+!5B3PhTr?E_3xw zQqe3)QgAzzR^d*6p+H3BwU8TW#k{3faTrC7qGXe$NOIrrsW~F=_02gZy&THSf?3x%{G~$i zhX`8UJ;by*O#j0zVBKp17`%`1U!(GF zt7jsniiB-ha$QBA+ifM}JD&+c``>&2eQNeIrUp%#@;8<$UMfV$rIYjoV(=n)V9dIy zAG%Mov>NC8JV8XOQqQ6U|M#0~3JNsn!800dpzA;<5-_R*W$+KgKuS7S&}E|suxRu^ zCA9TzsQN>>4BRma71JXwpf$wNtP%=4MWq_Ryf>jZ0#3++{3kmY0ywH-hea(XV~R%` z`i^}V*kXtpnNdg<$#SvHDfy>osrV-`vayS9vTe(;z5h0daqi`pSBs|K^+S89ix?zk4p0sx z$Pq@3BQlN4scxg6*+}-ch~ssCNX5g{9_cXP=_h$*U@3$qM<*lC+}{pse{~gSnwaYk_u6`IUih<{Bkj5&$`0(NaGr*{M;>$wbxm_bZ5AhYL>(| zScWpR5n)>D6ax?xGrW|ewfv7iMgpom)Bh;Gd!Em5{nnmeb1r7>W;AH2ZWiM%k1zM_ zMXl>IbSS+1!b9rfg9&f>c;D?}{I@s-3c+BV9WL*``T}>n_B$kYyFcQ9y>?z5c1+2= zy@8&w>;xjBFpdu1cyP@Rk1TTRA1C^n18Gq0Apd+1@ZatQ_gGRcW3@4z#k$?#hv{ zv{TuOAkldd?5c)ZE@@YuHjG7mBcqFY{sz3A(!`+?bXxW+lkdM8V`W=^``_QL1E+RF zL>{%parr;$ep-ZWjtEd%c2IG6YSI6&i>#Vkt5B7d8DH7c@JmLK#b7!&lML!CkT7OW zF_kiFeQJVO*4W)Cug5qOJ{mxi=fd_ndjk#OX%V@*>C9cKefq8qdI9(jEd+(|xF;ghTsELwLU&&#l| zMSV($Ew-5zZ()P$JJD<S$XMk2qVOjWT%JH>;E4_c`sC5Wj0%qnBQ4CLAqWj8FB zhN#{|p2Yoxjn2o&M~u8Subyv9^`RyCF`Vj|(6Vn1Q(HZ&>4`KcF1O(KS61TT?-?cP zE4Wp&(|S$w>@-RYbvz0WX*pt$bkLTF!A~w#SHe?GgJKDZ?`y`VG9(4X%s4%6ef43x z4xCq~;u9;8m=?-c(ILs3Rv~qyOyS1N%vt_AQd$3{yJ^4%U1h)R0Ar1%be#WMK=ZC( z8273>g7V+i;}QmPDV`mTH0zGztDQ}tk=p&tDXzkE`_-6Wp!}`7i3ryj<#Gi}C#hGE zAmZyE#f^BAB@G`m`Ja^`tnuJULFQ(lxxQ=t#&@*==3G9X-?x1K=z!n;qzfo|n{yBt zgyR~%PZ2U$3)W!WAp6{uDs!AOZQvJhwSK#?_gvj~?CIzD#>%4gsd|-PLZr*U*0IiT zFljHXVq`jd|E*-P@IJdS?lK3My4s&K;EAoiQ@-H!`{j`#{Pt5e&;Ap-JU91OP=$LP z^;GcnwNkqzOB#{jS?zQ_w`a0}nP=4FF>rNnA6ZY)@#V-R6~?=cBfcE4mSP!EK{zZL zd7;N$M1Hcn8Ya?7dehzidbZZ$2oRc(zb(fqx}UZ;8f+_#7i71!Wtt!F*icVOLyv5O z+dkMo)_yU6;J0=zz_!e6r5lI57J9|XsNYNyb!ktuMQl#9IGOsg_d}41sga_3NIwMg zlzn}zu!Y^)V$=bZRD`Z(>D#jQ-Y7v1*5YzEp}nI|J)6$e$A1#I zTq4yzHGB9AS&Nj+(P;kCLakXrmhoX>*WoORR2$k{@mu?oPhg%9C1*1v_%*<(5!n9P zyZ&SQGldb!2TP-FFs=w@0dA z<>kH8%=7M=dnAonWMU&~AbbzFP2}pa6yV2NzgUWoF>DnkKbw{KcqVdsE1jK&L?HJT z4GMwxlC6hRN0R2b?Q$8QNnoiKk319yDy%)`FSz~}U1xGz4dImD6BJ0C+6NssIv>5i ziD=XbmrllGP2v0Fn(lF~XsroQ?;-IHP~m;14vyc+w~IxY_5=dGb?}OxI>E53g=Wqs z?Ft(~r*Bbeaz~G8Yx{nHkzB7_YK!Z@ux)}%Tu^{zll1ot(=;~z}v0r?&5l_eh z#sct2R0EN~bud(L^~7T-P*!#Thx?>n1I^_95|H(Oz^fD$zH>M{?U^P718m0wKWAFK zOxFz;mfKxy6^Ul-i>r++$G&*~6EodbA5Th8oYu~_C-CajP4f=ImnS9SY$e&wQ8Iq? zG++5QxPdZ;D{fZ?1D@5nmD&uyEB!lc9Z0taAYc$WNjrJ+|90}Iq&5Ic#0j!9#;386 zs-ta|OEMj$!%&x%?~%?x5Khm!IO6bpKeCrTKWf|c`Cbaa$GkEP9N@! zb-MM^!nh=uprjVizH`}qI6^1&1ZXC4%i!!8Mk)-+d%XxYF?;k{(>#iJQZ7-YIqpAy z5~uj^-itV&qHSpy;DKl27k9X<)i{0XxQjQXrcT_{O?&S1eKN#`^f1@LRvtCQ9ST>4uUNYxRS#NP`->h7dc{0ZYHvL_!eSG>&*Z~J^xW#m<&2@}IWRH;qC z=Bt--Ho$?k9J>c%kEy-*KTI+QOcB5lG+6v_XJO5GU=&Q3BxAD6LkE~-(i6LP;AcXT z?aw!B%Hpwafl@*lEDllU;KIQP#uWVNs54G;I;kX2yF3Yj&}-+b-ACw zkIDF3zR%FFabP8d*+NhULYL&!amkc-6>C z#Ad7p0N>cgIx*J51516*luSsS3fb$&FdUeNhM*!KEC$TB(hosR{eu>kOd*Z1tE9gw z_b-Hev5G4 zATDTNUQ~KmGiZ3AnIYoo?3}$30FNlo?~=!Tg7gLPI&z(0$fumMlH zjkI*PbV)bTAp+7ZDJ6}hbV&%jNF&`P9ZGjMf`BwgOL^zZKKq>ixMSQg?uY%wEko8^ z&wAz)zpxF#&Lp<7%agl^*o;5_oeqe%wE*S=5F%herW}ToTWSyqvyBKULkH%>i@f7w zhUeAoCJ!d%)qHYisxs2#lVW`*Ay{HI^a@#fP0VL$khDJcq@>)Uz#516C9po%baj~eLD#(?TPu+292QF2cR2G? zXUwm%@3&RfkSrB7L}iS;SSXj07LpuAyY+Z1GhrY+XZUH(Km4Z`gf1drRV7?boKZdw@zI>t}~hn@9I2SBhsJe97M?l6*Ega>Y2cT^X@$5fNU+ zT13>l{Z3?@bnN>}gX3WiyVcTmDty_jA)<27L?I}-za3W9t z+n-OFEPkEde*-D-_OK)C09?*4fBmk8hB!MQGSuA^^eBW5q`Mdi0aZ?+v|ADlovU4= zEeC9WffO{`Db#E8DN2@ByMMt@AmL!<`3(Ls zofGzJjIvDVAn-LtuU(;{do^O^Bv8_k6`R-&cpFQc~1s&v<3B~o%AG+Ws5qo zccHlN$X-&U59!4|EPcvVb!fEvJ1)~iO?lUE`d*1(HV6T&zsd373x&%dV~GmOM!KRk$q(<~Lom*S4fA7An~o4K5}~6GiyARS-p_f*;(I z)ktdD!E&yGc19We@5&~U+y)2oDJPs>)nFAx2XHAqIFvlo?nxyJ|9&ZDYD0AiVjNh0ciEy(>5j0$`6L)(ni-s7)x9eG2g`x{6~1H z1j^$r35edIh9=p^NpL*$F8kwY!7dL{s9f{|XlUYP5qzg8WN;4X!Oj}M23BbxfzGq} z6C)?z5cL2Z5)=@pK&#vk>O4G@Eg6i^$GUOV)>;D?K>UOBki&q^htuD20n4!wE#PJt z0zd}5Ns>zpiK@=lZV2r9@PM_=adt2k9u6VYKXWwWKqak4RJD@FZmuF(Hj0p5{)(TM zf3uutey-cM>*@bdy!bv#a=>aTK5*4J=n;aJCQEhj4(TeP7ugPDn~zDAJ&LQ>eYJE?37uQ_Y^p9#n2-#}B_qN18(nVK)C>MA*+MSFFKy zr{pPS1e1|6`*IlObeFJAtSOYT#2?t+FA?n|MU*I-EwA~WsoX}6zZA+}q}Pjy z=|jhLOR58s<25#jsIPOrzb53Kl_=u>Q&cE*UE283dY=fanr4gG9(q;CEG@L)!5oME zpE*t%beS=XMNa;&u#I{nv>`ssVf^3L2MKQeMYlCjJX>qpAoB|0u}?o6q=0MBnm6p$ zTlV`1mGFuMIDQfVf5>y7sd;*GOp&5%{kf|>)kL>G_ zaXUA5jW;9lMoE*M3lVT}M#bD`NxU0w(tpGx$el*eCCHmS|M>h=iT*CG_e|mWs!3f| zstUIv&CQk!k)Ho*z zHSEihwfxu((dHmvN$5+Z1q-L;I+7Ge?QJnsYgA8V94ID*8w%=6BNrA{{illNeURwH<17<8KCf)Oq+W zEX;0tIy(E#KHDcR49RX*YFnNxcd29s{yjZ6Ejy_UIArUOXk__)85N zwF2E<*A~8g?>%YaP6a7<0BQ$Dr6%zJpz#A=OT_1+BZ_31>^omat|41CagRzy)mlHtfAU*Y5PKbYKVK#Yl>*Yjn^{5oaefYqSjnS zfcv}voH5=DJEhp0Ahz7PA3Jz5^(@iOK$4Zew{`r*n%V)7;PHv`G~+~(H&x9NGh|Rn zf1+TiR4DNy3i!fG8r=;=rFf9bJm=Y1=ygUg+(!~M(uyEug5QOcxBKk1omK+`NC}(! zd@-qhK(h|tvrh*>)bXFEsFs490rb?*aMkOI9H@;u$AVJ#L$J}l;U%Z@b?&w}-6sC% z4J!g>pfmp$7a%9uRc}YCmWn5f@FvvxrRYx?wKoKS`Y^&m1;GR%^Rh1kiWJZGhsKY8 zZ@TEeqYQP{$K0Ag_kVM1yX}#WA~+E9WtLsbuOMK8qdkEJ*dX&42n~QQx=RVd=j=ei z|Hu@c5}yN(yhI$aPIvuY$lQQ(9d%-qH^#;m@Ut2r-h0K|?~YzaUH2$>B|lwnZ`^`D zq@Ot9c{Vpn{x4pCD=&ci2l0zhmsSUqN*F%GsJ;u6xQWX68z0XAY2#EG6maccU=!7> z<;EQqJ}V~N1AjSG!-CQq9u{ZRYFJDwRzxvwdfH7KGgb7Bgg9tTFi#ec9veSo=wubS ztH2goIsYS#C=Y#nd6PrFJdlb>r|p6DM4+%-r@)9GX9NJ$W?+1RM=_aLL9vbp9VEcD zD~}5P#E4Sb=9gg7ueFM~1GPvJl+5#Rvtf)lF@i%dwymEbH?Wj1Z?~6_9_|8kSz;Ft z0Hxue?o9O*c!#J!?-vq5XOnwO8O*EsIT8Sr@QmLtv)dI?>DJ4{^iMbf@DhY4gqCDb z`8`0l{tgn%XYl_>dhUVk&UCv>YR;rxF7egYQ0kBvnr^p@QXizP6G6RVk$N$ z)B+PjYT@EPJ-$>6sQx3jpp9GFzx%t)$Ne26$iM$e1MuUVsJMc#=Q0vqr{9)%hidI` z)3+7>-Tz6rDS!tg0KV&LzXem@^EbA(T$3a;cNpGe|7rn1?l%GgsE1P6YDE=yBq$Gs zCWhVg&pk(C$k-+quqfvxE1LIM763l)5PRMyx1(<17UfzPj82Zt#|V9p0?#1~>t)u@ zMiX_wzt5|GUT=WnCM#&LvZ{7y1D<3*P%RYzQ*aLiUveK$ic40#V2wXer!1X{li^Uz zx6|5HU#kcsn+&UOB0t>G3a)W)!YfsziF~phj9UM?Cb!P4OcG_S`e*OJIJLByqEH;o zp@&u*JOgvv^V0N%`38*?Ha#=rOX?KetjT?UZU^ejqs5?GLuFRolotg0%5$5OU$x(f zq<$VM%;Zj}(<@^uW=mnpRiE?}xQL3Nx2$fX89_=&B!~z-Vwm*&8k_3gn9RFgR;@t` zQOt;@7zfxwET*i-3>*y|$H*baI{ph{FEW2b1^@Uzs9<>5HC|BEko}y&H!{OYQW0F5 z=8&gw&xiP3ESs}sx<_j?}G$L zzbZ?p268o?CVxH%`qs|GBb^ z0~OJeI*MVgB-w?e92+=&?=*-Pu7xS*VO%64UOLI7vdwp5+h?YS-Re7%b)>B|H#=JC z1BF{Tn10>AWKap-49q(+Ak^yTYsSCTsI}DNg(x9tHILiqeM45;krw)DqHIY`I3yKO zT+GWNEzF2}ua5a6*5y@$2ODzceBye=1+#q}e_a8~j!%;|!(t>N%adM=KQQpmLkwA~ z>n`XmX$7!@#N*}dwvClI4bd)8%+Fb;rIt03l$p@mxTW=p4$_wZ*Sa_l*0SG*^o@rt zO%tU9oXFfnh_DH9R|3-qF#{ictB0>}2PG<>8os0~@idT=pD3WaVxiOB!o!V1v7J{<&4%+{A$I;7G6D~6v8^-4(`wWW5}K?S za*>x%RjxnFzr+|RuRD?Rixt%zHj-^0uXMJl9)(|8& zjEu+#f1rsBdRa1wW?rniBQZbqyqBW^mn>T853R1v#9$E`a2RpbPL%u=}X zEL}!ONG-~9%E#n@0nNx^$Z-|_O%7U*$)WTAl^m|x@|T=C3@bVwy!G1czw+|#Q4;MR zdy|n>f1z7vl8I3YT_aKhtk{a{Z31p!eo4Y>6OTdsOv3q8_WI|&{duMl5}0AWU1@d) zdMzwpfhX@)?9y?3$}AyS0HyAM=uJZUIFwjwx!USg1JFKnt_UGqK!VKY`p4}&ZQOI3 zs}Ce9ECAkp!bzsJOu82ccFv&mEFxK{pXzI#ybF{;b8KSX>J7OZV zvGZh}_4Kcab&xni%_xM5=RS%KMXYZ*R}$xv)RpBb6#N)fe(8q3HqtZd_0i9d3CjJ! zjz)`u=yCd8M}NP%BEJ%%#9?`nemDmN#{2wT((e;k0&nAOH&h$YD&*M#b!RO8=*hPS zy3bH~E1=qs&e>{`Fy>1Mz)TW?OHHPEyFr^_&ky3rL{vGNPT-5A7sG)gfX@nQGw9MrKcwZnIvp(8vg+;T7V!f)ffBLCSJ33d z_=fR2aN_sb^B_vMq}eE??ZI=TnmNSR!*47AV;M*)3IjFOn06aZkt z6R$_}ggiddTj(QQ!%X`9_mRRMAIa_ia~eNhmte;3WMEk*6zBqK?Dx-_D5tCkFdH_UOV(Zo_memZ(u|HhcjA83pG7hz)n*M zhqg+;va3@bPXHEr0-aP7vj3id3>6eQeEOsRHNYSaW0H`AkA*52<+@M^=+-@LMf(~p zRP_~ibbBnz2@Ks3-q_pM#gQiz_n8HGZp*(30!V4THLqXfUJrkMZRs}NDS` z8|(b5M`M{HZ}5zzH`KYo|2CN662PbgLI9Ik%?2Kd>{;^*uzg~{eJsGag?tI#HiP8? z5M^KSwtt345^hFC6lk$+!4p6BZ0u!gqZBYoBlJ2QU}Rq;5H>wiS5t!hUG5;A0dW2L z2Hg|s`eEKOf*Je{=}-YKj}U8FVKv&ep!bzLpus^~w$CT}Cj)!Kp0GFth19Z8QZ?&} z!5QL@@)u7i2#Q=Cu!lPre&eg}DMiC6ipXKA#NSfX?INB@K$jB~ArGnj2zpCuc#zk~ zM_@gYLA$gF&1;KKbuT0TZ7+cB5=d$Cpq%`V>;x592SMN90&(T%6Fk>!Jn-%WrITvU zDd_vHs7Tf$@}B(u_Sy$1A8r8 zzPuaMZKnYwcX35)Q7#hi{=ztia@^6AW28-`^Ellx8W`?nV0PMmZJd*Hja)dGo`viEcazLn4Ds*^FWGsW%0za!f?Rc|G+3a!9D-H z&MrTfDj809Y>Iy)jkz?0h9o-@SAU{!(eRbr?q<3iUCmE4+ zHnaY_gYj@QJG~Z`t@9tR>rB;VroUwUjiH$~zz;nv%DzC`QrPF2ljlJ?+~hV$*TO!C zjm%|cw||CAZQ9$zBN*1blSera_=sKO47zfBvcQtwFSYxTb~ZMtF|iU9VtB$i zo0HL%dG{xBvNaJ)ZaOjb&(z2-yIzL8`teX1yfy*%zUjP_7EWTB{U;+GjSPZOKgoolyol{=}ZOqFTU`c+Pi*A{&;g}>}gyX|Yw2iZlz9s9hQH!d#^ zWIxfLI(%Y01*AoN_?y>&KGI4Symveo+@&0rQ~!aqRFHFn>Bk!7a|M+M=LXf)dFsIa z4g7wF&k=ANpJww=xG^lAkI|`{Q|+;TuGvY=VVdF$1MPU@gw$eQxniRhfpLOdcI4zP-)BzD=7OhO%Fz9P{^n zyGmTzh+?wIr|Za;d^yP_`|>zt3K_To)9<`AwV#mfNEo+Gw2mq1Idns)9SJm%=}}mm z^;shN>?J2j??_=P^h;;_zdg(2`MHiS$q9FBL^zgGPjZ1Y?d3FOO=~@#WmYh#sqQ+9 z9TIG}r`{>%wtGwDS5G33zx^HQ8CQth9OHFZ*>3S&^{`FNb;mid_?3A9vsMg%jwUHgZnp@6S#@5>;JS^+VT80Hbf?j^YTSNX^XVojqQ(6nYv6}ErXa_T3VC9j zradm4*X|iOSsTfaG^CurkSI)|#Ej_Imd!uAJrkU;Nnc1=Bgq-Q+cX2w$yVPkZI_g* z7kuJQHek?O@$ARP$_Vif-o@SXEC%^2x^EcNV_3Fi7--&jeSG^pgd^RY0lz6nr7u!L z_8~<0?f4jDNUla6&I^7x&qxH?905y~oY_GZb#Ey?&9An_x|p9L`2bXug?aHqYDgV( zAv?4kx3Z@_v2(dRYiObhFx@V`WrG#* z)0;Yegg0jBS3#bdO8U+v(;MvHci-?MaMA2FV9E5{jF*+Gt_7!_)8vI(c(j6z$Mr-X zC}YhF5<+UPm7hJ<0!&a1VglFI9X{wh5(>!ihf2Aaje)hu_s{=`baLTTskkn^0HQX; zXDJ1#4O#PS>jz+^=|cIZUw#>Y=k+87g993LQEv&ifb>gkku_Ax={BzPuXLWwg74JmV-|gu&PI**b;Ke%W$f^4hg!` z${m(DJ8otwn}L}bD{GMrBO>gbA8qqGK=d!){W;@XJM8##ozAe%W;uCN773Vf87Kb4 z6>Z}>QO=i>NT^1v(MTg!95`eF~UY5Bt$^V}N0YWlZ;M=i7F3GF~GlpMPR%+m!&N<;4Y=MYL zKqc$}BS>}L7jNF}(x*1t{XNrfbnIy^!5yXQ01VV-VC^*0_Ha=-XRh;hGgDTOhqt=j zehkkJ1DiHI&^PQ9fEf<_|3n!#L!nM_Od%D!)VZXmJ!D(2RH)O}apT-PKw%~F9Q3e- zeg++_+vu<@W9Mk=!@aIQV+2^%4~|-nM~QxjZe#XaQl&g>GKy<-lh8ocjiyE{>|pFb zx#&15BX$GMiSkT>lUt^9wN-~6QjWY1;J6{qm+RQx-Fj#PjFB%>CJT{DX6(#AeC9qk z%`eov*hv2EyIqw}tX5pI=D=6*=3<78pY`~Vt`0A>&VLUrpLwRtZqZC#VMnj4g3qUH zaz3d7#Q=rx&ie~nx?VDcX357)z$jh@8^nignRlOL1f~5YNWZAj0C1e|%SZ4WS}3@9 zL(@5gLACiG?+_GdH2RQ$J;MLZQUJw6d^Ur{S&dYcg#t&c`l1=2byjA?n-|-fw`agJ zu6Qh|pjtb54H@umsFSW^+$LX2B=MB-;P6T~_P}p}4y#!ltnZJ2-b(%LZn?NS*$08W zXYLM4`L!b%tpV}mtD6zlRM~;f+)fYAuR=L}yU3k8VI9F@@9_NxejbU=Bc3)1!Jb!D zh1H*(0IvKh#>uy51*{K9EyuFx=-!+5L>KcRbaSb6lG<+MM;xX9Jtl3O+rON=jlmsl zCfH?bFH{t?b13nV%m7QaPqMMpfiRBkO%pn_khUh()v|%8OmWS#V8@k3K2$%#aES+; z@Ts)ZU*w`nG%$$^Z>0p!w+kSTgTYP`Puc#vHg$LJ0Q!}1ejrC@>d^6-qA}X0INQ< z9ntW#{;?rYQ|sBmhcvLpv0P_7%wuVsk&2F>#-L363I4`BPsMTj@nGp8_k1)EcHnSjzBO`jK7bFjSyQN{2yvNU;D(Hs54P=> zR>o%$-xsTMfL%t>Zf-3!*Fkr_;qRdU#OhR7Xuj+4zjz?R#%yUErQDfYcU?x5>Es_# z6lDFeX}QTg3)64lF})S=_pW1Q;{{R6R=_KxkIwu6R&kXg(D9vJ*e(Hl!-ucgyGdTh zd>pM>J%L#t?zi6g`-fnx<=F49&k{(x0`Ps+C#L#ooC_Y&Q6 zXj?Nx=4oDpPY*OPrc+;1OA7AKu6g^afKlB{G|?Ouc(zOtZzIIOoe^EnuAU6OkS;@D zxMjc`M>HGgyd5`vxA{RFECzI(E!<;qB))04%MVS2OnV?YoyXqqcO;`&qc7VGfU?^B zR0eE^dU{{djJu*hT0Juj-0rAA%fnj>UedUQ3)8>HdV>I(=19>A=;!=!GjpqUzwiC- zz#smo(~BrRCy}+TWXDnyt`uFKm2| zS^-LZWdW>tq0X}-MFwpRqvxM#1J0VTurT-R@KLORZd*in!HV}tL?xIrdOK|KiBsT> zrh_%OkbI}nNwlezeCDZ*NK{g!FIYbPvtn-#Ko?Z5DcaGuLr+)btaJ-lG6_l(+d4&bMC7 zSl45}W1kh$dole;TOcWFc7{+-F zaYn)f2I0T)O!UENy^K$yb|$3a~b z&1>Kj)wg^zbc}uiLT!TXcudPD^knoizbCsOd>b)zGHq@BABxoKUXVS!$h?xzFZkh; z%AriC%YK8_FXLd=`K?TIXy>Wo%ETAqC+$w;CE+%4B~NaZXkr zFWp+vo~eF{TNR%!w!MbW+FFQ1)SagGvNq`pnaKC(A)Vka)+)=UK_x~AefCM6pB(qn zgDQsqW6Gk!j9ac;vX-saWPOFMMi)U_dPWxm2S1TyKN)_gr|4=_fx83aYhiTJ^zwH2!E6mb2NUj;`UB z|DPe_X2vW}%?RFxEAxHC{K`50)(2VR@CLnaUq|ausKQ45P0N>MTx;}Js$Y*D zG7v?h`6JF|b+CyPR@MbI=h{FHMq4R1v~y9mXF4|KbNtC2w|-zAFM zle7ZNWJkhWcxPYv@N&KUz?vw$Q@g00XDAx+z|pFDx1g&V!Sq){;fmL z@gED(25`=ocNe8sU_*BOdxcWb`o0-2lE;xYXx<1fZkla$*>a0{_B~?Nv+e~Tl(!89 zHnu>C95?W;I3o7PxVkb#!3VvjLW`QG$KG=-%RZ}T6aYGQ`?ag9tGQyKnP)OhRu>*x zCV)|RkO8nk@Hq)YUI9i)zfi4fL;~}kv+pI~&hjp%T?e&A19Gv=bUl>ggk>R!_kK@R zehJaO-6z5sNs|u6TKDzaP=f~hG)Bk(?zdM?uU!||m|Fza#kf2UUcFmyV|xGKR|h)J8Q{LK$)Sa!fN-Y4HN)5lY4i{GpmPlfobE zY&YFyX#$z>FN}=xwu-ndq%0$PkuMaAlH#Hau-|ee%OJLL{;6zugVUm9m`tMO^OjC3 z^cZGLgL`E1seVlOx0(bgnqYy3Nwx3Zmr`$PQUAUW`z8{IuR#C$gyDGp?|tJ@RbFfg zEni>LTXP&>lfx5-pA{kTmkNvyOs89hPx|aeKllFRi)uy#Cbu^hJqEft6Ewd+SDY2!(}&~GEu#P92FXDa=-9u7+a zrozqTa=W}Wyds>fPN||CMBVM)eG`durt%ue5;vyBo^c`OKQP_oGYFOw3|@hXQQTi% zt~Eo^?6XB6ses}=vx|r}9WXuT_?5o#QFp8(qzJkGY69LsOKpb^POcvrR7$qcmW1EG zU$OQMb+-Bj-grgpsnQi7tI31;_Nc`mQXqRq$a1sStGMZ>#3Da9T08Ma8`7JT@eIJP zj%2NGOAg95{l0iYmoS*s{PpiGV6l_w9Ss~4nL=vo1D#W9O)Y+?8uF`3`$Z2cMZQ!@ zewY15TKYj&s@Tdbx!OKoi*G8)S%!TgJn9lw~{aO%aLBi+s1RWnff=RL| zyWn>@6ZoSKpy~iNVs&I-5!9Ftta)OTX)MDfu&LF7Y1g=dFzY44|E5)swfYV2V^O*P zZ&B&eP2wD1*|(hg^TYCkPxP+tE5W0uv?ek*w71W?^@*oEkwxhGGzpYC^W?^M~TwNt@f)mf)^gfUc zDKQN&SL+xkKz+LI*tCvx)v|W4I#g)xPrI0gH};Jve)L(&NCtn4*V(VH0XJU)T#(7# z;-Tw~%kN)~s7$U%42;8HHv&bBt3K<$J8}aL6Yph^cybB13GC3hu$*P~x@FP%TqtJo zbAP#iw6L(l+IK%(toJ@R09_gU6{u=T{!)o>H|u%W4bNht6q+OVHJ^33wJ-U_t+-cK zFWGhZ_M`(9vxDu{qLhDv_P7M)2O_pWGaz97dZ>E|M;kLK;B0-@^C5!W0W7$NwXU)m#V5~UK)HwL?>)|*ZPzEsPbX2 zaygoWXAjT0>PvA*^*B(&-bOwMa%(t$C;*~V6|TVKV2*!hN1Qsh6B}o^D(oU$Ey5#= z^-lEqNAL>hxytfutwz%1yl4U| z0V+6$@}bsxY&Ja9i;D7b*tjtKE}F2(Ynra_3Y<4mW`;IjL-w5^?%arZhF4je{CPsb`s}lR&2p7a~l=G_NRg`i$tx4bYhLgu~(_6>8>)+53 ziqdoVbqx2FXb&RjwxC{YO|}t~)Dzb654u!10j^e*cqc)P0Q%9WNikM#6s(X>x_22J z=l=?@2j`{*(W ztir&ZJ*+*U@$bpXQ2o>3!~LhhZzo*G_=M44oF49WxxM1mYSmGZ&>91hw*x0A47MG@ zgnwdR5=bV&nj+DW(oR8zIgS8V)#LAR!{09x-w)=c%gEx}qjwNBiQV|RKfRuawXv@B zS=m2_62BUkDq6nUE-7o_5z`QU$G@5TAj8HpNo;Rs$daRSI{)+ah0%pn$Hw)n@)3_`D0T$aHcBofyIHOf_ z;mBarmlil2Sqk!!_}(mT;cq?=Padwu+Alz6bl={2r-R^3+Z%)Bln?eW5a-K29XsuJ zc@RtgUU}U0GKmnLXIst`CKZ|~s(j|9BnztWDb{NkfP!Ju&hJq!MB`A#Jn@PVQ-OK? zVCC(|eOB*0t+oHsDsQdS~*-6iYZsx~cO@E#`YdZ@JmSElNB_$v%hH^s4{THiNvoC=!( zD1b1}x!pIEX|QZ4X%JmMc+#l4n(csFSasU8DY`^B5ItG?EWG}Ge1j%3Z0nbgtQ@zeuvs|~5f~vo*!mR9-6}mxleLOM zxvBOv{EI=BVv4UK`l7(GA2+96U zy{^>DeQG>B{no;*R(ow$S#d_Op9Ocdm*jObBR}AJF#+E7%?@iFwVhz8b||y4Mmu44 z>gVD2bqpB2ZLD4ve)2})vE`|g!a?j8M%-O*zj_*aQlAkw$6%skrVr=!lTS+-vrtv; zBl2qtPkSY(Yc+o=FOp-a<8ZS-Y|@{!B&avX41?q4V^vG!Gu%`P$PhEe!_=fV-R)JE zp-)`+$bC@CHcL5ZgRYxA)7u}Edf{%wHLs;tYl--}-06$PNUT>Q*ivk2BocL}$p|j_ z@0Sy=vtbdP$z)^W*n~JE5qlu-D*Lar-D@uweL97LZM;5{-^l%UdYlgpI#QPJ#lW0i z&{bCL#bdJORkWrb{Om>W!ZizlXAM6VPug2tZz3=UcY@t{BF#@^mF3TZKNIiChZ#(d^V`2 zAr{G4`2IQwp(b*`Q!}z;NJJXPT_R(W?V6D~@P~!PlO3dP0{t0_Z2bts{KRy9&5k;8 zugD^a$@~qM3;)|_0gXNE*&qTZ62G2QO-B7VB0BBed7188^sCl}hEV;^AC@mPbKk^< zzZ@~ zx;@VDN6@xnQ>}_gxO2gWUi3|X`e`#ufxG_JVd+JGvcl}0&ZL(5xrau)|FkNqL7V`c z6f!lkb=^yZ^PUi1vUZB4Y*YWV*Thdteh!#B)>G(`yY1+5;UTWkOqz|E(-?(_nPC5v z4sz<3LK?W;~HShCEjDWm&PIZe-l_N2$ zU|p5>@WOa|s4zwfP3l|y5B$YnqiOqMe~Q(bp?VG~Y1{>^J}FBf&@Lj-xo{sNzZYi! z+gxgO_BoZZ!IDEz!FlC_A=2_eGv$?E;^)IctTVrb8 zG6Xcank!BjP(VELlFq z54mr$Caj;Ymp7NiT`%A0XB;a4!(gNFt(AnLFuS^!N;K)UmnPkj2}x-U9=73kRSi_p zG>P?$>8$5Nv!Qj2v!4XE?gdJl)rXt48HAsOI?wRqpUhbdZ07TO3voqo8)()M_As7O zU@CAm`i8!1vskQ9mxh8UZ9bDHc2;0)x5dS(@LSu2J`&dWe!ZdQ5O+Dx9I<@BY=Wu5J+q~ z@0k?<2MIBasX0vo`a@3?gFzIESzjF(7NLHOqljxi=y7Ch7D5ak6LlTA(9^qi-R8S75XWX#Yt*nTg@ zW~myuM)@eu8Q>8PWHC8CehY+n@-%7EKtyX@>Mw&pyO|%t6R_=#WH^O@+fM)|mou2G zr6R2LjSFCLX&~gMqqG+aBa(k`06hI8zS>)wad;G7pimNj^#m|MG94Oq-ia|?!W=k| zgR0LVG`06%DmqReKeQSolQ_&)#2Xbx;cpql2($tc8=lW(gnjU7GcEbCd8Y-z+RPEk zF5-(d;VAIU&{fn?r|(A<=V?@+N3zuN1q=D`v03_k~{+)pOm})bh%g zl|N-#1X3M&+xT-n+vZH&TkiQr^_Il8{Rtx!W!A0SllW?qF|=?C+r1Fn?OOa^_UyoBn|hn2V51#9Y&ObX83(?a--{OT(*=N zEe7zePNcC}VGrvuD`ho?_+QZRM0r{@l-Y+oO(IhaH$$q4kQIQNu&;v@X|i{xEy#$I z1}f211X=(cNI4OFJW>=Is?w)miA%a`zvg!}^HW$Qb?yMS>QH2dc= zF{8e_e%xoL#R;PuTL}gZhCGj(TJo`61Mnm}JPLnVaXO>Y+9GQI8Lj$TChz?SCC43| z9jL_n^!Utr)FvUd(y>gy>$84j_PHhx>wDV|C{GXq6M({6tCxsEK97*m$NXAf6@Kut z(y_fC#Yd^2V2f#}W5f;dmL5PoJ|DyKD-G)eop_bN69wx4#YsIrPrWrH1pBGw%S7V! zGE6F>;rv4`GY1pH_ZC(%)I-7VktTtJ)X=h%|2^3i2&2_D0F`0E_7k1e3nJL( zP1Objf28yE3b=A0aFNE)@`u`lv7#_iMK9Eowo*ZIC}my-{|&bfD{>&lufS(ibwej) zbA~!j1J{xj1>sU83#1E=tZH|Jnhofpi!$pzk0K@##e=XK%?`me^ec3#ZO~+s0~i4J z%j4tmWKW-JpXMaQayysP<2QdcsFB#R!s#Ds?fC`8YLunv9jHJhSY3^7y|q09dtx|% z9EZ}?`4C7ui2|=&^0#d%0Z_G47ads*p+;T;)%sYti?W!xLy+{TNH5qKi;mVQAe_cZ zfbuCCGoR<#`A7a>Okl2o;bQ^ zt{h>H3m`23@b|sGe>Pes54Ss{F0|fszX!_w4-Fj5wQ-l#N3_r)sr3BE{F(yVs z-Xj@H*xUe6#%}EMQw1pGDgAQt#>CJZ%pwZsz{D%&vu0bk#N*W&#k0Kp3E)hyIF*QC zU|SWtV=74353rR4U6llHhj%Gm_h-RS3%0O6GRjkT^-s+8(YZcikd)39vI`h*-&ed7 ziE`%Wr3$Ha_yWWxb7omG>EX(0$63}fD2CmipXMg54(MknAS~(`rkaD0?nS6V8!1KoK z8MVyt$x$5$isSUHX1i zgPZ7n#HCEvl1g#Ois{F4+6H5t>NnCopLH5>oVqH#s{AKksu!0w$raDK)Ei{zhr285 z#TE1CM1|xeEs(Hlj1abxh19+q{JD}}((?ErZVlo2wPDY`sM|f( zh!G%8FvpB0`Ex%V3g~O|&_Z&epHs*^mtFxiK$7p}Fwa;MW0vn^x&GI|a)U-y$_#-} zbAEXk?Q$Ed`r$5d88fm~-UCJJO3eL)7{LbIeTu#SM}Rjyg8FOt9N(_)1G5B>f!HPF z$E~hNGNO!xH1XAm(_9d$)C&~BEM_rZ-{9L%D7>ga;x!XssuH|u-m#~J_FM4c2%HC5 z05ag2huGcEWErCnjF)v11+NyBfNsQ9Z^vsZT5;->q-}@pulK`F8!Opp_Dn-UDG+i= zjif{5Q4ialI1?*HkWOdozC(tjBRgmy6%hY>4cMdA6fz{!?aTl5nS>@ecubbg8kJ3n2eUxpUV=2}uV?CCg%sWYPSo?d-q-`CGl$-0iC!En*k!Ev5 zzT%Q1HT&$y(q>bsGirxxanz1P{$1!8xR=3s?UX#oxV$!N`X@Eitlg^yd>gDIPnCoP zp2hxUU?nvTp~21hkt~ZP{G00y{9e3dOX?G!XOT72+1$~3>_}c-u&3T7bcrQqazXJ* z3K@j52I&dY#{DlsF9W^jB{Qvlmy!5s+UrOx>cBMoKV*GnR8U*Dwv>Q?64KHsASoe@ zGzik&Dcv0c0@B?rCEZ;jjY>*)H%K@5t<8Dk-uwMKW1Mpau=mAdx4O9h^zrh$>~u6Vq-9HPiZoGXHevW{c;6Bv zQy&W^vUIZ1>C^t;RFzE98`uTE z!;6zJ?L}+4!`m*wS+lgj0K`#1G7=^AP?|p{v>Wf!^VKq0V;fWF^V9Qj4Qv%3BYOh! zX4{G)fd+(|Hw&>bn(~|ONE-FT#Sdlrc|MoLRb-n=JtER9vQd1`p2Aljukyb+W?pvD z%=sb`Zv_ShrtAidvOiqbV13J3>3L=E0y4pxp!AN7%jqcs*eNCwPU|#cFCMFz*PNst z5%aPS7+Ae&&dnVdL+;*WA>vc$**q#1}IO7pr^s&^hp2`W&S#NIpcQzw6! zaE7hCGfG7fU-j;j%I~gUm@`p+nQiDE3C>h~Q|#tX#5$>pWc%YJ#3z#S{kd~l#mPr# zSG(yl^7xqDVq$2!6_fAOVkHx#DfcBg6_zC>vb7jr)SKEXs9JYo#Tayae`-mQH(Oq@ z5^$6@LmA&=O(!`qls6MnEJW&P7D@rz5w}k;DeX|L#i&!)q8(dTlt6w&tEgO~(JArz z%O@p6Nmt)P%iRLU+%u z5T-4EA5zyZvyg?&8|htwp0kqW?!_bUwPmx4vAgdfFpL}Aoz?G)cUDCnKUm6HHutT z|H^JAqzv~X<9!TJW!~H=)q^y5u?{h0?^RWVpwKnNSHkO7=OzP=pMpcA8#9oViW8l! zs+^lsh|8$Ty)07RY4NXN83e!0vlXpY3iD^cuGZ3Lok;k_!I<3jLsN3RPD(4GpQ`)1y4SBDSl07~>ZlTQowdSi*@&aD4t@CN~!unJG z+w|j~StHgT_S7&3QD5i;3zCpUu`ig%mPK(lp%a0 z*t8W06Q7aVez64Dm1Ca^`4^H)NVMJXv&|v-ATs-Dq)ckJz?%iE{c*TxvzWoE+ zA8$S`tgP=PCyLH%$>5t@I!Kb3x2WTwD^;Etvnx~&KwVE|SN5P(SNqu^X2oK@NOA2ehQY5TKX!a>9OK3ih1MTDbwfod=)MYB4X$foglmHGaHX}^qb zqQrPRv2P;8#H!sj4f9bf`s(SEtxiL8_(|N>FkFsAap{*CWaWf-A|~c8Q!W`M)17Df zy|#o&?0bnlN-zg2hQQ*4quR(K3p~2Ih{{1Ak?Y}XTRv#r)OqjSkwC1tf%M2hgs5dB z#bQp!V}G4mHVrR&VCCcQWEZVYJ?I&#z!mVDfiLhqbyH?wFd>t*qbu7#XLV8FZ+3?- zc#K|Jz_@x%LWUapLIuT;x{ysT1p_JAT~fgJDRwxDib0DCyAQ(H`~_y^$M* zs`pL+ESl(33ekVZXM6}F6ccpr2Qofw*o^r)e?cB(Q(H?`^7LrOiP0DGZ~pU$Bt6xN zA|6f1rg?s9i5?hE0QpUhde|ZuTQwA4NcN)3H0uYU_2&U_YR}hfHLpDn=0iCl6t}P@ zNRTjn%azLKQFqgh$iD>X>+gu-n2LAZA zjZX_kK93e?m&H`$ZR^M(b3!)Y{2`c54ZbaAc;4kb(0k>JaRGf+#WH|Vv0(S9(_ntK zf=h~N=Bv!!AmUH&l=opECc-<_E+X;Ya=s&oqd(;U(4Wf9a@|%N&SlgFSz@N>!8JrR z8>8H6Z*7z0h=2GP&nMcLO>$pRgimX9Eaa~5o8KhRQY6bZ`}lcLa6)$E2fMm|n>1_f zIYIi$V;>gmWm{PdZSSsm+1`t)( zEWm#w&S`#1%gJm)8bbZmfwM&(C)nZl*Y_?cxUV4nSSBF>f&HgXp9X`(N^cv2OYtWE31G<7MwRUS)Rc^Ku&r2MIqEqe62swiK zJ_$ks%WEWsfv=%bRt}@cYGo%);_#dvtXz$UU2qTU}R9k2yH=l!fO3{&?VY zutIW^Fqr*{H((A$IcqR?XN= zMsZuI*p|z6WKXj5>T>_eN6Rqxi;DdXZM&iltd|f+7mnv$mHdH@dY45WH~#ET5^liL zuC<+6TkOSk63=1Vg5gO2cQ~t^sY<%%u7XdIskW}yL`eyENOT@^)Ks8f*=lG;%`L>q6Ti;?~z-OT;6)__Et)UB^I_uBHlC}~PXH~ZkWK?(Q$ zaA!5=hK1a*u~itK=((3mk(qkAF%@n6aUYr6Tq$mxu#V@LR*9x2g@&8y`OM@ zUP)7xh_q6p21)b>N2yX6{Jf5Mjjb3UW4YrVa1x{RHe)!@@2bQh3;WAQHe*%nXAMPd zYqhWNLQoAXN2$j*5}jdZuXQ~GLZ}`sBMg_QTTS@!7yjKzkqV39oB&kob4JiaHMA@f zt_JpmJ2}(=slt;KR^yP2@(A7RDJK8JFX6#J&9oq#r3hBf7uV^Oqsg~TskznXWfb5H z{;%ZLX-FM>p5#1Aj%=DPlImp~AovyJd*J3)SNl7jKL*WCN?Tivp$Ixi#k0SjW&J=E zwAJRv8Oe%yPPL-?%IMa}E9*T)+QZ&@jwxdo$mj|Mq9VnA`=t@X(RC%L^Zc@>$#Bf( z*(u1Go6xiY_@SV=zBwAqQrJVKeLdG4e%WuL+k?o_;!AUnKNP^XnO9jzW8$sF1hgnvFu2b1rB(QWaw__)|fj z?5C^rm;Bu4zkLg6(tWik&av9mm==2V7?&-Kltk{hq!)@wwgh9;tHPL8r#Y{!Kb7J5 z1#0O{F08#J8IV}|3;*&@b9wxx9N|nY0mqu|Yu;q~ZgHbqUe}kr9^}y**>0oUN|K9h zL_K-vp7JEh-IEB^wS}3JWtCJEB{@fyADSR?P)|aJE507Mq8BZWl2$NMIe7Q4})(v~n zBwk^&8I50TWkCfm`dq$4(Ok6uFZK`ivYeiGoNo{gAKw==;08~Mcq9B77bf~Lb;>WL zj8|NQNGPXdPsV_#m9@w6wmc^Bqv7M8hMWk#h1OMOvNYsm%NEQ*lNj5_6-hy$TOXF} zz}LHTh#FiAJuk$vADd?fZ%>;u{Y+t1Q4$w(01bX(!Jm9${xjtQJ z_M7V6J_^mP-rKCwWiVhueg@8$aSspY;0N7zO(F|FoUJBDK2=1rqrF8ZPn);vB{~3p zj?N%`T0vnL?<%_G4vE+vWBFag81GRR)|rG1`k<{5$(jVIxz_-|7eNi0^62HNsPn%5 zo!Las_^pHQ{L=90?vq~g2;y!-HBDWXJdZ$8(ibl{x7wUW>f>w!k^F&Ig)gg>5yb3Q zCzIr91O@FKM0!8!-RxF<^9^)5x|hB08-SvUx{pGPF!&j*m@e}84_;q<$qbW~e%gMz z>Mxj7ETs_*Y*Z-0FZ{Foo+qBMqw#*eUx`sMo^;hbajS%NmQ5Rb9+o@vzSJ^zvcZ5? zS6H=Q@WZn{A+$oes0HgTGP*zcEqkTg4(l^P>>73X-t6^VRZE)W>dBRl%eSx!4Ss!3 zR?1YMZ%sco#SZ@|&0k4e!Vn^a3PP~+kt{5fZiM7cjNfC+U}MV*7^mbJgh*w3#D}4D zpHQ*&#n;F6tey%dVGh3TnEpVP7Vh9s2N-I>q|#^T4K7#t?&Ow3Z(uW<%)NT~D7Q;m zqX3+s<7t-upY>cHM0%_0t%=3CiMBoe`0-4GhRj6QN^ogmJ$1hIz0)}lTGSx=Hfm;Y za*$v(5*`Nd)G>+9@dW=k^6QiV-6y`f2^d%-lA)fo-pkJsMf_Mf_?>^jWV(t@3P6iK*5-V={2{W+3S(g z3kAL_!~3<9Is|jDK=DWHMyUJy@|Qix?QV1!zoYcokhh32CMUQHGSMG*pBIBF)nF* zt29Bs4SPSy5Gi40nOQRpvVm}uu-6o+6O$418Z~vPOxDVxg)!oVKo}H&cPh_|DPBRv#^R6P&*OY@Y%rsnNF_9ZRUS zay|@Vw3jy>?OJ;-4}u>L&SfY~>3>`0eQ0%ZZa6ncd>w5&vAw0XpC&MKzyKR`jy5{I z3&v)KL%!iuXr34a3jRMJw}G{Or4id2n0S}1TndqGE{QJyn|sI2441N5AmoEMpEYK( z*`m)FLj0bQwdKb{>wTP8T5BqQ&77z(;4uaP=6Vx0n|4aRya)6sj_-MoL~7~)CByoS z{yZrxS(z)}^+bK!`gsbIe|4cIV0a6GbMh%AK?4%%;lSj&-=i12=kkzE2V)(6<|)SD z411U(PUyKxPNDX~u=Oi=1sclQs)YF4+Ou}_BHx*bFSUAeSc5npU6Q;cmTNfk2JvnC~GR>0iB& zBpyK?C%^e_pi3I-Xy9s1>g3U zeR>qo&W=0QTUC&zr&EEIfJtukJ7%yq#lkat@ZtfpByh4P1t;GLeUHNtIE4@j=JmG; z>JdClgSZaq-K18KH50>dZa6O~Iz+Bq*B8?g>DmG8(SuZriIS1nuBgQtvsnI$_d{dc zuv{6>qX&Q}(Q-6b(%s*IIKr+DGK?u5a$r&e=LFXz+E9X24!oz+YJy+D#(ePB@f<`d zg&jXC?ux6B!-6SGN;#~VOM2B7uBO;A&C<&)!qrLGfebp1O?7bT@y2_ZfTIqS*c?w? zNC1AtG(+?3XkN0I8PrfZzp;ONB@4oHheNMTjCb}Pd_OqvHFlZc^JQWrOpoD81OO@G zKIIZ4YyzUw6T-ME^;5w(sr0^c_aYs|KW_N^^w~3j0gx2n&Z5{7 zRZMa*GiQzb5@b!~vdbpUdqe9H0n#3R--(C%Pi&6>kmZ04;GEm|eX1Y9)3V^-l6j0Y z2Cg@1e>Tb%QpV>U(=+Tq3Agt>-@3DBimK9b%~%PSm8%GAla5A#lCt)Q*I!QT37y$M zbZvn=t7a|^8nPFCL3fNi{fzSdR&0UCM;W%ApGH`iM-wq7_FW6s@3e1Go$N=n=EBPD zNSh;LuD9qOzkBsTG()5g3tuhJ2>CANGjWl>tGQ4s%B%5XD$!b-6ZGFtgr6vy>$v(h zC=;q~YgB2oC%%@$EG|fVax~Cv8t#l{ZY+92Sh_AV&h7U?!Axp3<)|f*o=;j6wV6^% zh%Vn6a6eG;j4H$!N2U65C*O)_ED9I2(WuK!qLkNzzlz(+S3AsN+_yD+$IuCP4 zvTv8gLry08yXWTUzFyAf(Sy_gb|5v_HUOqqs~sBu0U$}gK{jEFjb>e7m?sblW);ce zhMh%%KD_JdX9F|k)d_YV+g%0pwQG>YZ ziAmxYLlqcXk0S3jgR92&Rd|X&fa)L_n{%{%fIAMs2`^PmIJSnCxQ~?SnN@6pJij7q9;Obu)i4}0 z`bmmwpgi7%l~7;3g((b$m5OL|_oWgU7iGAFxmw(Fi`L6~Qld0fn-ZQeaWB$mi@&N) z$nwIdmD901c8`PBvilv*bym;H_nX^%d)TJ(6=I%Y6M=+#R=3Bi2!k3b!| zhJ0_K(dvG)AmKAGvwr*8N+gQ~UXYtdTEeuPV!MQ;{aWq^^quc^_Wv0n>xHnRt0be; zSRK+deI`7Aw$KlcS3rbnLA70sWx_yHr#Cl}Ns!~?82pHF_^V$MA4t=L@(KGl2p*OO zzXaQ^fk&|JiI5GeU>5Q69%OmyIwYD21CfTtB^$Iq1t~<=d?~3dK_XHjdznn7Rr@w$ z#Y#o>F2#VCx};PnH(|a{Dh^rf$4>>Ka@MN%CDNHPuU)}ZRjx;7O{-5@3gaY!t>Vd^ zp-kD=jW0qcUuZ)|6-uXe_p%$lNJ*D9V>m9G7&)(|yO+V#Sv*c7-S?AZESdS+FxG0i7`l`;a64 z`HJ+ewqU!?@Zk!e^-oAVuLgnejbIDdB%bN6(b|D>Lk?ZL`!jf@g}?M?BdwD1l=5x5 zgJLm_Jwu=Kef(O&KN@>&ZxT`4&n~nt#TKoSIQ$Hru2|8{O^OOTS+1eSDpdzR#?Jn! zNK_oEk!z2o8V3t?xq|!C$E}t|EH$YJSWHZsUen6mewK6kLCS3UE93^kM=BQXe?<2h zBdoBh>ARLAb)Qlh|J)vbB2@S54V&`n6F7-}`Ku%R z)f?Xd0omL(du2Biv^5#hsl^*tmfbw9Q? z?^XSr3)xg@kk1is1bX<9N@L(q@w!Ae^Z)AM3A}oc9*HC%IbuCj*^*Q?`JLH~TmWeu zdAaU&6HcQ7kJ0dq#%OXqj9+_CI`EHOOTX8q0v?`SW+PbD%)K$qT-9+sLO8SewZ-Bd z5Sd!vXYm#Nd8Mq$VA8fM@3u(2-gj6|tC^au+UnWWZR&*9VL&-+We$BI4C{M|lI|rD zgf&R4evcTh+A(d)U+Q1&4+H!wQZWj0X=@vWQ~uDxN^xo0hIs%Fmj2YX;NyT!MxGek zi4n;yU(itR`bz#fLcb@E8l+|(RadMA%PK`%c@P2hFI`L@WJ|ZMhpAx;;7^?P96AgM zAw6!W|4HzB_fd5^zj2=mwEv5{UvR}p9vUD+G0@rp4iBFzC1I?#ym%G9Tpu-zG=;u8 zFe2utcDMMMU%yd0)nW*-Re8Nz^TC3ke1Nt$hf{NIs|ZAUlPs6;z0N*Y+dqE$@>f<$ zklF;51PYvS;^DSYJqIqv^4^j#aG(I6<0O$OTZcLgO%CEUSPvjlvup~>jV1$Lr~Zor zop<37&5+`Mn&9_AlG1fEj+Bc45Y)qx^BAMxbNlH_%3Jb({Q_ zO+I zV`H&^jsvfxDH;sG)rFbmf${jiUjqj8tAAh>TY3U^%8$b7M6zJuO~8F0J-G2_cz(5j zz&4(o2-0+b;_+x|FPq{Y$_|lli!}`g4H#nSNtn;qlV?JGFotGkrB^2&_JBQhyUKut z>F#85#I3JvAskqdb%5M8O7L3LE~C^$|^4CfS66VH@a#b%lO?I4Rq#_E|WZ3m9kVmec#uL)hCt_0#n=`9FcFoRtk|$z8 z9T3MzS42lgu-nMO9$Qr`-Vd?01^0MIUM0jy+B<5}|x zDU66`g2-`L2;yQr^?>^iwW9OF#sQ2{{2QNiY9jvOtZ`@j4=tBmr0w}I);HDk@J_<{ z>CBrF&1RGE!vQh3m~dvVI?{SkbAu9+@W(~#l*rwom)vOhMc5S^u@m2$Dkn+l%tGVj zspdthQeq=&WHUT9<&f^bH^(@={9d_Ka9+6LdQAL5V7Ru3c2)GYpj8=(jPfh)(OrAn zc>Lk8tdguSxCx3A9MMpheff)pP7#M%%PyGKSmbhTeG{JQ80U@d)M_w1cKZOQx{u`* zG>2oRC{T-$5G53IO{&O>`m5H@T2NhVr>@QcH^DB|_v1a?l4~CWU8iZH=|QQZ7t`*0>a^XD!+V<@cg9T<&AELqQ(@w#oKejXyd_uM}toyk-5& zE6mQ7#BQq2`(D2YvUDe$DSRR(%>2wzVj*#p^tqM6xq zHv-JeOKr#T_EhH}0v0a8YW{pNOOti!4ZHX6lw z*vj_u1}N`}vL{)8FYxr4=II z3)k^^3v1>uo*LhV4W9KdGiD@bj+b=kHR0{?-P^evE!LMfuL=ZdW_$*f#Lvck)_g+UHVDV+qU&44Bz5PNt9@svV9kjgaBux#v=g_U z-BA=qrJO5<&{;*etFk;xdMVg8cHk8}=^C#${-m<)zPAtLsgc9QNoydyj;X6F;n8rYfaJB)1JRYarZ4~;TGvIdm zbd52CTF;J(jld?dR1Ujs85KRKAP@RSY#f#Us#V`WOS>2EW#P)e?fXdLITnkc_UrZc-)=dW82AWZhw10AcDhw8Tb8i(hF7yYnTxy8jEdCr&4uLoTc9y_@qo zaQ^@U6EP1^RRw3LM2K`E<9O?ZC+U2u&Qb=f0X_Lk>)LUZ#*o|~NF-@f)@&jMpcJ+b z@^$DBlYhgWBoU&upXtYC2S7hg8MBL*m-Y)Q508DP5FcKNY4NTe7fZs3K9Al4^9gpq zYm)d6&nZ^NYSqi+Mo{Q>Gz!O90wV!51J=~A(B=m9N1aJ(P`o^N0pM`pY56ye!!yZ-N!VhB+IFa z@AI~uXa|{puQKO_pYi9?`ttSy*j1C`Cu3lHNY3l><^yg{n+0A@{SJ5lYX&sZluUc& zCjvC)cWM3H4r9X-tjprv#Wt@~s^v6RXr3i{Lpj`Je=JRNaaq%keGtSVW%&wg*Oler zP?Do|8c)GRKCRY>GWhD~h7{izJ z@wrlstkEKsbxOt-qB14q2{NZFF>ixy&!1kMrGiWKX6_BJ0K#OxQ3E#7{28||Q13qO zA4E>R5Fb@H0E8du_V2L&d6sgIpl9ArIc^J9a0E-hkf8l?OW*nMl#J2zhMvmb9=UXGdCzE^dfej@-}E8h?HyN0OY7EllOH6 zW90<7&J)OZ%B;3_5xLAQ<76}rm)`$24-fOl(x~|C3$HG+-x(}lZ88A1h_2Z@zzlzW ztWa?tWOpeKPQ0)0`HgZnlLnaro-f;FUhSxU+kqmb)2a6icflHny7Z@Eg&>hL7=!on zQ>}k!Q;LEJn&Y?qk{6)MfvADbNK+`bF|0*RMd(~35>22#-h%?u4_JH}$_>Lx`FHRAe&ohh9Bu+Ivw7}`y6nT})>+5S}XW|iHA zW}Cso*1>!vh!pb2ah)jFE${M&(yk7o#fe9@xF}PNTOI{C*13hE->yB0nI8XZS1ZQF z){%^9bBJ=!nEP-WpHGX=&1j4p%In+>=+o?l|7=q{v6a1S~Dzp038$2zjtcEiaC z+`1gBa|Q8+@B}~?OzmIIgtW|XCo%BiS#%k|xT6f=fmhfpRY!8v5OqYNuGj%@BF3OS zuc@13I3T1AO8vb$H35jJu1n;Zfcu`nqaR}LAh{r)QWx&`)c{f*`ckVSaXLUG8DR>t zIQzt5`4ogVQIikT!G@PKt4uo2^@`mm93t=#&r89@v{!NVp?<4D2%K`e%?kV>tKThK{%m`p%f4t{Exo$Hgr@nh0|oLFnA?&E z-FM`;rHR>CT(@1y^?T+_W^ccn$>Mvk6eOF$KSvbIXy44S{^x-cvdK+@twA~1LzaJ6 zL$}){{CkY%GQ@ZB;Z!6hkZ!YWoIo!eo^_XfQfu{m#)4^paALk~NRJ3~EISOix|#mf zD??=Ba)H1b(eeM$hYx+)cHP*Z!;%l`2`8LXPJ0-o=&+`i%^zU1X*JuS7PtO^13wf- z+e=228-tXl&g#9xFy}I!rEvcO6UYcNM z^OMZ=gr9-%0PKc{3#U5;9X^O;>7utkLrQUtEdeY*A+A^?(_`s;Nh})h>Xcz-;xS;UrcY}^+V_3Ofj#0Q)jjSU}npE!sR`l#jA)L@J|2qge3%N8j6WW z0|>?ySQ>gsz_+$v<8yKi!t4UUZ@9o0u|eDbrx*$_O2Zui3=+{a_G>LH-(y@} z&%#!d0fGl_3rN&Us*&%ZniX!b_U6dA6gtib~v!S5a%c9fGVNrJq z^Ho?flJcT5+9L7~yLEwD$_~ne20f!ja-WqRzcqG}bgS|75zwMY15Om=Y5P@dkz=P4 z$BjauYFnVwhB%=@*WXbJoY(EDH6U-*&2cZtHzS#9TGhrsu)x6`wHvWLB2MCzsO#;f z8qDRCo-0U(aS3^e)B(%ZqPB0q$MpatXP3_`e1L0P(`vB&;(eqpvw1#iX1+zWO2PM#QQZVyG4LTd=R)<9oM80|GW>Dd2XK%lqO=!*o zXNZjqje8}>t{mc0^>aQKZ4(oez^g61?xek88P*-$BU@t^Y<@8tzTCq zWGK>{VwZr&NyE^b)1>x6TJ?|}Q_H2_!e2qhx&jx{0sR3z2C#Z}gqT2@6?(pqMG&TA zGf`=2dod+v8Sv^%EzF`&Cj)b|^WU7vmwynz3QpVAB*-?9=b|9UBgZJFog{eS<7cVi z(6L8Jol;psk~_KDnqqFE13yE35+DCm(;*;TrB_mXB{5y>%I8%_Lj@-Z(RNz*P&wU%ukmFPa^ZkkJOu4pwro9#Xv++Xy>vM59&DmAr08X=>0!NT(U zBjNO-@0iReO+s{2QBtS4w+fljs#pu$&zISX^l;tcYD$_k$hZ|&30nnXA{{-c3BGNo zlow^Gbn|oLHZQy%kP0O!?tOk2{B8>+uaEvnX!0{BnSqcjW6~c5_2bJ(XUQ1`Y<;C@ zvuELdBxZk!ZDH&$Tr3G465*6eN0$HTy7)x=GnS+@MUn8s=TG({ED~aIPGV7_3!c%% z=F;hOlvNlp94lh1|aNO zOs{dF+Kq05xP}zIX zR(HlE1PRh3(L$KuoRe6lH=Z>@3+&cRz8l!YtCZ{P3v%5YxP05aKm05plKOX^lIFSI z%h&7V69b3&`t@$VO{kKz9gjpYmQg|Sh#-JA6{F5;D*kZ;yZ8xZAdPN$NAjH~5#q6B z-ROgoS{+we9C9Zao=fXr3K$Dib&zdJX~U4?Nd_^5KpL}ciq-+tC)zZ!?JKztA+xn( z1M+OB4sXZwT_n=-J4~IaA#KNeP5O}`sh8;ba$hV7wAQ~E#R{ezy};6a_GXAfe&tDs zJmwoVeW!agY8^^BIN@W4x3m_L8J?+ieX;ER_JtVO7?Y2G48MPkNG=;zM8P^!`B5bx z3(@O!Kf5(W(1%uM@ksX(X^b;~Ouh8v*Lb)BFiA{VTN{?xg$?u-e3 z_pKs)3X?`Xt)z}uJWKKro&R-3*)2D!l>5<@+Qh7~!xx)1G9OB3De1ABl~?|4*!sm7 zS9nnq;;#dQN+`9Dxl5mtP(MqgV1MR$Gs~`ZMG1Byn`O;l1uKi1NdWCp#GC+7lkt2rp<-A*0^@z8SOp7g`N#(8o~1l~^|rQy;X zUx1POe+=^3w6EqV@%7)M-srgAJp!FdO*}C19PmkNk3)m4Lqz`78$r0sc&!40A;_eit5kfN!H_vsamT0h8dbD4CM*$J)v+==a$ z3t^;!m21R*MH4SgKE5N9t>Y0DYWMkFwPU#Fm@)cz4nbiUH5L1My40JLBqPm2C7!QD z%DiqwUMdrDt_s`l0#vH9$84(FJf9oe*FK`hLX)m8vOqo(Av=(-;m%{Fe*Uv3Z`Fvo z_NT3TM@C$yc z{dVaQ;2uuSAGccQQUP$>rKy@SZP}|G*q-g314FMSAHKWO-v=?$j3b6fJ5LmSkZOF| zp?)j&Q6uo&a#?WO3{OH0WQrT&J$ZNME6UFK5%RDExv*a}9C~*Eo{u-SM@s|k`(uOg zz7SHTg`;y7Vfb>83683#`s`P!gL@b38#upRh2Mba(Y*|>bM-46PbF0fYbUpm*-^d; zv~%*KN6ULCnScFeMb97EjuBC$)gm0fC9DNve2hUIuQTO~2lrFZ*B$_IjA{pdb{f!n z&ntA_A%N_nx*zhbxZb;O>nCqp=TzR1*MVMh-V=8k2t`CQ_^vZT1hv76nksw^*)&uE z$+Xa9Rtuk^w5E|J1rSb13E_+G#@#!nqk%Vq2>66Jk}hJg!C5h@3w<*Iw{az5dMCmS zddVbfruN)~BsrV&ubQ^am@_wDxQH?EQ!Ynp%fx!-<6c&u4h^*$gASsvxGlSd)I@{j>8 z@*|7Q?Xc(Z2pMh_`XW&};3qq5Kw9&W0o#_1!yf&5vlHriW41wBvuo7@x1{{JD<4C+ zUevIW)z=uykXyJE2T)Z3e~ZS{(gPl&i)3J8wd`$%@nHD~$bzg?wKJ6|^WrU>yzNaa zfDyzQlv^Nf4ut9AmIh2!n(*H)Q+iBs02C0;&N(U%=2wNaf<#UD9gM*PuL6*rFsdG8 z%P}gGVeS~KF`TLNfP=L2q?UtB z?MBKOL}LUIzk15}BFdLQ+aq7w8Vkyr6QB-97k*Qlg{U7O5dvp8Pa|+8$!U7z_TO)I z73;ya^mp)|b^S@2G`^C~UX+`}!MFa^z+_^^*MM+}Y2P-M1Gt`bFUc4hf*i9I`ocgNPfYCW>Z8;(MgS{D z4u_0l#1O?V$O;~|mi%L6fQU@k9SKl^0s>xqk=%IL#qZulzzEq3O1(xu~Ih{vgoQ zvX>k`cZKt=gq*#_1;*N&x<_7LhD9<}9tCieky>>y{usyyhoZJ^I3P1zEWDr|$4WSR z`bdblVIY-rNG6qIKqiA%{ae&y>q|ny-8~2H13uT+*LUx9U-@95;w<7X>0e-b>iYee1f09h zpwg&qv5;~zPm}*e|G2wnv~&D}^FFX=Ch3h~KF?7u*1R7A$|(;D#o+jLL@D8_Vdenz zMjc$`n;t#v#L+3z(qrFpg-S3aR9ZNEveBZK!E};P)}b9$YVrGhX!X)OKYLQ-`*TB8 zi%8}1pFd3T-HzyEN2SIjg^LEpey~z}Ng_{Ch%ZZJmM>G*Dbpu+ic(%nWhPWF$X+cj zel7L`wlXMgBP+yH%1KK37hTmE)sN@q&$}mZ7@jvKq{$>Q?bekf4Hu3lpj$RP3YF*= zwYr-c{~4DdFCg}Ij@Y>?8tB3g^UbSTkkh^)Ql!Pp8I)l_zy&3K7L8C_E zo27L1-5+4*d;l2I?<}OomZub7eAIa51-7e>Q1?~4QP4JV6qmGF?zs+JxiU8X08O@I zkfX}0<vWcbmOZx9Chv1#Y-5 z7YiQcH)rJ^=MUwxN`VaasnC{`G~{vEyyS;6nnJfzH*3w~(t~fRG^W{ITEDOZMYPa! zIC^ywE@XZDOX@XoJ&M1W?BjYZ@b^UE0KhRec+G)nS#k#u3ge^M+xl9z+1E^fF-EG-;K~$<$baHH=ZE;?gW-?IFfEfk9_F6+J)IJ!4I0{<{a8o; z_x^1CSY-+$5DgZC{jvTg11Wg|1)Y&0hn?)62?1K1y)gq}PKzNAz%ujGIW`Q@5#l2c zw0nMYFY5jKu(d_F&LQA@YXU0e0b3uyQU1Q!qj?5O`Pb=#*vbhY%(`p;6auevJDAAy zd$LTk*w`(!Km-q6Yy~+DuN1|SxG&x)ER38fBYSPVJ?F@Vi%*?OBDEl<+^?lC{H?A>Ycvret7qJ{C<^!8h;43ER)KbZb!S>X^_;OR)+}%aGm$@E(;Mgc zI`JuUCYj^v#ckq?F#qCmT&Aet*_iBzcZ6dihiEy-9Z}*6nhjE-N?*|w*6C{&N&RqY zvrTGnYtiN}2?+%ps4}I?5Q)yS;tHDdztt%gd|k@_IiA8ImJbCGONgmxbrXM=yLB{1&Ru) zT%XW>c-q8mB4P3En$l`T;P6RXw#R>VUB#WBeY)m-frs0qN zk+Qa-ioMjD!a|Kzz)W4SZj3@>G3d(50Q2TMpDm zqYrNUN?wJMgzjMLRQ_{@L38I`6T3C&piGf12)PBcrb>W=P3!*A}V0iIJbL`r7cq zF^@r;4A)Xz=d!7 zJC;~UhKQtyM6b4`_s71~s#U1V+zo>O{YZ>7=MFN(#QEkhq4Bxmx?R*=W{21z#g|cWT z*x|3kEgxrrLN#M=;n6z)!nn3z?aYAEgxyQHhV}*3ltfpFMj!zh16lS6BLAT;2FCQ! z6|zzK;@yEu?OeeJxc+sJnv>aCI}YD5yq>6AmN{88{dSGxc9i?o@(FQ}Ol{i~eB<_C z(?8N#`Ltzs(#wS(wt|e6c;q1}_4RFM-Fr10>dBP73*>X2x3f!*S^88Da4x2? zzFJup)KujdL!HTgyB$9giruj76#ufQ99-_=tM4;ps&H7A6wGs6HxO(fbsM_uec<7-!mF|`v5GkcWx94|C4hXWx6Tz4lsKg=8c= zy=lEV%H?!0Tp~24@bE4niqwlQ-J~}1ZUn3sHf_);Gw$>Lvw*M?i-b9r%Dfo%|bd^}FUjtiRNiU@Mpn2z+)n{xE--!d#XH zOHRcQcfwMx?2jkg%~2h%s(zWnM5Y8cJ^3Z+(LbP;~X|!;$}njFR}E^jt+~8Hn6297|7}Zj#oz&ZHAl;npKu5 zt6zvx-s-V31=UTHL@Qvhj>lMugqcC$gyvpit}SGpw7|Qvkl&d({gLnIW8l54JO8Yz z?Jc%(hO}R;)aH9@-MR|)Q;a23OV3d;uey7xiME3(-T;VcEYm#eQ(lGwVW_(nAd-8qH55AgZ`2oM(Okb?@Cdgp-X?`MVGqw~p0p#vC%s~yD9;JS| zynl1l<8VE!!dZd$406QadbenzAkDJ0#d~kl0IU_UxMa~8b-LL&E8`zoitT{-Hegjw z$j9N8=@`X??G;oBfv!)~t?|JVf`9-UsfYdYIh!#-5MUdUjgHFXs)w0^sK9KO?2Ed{ z@e_d{?=fH_gtkKylW+|r&>_M1xO%Z)-K%P;usye{azI1aCYte!`903Sa*sI1JD@!x z9a|StlKigkM#-ECt6B*NVTFu}kH?Kk#!G#lccR;(9db8pYZlDqL_<(|%;t|plXj}o za7qcTIR8PfRgS15H6onZQSZPT=TddmBjK>r>&SYNo5hYNGFb8Fk%gjDs;M58nO*(< zUHki^tWKOmMk#!CZ=#-QPb9d=A=C?aTI2RfR(=Ly@OLhlI%a-Dhm{TAtxqCHLo|%r zP&*e_;EDH$;;xii#WsW-*kK{$55dy1;rt|1{6X8fr5Ko&h&E{a7VRXfe3??~W9#^c zm$_#G0AKG*iN?PsF+ZeO({WKy+BI`+RtyAf)8o0V?APs!_IdG$+H?^(&&|1xsklI6 zES$?)hEWi#b%xV4Z6Ch=``Q0TvNB6<>*#lmOy^vrTTtSi_GwR$SHtGM;R5qlx&Ox3x7g3XTtRfY|>=w6c% zZ;_S1wz*5;_yG!C0jeY1L4Zw_ebU~{!emvqnW6Pk^k%%(1Ok^9m*&J+u)^G|Nw~Yj z5bSHd@4wV$I525B1#Zz}jitTtt$bPt9F}<095xW1H|plr+ABkc7BoWRg^fELFSB*D ztR7LfZQkGrQdou5v8J@^jZROr0KIzz0UUyOsy^zkYd5d?S;`>WFr~#z zPTa(PD23heWP4&&vx!e!CQGzaciD9!S8*4VDte%YC~}caR&*?@XjH>0l=};ANc`v6 zX%9X8rRcNm3D)ix21kv&jpVn$T4v^Iud3?RMGvFhaBn@ZIT5q_Leo+hdrdEDpVa-D zn)x$rE#h(;1rYCkYx}Hq_c!CqpF*c6Q(BSUpM&=G1p##oyraMrX&>+Q6Svm!z-`OL zTI>vP_86I|U0-hYoB`@H7X6=SaTo)t7vb4~&W8CYb(T}Wx=`{pSIvwimc7yLVsIie zoATM6Rz)ctJV0iFzuk-5i&tPP+XxnYe2uOpg! zM3it-Trm90h;n^q6?GF%_qqJf=d8+(RSpO^TaO7VJnxqa{z~~TPo@XO&oL(xQ1nQvQ|lKgMZ1%6iMHm~7|^tXAzu5Coyr3PjAxt**dstHCRxuEKvLHDdvT zJ3|T`9rngH=@YbGYM$VjIjF9+VtDtI6Muz(Dfh#Jmxj|^!WPWy1~0jJt#HklA7=nB z5{_TuLP|F5$F`f#ZUF^P?deNR#)4pPE1V#b+5&##Y9@5Lhg6LK{*!;IJ`5!Azs_O) zD$+>Lwe(#L7p4!Iy+=g7Dn&xuV7@Q`X$+2Y8*l!Zehz#vN`I{nc#1v0aci@c5-wGj#Z`cm`r2d^-V36~tNxLgRo2ON)Lj{jc32v}_PgcrdGPwej$1XE`NK8zSp zU0-73TQ?7Lj+N=YdHPbrs9yncUX3LvS1eO%7oqgzPIuK4)C24=p37wil>kfWnjVX` z2_~kqp-F2@@hI^qmb+o^!W)^-|9OVZEc&k*##HMd$TiZq?>NyT1(tw*)?8$el{(or z?_>&r@BDJ~Aw)xnT@!p-xu}I8KZZ&iI1#~|x-LeNfU+tB8aCqeyOW!m%aWq(G-;e`-K(79!&5c5QJ@zrs{{R3^lDso+ZyIYMhkG)VB5O#*d*xO&ziGTcPSHrdhJYx*D|j#^Vj;Xpk=ll`U@%ZwpALSf~IUR z=DQjz$mY7e;J;!N>)ttH!5$*Ofk6+L94{JpZA>8QDAywY?VL#6CCSw=a;}m0TBJ){ z&+aQy7ucqDpHepdz%=HIukPc32&zhKB(smc%h>_Ay3WhJ?U(kKdal_R~5vb zc868FQWHsSGc|zQyjZSsSyJ>t7sxln9<5PIU!Whk>c&fNrp#i#hE|r0qw$805MTbJbz=aSMcr6_e?ztv~a}Jx^7|TUlxCW z-;}}!iWpQmP5I(r+!OqG`6vIpF9Ad}a6Wr?@_{i!f_#ObW9!I_h}O3QNKUJDGq<=z z$qgjXQ(8XnZo~S1`s~hf<3ca$u{n9tlsOK1vINzYMi?N(?xCM$P<5JMaI$b};;-;= zk~4oTw!{=hL!J;HxOqj0aIQEP9N_!($0kQsD6|WdPy5)=30ptb_8vxPSR@0^+>&1o%pAYg#UhJ`K;FED+Q>EIjzcSDstZWozb~W^y2XMZw>0 zS$P#4G9}e-!z$C?D^_8+XlHxH?qYqw5@g8q>|7Y5ZJ^;noyTC2m$O4teE ziu3C}TBOt-0etC2k>dTo&lcs z)0;`QXnbFN*W>X%hKQo8K&_Y=*&wUlt5L)ZliNtzWVqP`yQ)M|U4l9)uv=GOWc(S5y9G$ds{JyxDOUnRKS@Og6 z);q%(yE_YJZo?<>&hx%HB@;HNqfpFWT;p1ra}*oa)8u~0^op2+QOB*_vb=H8@mt#1 z?gP`8;4znfn;3iY67)OY5h{s`w88qF3r?Q6yubN{5f%^3;V=<5Uz+7X9=3w>vO!s|JZI6UnaUMH?FcnLo0{4SYzpx`|fG;a` za5B|fHT&<=hVoCw;rpSBbIxy{YTOl7qBv0-(Y!8T75Ydn4*Bc;^sUjZ_(_KxvwftK zJu!*>} zQpm>f;?eeYalE>D{nO=DauLl+3_B9nNv==aJ3UcXME89o$&-YHZEsbrp;wyiLquv% z%Tfgb`XA+duO`WexXZJBIoV$?vrXfrIxrL`M2GR7DU74BMd#qtCh7fc%07}~G^~R= zHZ9Jx^dh4OcjcJ_CD_6gp^(+d+-ECnX}i;iNy%9O!?`4^H=WD0dlI85Q};V1{*Gjlwsx-v0y#9QH`(k>}f0&!cyOcqY3EZ1qbfd*bcKaGs= zDZ@@$VIDA@n!>bFjlt?vfvi}Sbu?a0fB~1)9VA-^`f&C3ddH_jz_!wscbuItZ zaO!R;YWJyt`Rb=vFDU-nJ&Tq^e=8br;m!%;E;T8wsF1&ev|cn`oVd6|%P{qj3qNSy z1_uP-?1T9c;iAU`DBw0Rmz#vAm|df?W2@si8;`wo7&gad1bd0i@;Taq57C)e$QM8- zU@j1S-i5d^m*q>R??<&mz z&+<;)ohdT{)Dn|AJMKBEn#Z29@qhBSp`NEo9sz&<_hLYn zLik`^CP>?bHOS%(IJ;r~>15~WbejLNTplO@_y1P_9w`%XN_|hwsC)7k{k^5rx`q&8 zYyF5CUH*wfy|i?c{BpoNmxwxEmqv>JMNZ#1ljl`bKPew8oPQll!;AArE>Y2m&YGTT zqBkWy#=82s%)MNU`_M(^jvt)%k*4FWLpTYG*5lacXuGyO*dlDAWZEm7^@N*}uxdjG zi2UDfSM2??cH%-=JLeNkTq$d*XdP|sli3k@6E4)GO%hPBt!xD1KPm&Kx6tB?i56QkPS=Dj7n`J;m3&MTWn#| zR$rmE2iXP0jYPsaU{2yM&a2xFh}x8m(6}ibW%>di;4J~m+yz=dVHrZ!N?c2zUYHRY zxJy3U9Q4T@B~lu$;pEZXz#`{Ju%?dD{M1!wKw!A%ruho%8z==UuTFN>$;%|Z2j~H@ z85tJYl^_f<`BdNPQ=s7f$azs$Ua1AOcz4pFoXFsXxy^?!CbIIAy5k~z@mVZ?>r==* z(UHbRclY_HktJvt*s#E-*X%(O;W3P;?EaDn)So{u-+ZG+;SIds5C(Yz|LIhHsX}Ws zvZTZ?5epEm&*t$L4W-1rw10o0Yo#GArt|a|3*uIqI+9LwwH~b5qSE{f>)MXX5keH4 z`_}&8-pzn}Q3?o+nGBc{_MVvT-#{I?-1E%JDKpDpX#8KfrmKzr6Lh0kAEb9$?xSBM zN0MhzyPuU}BOJOxy(YR6Cybz7#`~pSova&;DO8e8J+Jsh2Q7y=4H3ppYvd7kQG*qo zTl`F0U51S9q`c|+oC2t$&*eoDz^HLvwMp_B)lC2#m> zME8D8_{PtsnDL}_({)WTdp-6Xx}SDod@$1Y%jufkLsmv}^Ei3GTMCcqWiysS+8`HB zcFr59gl=?kFAo~q=wu?7ae$Ff>79z^lMz2V^-&Dah_&hC1a^VG|2GXT=L3*S)Dx5K zr#Zk~JsvQC!2irsSm<)E5X{km!&S@oVFlPh!|UW;zENd?OfNns7a*+G$DPNQC`7{6 z3EU(CK^Oi`>U3JK&Bt@|i_Tz#0e1n~2Nb2V7^Up{g_4|gIsN3e+*%6%3c zI4dKRTccykZGg$I;$R)VPhoMq#5ZHqPD3M%Z~ZBeaOLXNq@v$@S)-0l3~jO?RmRl} z|LyUdWap=Mm~EAz%SE1yWZFIEC92U!fcMv1TaWM7v&sTurcVbsPI+r z4NyrGf&{+b&J%!MO4If}Y_CKK<70-mB=a{&2-EfN9%5My!Q**aBpud4-CRAW<7=)! z9>sF+A6rFpMj^&Cyj86!iE; z?eS}v|La`{68oT|V^_A*2DUdzON(qwC4e8eIEsf>LG|@KAcn63 z(>=l|UiNlJYuCVB&$u3dAte~Z(&KAysR2b^y@hmZ37DXEyH7&Yfze7M9cUHCq`SWZ zNBfyEz}JJ0RbD{ggCSwgDl!U6LJ%9C0@ehW!s*`JnA7IsB2j`Do?%SAAU- zifp&=+jxhB)sBL1_R%unmttvQaK5=&axLQ=>Gw89a2nPdlsFNYi<|hSS@@)PKNP3l zz;aJb6&uHaVaSm&&H0bJy}mZ{Pp@KdQN06NWC+rcddc4xlp_yv`CL>m-;!0EI8~v) zvbXf=q^%6SSfH_N_YGKF-%T}h)p2V(r3%h9G%=7Bn)ry7!=orC`|9G=#yQ0?*e23j zZNU}v&55Jero;XHFT6@czwdkXcus17mn~j7Ld0enXC-b-Fu)l;*ee~?IeaKqZIIn@ zzv`>WgPSO{#x?0sn1t{w>ex=_GeScQ(b=~^j}jdPSAJ3+yF*{kS22=8kQoAa!c5UY zk3Q;@R_A8X(LSskhu@x?ak6!ZYasQt?#qDK7Q#$d3A)U^A( z<8?YRS21?_y^wWO;!nH5=bJ1xpJOpdT-v0+WPi=}y#x~DdSIQ&6G14wTb+=De*8LZojz4I| zDyd2PdFBO9T91*j9_j{+D?fM3svcuPQ>E)W-oE9oy#3bqunmMXI97Rfg}ZZ%n(?v+825m#}2l1~=gssxce!DjgQY5J#Z(5n+reVqB<;jv-q z?BTwlc_dCG9S|lUAQui4B>-m3$3R;0uM`=fvH);?tR86tz?0nLQEOun=^xD3HI{r* zTmqIJ)k?q8z~E~Gt8tD%a?V2p-c}JL0!e{r{7%jy{8F^O1`3K~N04CMqoX37pSM@g zLc}q)zbHb2RwHDCY#?v@aEYywE(s(Ks_DDHVilE9PG%8g*(wlX^P?p?GE(wreW{)F z@$tRawxM-2Z7)~Of#-GHD>pw4OMHLlU#{ZzOp<5wm zY~qWy*uSIbFd#(F+g3(gtG(WrtoYaX(RsE9aYij-f`NVXo@e}>P$YX%lK1w~VLSWE zD4kPz4#$m0$nMM%xVeXnbBUEsgZ?LrmQxq>!D*Y7O*eR_eSZU=&UTm5cf-lX`(W-a zOKmS-i1{8_c@Hv27FSw)o^0R?-P)Zvv~vyJ%HIwWS-!IlKN{pAvfA6$$(w1tJl?)R z2VplGC{oENGiT-ObkiY5;e*yW@)D9QcJVmhR6Q&OARm_c+p$XmZ2QnOS?X58*L05tgg%Oq7v=h)_-cYfMbxA26$dZH#3E+OpNZz-a7YCzjWw@PUQbgrXPg)Y-s> zDLPW-=IEi=^B*d%Jip#!pMbMJDTm~{GW9A0v{9p!3Z>+}I$J;UoBP!+s`{Og-JEgO z0i)o!!iyB>BUHC%FKyb~Pds{QG6@#bW-flM&(sp9kp|^jM<%Q`poe;%th0ynuSOCw z7NXcu;ZEcrGjUa8$x>r{+!zt;VC!6Hs#)JIh$o^K>r@GiD|S;MV46OFYPWWX&~DuO z`y#$AwsE=_f6Ev@$~$j1q^%IkefFTFG6n-+nP>m3__C;LTg9Ct77)!OjSrwJ1Dmd z{WO4L4kAM%pUe24e|<3nkmtfwK9e=Q9G3?*ZAFRIh91XLiXh_zf?wenhRppCo+5nG z^$iDxT8N)<{k*^FjDD_=ghog#6vl>r7X*O^TMTNRBebRIpY<1Y^?yC}#~6M;Jhp=% zR?}%-;jbOyRN4Zov#QkA4mM1Vai15b1g_+t!oPp}9{4xw$}$uQx%b)n3q2;RM}V_f zg3Q+%RfcX@lm14CVt`%JVu4VL=MYclWqg8I*iY%CzANQy(Gqkf_bbqizSg- zVhxN@YwwvsjxVD={jMose72E?nz+$>PwPAu63#o_of+Z&3=RfLVS6Dol?`;gHUM_k zhv(zhM`XU*W~_DK1o(|$$?GJS)?fX*%bsJWrB8|4FV^rES-us7(U^PB7=W`h^!aCBUp%QA4yU%vA<5*ndN$>0j1v0b*P&mTK`lRtz%HJPzv>{)BCM5On(&5n|4RjKWIO9Um;Dh~f zPINZi@)07-pQuV7W4gzORdAufuB&GLsIP_>RWvTfRHoBq?IZv7s>py(bNK&;LHCFt z$Pe!?F>S+o@f|^s+-2>YJ*!MjxjpyiRWrR0X{>Cg`zk0OQ74UN241U-DnkJ<>V60B zP#s8M^oOBc3nNA+uiu5bligJfs-v7x+1{e%6@%QNagL}wN~ZYL`LL*m5LbbuhHo@z ztUXRQRm+D)DxTKFCG5e;iCEVp)dEBGm_>Dj!=9()Giv>{)l~sIuc1oy0TotuABSFX zVq5XSZ!Bg>oJ)KxQJ?e%TD}&2G%?z3JQU1kYp94rz-10%1$8q^O7SK1>8nk^ZLjTq)QRWwk! zWSI<6p=ZDTBk7XkAQ6YERt<5VmMJ?orz`oTchtXnHp-G|X2#{zFXQay5BPijnN5J< zI)hm8|5}!hH9-FYu?Q^V+jvyXUN=&^i`xv6guBRFgk6(FS(B-x>`%^i5W6=5Z{yPS zMA;8t-_jp`K9msyW{kq^vB6)$2ZtkGaw?4en|Qo*{V!&+g~Ch#VErRUnE;bpY9%f^ zdbOF_k}4gD)?@%}*=}|EroYZ^gc*@d)NRBUgz@iLr^p$|mL)eI&%pO&PM*hrp}WlI znM2uZ_|dI-%`+?)MM7 z9wa#?U6IGNRCW7IPMc1NW$fcWzjui+gwPU{y3HQvoF|^bJDZ?~*r37(^ z?#zEJ&*;5y!Wq==&gpG-^b{M-Vv73S5i8sH?5lCqh9$bTamGT&+&zilv6CKTy`yCtX$Br z^gBJdGA6Hv(hP~M3r971gBeRPJ!S;robu56+-X{@WZQ?I;%{wnNjh4bgjaR&5< z)=Jhbat9wBBjl}_bV?NWv1Q_A2<%6DB$LH$IJM=Ml60=6rS!R_f{jklT+9N4EBm5+?6lACNgy>Ox}c*-Txg52mnsf`CxRc%a>81@6t)ThQUM$VRtwI zE9#n>0H@R87Yr(>7g|-iZnz3(F!wB&06m|ISt7?*wk%PHlnes>RX^;rQJ4%9B^Edg=1I!+0yMo!`oa`p?U6?kzi{6lbC2ONzpq5)%hSKfz zxe=;wG$J%XqY||I`R-wm7>2gdq+xl|wVXc4%>`({@!{A!v`Z4l?8(|TNX+G6;*4iD z@ocok9+=SX0v7!5glrf8lY30+y|^nz&KCG)cYiTTFB_5ayh6=be@gzBS&?0ZI*Feg zCl`IYM-rCJzIx^S|6}F#g}z=E|7I0RI1PAt_Md%Qz24#^kz&Xv`BBe3K*`lJzCNkB z+>g3mRxpT#RH|1q9JNAdk*{v|o|I{VYqpdg=`of24qKw1?WRn7lGA?CA*i5D(t$bV zMCVlW^8CQ@Xr=3IG;I)-`G~^Ui+!C2tHvN!0DfiydR0STP#rEgM=@A5n=nPIru-hC z|MCI@GcUf^SSyPGx1R>@>XT<&aOkAN5N?cD$7V!}v&UtO`f`bKiIShhjyetj>zAjQPEdoj%OccNN3W6raA-RSja z;kx_72RQy}HMo>&;ngMwoQ&!`^0?^QHsK_ZnqMX=n@@+hp6#skX5e3!7-*(b&NsF` z;p6U$xwd1|uQPGiu(_P92AFtd&swQ_6X6WQ8O_*)Tw1Knah=aco|IzD!6EGU$3NT6% z8&Za3W(E8nie9LP_{yJsIx-#E(l|Z``ZC^NT+dR3{z2yfbvrHcv_>zs{p0|U_oB@RXbw^g@! zi87@h{5EN;?uL+SN$Z^MFU^&S07A=dsF6Hocr>VH@I(xd33w}CXsMp6Ve+6R?L0&# z>a<4>k8?2qPJphB-XBb$#u#`)+C~xL4EL3Uu7^ENbt+Fag}Bi0VK_n{UAM27_2$Lr zS8Ybc=DuKEkQ}4PsT!{F8W+-2fHn)vV0Bk*yMT*VS}hlzTulo;bT0x8URK@`L-$}9 zg;6%(x7`CL`EI{iYXb`6c6#}gK9AT{&7E$+y(9U8Y_k<9@@44HGNI00TvX@==rYC7^iMK(kH zON&0~_usgCm5>XPz&0{UwxWj)66u+i2zP1)Woj|*vbL(Qv#><}fub2=(oF_(dvl)9 zp;1Jb8wD$ODyKb;rrx(emo1Fq!B$KEG|yLPEjk?8W9?B~S7c0}{5m^4cDrOu!O>etG?T@do6-9M&1R|P_e{L@AUM9%G?my$l+Udb{@UJzr+y8-@jMNn5qA--{a3foE3_`g+JqE`jnLKa!G}#C@8uVNY!(V^OQ8EjekpI zynl4;kfP=M-d^813ajQ3Sl-*mzzu0r%HxW$px}ds;Bi4G{~Wj09e!K>*7$kpGeC)O z&OYgV^Mh}uH}i{(wNKe(`KTe7w3VS&DpeMr4J%CQlmfxC+XZ`AdVr3h2X-xVz^;X% zVCzXrckz=p|934vQ`D!QXKn6fY|qbkB;O(5jMlEr@n}t!=5CCV|H8i*R#Ljc zyC_K{-KrYty(;R{rsL5qpXDfxW=O5UDKzt`*=VA5(F!KJ-+gvJJs{WF``wAege}gu zuhOT#F++J7OGW203N`|wI!kh#dFhV%kZ`6@wcEgpkn;7$Wv z$C)!$wg+18Fr^evE5cEa1#&1=+mkT7OEv}+I08?}Z+sJhFm!e7XUoUjk4-oBKRte>*XVi?=?!6c9ub)w_N*1vm_I&f82-fwzFno7ek$QcL#Mt6_*E^GDw$7OLU&j(&x6;)xP3 zkUIV%a+`v;_AiHnJvxfsYsLJMzc;(^fqek)>;h_5O&-f_CwWu9hc9+JSobp^mo~zI z#ifI$*$4nsbMNL+RS3|gtndL#zvHclRbT~@S2OVRS{e}V8o-EVLC5b8Z)KB0AgH|j zJbbKxS)x%sfe+7~j*n=Gnf=55q(w zM)&pU>2gv<`HF&pw?6)+ptfd)xXx&%SdpZ;P3ZEjSee6D=poEn;BH>y)N#A7mg7Qg zLASKqd|hSH_M^~Nfe4>+^Jd)&7!es{|T$9I;*W}2tUcY5DNLat~6<6pD^>q5Q1 z*2wGuw6W_sN4|M$x-nA0j+vo3)P2%-aK+Gk{rN24H4i1$^7Wnxnl*HQG6K;=@>@cRi8Jn+Zb0 zun$_8eofhXD=l6W!f=U!(#Lr_Y(<;$83f=;b9a6c9-t$;}j~Q-7j=p2hw8%e3x8 z^4mPY56#*mbm|Ksa+v5elTG6b5)U%n&u(G`lOtxX`d=*+Fo)GeJ?BaJITlc&qT{Eg z&OIlQ=w@T7ql#5(q^m1N_24ur=sT=8REJ@;+!|-b?Rr1l-7pd3V~7h`w+U94%e@%| zarMfcaQD=^sSZ>6z*davOy01k*YkbP=MfZ#i;B-!n<-Vin=HzLY1wL$*8%5QFTj@U zYl>*TGO?moK$A`63NcCq2ICnXWPDf%nUwsq7At5NOU1utwJqs#OKoVPVg3Oqs?|_a z$l0{~9_H=Qr^>(Gl<3fT;D5pub*{8R{|CHxh8CZF3WoIE{EaVAyRE2!W0gvCn25&J z2QY3H9ojRKBTY7j_J#y!$Fd+?0Cn}`zJ|Xp6+Sl2cjICTBSt8YK3uxx0(uwc+0*Tf zQ0PK=;|09L%Tlz88eG=tFNDJzq;C$R6TFrU@nyT$6ZS3IV@bI*I9!XH+OY$TeR%evEdA;>Cu-+%7C%UUV&-nv(kaE>+( ziy86m`*)K$rm(v5^N-UVkYM}ng;<+QC>65b(%+#vybgVoOdfh6@R7mRS`A-$nD=JWaU(O!`=tnuI;(EiYDZZ@kX6^!LNuslt=;@zKG^W#J1Wb@Zun zP&CsiSP@^O8MI{rOt7`|B)OaG_G#mlI57yx$kAhTVdx;E?|Dy`VON;DAy4;dT4I%m z#`JJ)xJW(Le&@lnrv8;Lugsm21j6cm58NY1x@WCA5y+7BHelndfqx-`%w?v<3ShZx<)7F>I^KL!^Bkb} zA;E{IA1>^Q%R%8YeV-ZB$!kr1xABD4{SeZO?N=j5-dnhdl)VXVRETn{V+`i32rB?Q zvAD*iPAcM!7JwEdHcs@i*_IrD8%A!@gEyS0s%2b=$BTi<;LJ?62~N#I_g-&@{f>`U z)=R^nWS8Pvh{yRbA46pm_3Z1x=fg@OnTwTYb;;7-*1<^E^|Xiak2Y$+ZiG`l)}@w~ ztsdfz-3D@X>!Mwz%S}vF+)J?YMsdIm+G<$x+>LNAGlKbhXE4SLkT!l(s|v4n#;&D} z%3j;WM`K3VewGD6zyIgW;v6;zv>EJkzO$Lvi{f z@uka~FJD!ki7$WvIJNY~3|OXTe<`?*|+W(=6Y$=;YBp%&!2(Dr^UY z1tam%F!t&FAPgeyP;5e;p+(w83tzC;I;cj)VKw3b_NCaIp36(g{JF692GnRAc0K7v zP6#*um#!9C=v@zsjO(!~!s0_vD=0tta#i4YS0$uc#VDqycc=Pk4@IMbqMrx7dk3f) zra@zq>od;ibpvW`;x94nO4%B$SV30A+!=pYj0y?+#VF{tsvyNNlk73fc>l1gYYt)h1U~+^3Hb#SM_I4GT{^Dj zhHqxfPn%Ou95n3Bw*1)ozvmJQ(!)%xOXYVy28f)3JjQ$~Tox-?{bTiRpx8Q{!md6I zCTKIBf;R&V?sic42sz(_CzRs9%0^jjUp;WV!$Tb2y6hLc*vM0SLZB~Hy!$v{ho9>K zXN-X03++6fJh5U?^0BZw`B0px9;>a#S-ssRFq@R!(9WF~el@I2A`~7uWioXR__7vcz|Y|D{X}z0tw$! z;MDh{kMhoTe)b!A3QRw~FQeJA+5LEpIX|MEoS8lTTyEjGj^gUhg%Zmce-!8Lrse$x z%H3&j(v|q1?)K?m!Hs8Lf)Fhsq{*bR{KjZf!g!T_ouSHn zH_{!D`3pf6G4f&X-Dse$V3q!MK8}o43r3uQwA}mA z(r53ffJ0N1omnIavi9q1-It7*7xHux;;&Q(fo`$fbm~lo_six+hM=lld$pGaq)sEd}aOW!Da z>XcqKCV;qk97cbLH*kWvVLNkm_rxP+iJhU`*w*IdGeaSjxZ&0AkTBS{)Vf>55^)>YV7;$48lGM-NHS3_&APH0V=wd%`(p|yJ;JzmsJICyr#GQ+r=dJL8X}#sE8xh zFwx{c0%yhWcT&um20nK|@~XXvIlGdFm^r&Ptw!M4sjr$f7I2Mtm68O{4x-lQ-UBYjA^N9-&z@ z;KCKeouYCKYLtHGN>;Jkrn~nuJ&h+IatY0j#zzQj{UEJYwQJ~KVwmzxNa7cJb9nen z?0L3LmOC)i%!yNl!;Y^7iYQ|f2XR<~_He6Mg@@wC1`W#g;vW=xWRc>-y8`E-+zK$K zFiY2J@wJEud<(8sjgb{>X!nA!CJaMjP&#BU(rvVxWfxy0hN~-|E;adKoc!C&iJ|x^ zy1T8aF}dDxcHR|J{+Td%T5dR7k#mtgz0%{}J&INuxu6@^)@N^1M)o|URf3LE-bC+1 z1_zF8&*#$$uq`fLn^W0YtKBj&Z6)$d-cYlryZD+3nB0oD@kj(Bd3;DJczA+#nS*7q zE76H6KX{UU!<=Jg*@bKIqxD@y@q@6kP|601SF9Bq-0ya?udOFl-w6L|dXcKWAtEQ8sNs+*!^C%0<(aE0Ja7@6`gP2Js|M)UU0 z+SsTeVAp5le{`vheM7NApwumIP{c~C+Pv7Da94iel+g?W$G3cwA*`O_cFsRbIxLP` zp(t5gPh`Jf(mO+bYLT z5p07hDQkTV%`!=lrSVN~(=h+@VZLZfCujOrdrPe_krS>8R#Bh1+5FRHvBV4tt?)%_ zQz*P)59hPZ!5e=jM2C_~mZUNf2bz`zxt<)sQ%AjzOTLVE4y`QyR;@na7Sh+V5NpFf2_Ske2 zki#WDLVjV?Da==~+|UsB{3K7-kawmTQC4|aS!~FD9t{}hSo<<x=Dt=&rdk~pL+E`5WgfLYKk-!)h!We0Qo{eL zYtdm~khM#alr-EYDOnM*O|x!%C)=}T5k5zco1lQurgU>*>OLKrgX_XZ!n*?RQWQB@SpwFJ{4WzDG`RUoXb8&;(#z{=)6xUBf}u^@J0>wpvS3>DJ% z4z4C8^D`3i$8x+wFNli51a>#kwalrNp%fJKjf0AOhmJL@@4X)R9Syvz^}E-?S4Qw7 z($m%@oMY&AgqEN5Y%@zT=}DQW^hi?#$vB2^|PL`DlibjKmmI1ECack4Vu;DP9c+Yg^R(a$y= z6)MOSEr%VYWu^;iabG2tXjImj#=-UiKXV30o+$qIN1>s1x@RCt?(^s+kBxjEqWuAr6i4Kw=rCuE5Ux6>y z6Uc8sjj8tV<7Ai!bLP8Q!HpjP50Ttfwfs-IL}krlEO5A%VeJmR;|Bd+HHGHPe?M+e zC3vTR4ZsSV-L#7*z{j@)P!1lhTBdN6!A`|1s~Mt+M#dl4g#n@-TcZ%Uo8{>JrI>YAyZz3_NIl zK3x~TdLUM4**6vL2=_*#1ouLIus!U+S>ryd5-PXI1GUElhysXpmx%Wu#e{x-%k#fj zFFKQL!D35$D*x@ik6st!sNtIh+xJX`6@wvDhe%B7ifOe1~yb4!N?~~ z5EI%6f?1Z;>bfCN^b~{nLEa*;WJ09isVt!q6(ytz^W8EhuxCS)9!gQe{3*mAR;TZy zW55gApVDsra03ifb2v37qdwCae>*%PD7esVBDTn43xZSbj0Bu0mW$mBEAlooSvX?N@@$QUY)7n%=wviz5R^C8CP;rr?J%lZ( z?@NJTZ1Fnaqe2e%>eu3_kRh6=?QA2#SJz^&AG{+n^Y4{d1>h`bDSq8s40~Gdw13MUaGnx9 zf)M*PjXxE%dZlXI=$vi}tUADoQjPyrI-G(5H{6{Hn!|>AG_(MMDMGU1266w%IM@_<6rkufpt;A-`FpK5FX3y#RHxh8b3x19#yOfi{h{ z(=#*ITPMvho}z5sG^0eccyWU=05t9b>pAVf>oaz6-*8qO!#|-q8YK&zW7&TEj+>xQ!}JYWC_50_AwuW{{KVO zSI0%!tz9EIba!``gn)o_NF&k>(lL|*(%s#HL3g)+bcb{!B?zd5bouV#dCqsv_y7E0 z=Dzp7VqI&k%U(ZlpMSb3g`-g{+T$GQ%l4oUq&JbZ1-@jcS@lcjlNW#kr-Q(WZjEq# zp+oufb$(UO5&Y7>VcC0eKqf>87I&T@Cc#Pq4$3WQkkx;nl!QLyo`(a`D-W=dLVAm{S`S-(Uj*4pV{~m>=SA9{0uo2+?FngueVaG+K-I5IcN7L={#%NW2s&0qtmUQ5Ne1<(`5d6>-$I# zeTI<5oqy+aCp}30{Rr{QzPrpCdKA_&p`6vYMa~~tp+KeKP&aXBjZ3GH!0G;vj*{9h zpyz0$l*WVTwJo?;SE~sJyg_|-9<%oG@~Ju7Eb~hqp+mO~q3z#2Qn%f|`#9y6Eh^T@ zuGC9mH;qNOF^SuIjYO{q3!lq>{57N|ur72Fu2>`PuZyGD_(4|1&=?KLm3Kva+^lR! zv84p5PXZaMf&tyS=VB_P&S)r}QvjTXqd{uPkDqW9qM%Hcb*C8QAPN?7=QI8HUgBVb zbKKX%TA>L-fc=C*d-uN|=1F1XR0PF<=s^fyd34REPypSj0!GctSn?;TWA~;^3v_@j*zO0)gGU|+T-Zi%Oz8I(F#RBkr*h_`LOa*9VW4Impm>pSv&6>Sa=7 zUFUB}A_mz@#n6n8S|?jc$z|N;!(IRMQRSo%3)B5RqiUU-d%W4{5g>Ahja}(p=)*F> zI$&lYT+-?nhxdHWzup2Ke@}*>2`N(MR5BZ1(d9Caa!=5B8gfoQR*x{dESk%Lmoa8O1}5W3ISZ%C@-s?P-|tTGinC&QKzmh6T7p*eEx`}U+xj)r^+1ku?}h3=eu&Kwy)j04rdgIgYG|`)2beD zcB-SRi?I+Nym`J^dQfk*WGSd8v^-0j?XgD}!Y9G0ietlrm_!4+t46_c;I8gtm&E?} zt^)dZNS#g06bBg6@L$Bua6$qZVD%Hal+Gr5AS{0Zq|VVyY6OfmXU^u0s;nm7t01y0 z{eIz$%f>~DI9DO?nZ>LfC8Um=c})RydLyWfI(Ml+8c$;9ShXz{= z6Ng#$FRd>D^uXTz9e=t~uxnc_&J0!xDS_1w-tzuLd7dv&&Z?Yy>9eA``{2JKL8V1^ zU#<5gdE-n*B&e=bwi1SZNE5KjitGgpY)dk2Z`l+M^M0}v;FLvGGvn*B`jc#o0n`?*d3 zw1_dzdypQ*y!|>hpRlYw|J@16-JoN&bZ%GT8Ex#o>uVrEdEy4Rm>2yyDHSa-j2aCGA#@$B>;aI<04?J3BRt5W9 z6|3ctx=JY_`uy#9ny)sH6eU=Kyq1drk$j$6lm5R65&{eI8<+WFFF}iNNL%uO8XnAD zLL7F&OM749!Gb(9SS0h!uMEt-ap~$ubNF-w?AMh287i8*+#Bc08Qx=lGT=rk*K3li zFsjq8Fly2cL_|DM{~+$G1xf(0*~Lh`x={-4bDhvd^a1ayZpZIDSj|A_;ma2p{t>Iar&ZWdY=aUK0nGG@xE zB862Fq4;LHvKudV=-i2*J{XU}>CYC+-gQ$9Su3f^U+vduCCv}-s|;h4OMYg(+6$%C z9@~{d?OiBo{ARM$D}_}F$B0jP`b51w$MnWg)@3u;Qa4_4z@)jkfm?gc=HeE!L_bc* z>icmbcf=H~qy3%^XQr9FjVCqa3u%^t?&ryDS+oeL-r$RaTCVc zY@RohL5_000D?bPt|v`24*i=U?%N6@@?LxX9tg9{N0kE_?U#AfWk$#mR-NQ&?FTPMqWGn#o z12=f%hCxvWHJ$F8D-81%Lx`TSHn^m^Y7%K}Kh)(?xdr6y-|$z{iMxrWt64vp9J5NQ z@H7yw64&pi`?dXj9SE4Jn+x~Ek@5>gedSXAF0r*;xh~eH7@#ntSX;^pz1?av(?Lds z8s^dup7;n#T~{$np|JK3kI%XtU_25^d@ zqb*4e)Vp#Nw@|NFSuD?c8&g}18{NogKO+(3$&yfOZ-vgpTC081SLM|cUPW3_c`e4V zDQBaWEku`%N&Q|;XqqWL6lz7!>IYC)T$a>_lpKc%4qKCRzSVzI@_Ia&q};j;Jue4+ z!PE;)HS$1B7zkRYLEtZdyAH~HuqSa0J%orUdlmRT?Qg|nf|coYJ0RCWP0dyUpkaIv z>#Zu%s>a`+1u7nyDvjwoy|szAjX|n`rkM_;7f0R5K&%8L&2QU zW3Zs}IXzA|G^`36tjx*lN75DgQd&bpX{I45Qm_co`Wb)X|Swg`x;geGvZ-4dPQ;dFc2!K@9DwP!}%(|$e;OuTMfgBNK*U7&^O9mZaSO{sK9IA9AY*?x8ADk zSbC#0rWLsL>XBEEZLtH|zE(S%R#kq|XF(^zt-(G}>E>RMvKwgy-Umh8UN_S|6aFH^ zjKGnN85~*{h(HNIi3VUsA05g^tbPoEj}4TlfwepZ{)Vung}w{|0Fw9*T z88{lgct!8*kNy>YarR?T+T+JpNr*M+@1^H2bE^121x3)E+qCpgMx2#<=KCG=JHEbj z0&Mt!t-!iFQogS?9LI@G+bS}OPiq#>U}0eqaQXq<{qvZmf*6q`&>WV{EoP>C!8>cc zk+)<1Z}S`qYlU!$!?smb6Ac)Ud@G!Zf!Gg@_l3$EGjDj z9G0RqE~C;nH+m`t&vGj^@gyXYM=$3O#}*SO-UME9~0iXirKgnOFS{L>qG!1+Sm4dL;4}Z{XOJf76PCS~br6 z%x#)kk>$u1SSe0NMu&HSU6GM-T@b%sUq83|9}EDDg|lD11uZFHR?HVS#6P60ytstZ zhIasc422>~jR10I5m=fb*~^zO0UEEpB#_Mz7RzwwXlh1FYgQaCU_@Ddt-Ve}!!Tn?3E?9}8#Q~@FHj-lFXO@d!NL|!R z6#{h7mm<(egw&b!%y0n>lMZdRfJBoLK2jh&;wG$#)`l}Er3Z&?|99dr>VI&)pa5*p zbN_k7D6Ihwpt`lE=y0B$19K9EH>u?An0Ta4}3!yU5}1#7JOC01#1;-Q6lH`kADvdXG7 zM$)*;o?_%nf{3R66cMbwH>wYR`W!K?W4r96fn_9c2+?Pxo0_>i3G% zxS&n@FafwF%@_FN3$>)<4rdr7^Jvy@mACYyII;=9eNo(9hvodK!Ke;>7))Sj4cnsQ zaqqQT#!cj>VWa4h)AD3Pe8#cI$!=ycZw55k#jWmpn@*B3_NC1~VKM9HdCrU|Tp-9< zTk)R_?2P$9`pR5;XSApdZQlqGndK1qbyh~obEW~6c6ZK}>rX5(PyY;hveC3GBeM_WCi&n2x2fh^{)3M zVCTGpzV5+%QhfDcLg!xw+nXk?K4F?Hp#6 z{1u))U_;6d?Z5_q2FT;3P_5rN)JBT@Zr+*^zxgS>MLw_AVU=dWKB>+Q;@_o+20VAY-uP7%1UYhZ< ze8whF%1_~I#`of^qgac|{KJhP;PQl#a^uDBGhS^twCPt)MS7!=4II+Q>n<|PTs!yBV;LbdD;$xY;8NhYTjYZB& z&pjzkNglh>{qXLEe1C6Vj#b*oxbZaixOz+$zz% zXhF2`ldQdiLd&tqD z3EQet9fc;m`%6b?)=Tms@rfsiuqyv!_3hpl$hR(_vQ_*#kfk9;saVc#=}|AVn1Gb2 zESM9{29b*In&zMQ5u80~carf%)i;v0z{1@xk1Qgimc}~!g&>kHGdvS^QC8i3^!@Cx zp+k<(3K5#76B|*GNc{K!<#RJidBWUhEK#l*kCiD^&}po`hZ3BXCHGm*j7@vHlPh9s z?HR^dZGR-mhF}%b<8lq)PI}YY?(zwfDkTq|nVU&#&a}kvu;vxkGBt_jglLsv=a9H< zeo?(4;nU%q!iDx@4zEj%K?ct#XBE@c&fw0sciowlk5hW>IDG0|f6PC={rt($rtrM^ zl?8_cQuVxFEmUyEV#DDh&N#o!CSqWiROds5bpH8^2IRrmXn*x{lijxtG0MxJT2K(2 zzB-K}WGBG#={*KMTvIh*NoVa-;_G0F_j@%+5zUtn(;9>cdOxf~2H^0OW0KU%7y{Q} zcQ{){tdb2VZ{AVZw}qy!0{rDKY3#r*wtb+o%!|*%V>6z4nA;`5BJ9cfpdmQunFo(c zs^uQOPbNlmJw2>u0YuB2_j=)v=f;~IS5UJ1&4knA;pT)hfCXl^klkMMHyEwQc#!+u z4Ppr2=ujK51lhm8In)7L8koCi03Bv_W+e=zaP_(x7Q6zyDSfP!F`QA5l#= zj{`Xr^F9E>b&{D?`q><4H%4jgx2Wm?ZNBW={dp72*E!eA8l3cY@iI$AZ_HJWOr2Vs zNlu6s{v!-q!ynxcYx{;#yLOcGOZ$HN?nPG>B9}eB)>7l{*q3k3Q@6Ws7ZkFI?$9Ez= zGplaZ6~rmN;!v8{(JwHdROP{Rm2PBKK7lnzKfZcPp^vV2gJK; z$vo~N5tBN#wCv*V>?wbG)p7jWxTy8iDt%I<@!P%I&77^Az0ET0M+E8x7W}~Uc;g3p z)cjC~;_LjaBK(0ZjDgh0w14Im?dR$<$wb_XmlhW_-R>%lW+iB|7ZoB+aBLum4zO^w zpEq)l37xC6R-66e??2n*Y;tX0n?Z%3w*!`QYw=l)($J~!deKHl>Sh>XyH5&<8W>+gD(A zl}p5BsH?14ov~FT114@2 z`i-{+bn(r-)J$nMB2kX99A8@W2bqjDa+4>AKMXvdjJ9=19oC^pG}RHNP=7%!^SG{W zW;v8K8Tl(c zRMe;bXqz~Jo0+G!I>Y%3^wYh%3$#Hw70Y*=!in1YmE08ybCTASYRxXKZm)xvw&2GS zMd;hE)m|8#t2>nJ;f>D$=KBVlKVo3NQ~<2WEBoMA1-4Pb=75h-C_VzEU{UQ;G?RMR zDsq@d>uEWq4(0$3`W7$zWM#RDoCJ{fuh6xJ<$H@iYdV>PipWY zLVxwQTL9jrj>SM6tq z#&~#wI3U&w8=GKFPL0U@<(91uT!^F~0xqZH{G{CPWz8jk>S?Mci9#CC(+ZUM_I)kO*v#ro@%;3?dm6l1l3dR0mXNwL00Mb3M@mW3AV30O zxD9GaBAc>1i*^*d@l-W9!x+TazMI|VKsA~VKG&W=OJ_oseF#{Srp$wb_*@cR(zzuL- zD(9h&5jMsdto~J|B1B*I!+yRP@NU&%glw7jz1jL;$GBlo79i;^=)F`2!+o{@4p}YY zK+h1qvBJ>My5ptpOs4EqC7$EPud}<}ZFsLj8dOJ)(&RO4a^)MJcQNoNNQF-u`Z#Qo z71h#K4_CJ}%INv|=Lj1Z^oyp=x9Zf$y0ZE-p4`ajm2Ah-ZjRwmoxl7+o>?^@-^Yyo;p@vz(z|ve% z6@T6-sPfQ0ZBB!;E^5`cvjBJkwzY47=|8)my@QwX+A46%<42`@nhbe(hFf>?&#n_; zz7c;q!~txCB&jCRfHRHJwWeX-o~l)-`%Fu=jU$1boVMGy6# zQS5UX!#34Z)VazUE=+TF@u}=5&hJ~l?-K6GLYz;3sXA6BR3E63Wsq`qo8-0>;QuLf_K5_Yu_MBbVVkj$#eX>&ORs9kp;#HnfTpDD`=*@ATghq@*&LbFu z5hoa>;Oq%VK{*~Z{PF-;CnE?1e2fkI+HbczPfiK-7(e5g*sZxwt0c+-NCPi77YRHo zeoNU`oeYOyfrF0Ld(BYZz$c(uJ_Bm#)KaMvRQOl)0YjTGLz!Rg7i*^i?K+=u#@izD z**O4|ekNbj0Dz2|h{(1F%%G3nG}#R>&W|q@<4*s%Ed_c088XP5n3a6hu8pwu5_^A{ zO8i$ZxEtJurrP4A{y?n1RWV2{$&owe!#1yYz|2rpb3|H*CZ{H)N@VG21MD&S4_~N> z6~N4|$i8GesYbPi@I_*5wJ6PVmHPj^-`DwaO&DvEFh&!P;L5SAAT;;F030K=c2-AP zL^@RKX@QMVvm|1`qln=Pl|XQN|9DpIOj|EuG;E(!z^I#)Bl{N{(fEGFhXOHx7k%}F z%%E@lgemiY55KPlWdt|R@hknB-+=7#5@;^b#=yi`-Shc#f^I#<>b~I_P-A!iQ&k-g zIt=_tPv^e*30O*|2|$XK3`RyG0@qf;Ml%s$Hd^k|L#g2H41HCUd+6NFVUKG+6dKld z2aroW@Q{7&ey_CD)Zlb&;&x`s0kPpMphZ;(uDZXdoqN%iX6g38P#iKhC>1PM{L{+6 zcI>k5{pWggem3XLTFIK{7UuJTB|M6293n6&G0xMr1JN^^)+u_;pm=gOrVu+w1elo2 zwnS9+-2-^hxp$O{Vq0nPSz>>b!OlE*1Pj!a=|GYixg z2D2YkRU}t-!=Qq_(n91626lyoMfr3OqlT0)2{6(TTrfrO00!$}stOx=WOG9>Z^|+V0hUCnPl09UK} z3>>C#{@XZ-6HlOIodI6eg|G;nrQCnMZ`A6pad3c*2jrmggm08o9bN=Pin;_zaN=C< zsri>8W02tJ0?Z!%lcH=8&ba0zCZmoN7*R+wlRh-Q5wQU(<0YUB;9yh(3I6#?)eFp*!@I*KvNgh8czXxWW#o$mG z`8M4?eba&E3-L-A?=ZO%(-Sgi@Fq+g2jwUEV^1iM+LVG#S)zmt1BjpSUb4yt^yl5v zc)`}Y04b9;%(!|cTiERsF(a23vZ))L#!1i^)zeY~8+R7a^KeZgZ3J;TWwD=FtTcc5d;d0T!Q>x05-Mkg$_FQib=hg? zu?T4I^IjCB1xbX#_={m8N~I5gwgfH&C}Vzp493y~#^Um(WxEp}s$d%9Gf-h84nw8D zbaZeEUpYU##bv8M)K3Fk0G`w5N`R?o+Z_76`_C%aeNYP89$Y3lTN($|%*UY+YqRbQ zEpj2~M{s!B2_kGXoPeiDZG#noLXt5q49UaQEi2T5SicNzr9%P{ldd{S^Wg@a;8YI@ z0|>dNbpgi5>LWHPw`wPK@({(wKqZKU+u>hovah-qJNt%4R3n*BlO#Ktl{E4nVpdC5 z1~lv&Ggxc6#$nW|HFjka*Nyeax#OzP01=E8#xdEg**i==Ro}p!1T9Phs#K_O2h1Sg z>?c@6f)|_h!7Vjf9)|_*QH=Z%hpE@!jd1G177HK4$m7M_7TT-^he=HV=gDk{bu+R# z6{ZE16Ofc?1Bt5ayuzsehDap#JA+0C#kooF`z*ur)xmU%^(R4zrWcIITBtIjCj{n~ zpvR+HN-_YkybZFx0kJas#VJsOgsDuiWmXFKSEoFEh34O7JEK2_Je*PmW_Zw?-pp3{ z5!z&8B%=VmjmaKDY_k{-%q9_Nroy70U2P}B{3!z;12$?%;Wi@R_+c2C_QM=t?Jkct z0{0X;Wym{DvVkFXarP73iHM?X*QouahO#HD+_ylGQw+q-?A)qg9v}ca7y{~*hNNH! zMA10=6&y0Vj~oy5!aP`Pt5t<560v}o?d>I_|7RK44xi^{_E;{2l5lQh)rNPVG$I69 zSyuFc&dWCSi;?fEkJ*2`09d~_*x!mv0x29Gb|bS7YzqhIXJ1exI~P;W|ELg{uhcH< zFp>RoaH!*}X8i+RuLxuVy9u!ih64>?kI8hRs+gLr93%k;pK$;``4fo2U&f1`M0nqw z&D*Y+8{t!Hno|V1_gnJCm{2s|n_*PVn~8aJK0+x5TUkF;rUDKTvr@ufc<^_GsPrE( zFv@a_dx{pHIK0nvzwFnE?!|iBc#;?S)zH_gXF$T|VKS-8b0-HSga^Y;9VVSzpqBxpjtfa*Y$&ME2v*zR(E1_ z>rFF-!6kh2Moo&N E(CXx;wiMmf~Npdg#(S3`rVCCF)ALjix_<~S8;1>8v+K8iV;VxSRnaVBO*wyNdXom?| zITxYU>o?g}04G9}P+_DP#nG}2vPc9kNnQM4iZk9yaG9v@SSqSy5w-yvjYzH3maeTT?$D;c!Rs zb|;vTz@;}nBBc-IfB|l2{!;~~3{R!uF8EoP?^D|IMRV5SUPO%HzCctd2ia>7Hj)a; z@`sVd!KMUOy+Y|l7GmmI4gi;bssQpvRY|W@5X#LV_&*j}ADKWcG%3+Z+eol#2^(K= z&YC5$+B_zp#@`T&UqvQ&_z0j1dF`%*Mw>}Gwi{4T6e#+qDQ#kuHONzWSr)VTjNJok z`jBB6MsvL1yU+f56_XJ0!gS&4nIC(fUr=9j+2rE*hWnj}3|iPD#jxI{@$N-@!)qk$ zvSh+iwqp8zl)YA_!yS5n!hyYY5#nv83b%C$R-$MCD|FVDQF9)N<;mR)cUduDL#4b$ zGSBU^K{fD6tXcdcr;lS0HGY#1sJDLg9P(8fJ3oabscTp9VpV{)uwEui+`PjaK8PLo zj8CM*Lk0b4)I7I2;(s~`RJqFX!dvjo8%J(m?GO4hRse2h6Jt)h3+A3$%~fU_MDt%b zgh0Cblpb_2c%iF6;Wo;<4_3or5~UG5yhg;pB%Jk|XXM-gud2bu@lvEe#KlZ1@4j}6 zJbNnf`i$(ZvcyE?2;=F$R9Lr)hLHB@6s;Dz#~YdN=KGH@fDr4Yj&Z@7@1c?RL^mH5 zmJ2HF$AQizj~yeVi5SrE$1ed;?uQuZ7Mw+)w&yZT)ce2238$bOv*o*3jUC{B|Lyae z!^(fH6BH*lJ*(Cm8A9}X%DS9WP^~>Dx^B4F*pE68T(KLwVM@%QDbl+_NrxHj@vqMY z!$pOllu{YrKJ%=BfuPi(nrD1>Hs-Y|a5F)JC($8E2Aa0i>+^TcwjJ=6@>vd&_L0jP`0$~>;JQ!W zrE=bLg~s~LxM3Kx2#s!Ws5x4m+m-N`FtLuYD(01LItuO+5eW*D*dOz`0a}#5UUr&^ zKZSR~$vT0jnoZoh%KRfO-inrcYtU<69zUVA=+SydgxJDA9_q#=xLY?JeAhQvS1#-} zIOT|m_vGcZDPD5DN2h>>crymU zqMSsh;vT`}CvaCet;kCEj2Z&TXYHLrux>W0YqMV)@Ko799-!uKU2RlNsOAZiJhGcE zjLp$nA8JIFQKB^zAQ~ecV2b64#_pUvm_|s-p<{;`Dj5E^ENna=(L* zoSoS4HDB)xyv|FLtjF1A-Xd-??Si*kPWCRd7)?YJ#u4Eq8gZdP3+AC1*jR=51`D2n zC54aCuOZu-hSo>REOV4V4({%IHgW;{yh?#JQbJO-BR|pd(b|p&5c4iYUlr5?)iZSG zSGz+fz>zBMla|mx$`!-~<)P>JAQ*ri;(EsQ-yAh;RTW%J7mm3aWWTCQpVB$-uou9E zgz&ZV!?BBdTCcWF%$44G{~EHxUOT*9(<__AgF$NLWDx`02nX}J@A?l@{L7kuujjxY z%vVkL7^z%c5D44^FaDN)^J7B5ejeu-!wXcEeC`^pZee~x3BJ|WEUGvP@Q-4Oti}W~ zM}?<}vb@*utb5(aOfI0mdJJbig;PGKt_03Lvy540SP(b3j0?*vhJ6+hyJ&#_*+qXz zF@A*D7Q(kud@So62Nj&Xo7w`#9#a@bcV|4`-caIU1dWEfOsnfeozkOz|Nb>-N88Cg z<4YxCYQgjwQd8tPpMONIM%XyICgIRDHNJJ5Q-#7*O0ZD!^7cr2enO<{iH$AgTp#(s zqd}p#Hjz~wnUU$>TKS@(;paluTttqcQo&T4v|DApk?~2Atu7teRpDqo_o5 zQ=3Xpyt6tcPva;iBU3hB1*+lPH4*Cnfb*q zEmEFho{t&f!_|Ra)jp5T<2@NHb;pEdQj@{NWI9gO6g4!) zKh*g>nI?h9ylE+&{;ok3$^L~wZ58#h^#Mpie+g(L^(Zx#)ik_tL$8k$_|1Eb>uunl zL9BJlJJT7Zgxn-jlONCtQkjobpw$SWaz|xqIeN(zAvWvkSs5cO)HA>35G3Zb$}{8# z_Qhp#%4}LqxmH^tqtmg}QJ~Z!acG(aHIblnBXzL#hbS9{zsUi1E$_-dGs+6}q=$!7 zxIz+tMCY)eoYR;SSj4hB_-tOF;C%(_#2;*w(a<}Pk-Z+HhKpZrQU4`rRfhrzeTpns z{Q+tcy@CSGWPn3Rr@67u>S z1zs;+7wlSc4(VRTt3KN7Pw_xa?`eXUZ1e?qn8=My&ds&`hCyRwk1Zi8DflJ=?iHOs^PzYZPsVU_S4;R=bAgX_a znq1}qphvq^gzu@`z!T&sdF3Hi_DXg#?ZTwZjR6TkyY2FY;!=mQ)nqZ);n4LflDb70 z->L^v9w;ns*W$7nds;b66dveNb2y1Q6Q%V4UnqXc%BH%{F~42U2|~5LWPyA!(rxbm z2Cp;S+(fM*k5g3B#`%vi4oerz|J=-enmv_S3B1-r}JdW48`D4uU{wAv1tlqr4#qjnJtgn#&1M%XjFR)m{Wj&YgUr2%!Y z@43Bn(=bzZ;705_0kyB|(HFMURG-2n23P%VDDBpE|L}LonMg*y%KMZ@&O6A36^M+z zF)A}@t5Q!9w3Ixu$>VP5jp?*6#~^P*OZnCL6mLTrWk@5FXmT`?Kc9cf@^sG=KZtBt zCaIMzC4he>n{t-W>^YANuT%63bi1Nt2iK(Lv;u`c@ImiT8x-bOf6Oq_SGRB-J^h)q7>%o!?J;_YuP`O)RpaJ#(LFH@dX^AUhP$+&L5@2&Tufku`W#99 zYr)^d^lSJQ^#qq4#sDD?tYP3 zm%+RoatpLVNE5fN1o2?ACNi7afI149mV!XvyGDsB{Cu>;l(aWrU1zrKGL(2za|tQ} zPFhlDN+HgJ>baA7=<*+2R@J#EUe&Ri*W!rJP%&d4$dI#e-`hNs-t66HkJ;3@~& zZn!~=dln_y;y@TT3#PbUqF2WbsYAdLlE9=C;m1?f)$j(s4@1LPOrP2IjvVl1g@Os+ zncSbGC#FT-?nk1E`dGr_X;Or9LR7(q6!kK5m(ywt4m2a*v)LL{_UghgN5VUf{p#_L zd`w_zQoMgc!As@em%=#H)ptfI!7m$5pjmHfe;gjry8PaG{oC@UoWPJ$}R zm1ZqeZ#H_^xd^xrcUVS_#MRwh!e4RwUqySF)Z^Rztdji;u4P~+0-YKw7IhNX3uPQA zia)TSN6zpPXJy!B!3y7iEmRND;!XqT5uDpJ?l!RfBN&#uG}mQ`jkI_whr8krY(7Zx zgr#y-l_76n01jCx%$D+u$i$rlQD-Hu#?gj?7(^cJKIUv_(0i0la+B(wIT@H;VK9UCMxXVP&u3srzrgI7?A!^h<(D85rVE`5oxN*d9EfP}0skFJ!`Swc8S@WX?Kpydz zQ9g=d2&W5ZfCr>*RB=!2fdPDcr|geMs)+XE5WXuqEsapTUoeFsTA*Q={%`6SN}O@% zx-apefWzL;XMeBay?6Gk2lua{QrT_V%vSMHPBwHK^qnh8cH3|UEcxHLTg@qY$lAu1 z#h@w8^-D>7LEbW25QmEOP@oUK8hf`__X^0BOG}0j{Hu)HXX2jz`te)!Q4nc))<$Oo zt{GF6WL5LE1enL4O~y~5p+9PE5yAHMZ0^S-!(Pxe>^i=*h~F2g!yLL>+Ux`#a8=Yb zXR%Aw^7TxHDrb}nJR0sKeo?c`^xO9W4g}#T=4@S$30$=@35sWM-Q5PLbJ zU6`VAT^KyDn3dUdS#4eM>c80tB9dD2vl+#wjvVjQ>f63%xczyeZ_S3tgtK3%Bq&}K zSspY|-tCAfPXK|!=d23izivg0t1f8%L-A7Rlj?i2MPZ*?k@V>ggCmq(oNneqeZz;_ z(pwGm!bE}^zfGF<72fba66XruWWn5vliHOLcF#(jx9$%j0e8jj?eDa6EVaU(Qq%Q*ioS9q)hKeb@%RJcl4XVi1$;|ppuH{e53gzM33YFKHqk&HRp^%nZU+ztptFYo8T&9GgB0?#+K+JFEY~isu{yYl zy+(zh9G#X+G5oC=38-T^SDv{Xa?EyV5)4C`m*ELEl(tR&UeBZD*D+9X9GSjtVVQ^Z zk-x}CGNF=x*C|;bRUiSa2e`nmBtvcff<4bvk^bNhrA?Xg5^XV#n$;2#g&$#MgaDV# z!@sP$`Dm^*je+ZFOgd%vz{V_ap&pppry~(IX0-u~iQZ%VNysm;CeV~AEDFIlhnX(V zu)sTkzqg!6_ZOHKlIk%u;^aJj_53}>VA?{5kYczj8jgAUvZi(x6nP}@(vi`tLe@ay zAfB>wpnfBd!xVGA1D z66f-5`gPTwed`R1G{p2qxzRA}Ey_3f!Gy3n6+X+$jv)8ZH=R*e*p1hZd^@8W)D5X! zKv40*-8rzRs7xf?4;$42>X^(uwh|jd#LKW~a!#i?;ED8Pk63{3cBHCfCg!MNzh&1% z9V|2{*T(FIM9>p$h0Q&t*o>0q0PHyVfb}4s>upVgY|)4=e`#qK1$`OUK;&)dESFw; zzRl}LT18owMdETOC!j;y+J5Eb+?MR1>J<3h>7ysdiUd3Xw* zNlG&gEe+%%aaF(4kYv=MZFlr~2HoN`jAjgSJIB32c3k_An%cQp)c-9y6FusbJ>R;uidY z!~ZNAkLArXMJIubt&}H6Ss)0E^}QGN>t0efjA8pLaKF=q>+MN(6ePm7gxTT1vfflY zY%&t?xY0G3utJl9sa-N9R|Bc|EVkok?Z^+M*Q$uT2f(2%*^hJMf=m@G3kP7+*Ym8#!smG7JXof( zuAu*IidOE8cM1FD{_OUsN8Q&fr0#2sHXYG#Q%=13i~6SkeP|EwK41>1gZhAs2`c~? za7=@c6{vwF$2*|s--rnML0ani1uG_bSFRr^CUHrucTBoak& zULjHX^&_zB1Qb;=>5*>?PVv{w0M~Vtw5Ea2FdC;nTxlWDw4a@C@FSW5|JLoy`~gL6 zCRH>+RZn&2ze%Ml`|-dW%QZph%!^kh6Da63#H3-Ck6aY2=bwz zF6eou1zOD>=-#+<**-1Zed!)B<;59Q$~7@f067~LXy1|o$oV@M`hy7})_m3-h1HzM zPQ44Io1mFvN>b<=BO=zJOCTWFpxzjy=Y8{SE>U~X@-&){_1r{f_-xy?$9i~B!%?XJ zRbTbv;ln%z9mUqjBWZbz;~=``Vo@nopOT(f{E>E|)vjOG_sx^0-hE1M6M10C zY$;otKG7IwNRe(?#I!q+YH@Jdzti2?()&ir%|=-h@g(TC^@|!;uNW)m3|!V2X0IQz zU-tf9dB+MKnuw}jo*~#F4v1N%Tiqx|(4S`dJ8P$u1zRHAH(zcS>=M+9KUpN-2tmkQ zb?~Bc{^Yd7bRAk7DgsuE^OuZfjIWu6`<kskFvF!T?J&*jzfWIe>HZNLH@} zE2IvfsRm}tz0J!jUL(DKWDLOz2FM)3F~Y?0r3jYEL|4=#APOA5%psN&cd#bjuKslf zCZj4Q%U44m9dx~W<_kQFIvJ>c9{~Gex+JoQ9WyKgBRc1Vrh3K+Ze?xJ{+b-9QD88ULctrCCe#1WL$ugXI zUE9320tyl1RHtU0D&td*ID4gpU?g{k@Ce>E_dQKE>iI;L$G-u!>yz=@Ukx-3Y^Z=2 z$PeuIM*a*79d#n?v2U~~um@aW;ZRRfjlMP2BqnMNko|jqhfrm&{^(>45CNYky~{lq z0$b@)iFF*{v?*P3jzn~Aqw55?DqE_ds0a4VbKO$b3E$;DW0<;tOg81Q6xVd82$WqB z-mgh*Pr!6SrCr0v2w>)-R!Pzim(3iPraR0}d<;M)zdEx>VtCd`i9`6>s5eLx8zrVg z>Z+}`&0Xx>y{Mkp*L$$8i`-`&+Z}WmOj86pb`=o`xvcWQJk6r1BDnAmp9)>RiSFor zl$7ZkMtvv)*mvq~;#qm(*>|+ReiX1;S+T0&kC={Oc&|gljNh7J))EjKaVFY85hGH{ zb;1{GLiuD`EcwuWoEiGA+Rryqjw$^HtT+B=pS|M8R#$V5?#Dy;E;FHWy#ipiuVVt6 zd30RA(9gqNCl`2T*wanwGzBKrjS|jn$E_`o4#*EH{QrKw-i!`?^$og(v&|`AgWpEB?J+U~T zg9y8dZUY7ps>!^Y&MjB;GdgU-aA`rCQZ8qv`U#7Xox%8G3HaJq0pG(A)hs1~7i$_K{?5aW=Au=3TF3uaI zSTo zv1Xype=1srC|qwbm9Hq%T=(jz;TG^hio&oc$(kc@8}2N@#e^_cYsSV$e7ykcNOcfU zu<{i-b(7ewZWkSot^*M`NJ3kpxjS@$UWr9H-_Ew-_m}1nJ{%(FmhVI+C$x|hP&Ha% zj>Xfw3Z`hl-2b^)M+o_-8}diwHVg|aMeO`W7f}g(Lp0V`BzZ&l5_Kg*n>V3=Lx7+5y#F3R~^0$MR zNzeb5EHa-aAs%k6>q;WZkrsbmcvQet9P1B@fkR7ZcX(U#xmg^Z^=Qe)I?s5xn3A63;{D7j@uL+5m-P*@Cl<;Wn>#V6;G};64%}xa{wBHa1e~_P@FL24f3FDh zk&1oq4D@~W*?%=^VC&OBYO&stBbel@*HI@^67h#hA(Sdh^% zRN#X4fz{xY`q?qhx>4+Fx#dGNPf-`kpBxF{TWRlb?vZUz`afIN8_s1V_S}Pg6JeD?{{--4}t%zr~m!RJu)2H1}xO?+<>LUrZKJ$BMo4tS)K)*GX zZKJ6ds79EwkCm8-+@KNvy3{B zfy>XSY4!CB0QW_h;Pq{+0^9vc4WL!ROp%0lm(a;Tr_cUXGeCtVgeaV_dVe+HkIeYO zXQ44aLr4s4<=qn-RZ|2mHcWbwrvagC%Vtw(X>++ewSQ!;B)I|wI(c>(_d|xvA3Fj6 z3JmnCVPn^kBIn!sVvd*JE`b6tjCRAM+8{DIN z&>wgolIqcRV0ypx$LquN(vvk>T6ABikJU<>SzLM@jVb+lyA^HBH~+ewu^I1Q^Lb|z zn1U-38$ieaZMJg%xsYfHZSQQrKKyyK6>~s~;k*uZ-P(`0lQbSua4DTSa0CkocVvkd+Jr1!3oG&*`(WA| zFI;mCo1M@_L5UhdsnT&9eo>UO!OWsKe!uN!fG|bX^ie~ogr1##l@9DhGC&7`<6dB_0R_$4Ymkbt8;iNJ+!GP3_17)!H=R;6+MLTGIf9oAO89l!i*+ z{ZagsgC(r}W`)d6$*@g6$7?6^RMta1uREgK1!4^h7GB1SU)v;HEBAp4=f&zvbv+QL zCFpP;@N0%RfP;03{I8KKK_h<8>y%rJo^1*r_PC3FUg9`fI?*+MJC(=XOGFCNG!+4s zVU0(mE}m-gt>uBKPeP75IvFrVsXu;`EbY8O-?{bTFddbsBpUy>+aSOW9?QP?XHe z-YeNVS=qn$*?6Ake*ONrzxVfixjE0zc+cZ~9NhK4(iV$V#jY();*+y$O|b3Cc-Lt- zwpQMbRnqY`DPGqxT#^>d$N5%Mt-QpWbEj21$2({nkEY3OVN}}wLvy3+al=+bvjy#o zPm%moO8if0Je_)fCcV%OTgoF{fpU)-D`OI$yE zX&Ex%HfMxq(nh}Xv7I37*{xOWw8bmEulwDYin>JaYzMPwSXWeZ(GpEqim7G88Kr)fageFG?t2%y;8K8 zzJopwi`>bC(|c_)Z2v++%`*ZT6_d9Nx2}d**_iU$_Oo#`?u7k>dD{o_G6)`iaG0y< zQ~f2j@H;JmL|mE-OzCnD`n&f7ia&S{;VFe)&YA{okW#+7n0 zDC$d*nUz?*9_1cqj<@bIRx}%Ce;Z1-ibedF0vbTcP>v?Xqo)J1X1ZX{DWVIM`#wdW zt%x7};IP}&h+yT13mmUY$G@nxdi;cUk|z!r-^mev6A!?5hM3wZ(hprmOYRzeF6L{0 zg4q60hkuNBeWL5P>vPmewQ(f8L$m4gQt7;Z>F++uCw??TAg&evRC&m$ zfoq)W=gcx;8271f#}B}<47h%~CXwe`xaV_6Kk52b6BAZN*{ANSqF0uxd~~>S{A{VI z`fs76&InLdT=91bJ!NKimguWzC+%3p=G#Z3 z1qr+d01ej?8+8{dqH5@GH+91f^?StBaEd&(eOiL`vNwC_Z_a|WR9vgkYCvjU%^;NP zeV3}aDuPF1G1VA;O|jna*-jW23^qOD2)aY1lqX$D?LH`Ri13D!R>M?9pYQSh#K%ny zVkgJ5W&|VEes=zISYqT-3I6c{y3KgbgBhD4tOYngU^-z}+};aP<|F=(B5>%&;Q#UB zjf|bH1cZ!~@+(mhCIL>-FkP%xLt3TXN~U_fQuYhdTjz(Jx}?dwEDB7#K0HtlU@LsA zHW=xY@a!nmW&)YgDFE5$G|QPULH6x<5@P3WWaGxDb-FD<<3DZX#s(f9VERhLiMNWk zk;=W>xq)JvHog;Qb@gTHmP>k6N{pPuU4p3&K%fPiq37c38HeoM>WRaMMG`HX#xsp_ zhC$nAmuGI|KCd|bWnX+*nxFM6p3&%@j;0E5ok|Jq7D1^&I4k3~`#8yowJoRts3l!; z^BBByH%*PsS$&?Qh{I(uK1Xjzh4tv-z53I$CvHqcf(n_1S;ryy8vzRI--9C*O1bH$ z4;yVq4wAoopuBJpVNXYCv7X%c$zz%O06@*ogdg25oTVyw`1pq3IV-9cp`Y9c2SgZ? zuZBInr>82g75!mB1}gLj$A(cb&W~_1KBLhWyOWgKm>W0SGK&c|bv~GSkt`@pJbB>K zNY^X0Rx4{Fz5&!A7zXzMU>RH_Jn7R}A8IjBV6@4b^z=?-JKvkLUYV0ua~`g-msk%e zkLL3XRZ(22h>D~s`4Ppag1F0*5^#lhuqNA9aYw6>Gy$P^5d3k+fSJQ)^2!e8M*Wcg zY5dzgD}o^1rSlRb8r-!~k3r)}lk>xm+s1}BPmb0B!-5U%Ga^4wR z6^RS&Aih zVmHke-PD2L&=!7T?NRX^wyQ3L*low{LkDc*Tv4%K^$ zxlel&$Rcfd3zYfAUo;R|2y__`7TkDf}E_qVD>v$F5cm#9tlU${0}E2&KhFF3yMeMg00eJ^;% z@b%&RkR)^Xa9|1;Twvk-40dPse)kaF&MfFI)gI`6codw(x5iy+`iXq;va(h~!w`@b zZnYabX1sOWb9$_Di%>%VqE9B=lZN=;lD>aZAZgGqxnaWUw%Us-TO7Lo+5s=L>e1Wp zwkNM3P~>dh&V5uvFWUvSqhES_Q)dfeSgxb~mSS@v#9u>&=fuO6;X8-l&Wh368q>=A zNl_2%5?Szbu9iauRrifwd=fuL$cL?eT1XKrF`sHEM_Bxux}nWM=k%@CBqSM`FM;w7 zevypcJufIW^&KBh9FtAshnz?yH+g(h*=fHSn+TJEBkEJ75MI5QlZ(hBXhgwAxkKgk z^SFMAxJNLN89l{YHp}FGrrX-1EPIEk-JQPbD_)yjvSfL^UmonQ)zwk*+UcgfrI|dZ zeFntl2R{nSGb*JD>xlsH_cROCfw>`6g?gKpM6Cv$IMx~FO9PBl6*7s74v?a6!8HFF zOU{Ri%A=g`&f7#iE!};$nf6#R=)#?|R8Hb*Tw6qYElc3PX=VIr^*F))46Vxl9t6+a z2G1XL9{UkB?vj`uoIiU9{8)8yt&{SBQyv$DE{nvj>VF?gp4+*p`s9>Z9)Z*2eEpAy z&`;Fg68D(-{xGCrRhIEpy6W!3_6NUerk*sQHm-#|U{N7tx_C0|s^QmfH=c>}76+Hm z#$OFvq8{HFio9`QdeYY7{k_n~yvW=o@4m7P^{JDiVGqSK0d^4wX~;g83a z#wYYk?nnFlGL{~(9bR0R?5yN>{$LxXwWL#Fa*b}jc13xrjjdzBXPv1tLGluYqX*Mb zB*U_kc+dIfk!ZoGH$|e3J_*H3e!^^4Z;CmD#v<@HibWINZ|$KUc&B3ZRUJ7^#7^o{*TG$TitYWEdJk{vI$c>qZCxAR-VdztSh?|Y7-PU^z_-8K zwyPTyj@!Nb=t8lNilp$@_hEdP%c?WUN*;O-l-D2QHZIt2Cg@+bFS%*2bIPBBr2U)Y zKmoPLqe+Q#1cr}JOa*a}0(;&cD|m>*BM4UT>gZQ-j!$=yF6uJ9$uW0_@arp{^H(Q$ zMhT$8^A}+3?`Z5upZ6ZoT^zs9N0sRGET7cu6X{6jq-bgY{xGEr;9%~E?KZ1og`RNu zSa|6y{@s^C54YQirTB#w-@d*tbZ0^dR~CWE;U%nn;DFI35E|D%@pCrU;unGMuTy%Z z_$;UTV@;)*BSPsczI2_W_!o8^gAxuR4t_Bg1!MfLC~w}CO3R~}v_l|?vQ!VGOjGGe z{F_{$IS$hs+9FWCWzac2plF!r4gzj;4Rk31)sXV^0WT}r+Z?FWl+}D*6KWzW9?Q-* zhhL}zX+juv{9MqMryY73tZ?yP#h%t=_F^+QsAWrUzY9MMKGEn^`n+^VM)|j;^A~@B zGnk2lU@z|1+kY>;>Ej?(RbfH&&BG^U$dTk$3_E)&@2W2{FUT6MabR!{ivq(h&eh<_ zw|B`(h}t@(L$p{dd?ij0jeoLMCr$guhMz5^$<({r;2UyfN*pA)mJ5qt^?QNZuvFZr zB@Yj)BKZ5qaB@C9_1J9}CS_&rd~(1HTIu(?PUnmn`2oYlrQxIQl9Y`0UMqRHShq>G zFI#Vod1M%M3k?jy#|It96|rxEby&o&Ufhwe!F2{P%V#=@z%0BB7mpF3}Q zF4eVLetiDJI~t&@K7>ez)soHJOUI6ca{2(r3gnj5K|1DKOU(VpQ8S<5_kzM)-?d0# z4GkmS-nGjRf6?pdv&l=yK}V!Yz6KIVKVsnPz31$jF6Tt_yXQhM#qySHt&6r~QT?Cq}VB4nVS%8yq0pXYC9A<#D2NXYd`6khso!glVP~`pBEF9y!dMD zZoqkq^CF;Y$wFT=S>c@=wcDL-(Z+}i8q_`pVk)O+f!u^0@4^|4nfUEzTF5MYy@GDl zUNrH2^0^V`ew`r?#W?|dtgWCE$NfVNCw$hFwQOLi_J z7=i0X8bKRmzsyCM9d*P1*sC2j1tmHvA>0R@RYY0I!sn`sh2ZV_DOUCQDkoN$+a~4Z>exT&gLF zc9y@saUD@_fo3Vm+bS?07jrSd93*~NY}KI+HqXe^Q|)%0&tY$e&vtQ|#TF>AowANHo2@$}?B{1vQ2aew)Fg@=9qH&K|1z(z5!=C9q7nWx z0>WYlger5+rtlmiQ@1$(sjhP{?iI9iQkUiF%w;Mlk20{fq|8B}nVBymVE5{(ki#gy zP{;vPXY&!>CcN(gbkA>mS{Ud4@!{9^&{OP~(?l3kWjy*IDc zXI!gG1GEa?_onAylp&7;QUzpE2WX9QkwGaxE{T9Krh3$?(={J&b49`GpL5laBZEJS z6+!*9X&Gk&4ApUxMt zztcWhC!PD~CEEI>iYpMdcyC3HjPuU_7|s${$i+>8h_7IInES$LHrQQoE_h>Ec0K)v%y6HC$MRwys(}1 z_<^q<#q6n7lo533Id2k^(IF`9p`eY-JMG9`d2hctq~BGK5`9hP4SZ69@5e&z6yu^l zL@5EkMPBi3K-(~f@VG*#B7>RJ(Jyzf2NklP$q!nAA%75vsJ<^Yu#h&Z2M+uKlv^_{Ey~V+>u{gPa`{ z)EPL_u@^fu^(U3)qHj=cU&)=wW%5t=Io3#V2ANSkNZ2F^a7hk;EHgL*=b176S$Eo$ znUAy#3v{5{#J}v1tk6}gy$fjIQ+xKrISKDNVM-5B`HNCftUdAZ3GwD#@x8dY@83~& zP#g9zfMk~XBCnoa-oA_zTapzCnag5Mcpxq#9D~Yqn zO7uyKa?QG@#=r6P7lF-K-Qq!V?jkec6LBA+AIG(BIzC2L=}g||W-b%KLdIN6XQQQ78(tRNZ)d)$!$ zU*7X182vIkYfyf3n16uG#{iKxhpR86J3R@ee(z=QQkI}>4wI{(v9&eek|(v^C{)&E z{@X22PWg*toSNowc7&R^R;nUfamp$DH;0(NZyPx`GxWp3sPMcqUvp=HCXvG^$*Hc; zZlK^LGOh2!?H~YZ%jDGT4qso9Wdqn>7ER0na;%R>K4Cl~D~U&`cgwv+6mjUUrM)VD%1}XVNKj^0^A49bg>3s5dE3~MWMAFpJ;iRl zyhpQ|){AHHgYS!0p^lxi&|+t4x=(YvRjA|eH!EG*lvW@(`Lr9A*g~ITZIpq+=C=3h zYHPI=42nR*Oe1Wo198*6Nu|#WSLJ2!Soe;+5cGOChOC^446UK@j0~OLTS-FZU8`N< z8f<%mUZ_4=88*B+~Kv!CxwPoG1GOORwTegCNt6ctQ!wewlr@h}$ks9&~JEE)4c zh>)P=-MG8}^qQi(lGCdrcBaD>m?fdfi!e|n^=7`@SeOyW?WeHqn9c7Z7hSQ*T{rvS zfdb3qZyYJ2$7}scqboKRozz?DiqS$tb4Y$Jemr^e3Yj*~b+?Fjbk^GHII;0s)+Tf9 z$#c_{7IQeOBW?pm%5;XhxL$>t`&#uV>=lbHwuS;!{ubn@f?Nw*&sEQpjpA5szb%WQ>Q$z;0StneZr{XxZMx1w^9 zZbrtda2}QHjHCJesL!{8xriV0yrI8ovoNr=p`8(<#2XU>_6%%K5JUYma{7-@BJb%_w*V**!-b*u8c&=W7duKZCQZP)K^MU4&5&KJWV?sdD2j@FQin7!>4TLgqx zf0X?bG^)8p-l=Lw2(9ujjn{bHj|JS^Fo1ocoqTh6UI;Rwgwqcd+Oy{^0VmOEc^{bV zT%YU}?&Gj7WYUj9db|wduNQ;@6-VCmdIT%*;#(~`vfS*Ph|Lr}yu1fBnivKDqj)d=O>ze8npH;is8x&$0l~bnC>+)K0jT-)a z!=<)U?pFHznkI3)TzT`}p3qJ;t2U~U;*|F3>0mXJUNGHS6Uw6k&Yd!_)X!Gj*X%tb z74KjK=BR8z{OCS6nlOR%7tBZpm33Kl1Yb|RA0yrnf~|+vF?xSjx}Fio%8l4Lol6LA z(!IHgPF7gVR3)2Ly+{dkfde`xNRP>+qo1%SXG*hHY5MqZQTjG~z9yYh3e0Ia&4j zM|_VDSAZDkLJgVpkq5{FefK%@dX~H^`Q}(=E^Qucz)YX0vYYzCj*8-iz zreLM_MD_G&roDvEFZi4q!OV@GP zf@!~^XXh_73JPl^vJ|}-pV$;7*A_iKa7lC?I5xHb8UptX5nj5xbC{R;z+_|g=E0U= zPsV3L?(V8xj<1hu$@H@?dJ6;NIE&W4mqW2lxCMoRt65cAAY5oN~}#~ zfFJV3-Xt{q(rW&GDR{a_>T5;p4>26XqcvG$?b4>m>qiZeCh)#zn_JQi{Ko}wMlPEc zf<;QtMLR+a{ZR_AeWZK$UKneD9<>#5nO5%_ul=y@+IWpU%u2YqH6OhpeXMYcLL1@D z&DaiD+1Uzd6LuUICTCVsUqXVA_GCU)%9@T5r8Lsl@ea zR-*`Nqal=UeHBBDnTXVm6X@?S#GWAYQ^eBM4Jc8J>dIRgTdT0#&a$Y1%Q^z>o7M-EEXv6W8z4Df-ha&Pnvj z@^%VC$vN{c5Hrd zu2>;iFTUjtogQ27_D*;mxitNaowJh1RjVfbAad42^F}LIyZz8`)|W4Z^KnJw=(XPg zY9*h~@2e(WKqSuJT1xb~j^a8YY0ClUeNA@maaolJ(+58zc&aH5xm_s%I_++70A@-kT(Z=NA$x_z*lLgek& zHHeCd8SnH$6I^;*fx&X#3LAaDM-oTcLy=MEi{Y?6i|30Y8_~QJptV~ylO8;BU5Z!X z$a93y4wo7k#qVaaTG+les8aW%7A%-yNujB?lL){Vw{*IPK<{;#9&LKu>S@GGfGlgQ zDXBJ#^Fa})UM2e+?g`*>g=xw!F}N>yv**P@h{sa(Vhsp%=pV~(D_lf(mJ@IQjr7u$ zyfaJwOWETFPwfx)!0elcoFO;Z6!n@6B+m+~y69ynI6{09k($<&YPL9y?9I?YQ<*o! zBD{2)XW^*JVo3;`on)htt@yFZ%>RZ6gXefqX^vcj86>{{5bRQu$3V!43UnJRCm2x` zfwkPn_NUWZ9{c6(lYRJUYjE2RB2gp?Bxxd_RC-N&GZW}9tYyEjG&d=y3J+ZYdKS6Q zFWoy6y(O8?DiNn2+0!d-ph`4-SCzJpc)YglG^E? z5ZmARLcLyXgtOV0j+4r@6tez7mvZFLvG|{;*OOM8OA?Tz{e^${%@Z?vf;#X7HkSe| zQ{3qRZcic=Bv?3sr6*7ODuDZ>!$?{FAyhzF6U3u1ALoUNrB0zWH8b6LR$Dp{hh47(ySoTY?{`T4?-7~{$vLiZ4QH9~qfXWHY_KEd@iN|Fp#F=OUwP%$8y zN>r{#Rmi;%-d`?OcGk33%)v1|I=FW=Wnq-E0}w>l$@INxh}jufS;$t^WvS!umy=LH zjn0$&(@nFd87%<(EClMyj(1}Zwg7DV4oO!F#&ww7vgu9GJRJ6V!15yksC(ey`d*>} zyi-mI{Zh>_P?px?B6K|khtl@Xp)3rlGIikieJCvlE7g&;B9nnbS>Nem_S;Y+SNjAK zaOvPoCp$5&#sni|4olr=MH!9ToQ4&hab?b!6ai}jTV7nN)UUwgt)KT64al5QWJXz!~c#Q3_?;t zJg%Cf&`hHs@j~VE)t9UT0o=aFNF^y2^O^aMzP7JY5jyiD48#sLM$$OOU=;LQybg*0 z8_Xx~SL3wCAUQvUb)D!}^QUI&B; z;t2m|eaUgAUpviKDk?3E*Hn{qKx=P4w@06)XaHKqj*y^n2kU$Z$4h}@Q)h-NMf^31 z&v7;Pd&1u7RgkbV1@4nFkaJ@#GKrodrC`eoqrAnEf8Ru~7W#SaKy*XD@)ZQf?dk(? zRn#l>6RK}7W7DDjv;KM|$E=bhzS^9emR9toPcB69ZQFHV|d9aBQ@v~}G(o?(- zR}FU@<&Gnc){c)j$0U1fbd0AG!|PFJd($LNjR2LC(rsNxalGB;f5NsG0UZ=z_PrA{ z2+yZxvITM*$u8X58F3UFfj{t4iTs1RlMz8Y{at1*12z4)8~r(Va!q@hs%FRR%f~Jb zsg%%Bls#M;ZqJMzszI>CCvXFuQTmHKqq z_lp7C0ma`js1UnIC++CXN?r&!nf@4#^e|7K2`j{K%Fa2*OZT>Xr6W?g+oEk?Khtf% zYt!2_mgS?z`bpw$ly01CH1=Svx0j~HHC&piRS>J7_6-MS7ln8h0X=D@UV<9~)!hwo|6OCMYE7%N2Qpna{4Jv_v_!}`f6)CkI2s&4okGmK`+lk8l1vzz}$r ztLMYkhVGi}mw2mI&jbHC9iAJ9LcNM>1W6{Is{JWRB3@-6?op?-&ncmlt+3T?71;eN zBz?A+?s5)+@sso0LNDFILoG~>9c2{J`2D+EF)5YzyE3?H>Rk6wiuUv*d#sYUKXnuGLSR$Wt5ZzC}q(VsIddnVk0kpY5*J&v2!-SpKzLs;!+3R%Yqx(Y8z>_2tm--rMC zqfU_Il3{dR-uzv($MEM8ysn|-crQaAW(C5<%7@&4qO}WzI4uG=RsuInU2ihLyKmPI zb+QHxbF%urZ;59$m{E`}%s__>AJwiu9I1Hl`j+GPC_rd_Zw@jird?`tXP`g>t-xF; zf9lUG8Nv<)3T=-H{AY)v7~xHMB)ZQ1HC=vr$UEJVjMkQoYVWR>x`zD66byuME+N-a zZdrGKE;t-0MdHm7P!9rVn2ic%2!L2z!(u9D{J#H5>c4lthM)%)mNXHT??G_amH)Mi zuy9QL@Fu7Erblo5c@x?>q;i8Hq)VQ`=yqxTPC8-ZaIKI!qX>Xv%PiK3NAkKYwb9OG z8$_v(dEScg9-6r=^_MOc)L{@}Dj=0-tk?5V4o#^|{+41~>5N7gO5NLU4GhBVjFI67 zk5q~v4n4P*7iRg0FwUNoogT=i%^==!RL$k;e|Cyi2YIE%rY^@nkHsg6A$SgdFU#8+ z{vaa}sml_>{t#mj;>MniV;9Q~Trv!={ip?KP?hHvRt^I;#SBtFINBWx(XkK7QhHhw zIm-dM^cplfUFj`FpKeQefp!*Jl5-a8JYw z;7YV^5oI3#JwWx>V6zU*BkbTGQUpk`pAmM7KE)e~hTGF5s>9?aK4uD$C}^)WPViNE zKgCd3AboZa4uFzq-5Yt8F(W4xPNS+`pf$Ru8s<^&Gx5vo?&R{$gnbW9vZ3c7Fkx2S zTNy6BhUAKGp1r~#H-qUR?BGm93jW_~^7qPD!dkKo$b=pJdkil(=5k_)t@>-wxWoyp zKnYr=IXV7)sDdm*)yhddU5ZwTUsJz*Y-eljSy=CQkz$LehozlGQBGjgLhNU(krA684;9Z7Qv*3;C z$2sD3ad!}kiP>W!)$r@bMO<;m60&mqBl zpLOjrEYPk=f`FnG7%EMnUhWQvM4LXPuJ-3n+FjRLwh+@}!3WoG`3jBa+5a_~$m5gn znC%jF75w{eTPpq@F&+3imA7PgS^0Vmagf(lz>=7>gT2Wj4Vd~GSq|;a0quPKd-1Gy zB5?R@a!UeYJm}xr4_7o}s2>+>t&WW#)N6Dp=uG9OKWV?sT1QKjL2G>U)o2zisQD!7z+PENOfO*@pp}{yYjGsq8I+VjN-|#V^4t-r(x}oyq*K z)&CWHqMkw%BBnoX!4Ns1-q)VlC3u_Pjzqc?jNbyeYs6UVhd}Ro1t>L3&2}yngC@jK zmKd+Vf>PY$k@Izq?Cp-RS3N=SjCya^|a#ry|8h4WX3q zW=yz$=TplamSm}7FpRRF3jGWDTPXKCyJIPo6VP6!o58t~=HnF~#{C);Pfgp!ZUBpN}6 z^&=fJJ4}wF@mRYuE^so_ZPe?bTjKK|#@mMBmN6tG%t)5!&($x@30)0xO8E3?QVUtw zpHW~Pu3TU#$NulBNr1PF{Y>hJ(kfxEu>Yn$z=S}HcF zo8x4CAh6MN46T%ptx72boA>1&tMbPZE(CHnC$N{gdmi(tKIDpKVjr0&95m0 zR5Vl)1yFy7VlGnc9e>gFJ8S-TBgNBFU{B(?ojCqo2Jvj<0i~IY$`iW;qOcTf$Zx9z&OuN>m5Brc(_Yeg9mV4)3f-s^8G%(Ay%S(yYW$IDn zy`;)EhgDl;W>dcCVwi3n4;BK|=2Fl>uVPYfE@&6UvG9K~I|W5UJ7Aebz+B`g$4UP{ zj_%ZqIOcl2_?$b2HoCZ$5|c_@yDEaa^dnGq%ZF+Saw`VLWu>l(*fGjHcR}uSlOw$1 zu1M`Da!~&FWJR5X#}rSRnEyRw(m~9Bh|ucYCd&tMZ;_iG6^~87EJq(JJ{wpJE^q{Z zyUWo*H1C1Es72m#@dG)AR9S%&-50m~+PqXtp&q(ay~vQeEp+-w9+5JKVEGl%rmayb zoD_mlgX_s6V0k&lY-d>FjpAT0a`a0V`%>(*E1hSa+huvMq4J<4vRzJcJc}*gYzZm5 zQ${BJt%+K;y>@}$3r_2coUjgE=j;FWne+G&R43xIzea+NMLu?Fm zl)smO66|W5G&^g|SNfDgXt`|zbY!pzsmj z%JX=;9@TAApjX@or^&KI_%IAOZ^prc%miGK7Za2rLIRrU0B&i)p7k8fOzvjE822O| zYXnbN2GSdt;c`buNI2cLVtkkgnK(|icYHZ6AFO@3olnxn=zD8umh;6LJx_ZxOMIw? zl>&dRcAQM{{jp&4cvfk;l-;GiY=f^z>?Q<+bot-_RCo|+ASwPH0s3t9zh9FLpMs|Q z?DNl{{eGPG4Xgs$0qKdqUWaxL`QYQItI6?VE(K7}6sIM@reB&DF)V}_5f#9FtU;Qc zX#BXjc6au2x$3}1JxVp7xw(zHZvEkP=+D}3I(KCqg;An$G&ow~0BLx46mNwp`Dk!@ zHf?skapIVjk3}Cf8;8wJ)0Jm4LpPbX=!<@@AYG5jNZxJow5bMy4c(owDg&r;8&VE} zUyD^K?qnp{!AD+25*D8eJErH~Zv44eyNVDh`F>3hf?_Z7QPZs8O@C7BHJ=U%c#Y-+ z58y`NcF3#v`@@@5uq6DHe!j5aMssMzaia%PgdP^u0L>q-$4`svd-EKm3vG3A+L~*} zqF^z)NKY!qL?w2FjHVqI*al+x-6eJ^x*L7tq9F@0G(4plTnfd$aQ7+RMnY<$AtmA@ zjhZXKyUBsh>b}oXgrZ9a&9Dag)CH=egD=Zh+W(xj6Udr>OLJ-c?|oWFj$kBt{y!`C z_7P$l_|7z0&U{T`g_w_!2_K8N{)2GVjw`N}f(g@*(o)^gUM|!&T8}_SVepuJPpn*3 z0~Hd{Q#H*+U2QgXhJ2>&m8v3V$}U=naIVelo3AzTFK#AdbUVwT4$i(N{YF6lUs zDnUhjUz2y^=ZlP`QirkO3nI&EBQW#)()1!Jrexwx_hwgT_cBHMpEL6WdH)~ZT$cX( z{+_Tq-BVJ~=|U_r6cUgGbvNvEk>0M;EDQ3aCFu$IXZlEa-X29F%;S>ccvhV%K;wQm zj9`aqS>MtnGWM6Q+9kbn{>GR2>`NdsnM?qD#>yBl0ADzDtoT%%_Eau57p|H9-n?ZD zCP$l@7?VRPEySvj&djX5|G_S6>jJfqZn?u)o?;@0cDaKQIT9mo#$+;1!a96ar;WnL zPGHmfj%~7aXQRwxW0q`q#nhMHiPvFta4Yw=(3dY))f(Y5S}xsq^WXH_`4xQ3YUI_2 ze~md44{XjovF}Zm_c~s%B45nJi4`K6MfX6-`qmb}bzX)7ltIHzK{Q#+x7K@eQkECS z26g6MCs@OEX|)}08i^6Yk<-H|;dlla(bE~qEZ%C-BAy#@fyH*s_3`gGmURJ{AK=^?~83rxw1%pu{Pf9a7m%z4VNYw~r>- zUMTe7T7AMBh&h&^O%hjcRwF zdv9>L0PZ7L5mGR&Cy7U=cVEo{QEEJ$OIJW*Z8iflMPL0zy0mR!LY=f9(X1$4Iy1F(@#V4zaib9ZD^tZTPMJ+8@ zjJZk=Dqbn#vpsheczS%;8D4lywkcW~i*DHWUg%;QIfFVPMf}Hu`&}2Y?nxrHsKQzM z{~mL3+DN#zGhL?6tbbPccf~OQz@Uiq%VTYE2 zo?p0l0`3vkB?||l|11v^H9YQKpSu6=?OG6Dv4Fak9G`mC3N9OM6ahAU$O4(ttLzRO zc`Ki6_~rE|$3pbPMSOcrS~X%UaB@X2BJaWw`_X$2+wS!;oa@%}RV9WUJD`{6zva!O5fl3Xs_xh0Ud1dF%maIxM zTbL^XnBei}=0srH8X8NU=odM&uUY5gZ980&Jz!swVnzLBsJ@emwCPST=_WN#({Oe zaT~@Bf465+Ze_(CAeCA%^X8ZT$Bx3k-jjlZ7xvLv|9`eN4B6UFy5#r>8rk2KYgqv7 z^k}>y@s!W|@m=p7ro2n?QK@>D;At%M)o;=_*%^xD7<)80X}a3*Esc3&N7HqF40dn& znTM*1ubqdEe+kKF*Agk_;Cn2@)j(cj3^ep}cAxxD7pQ1p-kCKl!;Eh7$o4VJM^qv- z2hdPq+6U->lC>8JS+f+|T#~im1kTK{m7k4x_5iyx}AL`!k*Ck}DI(N97{}Z`H4P0T5{U z`L&4Ww#iETLQpzh>C-U@7h+ToT%AO)HGl%YKK#;s48iOm`jWPsdP+;gC(T1eY3NZ% zM7@2m(IIq|ZFxiDD?k8NBg!ijM-}h5a>!d!b^xJwJ-CMC&JLRJ>`Y!<4`We9_aj88 z!{KqSOnvI?8t@Y?%+A2Mn^{$)Gs>W&K#(GC=FEN(F*IOUf0|QO;{V%~KVj)P#A#nO zR5}0Gc|dgoOs#;n2&o}hP7>S$z0Gg7z(2H}3{Y2PP3#X+(%(nKj@AjTpF{80 zW_n~Y**qNgcoU1c+Y#zy$swpmH>vdMlw-|Sg83@CLDwg^Y-_C&&xmI72kEupvq3x45aXzBIh{g*%w?=*4ZbM(>0OaW!Hv ztEbUifRX3uQT)<+zt(%f^HUf4Dpbuh_uBkUogIdvCH3{RMm#Hgat>FbhPc~3b#-sW zLVVCTmI{?`<6Z(5{dsIx9Oeg<084=VmX_|yo|n4RE$M9ylg%qLp#fRYdLBUQYw620 zJH&i{te{rfBfP(4a%@8&vKa-X`A(X}1RSf$dK7&FFdg+ti}hIfaZ)y{j#XD%4T+x5 z`uzUeBSp$Msz<=TA%i}(HP@-Jz@{p^{<*>eZO=8O*)&ckMTL}= z177sB>Seu1G7Npv0)dOq(XQ2i%z|coPLFJV>?Y%Tj!RogJL#2siFF4I&c@o#Nb6=b zf?6a<97Phf#nD>DFHNJn_p`f*N%axnhBJ)!qL@SJgbWU~?rY7YZC|xAFGNSB?%n7M zLf7gdqtDTtCYltqC~68Eu{2K~lG} z6C5j+C7gPSbg^Q3!~5)9_=Gd>rom?semfy*n6SzljT-= zstHS}8BDAr1;VV&Smh+12h3g!yQ2UeR}t;zXEmAvUSA8KR?JdFFRrP%NLf>Y1=mjo z@=<#I!s%ClS89GaP~PVZEM6Qo35iLYk9PoZ>b^i`ek}tDMB}&~OgBXnRm#n&wKPg) zi?Ir#v;1SRH6vKi3yA2^J)Jlitx<3F5Lppf4rTdx4b2o}vb`FRPowvE5>u6JYrG)mb_J+p*Kj?x^8}p7 zJ?2pK6-bl2$GT3#mL)v98`YSQ=9EfcKAJ2 z)4i-VO`ej~dI+dIUJjXbPd)vL6p;{0hxhG z`wTf;sMYQB|L*Y+f}wge_0&#V*tp~wlHB+m6Ll8jah_V3tm@yQ9WUHJtPPJiS8%+h zlqW%Z^JM$=8m$j;#Cbhci{6*(lpgciv?hz^Ib~U8h2U+&P+0|=a)>Lh>Bkf0S4PSa z5U9x4O=}*8Tjyy+N2PNVNTqJCu@gs<;wolDa^i700|Cdra{2N|d0{N3B;Axk$M8X* zkAlr1jXbl4-M7$UBW|;2C_A;?|cyHDz=)K1t6Y# zM;1A|{&bG(TG9x$pz9B1-B~8ML{!xLRpVTvt)riRyG0@!WJWiQ3dT^@SZpia99#BInVeD!4#z65oZcI9>PRS8S`(fQhKu z-&~vUIbcdJb_7uVGq**_U@`V!;}S$!^?!E&#rgfxqyQGYGVqi*$^uMv_ti8Hk`y7q zM#mub9sy4KmPpvB*+RD6c~Ihf$fcDbr|P*jh#-5xNw@^bv1pa#!k8%+*V%*0yNMwY zrH<*BxfN`W;aoyHcxJ@GJbMT=5tX#5NNUQ8yZu30ws)485R`(sI? zccSh_G7A_x~zeOcZ{Db0qIa7=&15$>>gUChmGgj%{zT6-SDL z|8|bZP-hN_i_|9{aSLFo23$2}`FLY4J#`ighye+A zXF@wUtI7FOhU*)0Lnn#M9BAUMjzUru(#v2#!RL^IAMkxYYbOywg1JM0_O{_}U@;K1 zwBA$fDW21s0V(c>)_i>`64pG&`#Wo|7H@1@A$Slr8rrI}ej)VWq(^`-ZUxlI&aWgK zvB=U5Lf~!>U83k8$-x(3#L5X3Q`XIncM2)m(hC>XMnO--$Gz?5*9uP4mInooNAUP^ zYUk0eF$yG$bt$I{S#wQxtG7mr+e-4 zNVy}8r?d3i80^_lx+sM5#D{yv8FbFezHo#zZS%lCX@0l%vqbw%lY*+noVUAg_P|XXy#iO6<(je(hxj>ZGfe|&BS0)K*#V?} zfl48njS;esFHa>0Xw|x~iE*vC!jZwRlIpDc77FPLV$*jCv?}@mAI4WEI9XO31(o=A zgd(-#(08}hM9Fd!ycuOVXW8wqCY}puj(YuN(V0Z;g7tTGjL+>~$?^A)OU;O=zG+hM zh19^Pq6=_+!mEVd>Bu+=bDTY62tTL-=%l~tCchkL^k=9Fd<7v=m`FNn@UMJfAqIzI z+C0E6A%(!}Uug-#2QEokZGKHxZt2%C%VY`9)UcxHOqgG0gkV#nd+eYNG5m55z=U>y z7vF~SQDt5Rq#H;ElQNhY0`R;}6@eMjWTA%+(AqA8gfJWi zx*26=ZtvB*<6`>W7;b&V4lMp2VvoRJs9|kawL;!rPgm}~#!!xw1h;>6dEvw{`zhgA z)jkceV64`!tbs&kv%9&A%J%2}#(aZ(xAp_kM{apaD=-4|t4AuK3Io~FwUK6R1=f_) z9n>WqPq(-lI)pdcOz#BuGYl`ZD!gL~+JRQ&#k?Y162=!tMgIRnvX945xHybFZToL& zr_aFB{&;X3CR`e>{PO|etlNm6lVM0LzC-%p)RkUekond^L5Ck26$4U!Tu>%e&$CJ= zPJz)}yOs}`bb87}82YH}JfrGW*2f>oGwyg?unyIXJEy# z3J&DHNs6UE*qDH;yk)7J0poPyGVuN131ExCPD=YiHeoT-5=}9(2{*9DP zyM34?bMgCsjJ-a8rl`Q!7*VT{lM-H)I3z=S$dL6e#@DZ%r*24!J|Bg42Jw3d#pCx^{ zB0sX`!2G^iQT#X_=PWCV%xw4d`G*c_eZK?d3G9VwNjx?d3Cpw;iV|Tw`wj3{uZAdD z_O~WpKKv4XuK+3&Fr&7iKT1v40({z^9@_x>XicswH2gsXtG_222BDy91Jrg|{4P%k zWp;0fjf;S#I=aTtdyRiV^`o*L0O&0*TxBS5Ld)JVDv3<}0o4@os8HCyPZF$a7(bg| z>CeI2cf&)7-CSsZFQ+Bj0s;z&Yi^-Y!^geuhrEGwYX-NCwDhc>h;FwKIJ_?rRcMVA zV^v4+RWZE$>M`d0U0?bEQ)A$@vldVhSp_k^r9=}XngrK6mG!VFKMX_K(HeL;QOif; zrdHxV2EYe?G0T;c=no3X=gm3**mJeTkMUp6|IbYv`gmx_e-qf5y#?KU9rJTe!L@fv zmBPo;9ut5d?`8hmJ(u{+y4gCWTjHK`3* z^i7pTVMv`;VAa4M6Y&*LjyzCan9~-%FHE`RHXU)XZVwR`NEKe+SDtSMWH(G)07=n%<`ie)O_jsE<^{%Y#yk6;E_E`yzuM|Qu?dv-fW zs-$xrkKm~*|MY94ahEwiO@)t=TDH~uE)}!MLCLZy%(MsjXOmrOt&M^86>~1FCz!El z01U`N-+W2gWq~by1bKdIX*~vKCpiNryJ*On>@!X|Pyq5j!Dx)|jhtg6ft3K)IjQZ` zGTBin7YO?}tp}W@-%S5)IN&~S<3T(@iz9Z&%OIJQc4{0#=y4Z`3`NRpa9SzG0#(;? zRPT#Z)=CIgfPCyCsEayGj1AIu4g-IiqOHd{n3C=7f#NQ~DHI(0L0Cz2%7d5htGsr_ z2Iwr`_N;=Yqoj+90`7<|Am*?%<5D!4`an?5Zmts zIc)d>&E-8X3pe_s!#Dt(<*+|aRL*r$Z)c!{u;rOcC#t#tqrN+mU~9#640ymmCqzvK}7q(Dp}mts6Mz5ikDl$LNR!Q2m*#$ zBU3>D(`?Bh)}j6PcSl;gCL#aOG>^7*D9sDJ3P-sbifaghYD(7ngNmDnN{i66Qp+6 z#7&a3x_}g^5s<8R7bAhJ3YjH=4l85)oZsHtzk6ogHna(|m2Rua^MS=lI2Gdr3_DGa zq4Xt|wT7dkNBo3J>-xFfTAm}f^e!2mgbKb}-bqIfdOGCsu^ul_=DVMtTL6@y6o}#J zsp{O9{@sG8qmWnpxrnp@gfDSZIxu3)LT8j0*7+@!;XF%ks>nrdp`@Ajh&h;( zdL}SrIO?wJHbsCG^oV+~y)d(ZccGjif4B480-z~x^j1~Uv(m-~1pFy{2^tFX9HF3i zi`2hU<~xVmxv91g+K0mnp-j3j@eX#o$qy}tK0H)rvWM-sCc9sE2IFWG7~v98BzH0eLEOkdAiu+h?)v{Wu|Z37b*LP&Uj@Ho#$UQLGH`>6Zo zGKoYfQb`;Dai4|I1c~3P8l0*JXN4A>Ok%d zJ8kZr;(%q9KbqDb}WD)1O;I0~+D`0lYF>oK4hlx8dHw z&jA{hbVWa1P^?G2yKW$w$wlyV339bi_V1DQKR|bR7p^wNOjPr=i_N(GxatZUw4w)C zXB8}<0}RDJhW2@%Z1=?6nE^Vi`J{EAy^9dh#Kjc3Z~uY9Zk+(AvPBwYZU6z^(L!>+ zP%<|QKf7wyth!6?-Y`CaoM=QpXb8+J8ty2kSJ^Boc1+^i$e_hBBND9%cRoP#W0$T> zge&OFXEEH<{%okzka9^vWuq8zqM#T|>MY~cM;D*%AlVQN0T1=XLb_sph26f1CU@Q| zfI$zru>?OKq~g!U{_qnuFj}uGOq~Kwp0k0aUqdzA7#DEr=-ERbCyV;*%Ms}WM6TkY zO(xQj1fz_Ppa+C*JX}6*_5!gIONC3AqiMgqiR2`fKIQ94Ay4$vdag4K^0wXbwE#m3 zHz|r(iWd@>5Qv#FI(>jC)b80u*^m+kvDo(qfllkthInAWKM~Bf2oBLnC~ANvcN*B| zhKMkL5T<0C=}dF%M6weUBdo$5X>UQ(6QfCr`I9kE^~=`V5>_x!`aO8-3W44C5#{(% zQmlPrpZ$in*Z0(#DN9Twxq`f=_o25GM_FRolf~`*(KP2``m7P>u+yuBpp|@z0*QJ_ zW#DX%mjNrTKSs}c?vfghbXra&q|7oDaXWYnhAA(MBDO)Q;6_x7B-)!q`rYLBe?hB$ zv2wMQt%Xp}iCT}OImoJcey@k6sBitkj7U-9PDu;L;D`N# zJ;PjP)DHc*nc_$Pl;0OLMXnT~tJ8#FVe1@#q1g{&QfLhcRlP3Sw=Ah`7l@ z0k548k?$nG(*BfaBX`#2zZ@NpqOl0vTZgINA7|oX@mi5d0dH7kwv%Tn#uWj#Mg5yL zEo0{@L+sU7_GrYXSR0AH|H z`c#E&wlHkQ2Cx_jNc>&PQEcNd1;5~$5?@GB2HW4oY0&FDaplil>piOPD{?jq z96(07?z<0+%$8Gr{iDDr^TbZF!JG6X)mcV>An-N9-lAj4AaAnGtyu@!ZY5;bAD?Ie z93QwgHM~#h&Ab}CK;5)cq>a}J7$TkkFF#Umjhq*0KOfX*sm zw$p!8ow8t#Ft0WHF#+B?{8^#SNal!v&UY!$9YPsFwCFQ<+ZO_PC!Zz1?tWH;D0G0g>TJ%w(!aCazyYB~qAr@}|?|HgA4t zM|so_M-c0DrDcuErPYXKw}=3*{*VdhLD7L^i6RFB{}ol9h6_v|$Wy)?mMe`5&yhYN z|HJ0D@^U#7W+4~i8HwUnz?gveErQ4`yuxmENg*X3MGQ=dyLL``fq^bCxU*trF0(cN z_u@cl_LXIP9VQLfRk8Ftn;^hA{lT#pM?46M*4AJaoNmg0m3?=)5EPdOg9+J&tmeBj zTixZlb(56IKsa?Xjk?3T2Wj(w07`tuu+2b8VJw@OSdU7#80slfH5drHDJ;EsCGC^+ zrT~GrYLc3t2}GQZMtC_=C3!=OSRF(Rjnt&{$rzi(myU|--w@{0w!oJmvZLUa{O@`E zKzpFK_&CP!BA}w9ecl!k#7#r!LP}Ut1fmfU+C9+kGaNUnhtJ8326 znAa^wD$AQloq@8&d!Hd8h=#rk@hfuo_Deik7gDS;bro8Hd^4vj*_4@KB6*rtf+67# z^tJjDTt_S-+SVRW1X`n#X*2^{Q8m{zmtve;Fx)w#cZc^_6TW12Y@k!Im&| zznx0~ZeLEd)&kVQHlXP;2z6EKz_X~`uDei*C)woNfx?nAE%jaRD)FtZeQG*)rtFzw zI)HH5FBpV>=@zq)j51se;T$PQlZp`&Wxt};_F#OBrGeN8wBIFmq;+O4UQMG{;Xx|D zy$?IJ74Z352om%hMgI)I6iGD%lJUxP<$A;j^$^0*sxW%cc2j;DsWHcX%y78ZB;Wg7vq%wy+{U#eQl8RMr3Eh= zQ*{B4I=bp&V^Qz5_z3)|zpA0j`sHwSm$I_+uvDqEtEZy1;@3%sxF7w}M^xwiRC10( zWUN3)tM6lTavtX|-!7m%C65PPwL0ag+b3BfyskhA!Q^g8inRS1B+T7MG!bx6UqRB9 z=aNL-hENQX#d;N-RuU9ZLK!DiCW z@oDNL*CjLzL8O`FW&d4Q0A{sy?$}FH#zW@VX#6!@2|rT`w?a}evdv>Ota9SOGr+$X ziR~g=Sv5M&eOxwxJc`4l5x*?hCcpZ67;WC2iaOdIlLd}J>xrrl7%!ESxkR5!sTfyU z#Xq^Jy8cv553Ps##BbvuetSH_vF*ff-ws(iOYBa3zq_s$(iaD31Ib#q@ zL@TnrU4H99`#+RfMWm!M=ckawt3l3b8Vj14(2y!)-q(PhIi>g^}b|7Q@=}N=&OxLFgaUEta9D@E&T%{KHD^ z$Jue18YFgqEV;@Fk2X~BKF&9_pBMXc$WhFkpr-UcyBvSB0Q7DaK%&&cJ6CqWEg#JT zpZdI1>ktFUQ^Ciu_)}-IAv0z71u-y)^OhiR3AKH#FNIrfIF)*dga-l6O*sftb5R=f zKxHx)ctsC)1-2)z02XI>eBOv2=ylYHKDkl$#CzuQNxUv*2EVja+PgtFelk320r`pn zgukJ*ysqyC>LiFD(dR7xg?k5EDK?+he<(E2W23yk=V%YjU7hlXX-}{GSw%B20D7)N z+g>D4nKuJl{}AU zGXx9+7l?#Jm$J0V@^}TZSVKaP(k@_hQ#5s5hn-yjx%i-`@buA1(V&CBczUOC=+?2n zs77Qrz+dRCzN$EZZXa$0)0GcEP+Nl0R1e#}tn+z*MOk*Cj78j+?7e-bK_FQ>Y+s$@ zMOOqcu2jH_vp6)`XiyUsS&CGWg~Ead1C~C3L2T-S34z*`o3UejL(8^2t@x{-&B6YP z&matXZqrxE{822y$#xT@k-$M={vSf4y zZ5shVlRb#?cp~`hd?d8rc0|*@YJ(M3ewkObV~NP-C(lRNEJEF31Fovr-MQZUw--c- zR5SuZtnpeQXi4$M{f-||3LXC>SMG=7B8WcQ?1A*sbzYYogghV1Lo#!`r<3S#1>&#Y z<~eyWf{z8O&YOKR^}TOGa1<;Zd1{<*SZHu+G1!ExwQH+^Mb3ADR&j~%m*5R9`**M{ z0b?W@ERs&iIbi#u?i#dS_9eb855A|&`!mM1*8=u_q8%;xQ_>x+owSIm94weKE3=ga z5y+X$myU!lew9Hztlj-D@W^rj)|gZlM?+v{KB2fc3lNjJB;J#HjyO`!u_!55JAN46 zIN#2}vuC(BP2qZcnUD|AcS7EJDg?G)UhK^-qAm-&J$czz`x+Ef{+KcSFIpb_J;+Ug*|U=_zjfX?Sn#$# zoAdO5Rep$6QYP^agro_G7~0$e5F|X7vT;-z1mh3zAs(_Wf(?KaxIV-*whJ1tH6}?LwE3+ z3U$XyBYl2w7XA177S2)Yz}wdtpqO0&)I0(8y5o{7c|P36sr6qDj>n$3k`hE-AcNMG zePE0ApX&{G<`07ZO~j@tJL4;j_bt*MB>+|a=2ob0epp5z0=G5jR63Y%P;mnSjy5YZ zIi40{3fw8Nir!F&HE!#O_pS}^>^r|jC}F?&_!Pv927!}EqB6sYKa~sF%fp6dAanKX zx=`sb0V4`w!ocMW=^k_`7iWVQpaJ(ocr!>HSB6tX`=uO?;QkQ0td3qY^B%B7^@n@fY7SxgDo zqK~`wAxfjk+lCn05DcM~_9wbS!vy03lQ-tPDaTE+h#}Ot&3#Ax8I+m>y~R$_Ff_Og zLf1`}k1_UPdgu;^8F2n~!|*@;`q*^~dLtKY<@eMwxjfxruQ?n676ZZkN^d&QG5TT1 z3z6E}jPYkTZ11FX1T!L596{dvs0QpecmYFK>Y-@;4S=}x1JuO~{C|+uEd!}>)uISU zHd#l&ep*TTmK)XuerO``TI<*Vyl;I^worIGv@1*?$O##^=ecE%u;bxP6th4VoGTy? zrps`ESwWc17P2jc$M-58Au(V1Dnfl?>CfYQqe86G9s8Vr^-7M{=zs(rmoDPz_yZ~9 zTZ#Q`?ku3RZ2*!&Y`7Cj>o#5Ka-3$KHtGaJG%NlNvkmLDH-n4@f?qS~NMeiQlX?UA1 z;^S@U*ZLo2lan)y0tK!zz@uc_S1bpz^XGZRTmdX|D(xm-hST>YP{L zh09$pXUH>9crxI=FaCF3y1t(90D@B#2bgWuJj27pT znjxAi$%K$XiXf{b#b64_f&jH4vLrD6;ufNo@gl3K_F1PtU8hDtWKZdnUj37t9~%s3 zkpcefnUk+scQfn)VhkRNMgEt^bLc)65!L{|(S14cFlVHe-X*gP2&fHcqlgaYPi8@a zTrWx$WQkv#blwCY3mj>w%_5m9+0h6L4T){E(j=ogc}xJF_VImk32mJz;EsfMlh)$E z!<2#-5-W8BRR)B*90K21EZN*rC0n%=Ca<`s6{|BpXGik^>T)C~MYZ zQHoROU%)axEjGA}j+Kb?y#P8N3pR=|bmiTBzX&nLAl`QwLNkYx$VHg<#}p&RG4i3C z*@~%M$Nd0`0uWJD?9Ks7rqLEb^wlz`L!x39IEl)zjb++NDi9623dE@|8A_w@A`j!h ze?q-)oJeVaBYEiZ(Ls>^#WXHPLcPDMFL{q$;ETsag`R+0y7Wwh4Ffj=YK^Kj;pQ7) z==;F8+8{0Pxr#3Ml`?G_k(o2iY;Dbf-kq4oJN6#gu^-_{dN|+E&`hY}=YE9VJ9wl- znWx{O7Xnf2S5U_!%#pRyQFt#FN^F7nI0TzO`x1yYp}>M^LgJcNlkhCEBO53!k+#sB z0F;MqpfVCr$>XwUjwJC;vHs40XDS7YAi8wpj{ajK9F^G5U>{$NU1Q)~DtRRn%S!b> zu4o0Z)5Bp7!Y8hXIxH8teIs)Gwm3(w2)bgAUo8y z1aD#kMZAQ%v+QB^8&FeLmS6e6aJ-3}$d9k*>VSOV-F0-z_?B2U>OY#m`jhzeCm;F= zag$7#w*Q*yZ=aX4P^ zJtoAL8UhBhq(|TV-Np0x54ez~jsbMXv{J7@y{!O1{14u_~=Dp3} z065&miUMxrlQ>vFb}^&C0j|4m+(w?zlF zO<@lDvYRtP#|NHs3w9XcDk2Fat||;;A4&CdFQ4aEQ9t#E?$MR`$>1b>Alpb*BvL+#rL+65e)!_xY8AL#8_b!UVlZPoXEhC%wPCB15(z_3&B=(pi6ybv3hq4*wZ zS07_qk-0ir@bi@t<@Pn8-;k;UZ#oxr=)#wb=^^AQf*6vqQ>0;Ts}JPDuUN7CnS!QP z8SGq>-2l`A?ci01SlhktH&un-{>|khJy99l3XwNHVAdhWefe#}B0a$rVkB2%E&7k4 zdFe&K)oV~+05e+zoka7yinYIC_Z=ODSI1%aW{~q@A4}0amnkS2xyat<&k-KX4!o?y(>ioGb0A!W$>=JLi_X0`k6p$YAgsso_}Qg*mZ9fYYm4 z)DNNDxCpt5IP0s6$b7Un{x`oXBKlXzbgGADF=J~0mMdou;gKzRRZG_+#n6e6mVT7k z>1{^%N6Tsee=UTNt$vLc%d1uDYM0bZls+mZH6IGE{m>XO|KzbLOP9zGfz<8qO))p@ zDmy}VgHSNX4d07q(wVR2^H~JlTq}UU zm5-KOWz_6LhGX^1x|)2CpmOX{>jNN3DN z3f~jqgUH<58asoqMyx^N)HePuv+akUXOZ~smd#$^^yk@*l;JH#^@5rfOlEFL<5wl= zdd;-aLn|~xA0pCR6;hgYnzI4z7nqmG)~PB)M82~Ph$%J19rH$ zWoFtv%c1s_ur=NG9q;H6!jSxem1pr-%Hky8l?NVTu*Gqf4bo+}8RwB#(`$;kf28k6 z?wch1e4rX;-m9qqQ&MS-RzeWEK8cW*)c`i^nzb!;f>Q*{!7~cy5e~Zp*xxG|ojt{h2_w{lluHG~Y_)Dn)waFJ%7E z0LC@#vE)!umsE8mu#>>*=0XMXrGEplO}DfwpCwuV zv`}^7YUBmkU-fvV{!?0^fC)-@)X!<>Uk6gMinasxAXht;R&-F&pS@bohLVF;ap3eh z;;=g*$vfn}a{!c6bX3D{kya?DJV#5Aoa{X(L1(6nvSg6-fXrPRv9Y|mf#wy zmw_#?c#=%`pPhkO08jvNT`UI?$ir>O!=sTy?w*<%=Kf)5=}YblThSS=$g<-U81V>7 z2Brx-&JGkWDdcDwQGo)k$o%eW`;AS;&@lA7;tRqDWB^p&mrmg6Ni*nXsiJZC0o}kE z3iE9qMzGf=z<^En0_sq>Wm>ax30s1mV)(Vzt4!LnlPIu3tQ3h7ri$#@AeQ_kKqmYZ zel9p8l(<1eD)bSj5ZT&7(M$%faO3oEc2(j-5M*`jnGQ&?D6eslZA;;{MK*TDKy{&o zQVSzUF+%dJQb343D#JyRdj8f(Ky`9HZJ;JNpTj~RWPgJsyik5ZxWzhwQYslXFG=}_ z))6EG%}?o~SVgC-{Lke{G!G>==XWP$_h$ZfpL?i?epe44TLx{w_Z;c$bhe_!0rZAY zyO3tW{3otCY1RSIRvQ)|39_HYJRBBu%GU`3?KG)Cm!rFQlH7iV5S6|!X{P?zfpbf} zgRY-(aeM-W6C$w(g8%1hvCQ^&H_HsZKI6VANy|Y{K40+lQZdB9Ljl5ptf&})3o7~I zs=zxAV+WP%K?t-bI&WruN>;=sSZv`w5qFa|-OY!}FVCr<+V5OU59EfLK z8KGpx2pRyrLnw0DeY{4tBrO!>osTQ+n>j%TlM{SCS)&eU_|uXLw^>2k1Sy^ajjPWi zbRgDP*n-hZ0qgsi8&wQ#pn3s%{uTUI;EyjRHzGL9yz+WAVycUDma(C$Qg=T9R%Hod zw%~Zp@J32>q^VOHvbg1Ff;56a((ni1{yu01Sc5qTy$f<5S>?-fARh71LV_wXQ=CaD zOR+^K_gv5;`p(#Cv9Mx0KSX!qFmC(*zp;G&#edCKWXDXh{@x4qIo~hzV^Pa^G-mOp zL}Vb1WHkQ4H)c^PT8SbNz>svxVT#gHMes}XH%5o)C>!hHjr-hdu&d{0bDU0$<=q^K z_0Sz{NSTgb*-m7e;cJr~_L$*+%WP2m+RM#;{5<*NQ)s8rn|;~ZOmnZQ6r0N2Vpj$q zFIP785X7Q8^K^La(rB_O@{CY?QGVK4Pn8wxjfmYJ?Lisla>w!=yJ-LEHcuBwlk zV3%0jX(6t?Wj~!=3#x78j|Dx^&|Al^81f>2ICcC#A9Nqr#4hNeojUpm#}O@ngzJ{| zRYzt_+3Tm#aRkXpeFc4si%gmtp*by*UU;!v_N@RTkvihZT;B*`JIpD^|Gt~MoJP=H zf)!1@m7=6)Y5uLtiANgytlOJs0ZwoAWnO0Y^h|?yl&A zo9*oykb9eSrb;*fCR&GI{ISoQk(z-srx9AdkZ#z6sA~x16^%)IW>|^26 zXl9p^u0K0jIlZ_TYh%7%lAFK7o;qqS#zNL@0Yd`D$qB^dUvn{vf5*Io5lZIDept;e^xzqOK`0DoWBa3E%I+(xck;(#p7}W?P#@OQXsqFPdj#l ztLyO(fXa%bvzH*V9|Bkyr@z0V4!OxQHT44b7A1rhE)`qroYsBv(ad_wOT9G;iF|Is z>o7F2CnI7i(=aUoaBFO65=76kAs}|GmcMpBUWeeqU3*j#OF}eT6|Iu+Co&#g;U_$~ z8LFjSCdlf5q(G=tHh(?WFkQ(?HA`7SjieO7oq7Q7WTcjjzf*sO%dxhe{P26fI_1lD z8Ypo~cY(FCyEnUt`c&6BQ2B#E^-B zAnZ~ck$5-ibOS(gFt8uId70S+)QLM^B6iqPIMB?_p7|?; zEa&|F$2A(~)h~)3t?h7ur(LmdxEyD^X0mXPPSV6z{9Nn#xVk^tzK)0IZ%=`oo;3nA?Ime(x8$V6YQUcnnMYlh4yOF38Qy-~xZTly#imR!^H z`$CQwyrcVPFj}1?n{zaZ0}nh=GU?Ji6SLiKWCuSwF9ciM-(2iBS`-><&Lo~VyoP*W zn)vYe4)5PT#)$(ywAP;$J3h}0f)~bP!37%KBc`$!ksTm^f>z~Q43rV~>=+F9H_%nl zf@yFJu#?y&+>2F!K$1cPjQjQe>Q%a~=*F^|1U4t|s)1dgBQh+qB4jlVL&Lxlhd!f7 z`gSEa;=N~hTosadmSc$8ieicw_C2|Z(51n$CXnnf{yknNRGILBq9<#REBdm1G41e-;pRNN%HmjCXIq_kWD-=n@+{H{A-$7$E8rD; zGgC1QFQHX2TZSstd1Zv(yj|w4>)?AER)ElML%}@QQe)?@Rc^%84WX_;H6UrcQy`J~ zDm^VWwwNWB;mejwIc;U(a+lePpct^!-wyV7KMnKjo50K($_Jsuih0`Hx42Z3gfQUO z1hOz@+1i2rv_g-pD;7Ri7o7#dD%gAtdNu5*nTu&(`oTwl+6iem7$>5JoA%RFp#KZs zz9tFY>T~&v$ItR;jho2TUHLTAczktLe8tg`t2;kDQ+fE&7u;Xi@m>Hj0!-rYyXT0v z;e#B?VY-x z-9qW|*I_lWs-syN#i&j0MIkK`UKjlRxIhkUVGH>zm_{+O-+aM66q2hyE_3Uz-+ryQ z>3hKv>BO%nEVd4_y@L%ea+*_Y%Q83Ny&IChG*sS|K@L9V_KG}qv;_tmziC|{K}c_~ zYl_9qs+!mlJ!}q;@2=~U((?fC9id(B5F3i;`5OrIc-WlaDY~zVk1>3>rggZ~J&g15 z+i~ry_fGF$tp(5&C2%tfsHiUE9C|Ge46uJTdk>T4lZ2K<$6&*1qTmi`yA76`BX{F8 zx}hd{5^`NwymqQ#?}v!Ey2#br68TN`wtnm0%8gD>s)q16 zNcHQ5ALAF|=e&AomDXqY53!yMdR2TQk5gy3o&Sd4dO&pg`|3>FjbV7vJV!5@3-LqB zjV2Ro=OQ7k5=V5-2sx}@uYs1i37R(3;X?CkpsE;R_jZnq8U=%vO5!V5s&WvwT*xFi z#_(f>eCFk$4|}|ej)w<6db+Cjf?6>05jmbL#87HbeYy`yWacn%k0RL^LI&vSZv(-zx#QM=6)c_xo8CyYw2cw8WvNC6ey5#-1Sb8!8#AIgQX( zG55SWJJ)_n6XY=YvhS05WryQ(6+6te89*T{5;_D#H`YcNHHt*C)bedfuItOJxI*$G zO224vfjteiF$!JK`pB%0Ij1>3Q4_g1b$WxazZ-O$C!uDvd5PM7I!*uIufvuIb+Z^0 z^}d0D4r!_a<-jSVMs?+MZy>Tl96@~;eaH&(4@N-;XiyVrYXmn%v_i&(MjXflXHGs1 zCZukNal-%TLD_#2{D9nvJfFE!h)@5ZN*-Smd-fx#+axYBl=dPgwm-NxDMT;sn}V>3 zNn<4Leo_8=JSxfSU9$5<8+Xmy1-iy1v4rmnUPChg?_RQRX>^^}HB zt5~akc_SY&t59K`@FMBMXf1Kq1V>z}R41!P#v_AL6$r|oTNsCacMO0H+@9lcttu8A z254jQ5UvW-lwBMwG70L=)6s%pbm`M1;)hLKoNb+nj^B)DQ)+fCG)sc#VurK7Ky>$5 z3pS{HO4c#qC}v0{_O(Yd^KVgX-Jg*>d`3do&Qsf?+LD#7z^JlShu`C(6B1;ezwUQf z8i1!b?~cc_G!9uX1&{7uvYeK^%^%Q&lmNh5aNDet7T{Gv0yT)_;d`?O5ZxXuFz|Do zg$@P7X|`O4sCz!%cKUOWjB?VM&aL*{*fZA^BnQY9G}NVj=?T8BVLU7*tp+Ya>lF$&OUq?>$RZe2 zv*Y-n$}M)DvCm2_-3Lfu$P5{ydu`Ro={*ubJg+?+#EY4c7wRoUkpYA+qLM_3H_IZi=>* zzr`j<7s^Q2p8qnpDRdu^4co zRke@Wf^Trjf+bwy7tM9r0CIzR72(V)4R<&z+Esz*CHfbYEOSPm*JUwm zye=rtb)BgihW*qR@{>UShP}0}m~>X*WjfnKV(=9e8X@eJ(cF%vc5mHo(xRD}wZ?}7 zpF@eAJzXYUyl1yBSBrBsQ$6}FU%HPE8O--Ezuw-}iISn$TyNe?TriWgMExuQrpWcL z>ugkBBnu04@?Dv#>%{DPrkCdC)FEcBJ&Kvn-v&ebIyC6L1(BZ5bq=|N%+av+bzl;U zhzQLP>y?;?R|4j=K7I@uz~K&4jiQr%nEWpee({3>bSzAH`7Qh*`pl*7wR5FV_`Gq? z>5$12ma#Ng?Ng1^oVHv75ANjlE)K3e_hKy19(^$jk+U*7w$XZ}m(9|T^=l;vm+R3! z(rVb1@?F^i&_LY>W)On8BdM}$D>v`VS!wnpcE0S(TD1vLNvECT`eHv@-!IDHX#5qz z{ikKni5ywzy!Xx=a~W#R9=qy@dsKXL%C|VuS- z?U?k-9D)L9aExRBhLN(6-qkPSAkAR^w`(g4JLNTSheJ+9=Pk^zL5(c}APyQ>XUqjN zU>lO(Qki#6Kx(R~`{*Zz(ST@zZO72ss=z)lzQl{0i5_~qofEcG?HWyz2+ zTPcR|vQrkeL<3eG$xmn7_!Ci4b&!8rKU}l!YDhgFKesvyljCvuWb*HSH6Z%JrWW2+ z&Zp7A~szy(d?d=HpEB>2Ytwd!RnjBR~YDEyJp)pZ>4!RA1-PAOaYO4*KR67 z%bKp0xSZZKO(&6ia_trdrMO(Hct!s*!5NVP(!vZoUvtyv%C4_HkK2;0aIZh-(H)Vu zb0&jS&xd@y1a?nU)Ltwdw}-uYM(b?iYS4;;&%<^*B6A#?H*p90JOGV#gQm@=Dc6Mf zd{v$%Vdu=JF4U%wUOxC3T(7scA4tuu5b`SRr&)o~ zt!I8Q%{mEA;;y9f;8Kk8zQlfFhu%0cmiJR7z?>T)-JsN~@*EMk3P#>Hk`it>)vyfc z_a9yc*YjN1ko!6|Ecj+08oL*IH{aG%U%b8@4!dRZ;gD9DC?VQrK3_d+5EoQAx=#m%8JIY?pwSZSGcDPbr#LV%k$` zcJ23OZKNc&?AX#VOV@S`b6ADF-ed+mGa)YPc zs|L%^{cG96{hpJ% z%6dnZ>ojUp{@zT_#3Qt4iAzKdU8e@-y3+Y|>QfTGLeyYc$+UJi1%cAB0{U?9T_Vd1 zpPPp#tB*%+Z~Dc4Vf@R}3!>FJ2DUtAdk9-(8OvfDIONG`ws*^6hRNu;Ax)sfkjRnto^L0gLKMuw<2tyABanN29|}j(st|1Npefq+)gc75QM$NTIrg ze!Wp(x=?dklU{O;*+rsXamS;+S5a7dr>XLwO!}+J$K}D~oA%MfD}>lM3!H0Cn!0ib zcgPcn1&J!85Z(CVQ^6sM4$Cr~Jvv0S^IVw7A=XLx1|4}Uny~ho*k{k!U!?i4{V=>J z`{r4w(^(2eBKbnYy2ll>qduh>erE^Jo(7^V_Kz1#rmTZOUt&22D^7UC4aMtpMnXx5eNiR{i_{s&Nb)@J%#f*` z8jEFBN>A^O`1o1V;an^G@BGp09o(%;Z*K3cpBsV~(T~mIJ=;~@O0SY-%zK-gV|Ks& zR|9xx>)ka?u`>P7Pl_NXwu_w+^h4Sex9qQ&Yo+cUKD>544ygHje^-ITYFqCW3pT^6 zTuXzCR8yn^qdQ00OobWm<=!lYOVzL&qz}J5VwIl_W^PSfK4r>_Ei}*kg&he=(=c(k zq%2$`7w#V>@Z?h$YOg`u9{oq}jP}jL<(Z>lV#m)F&yB8?V#OSg+)uwV^vT1u=2xV* zK+1i#Ts&1s!R?}0E7%SI^cg(iUPW$mpxx$5w%;sxL1wljP?JFF-Z`uvYw_!L<7duY zilmZ?M&GLxPk-Fq7SE1@YTyHcun3PmY9@m%#1kOkNM3wPJN<2&n?OTQ9mK=+YiuG3 zOv-wf-O0S-s;;gz4MTT13h6a}(A#m;1Po+I=KXcHQ#P8pvR|!;_%3!MVG0mP?Sf~v zp*hA7wNc{mM)gCN$0K+cH$LA_&@2X0Uvk%b%Px;>CGQ>CE~44;Jd-7$X$xaJ0 zA4oNBaiw)pMboRbp5L1Uy(

{=8@YYh<~PP;Xo2g86o0qYqNv-uGwY4OAuQYHI<9 zO9#W?*iZ(|G0bnxMG*{Dz_V8KpBKzAxV`vi%|hMVNx8M>c5n58iS*~p zBz8u{-mw;p$`CgEmC-?kn zmZ>gGR%#ysfcy0F<0efW&2-=O+^Qak&r)(6Y7!{g=irJzv-A7pQ;NQQ<}x3-)_ZFf zy4F~YS8VoIu_4H`OFm1w{+BEfHQL-|_hlGVHWQf+z0&JzsbQdAA3@OH`i;5zcsJ+bIKc!Bq}!nCjd8B+0+)qdm<2J zv>^f~H_&uU-hCbSELmW8k80>#DacLbkZBC#YK0gw-ZB}rrMYXo(7kxdrP zeNjQZYE4>3mNz#>*kHNynSXQJ|3=R-Yk+Uy%xaJI@l{%xycjRq&p-_FJO<{WI4!AgZ_nu^G!!YD0&Y~bb3jmFz55F+%eRs_#MvNkyMqwc#eG+zY)$= zvYSY}&h5EXz>;6tHNi!44b1J@j*~WYMNA-BkJdpz7n`~z7ta?YCyDKgei=fL!qwzy zZXQC>O7LK9nth9*WJ36t5g9IIZX)Q|Wg(YBWq(y>B^If!{rz-Du1*W4A61WK_eYo@ z=Jp4?xw-+83D4u}R>Nt(0fCIhL91|IaPRq({llq11N|o!tj8cvJo4cM(!79{PAX`t zB)^@(u!P#jOQ2N>14+SqWk!YPsVP)uj1gOg8!oup@%-orj#@3+?P`GF>R9`jZZSoQ zechz_bw?M$O3K+*fFev|n@+aG^}4(@$BKtmmh{^5JC0ff`o8Qo1KmReD}Nxy+}|ia zE_4OCj@A;tBYWnu*w^s~|7W5aSm6CUYGV4P?|nJ7^KOIMp?aH8o)%4ieV5Z~)WZDR z_cEF0U4Hnoe}G^STQLcfUDp?iFN)VlO8T_Xv;g<(HstV}2Og?bo^D#X@FheCJ;ctq zLq*58PWDVR&+u*JnTcb0rAUvNO2QAfp>tDLFG z3NQv*B~);F(1RlbN(^z4V{K&7Sqq0@&QtF`Bg|jwfu~|pW70ljowWB_da8?zzZ*(Z zzf&0yve?O$=a2TjQPbt5L%)|Q8P(VoHTe?3npb_+_cq)Y`wCbUn`MU*Vsk8jK0S~e zt@0vSB5Qo7&Tp6Ao3WA)skqj&g&k0(GnjKM!ZQzS0*KZ(IQ4`@($4FUPz#+7K-FlJ zd@?Kf6SwnP%kb>YmbR}*_!))|dlMl95#nL|cFI>A>-8D4=?OdzHmice!YlF~Yx8O# z<++IflRTL+-YQsz<1pTShjF=NAQoB{IjzqoW1lGHe}vX7l5QAX&F)iyZV2)tBc$N% zJA9{}Se-9<3HG@-{xcxQs6fHBU`^D4N)ydYQ&n~5CMRdYv^_u^rkNU=eCh5j4OjH$ zM?_7W5z`+7%zzqrs>;Nop3W&d@i|IXGF!I`PE~r*AC$X%{s+VlbOW21; zWoqm9L;~p-UMBfc_WiV)GKaUW&(Rq^{N%raCC84@n!LoHVlW8B`tOi&eM;r|lwYMQ zjJ@W5xdtgS;fSJ5*J$7;TupD==o3HD0(f$KJo|bOqd<~S9I%KHrD0;8Gv8g1JK;ME1@fz(Hg9*dB3THM4K)1*N(VI< z?xN$p0MSA2berEl;O5;iy)_`l8g+4$ySmQq9u^QiU8h`#!h^Wc9o}w z6OXK8!2@uWqw*aG;vC2&`5g$CguTja`NPDe4lK{<_wMVKnJ|v9Jm}E&+=ke;ZqjBV z^~CnTZZ!LUrZ_P=VPDYKitjUCPhwjj=5>(63QEu+LfIy6Q>HUoswtC6sl>d z5P;W5#?c1*HXR0B;zFm<%p_irV9TYMC^}c!uCj*#VIl^ySE~6rQK_*-U~&J7yQFxA z5~R2D`?llUOMIc|kC}{JY1*e2>~Y)0IqSJs2ZeuvcQ`E0byMuZFA;<|byU zu(tr^>|rP9aX4N>f*!$UEWY(_C1lL>YknVD;$w$4LR(~do21x>VGB%3R#g5r56#fq)QQ){TK9U6ZutJZ$ z)LfL=vFmcH)5q#s>kd3F&>Sn!6U~I`v?A0*uKbt+49Wy6WMoZMn3%X4m++?pWp&a= zRqvF+gw5ycj8CCODggl2mglT!GDOyaPC-4ZF7xd$dZ1d;9VF{Ahae}Fy=*|3UX6n! ztkG5Weov$QwsPR9OgMhOua2PwfME=v_JBb#LjK}eA;D{3< z2DjeJ$&=vg^UE<{)3xfUMb~4^G2=m2PY%J+l^+12az|yd@Z$NRWP`g2*M^6>w*chH z;!4$DB%7Hrs(7aiaEpYax_36Yb$-0l1U5M1R~R@?1Jeyl*QlJ3`yILPgBAmX`>gs+ zAdXf9M=ABTi(ucS$j;;oq$}dk-~~(2@gedc@u9yy5k8}DvAI{<;%K!V)-@(;yeqUS zS&@4H(u@{RJ@O34M(|`E|NWz(8kn-hk&1%WN(ibEiw`@WS!B#D9~K2G+aLYA|27!L z(m9|YG+B!??P4YW7I?PF$;S~1a1iVq2eP>6?=1Dv4z6FS8leF#OHkp7)jMjmLqB%e&K9UjQ2 zt8Yk%&(=A-Bd3Rpw{UDh98lqx;*rS{P{htxPrQ|>dWn>e1_KMffKSH|y%nV5@H@7L zA)aJx|D6*_qacK|9|)tuHgy490|oFjcr?*7;kEY{O7aUDkQ+Gejb`X23+O{4javm2adZ?v!a4DVLI;x^A)ZjE{mEo zVvgsj#xVcpv%yvAySr_B%icwWQeLHI@}kAgNcz?f%=kKL%1#&%jVQPf?02I2=}~9H=bpBFruGn)pDA1^O^8Ou@6Us?)h0h56O@ zp+>-iwSydrnCC7-!Wr8tgVdGdh5(GX+8x7l?HTZ*9fCR=n6pR8gu#>-5Ezh^2xe$Dseu#*?$J3b2=Bco6B zcT@X9cJLk{F%8Oqf#CM(Put}=y3+$lAo7hQ;t2Ig-{&l+J3-W?h-WG~LrzGQx+{4{fjM7@~gHFSmhOnz8Tjq#5_jbrI^>V) zmbQaaVM6?(aVlo7c$MPy>F`SZn~~LSZLolc{vb46N7 zr%l+fRUC-VbHsM!t}K*a2sc*bpI(JX+xhWZu3{wr`MH29TxW7>Xn_b-JinHNf%J{% zW6jG$c0$r>xgoFM`IwsMH5O%7uMy7D!em_pE9H-cTS5Zn<3@F4QRgvqVWx|;WfG0^ zWWnuwnYLhfG5Ic27m1P|H+tdX3&8ZAc>#BxEdVTscaFe`Ty!^lx?XYLtM4KEQMhuh z^qJc6GSV#|X|=ZtL{^|PG5?=kK=qc1IyOB|Lpm}cIl6L3T)-`%=cQz;vk3u6*$1$g zlOM!3gu=b6NYffK*UUa`fXfVWT@#~M#p}UKS(ISD+O0EkF6qj!WsvC+Ji53_9XM^O zC^}z?lqreJ^dl7~mA=A2{t=1=ehemQ6K`~h^Oy%D{51jXzH0K(*2v%5pc{J=_ z^UblZC9^OZQ9+FRhhaa$H8TD)0Ypz^Wz!3K|9lN<8d1F3xAhV9GYll|&)EvSg3G%L zJE0C7g!|0L>Eg+=@|3nCV$D}?oi;)h{_@qcd&){uv0Cm!FI9^#iN_k}I2Y%ZLOyjB zFot5+?|aBCO(zLSN&6I52W$Qn>|@?lHc#W8&A0mipwdeyOXFvsIspJc9R6tcAVRB)oX5PIEprXr;VQ2ibj)*z zmKYX0DpJ(rF(r5Ta3*wr-!RQ43e*H4;i3Pe9c9iL-74|5>_b*Fx5bue_>!rLP8K>; z-7;I~2RnePQNCHT$Xx^#$nFQ?u`Ry86FU?TJ1X6@W{>)&hg4xdTX;-o54G+R7BfOT zO5b~dxr|J%O8JRA>EY_RhRhdr)A<^SL|e}+5; zgC^nGgRd_N)0ugm5?Mfjqr@;NGnr-LkD~GPO~)6|6^{=zcSpuzj_BimeOiyL-X<^3 zzr@#uZyZJHz1B_1xdFZN*gci=q|am`_=l4V%}83#xr+zsb;NIGT%<>mJ1%-dd_5ra zVu=k1DEaT}ZW!p?2KICCz+T~Hz_p@;pTllM&K%1@G@MPg(uBLaMozGEEtmZmy^l-6 zDqwd&nVYZXrTF0Mhv;AXO|SYhZ`36k)jd9dhOu^x`ohxe<4FiPTbC+2K%{>eC^y`f5@~3^fNZQWa3a%bO=%04T@$cpQ`|t<>ko>#VW4DR`S^+?iNi zViQo>1XE%W5y2d|V$-2%19>W-yWxP~y-ios;?-^*}&2gJbosO5^3*kfI1VCJPR( zcccaj0e_^n!+V{_ko)Dw!%dDG5^i1|CPF0Iaoc!H#ZXAQn`5x%@4s+<%dOe-Z>RPR z$|Pr(?9qeUTBnfXm35^l&l&XfCPtIJn3@Dcn_Sw_%cZ@h>XHkp;A56u!)z<7qwk60 zN&2ac&Mxa$+6Vj3uEM~9O1JQlydGGL|W5Fnl@cpyd#z$oPt|c z97&2I1UQX>qv>h&#J@U?M9R{%jJG|C)33qwMC(GoDN^VUf?*Gs0-^vh%6xx3msfPp zG9<1X8LUs!*XW|iUM6~E1>_7k64PUJ1ek3MKdK4Y%EW6E?PUWf2DJX|zet7;Frks& z@qQKm;o8%tw`(4Ze2g4CbKgpqJ3+J5rY@bq#8`TJI$s08Km4N87Er9n?6@xIcENMo zJj>^EoHFIb@FkufTpmEv)PWDX!>RX`k8)*17YHNE^gA8zlP~aq^gMr2{wWgV9skKy zQf@67wilY92}uhYBrO4C$5r!-gF#1toU$lJspE1F02M7J^7nQ7>aJAR13{}}@)}sO-i}Q%)an`5>?ypoE6r!A3ui6G2wUBE&*f@sGmpo|h3uPGRp9`6KGxrv zl+^8|Ra4X?(Y6liiC=nlzbY}s2=w^jzCGEo$u-%q?ReIluSjJ`#WlZ$<3^sG_syBz zg$7L{wApL@qFWc{O}y)^O5VM`U}6uqWA)0dpe&l<9=YT5)?R2=UN*6xJBO|Q#y+h# zZdZQJ4Wq1z)VB~=4qUZcL2dUkJw!RE*3b!b^95WQ^s4#syqi28-m;F9Og}p{%Q1AD7iUxf_t0;8q=Q>Au0ht3DfEP zR$kg+<0!)0(M3!NJE18j2P8Eq@;)G-UfocP({xglo`I!3&8!3wCShV>*?cqdBjAQZ z(v24vU<)03{AO>rSL^AyEBWBrHSSOqzp?y|#UL}mmHn*FBtPA2i1wg*>+M%syEbXz zJpoi0B4saV*e^tDgjS0=>X|@bgrQ?>#OKsw*sYNez^!mdA5GjDFD`lvy`1$i7JA#- z*_3BDBP?s_KrlTp$xv_&)V& zP7cMW_#9%DHzmDrlgDF5SJiU+lTaL4J zL#BP`_%aDftj>{+N9XR*i#ArVy^2shFig3;J%P_=bhB3V%$pORmO-%k%RK&9uSEMK zcJExLrT1?6!aDGzZhh6(tr0|AoA|d`1{blgV&f^LKRT`NKWIhNRyjJ2oaoN4jKxOfy7V+X2i+j>+b}F#j_oRnJ0Lw37-FiFUGWhG%?`PYD zI+x%QnVQQeqUI1k7y@0{-Aw~D7qM_N5KHnsg_jvC3Vl2;Vx)^!>PvZjVA0y4g)yF# z7O;*1&G!HToE>^;7~c&xZ~90|U^+vdWnirw@8dDJ13V&OSAH|Do|n+);YbWo{f&%L z#RHu^?G}0V?$hblb+Et8F`{ev$in8nk}vZ~j*BhHXUE-9HcQ>LAk5gYlzRiP(_edmiBuRdxHKHk_s3+%QE;*aHX8T|S*&s3R3mgc^_ z`WvN@e!5;d8^Xgtc@6%$CVRSfJEXm`_FfIuxox~m^u~xosFb?$(x|3 zTL;;7-kI|rhwAm(Q?v*Y(bGdN#X`eF35W{0e;Z)`d@F-4Ane^zPZ5P7xDFY-5R{wS zICe!Tl4yPaYQ8;HlhA5(o4V^$u92EhF&iz5&G)F-7yDPS>XE6}&-0ILU`H;UamWl( zYX;#C)|Q+67t8l>C-J7ER@sVj1n1`SZ^+i#gSvhQG8?7%N#1;-$!prUXWx|H+Ib{) zpT3>{%%ll?zGw_vQf=!*U;KrbFth*K=aEFV(H>~04-p;*3+V&G)kA~$vbbHNjQ-*o zKuaz~4t1-~M=rk!3Xx>U@DdaKU^rVpIx=Z;*g%OiNmQwG1u1QXlUAZb^Ro1U7HMxm zjl+R8Ut5&zSK9-CElsRc__U8cM>K{tuq9APPgLS(FZS!F z7_R^u#CvVH+HbU2zaOQ$vrxMEbH4xi`~3HxilV{ZaW}Mr|2Nvy-~Z+_mYLFQTl_qK z<3YavFY^f%@tH^odUU-q@*cX84_b>g^_x0x(eU55>juH0epJVj-ko`GEB(VRvjYx?9a~qM%=Fe^P zIaB(R)Zs+@|DDto3Q{1ygHik055V{bqG7!fu%?&AL0flG~8Hag^V$3&Vk*dK!4$xC22r>7TpkvS+Dh!1m3lCyvTY#Dv2|K_pR#8T24Q5Xd# z;U%OU$C6Iu1c;i*F6Frh;r2=|U=4*)?*;*L?0sN`=?4JaqP#v$WWEQ6X*Z!^(EDAv zD#GOiFf`1f+l@R9lhHXnf7Jl@FrbO07`x>6s^)J4Vg?>o+S=3!2< ze`h2rLxp$cBjAC|fca0wfGc2zF{T$DXl%6;`2`89)OfV z&zo03i>IuEIArFF42Bj+e%NQx?WH~6rJEH8K5$5@ECBdMUvT_1ARetFPOjRV?8~7lkmH@}{=F5GFMi9?H%mg$t7ze_cjV=Zf5f0+Ka` zKo~uW@2!iU6a!DHrq>do_>>VEhmU$!rQl-A__!Y?A}$XUjUq?DP`xfXY3 zGq4zAxcN}%vm(gP3kaSEtinc_>JY)_2U+y&8%fCUZf}7rz%?hsLsJAO793lXKuYyy zj)OV9Fz6;}5$CP_3L7M+F<90K!&97^fa=hl|2OKIC>1K$>8iJ!+6f)+l&dG{ZJ?)o zS~G3o4Tz!iECWyRaTH`QhR}7?*D6WJ{FRfLnr1ssnDtGhSjY3*~H z-8Mn^!wfQ>x0@kRbL0%E?B8_k@BU?Fueg=tBBVRk!3Ji-Dy_+Tu}Si*A_dfF!iP;C zZA-fA~12s^X3C5IE8zT-L;^? z1~j#fa&H`2Y~PTo1Su~Of{wwEo{2lTj>y|EPAefGo-QTE`ncqSo*(JNd!`uhz<#M} z5-9Hr;Xv);WB!dTaX+vS0yWV%>P}EirdL-4@E+U#Q9YK;H`%@=&j5r29_-32?&~|N z&;Va%*e$eeDP^8YyW#F-*na-YQ^63m#*95LxBTlk0&29DpXISm_hat7xax%?(3wR4ZhRuheg7fyAzU=I!lo+iiX#Elmvi4hLLYO$6 z$7{Nn>ub(nd3c#Kd8%DFFuPcPYo27X2c2&xBJz^&`xTLK)PHKaBiXaDyCu%nTsOqn zM$BztDM9B7bD!dn4h2N732j|k=a30L-Pf$-d~uaVTiM+^5ToYVgASd=vpv59!=o{M zo#|%fAIB!-+W82R@jY@69qs{;5xx1;?9_wf4!OKlv06(O${$E?btMl-^M}Zwed}!# zg1GY#Z4E}WRRM}!1S(O&i#~&@=i%|^P5&*ly+I_dXWS;7bhVpo#~TLP(*DL-8G;P< z+mT&HB0ymE(X$2(i$2l5F24}P6}NjafMo5r4(SmP7CVmj>!djq)9Mn8J?vKfjaf}qhOHjG|F!+a^VwE$zU~-^mlMF?y+)ZI(o#|Y56kPi*zFPre^M;_c{)*1syibh;0uJmW%#VCe=pA+ z1N}=CpC=MJ74P$dF3z14M!gGVo#R^fUy{R&df6w~h?Sc>TtgLKKU}2#7S23}% zWN#rdZ%#m|(b*0Ixk*yMOga!lJhFp%CI?~kp!8{KR`>09lcP;~TMG(E4l`*jK(r>x zXBTr*LW49#QVv1FR27HN&kOsUpWu;^Ps%kk?^6fq(=V|2FxN!gYrqrLAbHv^MISr6 zP+U1iBh(@{AA&M#AopJCn(q${d&75uPVKNm3xa$Wf4k0*@lP(6wl!2F#_yT(%NXBX zdJGt&`2ef!EvUsgVPHV_Eb`+LfHHNu9dd(buZ&Kawe?|gcma{H7+l5|sa|ynIXo9bN2>lvSiX zhLo9`Bxcs(s~@4aVjk8_&)QdL|IF2*BFtdm_In2eeZSZ4NCssTc94VuN>!=cpJ3Ob z3Vl2~WPnd>pW0^FVnn&7BY5q;I-c-~^_Pu5!*1MPwY@xWAY%ScTEE6TtC5>Tde3qe zR1a4WyOM}YG8O84^m|s7R4^YaQ^P->#s{ixWl1Z%7xxh+Cl+PFC(Oql4!)tvLQ}!= z{jFuxG{wCmWTvBs8~ycGQguNNS4TUsj~FKVM043w#fOz38fpS$BOh?%lK+u0{%{Ki zNHbv0e4dyjjkTR)4a4Q2O2q2kuY<;=wcT1sm^2J5dgwzS@<20e9yE_sZW9h*^~9V1 zmP%2sC~7lSoOh|bs^o-&a6))dIk2-AaT}lViH-1@@Icz*@}w7B7QYk7_}FMaiDf*F zj8NGV1f(~v|IRn=RP%uICh*Pxdq14QtpG%$2WIrb^!U#^N42EzQiYX->BDKJK1&Pc zyxl#8DlWJ19hsWULZ-($lYBj<3Sysk?fgyV^o`b)i=~H+)qvPFM|37KW&fJ-{xI8E z=3ht{K~Q7YnCCKu#hT;wAy8bd@KGu zqG3O{sEnJjqKxx1y+IduitdcsnG|pG)#mO8xCA14M*@sZ`6)5;N5AdArqQ*d5(tB6P$^w+hnyn!RZulyo@iUtTa<3qFuF7@W z*XmIFv(Xmy(ne{lq#U8`bXw2fgUqv@(v3(;d*(N^?VtH1~yp1$93E&ih^*5b5=)p zlmx2@>Lm$F+B^-w(h|)&`Fo4j&z9b={y-7Rci+cO)TrrU@!8B-uqNteS$_*G=%5cE zVniX!-r%*e(xYX@B559ERTBT2&Hiuzq_TsGNq8xG0(O_Lq1<0%SHJpm4grZUs|?{> z*L&=1A0fl|AZ5!U@}yI>g9=R2c^rz++(ja`**3WGp|^W>$tn~^SJX zI@{b-wwz;+l?AA2sO#VK1-mC09=d{dM5fj@V8>sGZ=_F>;vzV+w00Lg z%iOZWa|fbI@coWSu&OTC%4rR#jC8}Koj)NM_|SvRnt1Uvoy6?#$gkLI7uRrxoI&G zUcAbDyC1EGS&4UDfVgSZXc+}BTMe?h`MsL+B_734SLK55B~08<`=pBKgAWUd1}!;* znNkFoa$2`bo;{jWMbM;N;(P&w(=)o(Y@ofd>G^*|>Jg8tKUUj!6A#nWWn0?R-xAw+ z54ava5BAM0Y`};4EqMn_zr(?Scl|zu{jg8&d)n}$rT}#_hklxc-&2&OUHd^@&_R_r z$5}m+D6O4GR>0KwvcLMVZ=axPvvak^5TH5kMejtdM$_@9rYyOxoS+hG&WN1;O#0Eq z0`0z51hJ$=cgX%C3q~VzP3Ostlpm)z~tMtgIN&(2tqlE3Y5#Rc9*L$R$9j z*A&C**Td*;hNrTQH%l*dDv#6%|cNJt^Fk)V;bTRz|GbRp+(p~rpN{4 zCqK%Osp-Y^EObeTy0n}8QzkrGn7Zd%IEJVY_qFG~`&^@eeJ+MK<0jdd!95I?xA{aDZ){Y0ef$yA7bJ4ZC8#0 z7~D|Xhpy!XFqRrfAcradfz!mrnpD4+0RSeq-2dIwFqyr3A^5vY5pR&;RXqn)pF^du z@Dj~~U26OTl7c9-Vlod^E|3OXDI(pf^6pp|Ve%sC&(3$*G69xHlGcvB*eEMtR z;+VG70`1N743BP73bLN9q&wH#OzD`p{tgSng?@9rubOXHo)cKEB(+^4$m=|#(y-Yz zYYF{E#mYU=5((>TJ-q7H=DS@!nB-RP);!$6QAK?ZYPN)?D43x4_Qahp4A< zeFP_bf=9BU0eXGR+UgM4(LyegADW(au8a+v4eY$7j)r7wLLE-${N0Q6Ok8&EjHNAl zd;pNpA}E$$zdZINzhZN_gOjNuWKNal5k*@j#tK8BE)Rx2&NeBFIt7J?^-zB72X|xTD2{ing-bD187ht%#I$M z3P;;U72f1@RV5w0#_Z<+u_hlcuPUHkwGGmcieJPO+R_lB)^r1cJJc9;zzLjhWFla?bnbJ)SY_`|mb6(d0F)}V8-5XY2 zjV~?VP1pkhch3L5-GGh5wPQY=pc!y=%Vjt-zJONmUSOL2fdO8o0N|_2-)Uf+re3jq z)`^UQt2;NhcGoDUw3-8QJhOH9AIfLPyf6S3*2o|MV!FDnb!xW3{wn!k0Y4|EDP!C# zm&qXLSiG04+t7AHhPR)QC0$ps?MBAY2!Vhridz^fs`gos$39oVyRWAQ=REKALoPAk zkCn^AMJSnQN0%5w0U1`1Qccq6*S$am;PA$AG`6t8hSiJr#~UJBvGIxxNOk4R4-Jj# zVP#`9m8)hbV(xJ@-c-nVrPBs_n*=nNyL_4d=0UKt8)qiMHXflX0bX6;&e z;{UqyuC?`rwAx(^mWdoA>n1UjPh|;{6o?8}oIkhgWTC-3`%~#kj$5D2H|btKo}4$d zHOij)$2Kz|qPlI*QPqc>yfxhj9r(uldrpo1ZiP_dSeV9l41K^o?5S;WBZ$ zh_h>jOy>@NHM`;{AkFU8V=KQPVq?XoB0X{tf0cgWD{eTVzPAWj%NJI~0dSP|!3Sjqo|aRKdCI&}^P3yBHWO`}uqG8D&!xw=h*y4OrmmYzo31 zOGUEoAD=p>_DRn)Y8qx;ddJqKW0|2*>8bis9EagyE;YV`(p*|#=tub*YF zJcYIz9=Sc)u&}D7Sf5!^iz>QAbt~jn+>P>*hNe@~q^^zdF)*?+GFGteXZ14{Fp(E2zVt#%=E>L84-_1?NY)s+jCoXh{^L#bo4 zXG+_yG8p84a*9cI4TZMAU#8=tI|f>;N}9O+ve91B&B}s|-6)U!2x%8QKuqAX-%o#v z-*Eo)!T%NuKSxKZ6JymZ+R6YIv?q&@7&brW(|#iCnHB6`*@Q1*^x=@sj1QPC1(&mH zYxL$!Ce}#vPjI{H8-v9thH5{?$GS$In)d0nC!+c}*8sIvuL%?spKPt}K>CByl%P|ZVq63^*VH_GBmI{JGTjQuf0XW7>bWj3G{=EqC}2W%ZzrH z@Kb26YHBWe@3nFZ4T97LmjC!b?4Zt_yES)b^inZQCTjFEY61pL*yVb^@LX=LVt! zwBMy+bn4QR^@1~=m+zpcZ8ZY!Zm4a6J?Vfyd9Nv z3_N$Y*-3gA{LdTrKmUt~B7QfU@YR(wt>P6IQqy+4fHSiG)F2Lf)v&|jX1H-2agpo! zUl$GmTQKdpuXu%3>f4;&q%+u61;v@QP3i^ql)jXrhX9H1dfR`@u$7;pn{fW{54ZTY zO`|v+yH4gtAm`Nd6h#ASnuH&OJ7m!B8o0ZVdw-eOYg~>QMs333=9m5Zg>$rT3ilh+ zCeXeRKngn#<;YoFC3mY2w4ln=VYy0-lPV;6;mwJpo)!f6)R5C5h6SAXX zR!<^J42G53$9rmos$@Eq^lV+Fxy1HvUWYE7QnJJFdswmQlwDq5@G9S}$#Dym)RS)J z$2~(PkhrMV;Hd&kQrx?)oH!j~o70-SyFedm$A7n?CX|2?9QTA#RzF zFvI;*!!nc~r~T@)B5he&!d%K-B!w7uGL=jZOEkWqGsYkW*a` z)0XFOf?E+{^Xy}Xt7fRIiw)rKiLc%o;*8jAuR8IA6Y_1TS)N`%GVYm})i67b#R!CI2;%{Qj{sdA9Awz8cMFu~3WTd$N$hGCm-!LRJXk`48tMZ`J%v z!4~$H?TNc9W3Tq0UC<^t~DD*P*$*5M_}_-#AjUhs1FniLv3|m!DFK z`7hk(&@_j}36oUk|LQGlv&1sEWVXeq@l16_7lNb!7ff8OrPOO~9=OF+q1_BWQSCBl z^#0A-Kju*`S$7{MXsyc!k>JfyUiIU=^y`zFY{i4BiNqGqjMIor!|$s>gWS>BrDmt3 z22auF4$9v9CuhCJ-hL`k$wXbGw^BZUvXQg zD6P=0ZrzJZK5h|jLsQ7h1=4E!-=QWp#S1%)x>$1UgLzql*e|b1A;`>S_vE?dnZ+}u zg#sNf|7gE_tHhorQiQGtT!w?*l2_Z9n^jrw4@0$BjbAGi!Jy zOT5$cFiG`3teG#|G5N?l_c0QgXo7mWDG1_XPA?zLTW?f4`G2t&_k$RA)3wR5=ncvm zHoXLBDpkD%6?J<v09fGAnrqHOaY5q;6v{pT1lxDy|HWS zuU=OwxmFImovU?3QEwK>b%gbA@(>V`jH@_z{w=g+x{z^=peWW>p=33?o`GykN*7npQH?b7U$36{5e)X z*#mzT=g;E&TgCAwV*X@9{8^koi}NR9{$vmQS)4zM^XEGGf3|dc<#myIVo~g?0gu1% OkE)X9=A?~#j{YCJ$?_Zk literal 0 HcmV?d00001 From 85430551b2b7635c8e1cf4ed1ce171f1a875d1fe Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Mon, 9 Sep 2019 19:50:22 +0100 Subject: [PATCH 126/180] gfauto: update signature_util.py (#734) --- gfauto/gfauto/signature_util.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/gfauto/gfauto/signature_util.py b/gfauto/gfauto/signature_util.py index 100fe2570..cc226ecc7 100644 --- a/gfauto/gfauto/signature_util.py +++ b/gfauto/gfauto/signature_util.py @@ -106,10 +106,17 @@ PATTERN_CATCH_ALL_ERROR = re.compile(r"\nERROR: (.*)") +# [\s\S] matches anything, including newlines. PATTERN_LLVM_FATAL_ERROR = re.compile( r"LLVM FATAL ERROR:Broken function found, compilation aborted![\s\S]*STDERR:\n(.*)" ) +PATTERN_LLVM_MACHINE_CODE_ERROR = re.compile( + r"ERROR: LLVM FATAL ERROR:Found .* machine code error[\s\S]*Bad machine code: (.*)" +) + +PATTERN_LLVM_ERROR_DIAGNOSIS = re.compile(r"ERROR: LLVM DIAGNOSIS INFO: (.*)") + def remove_hex_like(string: str) -> str: temp = string @@ -157,12 +164,21 @@ def get_signature_from_log_contents( # pylint: disable=too-many-return-statemen # noinspection PyUnusedLocal group: Optional[str] - # LLVM FATAL ERROR. - # [\s\S] matches anything, including newlines. + # LLVM FATAL ERROR (special override). group = basic_match(PATTERN_LLVM_FATAL_ERROR, log_contents) if group: return group + # LLVM MACHINE CODE ERROR (special override). + group = basic_match(PATTERN_LLVM_MACHINE_CODE_ERROR, log_contents) + if group: + return group + + # LLVM ERROR DIAGNOSIS: should come before PATTERN_ASSERTION_FAILURE. + group = basic_match(PATTERN_LLVM_ERROR_DIAGNOSIS, log_contents) + if group: + return group + # glslang error. group = basic_match(PATTERN_GLSLANG_ERROR, log_contents) if group: From 5f5563f697eed68d70711135d49f6b1323a38024 Mon Sep 17 00:00:00 2001 From: asuonpaa <34128694+asuonpaa@users.noreply.github.com> Date: Tue, 10 Sep 2019 13:39:22 +0300 Subject: [PATCH 127/180] gfauto: Add multiple pipeline amber generation for compute (#735) --- gfauto/.gitignore | 1 + gfauto/gfauto/amber_converter.py | 65 ++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/gfauto/.gitignore b/gfauto/.gitignore index a082d4075..39ec0a6a5 100644 --- a/gfauto/.gitignore +++ b/gfauto/.gitignore @@ -1,3 +1,4 @@ +/dev_shell.sh .idea/misc.xml .idea/modules.xml diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py index f973e11e7..8d81ba1f3 100644 --- a/gfauto/gfauto/amber_converter.py +++ b/gfauto/gfauto/amber_converter.py @@ -565,7 +565,7 @@ def graphics_shader_job_amber_test_to_amber_script( result += f"RUN {prefix}_pipeline DRAW_RECT POS 0 0 SIZE 256 256\n" result += "\n" - # Add fuzzy compare of framebuffers if there's more than one pipeline + # Add fuzzy compare of framebuffers if there's more than one pipeline. for pipeline_index in range(1, len(jobs)): prefix_0 = jobs[0].name_prefix @@ -583,48 +583,57 @@ def graphics_shader_job_amber_test_to_amber_script( def compute_shader_job_amber_test_to_amber_script( shader_job_amber_test: ShaderJobBasedAmberTest, amberfy_settings: AmberfySettings ) -> str: - # TODO: handle reference, if present. + + jobs = shader_job_amber_test.variants.copy() + + if shader_job_amber_test.reference: + assert isinstance(shader_job_amber_test.reference, ComputeShaderJob) # noqa + jobs.insert(0, shader_job_amber_test.reference) result = get_amber_script_header(amberfy_settings) - variant = shader_job_amber_test.variants[0] + for job in jobs: + # Guaranteed, and needed for type checker. + assert isinstance(job, ComputeShaderJob) # noqa - # Guaranteed, and needed for type checker. - assert isinstance(variant, ComputeShaderJob) # noqa + prefix = job.name_prefix - variant_compute_shader_name = f"{variant.name_prefix}_compute_shader" - variant_ssbo_name = f"{variant.name_prefix}_ssbo" + compute_shader_name = f"{prefix}_compute_shader" + ssbo_name = f"{prefix}_ssbo" - # Define shaders. + # Define shaders. - result += get_amber_script_shader_def( - variant.compute_shader, variant_compute_shader_name - ) + result += get_amber_script_shader_def(job.compute_shader, compute_shader_name) - # Define uniforms for variant shader job. + # Define uniforms for variant shader job. - result += "\n" - result += variant.uniform_definitions + result += "\n" + result += job.uniform_definitions - # Define in/out buffer for variant shader job. - # Note that |initial_buffer_definition_template| is a string template that takes the buffer name as an argument. + # Define in/out buffer for variant shader job. + # Note that |initial_buffer_definition_template| is a string template that takes the buffer name as an argument. - result += "\n" - result += variant.initial_buffer_definition_template.format(variant_ssbo_name) + result += "\n" + result += job.initial_buffer_definition_template.format(ssbo_name) - # Create a pipeline that uses the variant compute shader and binds |variant_ssbo_name|. + # Create a pipeline that uses the variant compute shader and binds |variant_ssbo_name|. - result += "\nPIPELINE compute gfz_pipeline\n" - result += f" ATTACH {variant_compute_shader_name}\n" - result += ( - f" BIND BUFFER {variant_ssbo_name} AS storage DESCRIPTOR_SET 0 BINDING 0\n" - ) - result += variant.uniform_bindings - result += "END\n" + result += f"\nPIPELINE compute {prefix}_pipeline\n" + result += f" ATTACH {compute_shader_name}\n" + result += f" BIND BUFFER {ssbo_name} AS storage DESCRIPTOR_SET 0 BINDING 0\n" + result += job.uniform_bindings + result += "END\n" - # Run the pipeline. + # Run the pipeline. + + result += f"\nRUN {prefix}_pipeline {job.num_groups_def}\n\n" + + # Add fuzzy compare of result SSBOs if there's more than one pipeline. - result += f"\nRUN gfz_pipeline {variant.num_groups_def}\n" + for pipeline_index in range(1, len(jobs)): + prefix_0 = jobs[0].name_prefix + prefix_1 = jobs[pipeline_index].name_prefix + result += f"EXPECT {prefix_0}_ssbo RMSE_BUFFER {prefix_1}_ssbo TOLERANCE 7\n" if amberfy_settings.extra_commands: result += amberfy_settings.extra_commands From f3a0924edca9c43a58b0c34e6ef03f74a11b7fc4 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 10 Sep 2019 16:31:56 +0100 Subject: [PATCH 128/180] gfauto: fix dependency resolution and update dependencies (#736) Dependencies would possibly get updated if you changed `Pipfile`, but the intention was they should always get updated when executing `./dev_shell.sh.template`. Now, dependencies never get updated unless you delete `Pipfile.lock`. Fixed an issue with dependency resolution by fixing `flake8-black` to version `==0.1.0`. Updated dependencies. --- gfauto/Pipfile | 2 +- gfauto/Pipfile.lock | 110 ++++++++++++++++------------------- gfauto/azure-pipelines.yml | 1 - gfauto/dev_shell.sh.template | 25 ++++---- 4 files changed, 65 insertions(+), 73 deletions(-) diff --git a/gfauto/Pipfile b/gfauto/Pipfile index 70071fa91..233bfabcb 100644 --- a/gfauto/Pipfile +++ b/gfauto/Pipfile @@ -59,7 +59,7 @@ flake8 = "*" # flake8-strict = "*" # Just contains two redundant checks. # flake8-eradicate = "*" # Disallows commented out code, but has false-positives. flake8-bandit = "*" -flake8-black = "*" +flake8-black = "==0.1.0" # Fix to 0.1.0 because otherwise it requires black =>19.3b0 (pre-release) which messes up dependency resolution for some reason. flake8-breakpoint = "*" flake8-broken-line = "*" flake8-bugbear = "*" diff --git a/gfauto/Pipfile.lock b/gfauto/Pipfile.lock index 4d718af10..b5cd27a8b 100644 --- a/gfauto/Pipfile.lock +++ b/gfauto/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3aa0659fa85d176cb557e1d584e9a4fbd383af08b47a975ef92b6e6df14264a8" + "sha256": "8427f9b747e8467d42d63c04936cbc6235900a98a6d97686ac8456cf4a3084b6" }, "pipfile-spec": 6, "requires": {}, @@ -105,13 +105,6 @@ ], "version": "==7.0" }, - "ddt": { - "hashes": [ - "sha256:474546b4020ce8a2f9550ba8899c28aa2c284c7bbf175bddede98be949d1ca7c", - "sha256:d13e6af8f36238e89d00f4ebccf2bda4f6d1878be560a6600689e42077e164e3" - ], - "version": "==1.2.1" - }, "decorator": { "hashes": [ "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", @@ -213,11 +206,11 @@ }, "flake8-docstrings": { "hashes": [ - "sha256:3ad372b641f4c8e70c7465f067aed4ff8bf1e9347fce14f9eb71ed816db36257", - "sha256:d8d72ccd5807c1ab9ff1466cb9bece0c4d94b8669e9bc4f472abc80dbc5d399e" + "sha256:1666dd069c9c457ee57e80af3c1a6b37b00cc1801c6fde88e455131bb2e186cd", + "sha256:9c0db5a79a1affd70fdf53b8765c8a26bf968e59e0252d7f2fc546b41c0cda06" ], "index": "pypi", - "version": "==1.3.1" + "version": "==1.4.0" }, "flake8-isort": { "hashes": [ @@ -287,11 +280,10 @@ }, "flake8-spellcheck": { "hashes": [ - "sha256:adbf59c4491d18a20809b8bec452245b7aaf0c0e7d9e2db646a3b0a2867570ab", - "sha256:c33e097e4abd0586cb30dcd08af0744a8787d8cfaa0dae61e8cef5a50556d696" + "sha256:0805f1d7ac6ae8d6ab84ef68244042cd6adbdee0b7a3dc653a39f3e9780d6a07" ], "index": "pypi", - "version": "==0.8.1" + "version": "==0.9.0" }, "flake8-string-format": { "hashes": [ @@ -311,10 +303,10 @@ }, "flake8-variables-names": { "hashes": [ - "sha256:728cfe7ca01fd2458fde22563e0bf53ff88018ada96471c73f6072f782218597" + "sha256:f9ee2edf0892a73fff33ef3eda37a7182cb91cd4c3a19a592ad432b68b261927" ], "index": "pypi", - "version": "==0.0.1" + "version": "==0.0.2" }, "gitdb2": { "hashes": [ @@ -325,10 +317,10 @@ }, "gitpython": { "hashes": [ - "sha256:259a8b6d6a4a118738c4a65fa990f8c8c91525bb43970aed2868952ebb86ceb8", - "sha256:73aa7b59e58dd3435121421c33c284e5ef51bc7b2f4373e1a1e4cc06e9c928ec" + "sha256:947cc75913e7b6da108458136607e2ee0e40c20be1e12d4284e7c6c12956c276", + "sha256:d2f4945f8260f6981d724f5957bc076398ada55cb5d25aaee10108bcdc894100" ], - "version": "==3.0.1" + "version": "==3.0.2" }, "grpcio": { "hashes": [ @@ -407,19 +399,19 @@ }, "importlib-metadata": { "hashes": [ - "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8", - "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3" + "sha256:0c505102757e7fa28b9f0958d8bc81301159dea16e2649858c92edc158b78a83", + "sha256:9a9f75ce32e78170905888acbf2376a81d3f21ecb3bb4867050413411d3ca7a9" ], "markers": "python_version < '3.8'", - "version": "==0.19" + "version": "==0.21" }, "ipython": { "hashes": [ - "sha256:1d3a1692921e932751bc1a1f7bb96dc38671eeefdc66ed33ee4cbc57e92a410e", - "sha256:537cd0176ff6abd06ef3e23f2d0c4c2c8a4d9277b7451544c6cbf56d1c79a83d" + "sha256:c4ab005921641e40a68e405e286e7a1fcc464497e14d81b6914b4fd95e5dee9b", + "sha256:dd76831f065f17bddd7eaa5c781f5ea32de5ef217592cf019e34043b56895aa1" ], "index": "pypi", - "version": "==7.7.0" + "version": "==7.8.0" }, "ipython-genutils": { "hashes": [ @@ -445,26 +437,26 @@ }, "lazy-object-proxy": { "hashes": [ - "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661", - "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f", - "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13", - "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821", - "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71", - "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e", - "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea", - "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229", - "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4", - "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e", - "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20", - "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16", - "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b", - "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7", - "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c", - "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a", - "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e", - "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1" - ], - "version": "==1.4.1" + "sha256:02b260c8deb80db09325b99edf62ae344ce9bc64d68b7a634410b8e9a568edbf", + "sha256:18f9c401083a4ba6e162355873f906315332ea7035803d0fd8166051e3d402e3", + "sha256:1f2c6209a8917c525c1e2b55a716135ca4658a3042b5122d4e3413a4030c26ce", + "sha256:2f06d97f0ca0f414f6b707c974aaf8829c2292c1c497642f63824119d770226f", + "sha256:616c94f8176808f4018b39f9638080ed86f96b55370b5a9463b2ee5c926f6c5f", + "sha256:63b91e30ef47ef68a30f0c3c278fbfe9822319c15f34b7538a829515b84ca2a0", + "sha256:77b454f03860b844f758c5d5c6e5f18d27de899a3db367f4af06bec2e6013a8e", + "sha256:83fe27ba321e4cfac466178606147d3c0aa18e8087507caec78ed5a966a64905", + "sha256:84742532d39f72df959d237912344d8a1764c2d03fe58beba96a87bfa11a76d8", + "sha256:874ebf3caaf55a020aeb08acead813baf5a305927a71ce88c9377970fe7ad3c2", + "sha256:9f5caf2c7436d44f3cec97c2fa7791f8a675170badbfa86e1992ca1b84c37009", + "sha256:a0c8758d01fcdfe7ae8e4b4017b13552efa7f1197dd7358dc9da0576f9d0328a", + "sha256:a4def978d9d28cda2d960c279318d46b327632686d82b4917516c36d4c274512", + "sha256:ad4f4be843dace866af5fc142509e9b9817ca0c59342fdb176ab6ad552c927f5", + "sha256:ae33dd198f772f714420c5ab698ff05ff900150486c648d29951e9c70694338e", + "sha256:b4a2b782b8a8c5522ad35c93e04d60e2ba7f7dcb9271ec8e8c3e08239be6c7b4", + "sha256:c462eb33f6abca3b34cdedbe84d761f31a60b814e173b98ede3c81bb48967c4f", + "sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1" + ], + "version": "==1.4.2" }, "mccabe": { "hashes": [ @@ -529,10 +521,10 @@ }, "pbr": { "hashes": [ - "sha256:56e52299170b9492513c64be44736d27a512fa7e606f21942160b68ce510b4bc", - "sha256:9b321c204a88d8ab5082699469f52cc94c5da45c51f114113d01b3d993c24cdf" + "sha256:2c8e420cd4ed4cec4e7999ee47409e876af575d4c35a45840d59e8b5f3155ab8", + "sha256:b32c8ccaac7b1a20c0ce00ce317642e6cf231cf038f9875e0280e28af5bf7ac9" ], - "version": "==5.4.2" + "version": "==5.4.3" }, "pep8-naming": { "hashes": [ @@ -654,11 +646,11 @@ }, "pytest": { "hashes": [ - "sha256:3805d095f1ea279b9870c3eeae5dddf8a81b10952c8835cd628cf1875b0ef031", - "sha256:abc562321c2d190dd63c2faadf70b86b7af21a553b61f0df5f5e1270717dc5a3" + "sha256:95d13143cc14174ca1a01ec68e84d76ba5d9d493ac02716fd9706c949a505210", + "sha256:b78fe2881323bd44fd9bd76e5317173d4316577e7b1cddebae9136a4495ec865" ], "index": "pypi", - "version": "==5.1.0" + "version": "==5.1.2" }, "pyyaml": { "hashes": [ @@ -694,16 +686,16 @@ }, "snowballstemmer": { "hashes": [ - "sha256:9f3b9ffe0809d174f7047e121431acf99c89a7040f0ca84f94ba53a498e6d0c9" + "sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e" ], - "version": "==1.9.0" + "version": "==1.9.1" }, "stevedore": { "hashes": [ - "sha256:7be098ff53d87f23d798a7ce7ae5c31f094f3deb92ba18059b1aeb1ca9fec0a0", - "sha256:7d1ce610a87d26f53c087da61f06f9b7f7e552efad2a7f6d2322632b5f932ea2" + "sha256:01d9f4beecf0fbd070ddb18e5efb10567801ba7ef3ddab0074f54e3cd4e91730", + "sha256:e0739f9739a681c7a1fda76a102b65295e96a144ccdb552f2ae03c5f0abe8a14" ], - "version": "==1.30.1" + "version": "==1.31.0" }, "testfixtures": { "hashes": [ @@ -770,10 +762,10 @@ }, "zipp": { "hashes": [ - "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a", - "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec" + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" ], - "version": "==0.5.2" + "version": "==0.6.0" } } } diff --git a/gfauto/azure-pipelines.yml b/gfauto/azure-pipelines.yml index 096efbfc9..b19f52479 100644 --- a/gfauto/azure-pipelines.yml +++ b/gfauto/azure-pipelines.yml @@ -37,7 +37,6 @@ steps: - script: | cd gfauto export PYTHON=python - export PIPENV_IGNORE_PIPFILE=1 export SKIP_SHELL=1 ./dev_shell.sh.template source .venv/bin/activate diff --git a/gfauto/dev_shell.sh.template b/gfauto/dev_shell.sh.template index ed3ace4d8..2374e0b93 100755 --- a/gfauto/dev_shell.sh.template +++ b/gfauto/dev_shell.sh.template @@ -18,30 +18,31 @@ set -x set -e set -u +# Check for some known files for sanity. +test -f ./Pipfile +test -f ./dev_shell.sh.template + +# Sets PYTHON to python3.6, unless already defined. # Modify if needed; this should be a Python 3.6 binary. -# E.g. python, python3, python3.6. +# E.g. PYTHON=python3.6 +# Or, do `export PYTHON=python3.6` before executing this script. PYTHON=${PYTHON-python3.6} # Upgrade/install pip and pipenv if needed. -"${PYTHON}" -m pip install --upgrade --user 'pip>=19.1.1' 'pipenv>=2018.11.26' +"${PYTHON}" -m pip install --upgrade --user --no-warn-script-location 'pip>=19.2.3' 'pipenv>=2018.11.26' -# The following (optional) line causes the virtual environment to be placed at -# `gfauto/.venv`. You may wish to add this environment variable permanently, -# such as by adding to your .bashrc file. +# Place the virtual environment at `gfauto/.venv`. export PIPENV_VENV_IN_PROJECT=1 -# The following (optional) line causes the hard-coded versions of packages (in -# Pipfile.lock) to be used, for better reproducibility. Enabled during CI but -# disabled otherwise so that packages are updated automatically during -# development. -# export PIPENV_IGNORE_PIPFILE=1 +# Use the hard-coded versions of packages in Pipfile.lock. +export PIPENV_IGNORE_PIPFILE=1 -# Install project dependencies, including development dependencies, into a +# Install project dependencies, including development dependencies, into the # virtual environment using pipenv. "${PYTHON}" -m pipenv install --dev if [ -z ${SKIP_SHELL+x} ]; then - # Enter the virtual environment. + # Enter the virtual environment, unless SKIP_SHELL is defined. # `python` should now point to the correct version of Python. "${PYTHON}" -m pipenv shell fi From c80ee59b94dc14cc64c8ec6016b6b286517d3474 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 12 Sep 2019 16:11:35 +0100 Subject: [PATCH 129/180] gfauto: add {download,run}_cts_gf_tests (#737) --- gfauto/Pipfile.lock | 42 +++++++ gfauto/gfauto/amber_converter.py | 162 +++++++++++++++++++++++- gfauto/gfauto/binaries_util.py | 1 + gfauto/gfauto/download_cts_gf_tests.py | 149 ++++++++++++++++++++++ gfauto/gfauto/fuzz.py | 16 +-- gfauto/gfauto/gerrit_util.py | 163 +++++++++++++++++++++++++ gfauto/gfauto/gflogging.py | 10 +- gfauto/gfauto/run_cts_gf_tests.py | 152 +++++++++++++++++++++++ gfauto/gfauto/settings_util.py | 18 +++ gfauto/gfauto/shader_compiler_util.py | 35 ++++-- gfauto/gfauto/subprocess_util.py | 6 +- gfauto/gfauto/util.py | 20 ++- gfauto/gfautotests/test_gerrit_util.py | 31 +++++ gfauto/setup.py | 4 +- gfauto/whitelist.dic | 7 ++ 15 files changed, 781 insertions(+), 35 deletions(-) create mode 100644 gfauto/gfauto/download_cts_gf_tests.py create mode 100644 gfauto/gfauto/gerrit_util.py create mode 100644 gfauto/gfauto/run_cts_gf_tests.py create mode 100644 gfauto/gfautotests/test_gerrit_util.py diff --git a/gfauto/Pipfile.lock b/gfauto/Pipfile.lock index b5cd27a8b..df11525aa 100644 --- a/gfauto/Pipfile.lock +++ b/gfauto/Pipfile.lock @@ -14,10 +14,31 @@ ] }, "default": { + "certifi": { + "hashes": [ + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" + ], + "version": "==2019.6.16" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, "gfauto": { "editable": true, "path": "." }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, "protobuf": { "hashes": [ "sha256:00a1b0b352dc7c809749526d1688a64b62ea400c5b05416f93cfb1b11a036295", @@ -39,12 +60,33 @@ ], "version": "==3.9.1" }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "version": "==2.8.0" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "version": "==2.22.0" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], "version": "==1.12.0" + }, + "urllib3": { + "hashes": [ + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + ], + "version": "==1.25.3" } }, "develop": { diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py index 8d81ba1f3..994034c0d 100644 --- a/gfauto/gfauto/amber_converter.py +++ b/gfauto/gfauto/amber_converter.py @@ -22,14 +22,15 @@ import itertools import json import pathlib +import re from copy import copy from enum import Enum from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Match, Optional import attr -from gfauto import shader_job_util, util +from gfauto import binaries_util, shader_job_util, subprocess_util, util from gfauto.gflogging import log from gfauto.util import check @@ -675,3 +676,160 @@ def spirv_asm_shader_job_to_amber_script( util.file_write_text(output_amber_script_file_path, result) return output_amber_script_file_path + + +def write_shader( + shader_asm: str, + amber_file: Path, + output_dir: Path, + shader_type: str, + shader_name: str, + binaries: binaries_util.BinaryManager, +) -> List[Path]: + + files_written: List[Path] = [] + + shader_type_to_suffix = { + "fragment": shader_job_util.EXT_FRAG, + "vertex": shader_job_util.EXT_VERT, + "compute": shader_job_util.EXT_COMP, + } + + shader_type_suffix = shader_type_to_suffix[shader_type] + + # E.g. ifs-and-whiles.variant_fragment_shader.frag.asm + shader_asm_file_path = output_dir / ( + f"{amber_file.stem}.{shader_name}{shader_type_suffix}{shader_job_util.SUFFIX_ASM_SPIRV}" + ) + + # E.g. ifs-and-whiles.variant_fragment_shader.frag.spv + shader_spirv_file_path = output_dir / ( + f"{amber_file.stem}.{shader_name}{shader_type_suffix}{shader_job_util.SUFFIX_SPIRV}" + ) + + util.file_write_text(shader_asm_file_path, shader_asm) + files_written.append(shader_asm_file_path) + + spirv_as_path = binaries.get_binary_path_by_name("spirv-as").path + + subprocess_util.run( + [ + str(spirv_as_path), + "-o", + str(shader_spirv_file_path), + str(shader_asm_file_path), + "--target-env", + "spv1.0", + ], + verbose=True, + ) + + files_written.append(shader_spirv_file_path) + + return files_written + + +def extract_shaders_amber_script( + amber_file: Path, + lines: List[str], + output_dir: Path, + binaries: binaries_util.BinaryManager, +) -> List[Path]: + files_written: List[Path] = [] + i = -1 + while i < len(lines) - 1: + i += 1 + line = lines[i] + if not line.strip().startswith("SHADER"): + continue + parts = line.strip().split() + shader_type = parts[1] + shader_name = parts[2] + shader_language = parts[3] + if shader_language == "PASSTHROUGH": + continue + check( + shader_language == "SPIRV-ASM", + AssertionError( + f"For {str(amber_file)}: unsupported shader language: {shader_language}" + ), + ) + i += 1 + shader_asm = "" + while not lines[i].strip().startswith("END"): + shader_asm += lines[i] + i += 1 + + files_written += write_shader( + shader_asm=shader_asm, + amber_file=amber_file, + output_dir=output_dir, + shader_type=shader_type, + shader_name=shader_name, + binaries=binaries, + ) + + return files_written + + +# E.g. [compute shader spirv] +VK_SCRIPT_SHADER_REGEX = re.compile(r"\[(compute|fragment|vertex) shader (\w*)\]") + + +def extract_shaders_vkscript( + amber_file: Path, + lines: List[str], + output_dir: Path, + binaries: binaries_util.BinaryManager, +) -> List[Path]: + files_written: List[Path] = [] + i = -1 + while i < len(lines) - 1: + i += 1 + line = lines[i] + match: Optional[Match[str]] = re.match(VK_SCRIPT_SHADER_REGEX, line.strip()) + if not match: + continue + shader_type = match.group(1) + shader_language = match.group(2) + if shader_language == "passthrough": + continue + check( + shader_language == "spirv", + AssertionError( + f"For {str(amber_file)}: unsupported shader language: {shader_language}" + ), + ) + i += 1 + shader_asm = "" + while not lines[i].strip().startswith("["): + shader_asm += lines[i] + i += 1 + files_written += write_shader( + shader_asm=shader_asm, + amber_file=amber_file, + output_dir=output_dir, + shader_type=shader_type, + shader_name="shader", + binaries=binaries, + ) + return files_written + + +def extract_shaders( + amber_file: Path, output_dir: Path, binaries: binaries_util.BinaryManager +) -> List[Path]: + files_written: List[Path] = [] + with util.file_open_text(amber_file, "r") as file_handle: + lines = file_handle.readlines() + if lines[0].startswith("#!amber"): + files_written += extract_shaders_amber_script( + amber_file, lines, output_dir, binaries + ) + else: + log(f"Skipping VkScript file {str(amber_file)} for now.") + files_written += extract_shaders_vkscript( + amber_file, lines, output_dir, binaries + ) + + return files_written diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py index 25c270a28..2af6ef9c9 100644 --- a/gfauto/gfauto/binaries_util.py +++ b/gfauto/gfauto/binaries_util.py @@ -66,6 +66,7 @@ ), Binary(name="spirv-opt", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), Binary(name="spirv-dis", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-as", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), Binary(name="spirv-val", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), Binary(name="spirv-fuzz", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), Binary( diff --git a/gfauto/gfauto/download_cts_gf_tests.py b/gfauto/gfauto/download_cts_gf_tests.py new file mode 100644 index 000000000..85d5f22c6 --- /dev/null +++ b/gfauto/gfauto/download_cts_gf_tests.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Downloads the latest GraphicsFuzz AmberScript tests from vk-gl-cts. + +Downloads the latest tests, including those that are pending. +""" + +import argparse +import sys +from pathlib import Path + +from gfauto import ( + amber_converter, + artifact_util, + binaries_util, + fuzz, + gerrit_util, + subprocess_util, + util, +) + + +def download_cts_graphicsfuzz_tests( # pylint: disable=too-many-locals; + git_tool: Path, cookie: str, binaries: binaries_util.BinaryManager +) -> None: + work_dir = Path() / "temp" / ("cts_" + fuzz.get_random_name()) + + latest_change = gerrit_util.get_latest_deqp_change(cookie) + latest_change_number = latest_change["_number"] + latest_change_details = gerrit_util.get_gerrit_change_details( + change_number=latest_change_number, cookie=cookie + ) + current_revision = latest_change_details["current_revision"] + cts_archive_path = gerrit_util.download_gerrit_revision( + output_path=work_dir / "cts.tgz", + change_number=latest_change_number, + revision=current_revision, + download_type=gerrit_util.DownloadType.Archive, + cookie=cookie, + ) + + cts_dir_name = "cts_temp" + cts_out = util.extract_archive(cts_archive_path, work_dir / cts_dir_name) + + pending_graphicsfuzz_changes = gerrit_util.get_deqp_graphicsfuzz_pending_changes( + cookie + ) + + for pending_change in pending_graphicsfuzz_changes: + change_number = pending_change["_number"] + change_details = gerrit_util.get_gerrit_change_details( + change_number=change_number, cookie=cookie + ) + current_revision = change_details["current_revision"] + patch_zip = gerrit_util.download_gerrit_revision( + output_path=work_dir / f"{change_number}.zip", + change_number=change_number, + revision=current_revision, + download_type=gerrit_util.DownloadType.Patch, + cookie=cookie, + ) + util.extract_archive(patch_zip, work_dir) + + # Create a dummy git repo in the work directory, otherwise "git apply" can fail silently. + # --unsafe-paths is possibly supposed to address this, but it doesn't seem to work if we + # are already in a git repo. + subprocess_util.run( + [str(git_tool), "init", "."], verbose=True, working_dir=work_dir + ) + + cmd = [str(git_tool), "apply"] + + patch_names = [p.name for p in work_dir.glob("*.diff")] + + cmd += patch_names + + # Use unix-style path for git. + cmd += [ + "--verbose", + "--unsafe-paths", + f"--directory={cts_dir_name}", + f"--include={cts_dir_name}/external/vulkancts/data/vulkan/amber/graphicsfuzz/*", + ] + + subprocess_util.run(cmd, verbose=True, working_dir=work_dir) + + shader_dir = util.copy_dir( + cts_out + / "external" + / "vulkancts" + / "data" + / "vulkan" + / "amber" + / "graphicsfuzz", + Path() / "graphicsfuzz", + ) + + for amber_file in shader_dir.glob("*.amber"): + amber_converter.extract_shaders( + amber_file, output_dir=amber_file.parent, binaries=binaries + ) + + +GERRIT_COOKIE_ARGUMENT_DESCRIPTION = ( + "The Gerrit cookie used for authentication. To get this, log in to the Khronos Gerrit page in your " + "browser and paste the following into the JavaScript console (F12) to copy the cookie to your clipboard: " + "copy(document.cookie.match(/GerritAccount=([^;]*)/)[1])" +) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Downloads the latest GraphicsFuzz AmberScript tests from vk-gl-cts, " + "including those in pending CLs. " + "Requires Git." + ) + + parser.add_argument("gerrit_cookie", help=GERRIT_COOKIE_ARGUMENT_DESCRIPTION) + + parsed_args = parser.parse_args(sys.argv[1:]) + + cookie: str = parsed_args.gerrit_cookie + + # Need git. + git_tool = util.tool_on_path("git") + + artifact_util.recipes_write_built_in() + + binaries = binaries_util.BinaryManager() + + download_cts_graphicsfuzz_tests(git_tool, cookie, binaries) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index d9ebceef1..162f49c25 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -124,10 +124,6 @@ def get_random_name() -> str: return uuid.uuid4().hex -class NoSettingsFile(Exception): - pass - - def main() -> None: parser = argparse.ArgumentParser(description="Fuzz") @@ -171,17 +167,7 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many skip_writing_binary_recipes: bool, ) -> None: - try: - settings = settings_util.read(settings_path) - except FileNotFoundError as exception: - message = f"Could not find settings file at: {settings_path}" - if not settings_util.DEFAULT_SETTINGS_FILE_PATH.exists(): - settings_util.write_default(settings_util.DEFAULT_SETTINGS_FILE_PATH) - message += ( - f"; a default settings file has been created for you at {str(settings_util.DEFAULT_SETTINGS_FILE_PATH)}. " - f"Please review it and then run fuzz again. " - ) - raise NoSettingsFile(message) from exception + settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) diff --git a/gfauto/gfauto/gerrit_util.py b/gfauto/gfauto/gerrit_util.py new file mode 100644 index 000000000..a176fd8e7 --- /dev/null +++ b/gfauto/gfauto/gerrit_util.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gerrit utility module. + +Provides functions for interacting with Gerrit via its REST API. +""" + +import json +from enum import Enum +from pathlib import Path +from typing import Any, Dict, Optional + +import requests +from dateutil.parser import parse as parse_date +from requests import Response + +from gfauto import util +from gfauto.gflogging import log +from gfauto.util import check + +KHRONOS_GERRIT_URL = "https://gerrit.khronos.org" + +KHRONOS_GERRIT_LOGIN_PAGE_START = "" + +RESPONSE_PREFIX = ")]}'\n" + + +class BadCookieError(Exception): + pass + + +def gerrit_get_stream( + url: str, path: str, params: Optional[Dict[str, str]], cookie: str +) -> Response: + return requests.get( + url + path, params=params, stream=True, cookies={"GerritAccount": cookie} + ) + + +def gerrit_get(url: str, path: str, params: Dict[str, str], cookie: str) -> Any: + response = requests.get( + url + path, params=params, cookies={"GerritAccount": cookie} + ).text + + check( + response.startswith(RESPONSE_PREFIX), + AssertionError(f"Unexpected response from Gerrit: {response}"), + ) + response = util.remove_start(response, RESPONSE_PREFIX) + return json.loads(response) + + +def find_latest_change(changes: Any) -> Any: + check( + len(changes) > 0, AssertionError(f"Expected at least one CL but got: {changes}") + ) + + # Find the latest submit date (the default order is based on when the CL was last updated). + + latest_change = changes[0] + latest_date = parse_date(latest_change["submitted"]) + + for i in range(1, len(changes)): + change = changes[i] + submitted_date = parse_date(change["submitted"]) + if submitted_date > latest_date: + latest_change = change + latest_date = submitted_date + + return latest_change + + +def get_latest_deqp_change(cookie: str) -> Any: + log("Getting latest deqp change") + changes = gerrit_get( + KHRONOS_GERRIT_URL, + "/changes/", + params={"q": "project:vk-gl-cts status:merged branch:master", "n": "1000"}, + cookie=cookie, + ) + + return find_latest_change(changes) + + +def get_gerrit_change_details(change_number: str, cookie: str) -> Any: + log(f"Getting change details for change number: {change_number}") + return gerrit_get( + KHRONOS_GERRIT_URL, + f"/changes/{change_number}/detail", + params={"O": "10004"}, + cookie=cookie, + ) + + +class DownloadType(Enum): + """For download_gerrit_revision, specifies what to download.""" + + # Downloads the entire repo as a .tgz file. + Archive = "archive" + + # Downloads the patch in a .zip file. + Patch = "patch" + + +def download_gerrit_revision( + output_path: Path, + change_number: str, + revision: str, + download_type: DownloadType, + cookie: str, +) -> Path: + path = f"/changes/{change_number}/revisions/{revision}/{download_type.value}" + log(f"Downloading revision from: {path}\n to: {str(output_path)}") + + params = {"format": "tgz"} if download_type == DownloadType.Archive else {"zip": ""} + + response = gerrit_get_stream(KHRONOS_GERRIT_URL, path, params=params, cookie=cookie) + + counter = 0 + with util.file_open_binary(output_path, "wb") as output_stream: + for chunk in response.iter_content(chunk_size=None): + log(".", skip_newline=True) + counter += 1 + if counter > 80: + counter = 0 + log("") # new line + output_stream.write(chunk) + log("") # new line + + with util.file_open_text(output_path, "r") as input_stream: + line = input_stream.readline(len(KHRONOS_GERRIT_LOGIN_PAGE_START) * 2) + if line.startswith(KHRONOS_GERRIT_LOGIN_PAGE_START) or line.startswith( + "Not found" + ): + raise BadCookieError() + + return output_path + + +def get_deqp_graphicsfuzz_pending_changes(cookie: str) -> Any: + return gerrit_get( + KHRONOS_GERRIT_URL, + "/changes/", + params={ + "q": "project:vk-gl-cts status:pending branch:master dEQP-VK.graphicsfuzz.", + "n": "1000", + }, + cookie=cookie, + ) diff --git a/gfauto/gfauto/gflogging.py b/gfauto/gfauto/gflogging.py index f1f553fe2..0bc5949d5 100644 --- a/gfauto/gfauto/gflogging.py +++ b/gfauto/gfauto/gflogging.py @@ -36,12 +36,16 @@ def pop_stream_for_logging() -> None: _LOG_TO_STREAM.pop() -def log(message: str) -> None: +def log(message: str, skip_newline: bool = False) -> None: if _LOG_TO_STDOUT: - print(message, flush=True) # noqa T001 + if not skip_newline: + print(message, flush=True) # noqa T001 + else: + print(message, end="", flush=True) # noqa T001 for stream in _LOG_TO_STREAM: stream.write(message) - stream.write("\n") + if not skip_newline: + stream.write("\n") stream.flush() diff --git a/gfauto/gfauto/run_cts_gf_tests.py b/gfauto/gfauto/run_cts_gf_tests.py new file mode 100644 index 000000000..933948841 --- /dev/null +++ b/gfauto/gfauto/run_cts_gf_tests.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runs GraphicsFuzz AmberScript tests.""" + +import argparse +import sys +from pathlib import Path +from subprocess import CalledProcessError, TimeoutExpired +from typing import Optional + +from gfauto import ( + artifact_util, + binaries_util, + devices_util, + fuzz, + host_device_util, + settings_util, + shader_compiler_util, + spirv_opt_util, + util, +) + +DEFAULT_TIMEOUT = 30 + + +def main() -> None: # pylint: disable=too-many-locals,too-many-branches; + parser = argparse.ArgumentParser( + description="Runs GraphicsFuzz AmberScript tests on the active devices listed in " + "the settings.json file." + ) + + parser.add_argument( + "--settings", + help="Path to the settings JSON file for this fuzzing instance.", + default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), + ) + + parser.add_argument( + "--tests", + help="Path to the directory of AmberScript tests with shaders extracted.", + default=str("graphicsfuzz"), + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + # Args. + tests_dir: Path = Path(parsed_args.tests) + settings_path: Path = Path(parsed_args.settings) + + # Settings and devices. + settings = settings_util.read_or_create(settings_path) + active_devices = devices_util.get_active_devices(settings.device_list) + + # Binaries. + artifact_util.recipes_write_built_in() + binaries = binaries_util.BinaryManager() + + work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" + + with util.file_open_text(Path("results.txt"), "w") as log_handle: + + def write_entry(entry: str) -> None: + log_handle.write(entry) + log_handle.write(", ") + log_handle.flush() + + def write_newline() -> None: + log_handle.write("\n") + log_handle.flush() + + spirv_opt_path: Optional[Path] = None + swift_shader_path: Optional[Path] = None + + # Enumerate active devices, writing their name and storing binary paths if needed. + write_entry("test") + for device in active_devices: + if device.HasField("preprocess"): + write_entry("spirv-opt") + spirv_opt_path = binaries.get_binary_path_by_name( + binaries_util.SPIRV_OPT_NAME + ).path + elif device.HasField("swift_shader"): + write_entry("SwiftShader") + swift_shader_path = binaries.get_binary_path_by_name( + binaries_util.SWIFT_SHADER_NAME + ).path + else: + write_entry(device.name) + + write_newline() + + # Enumerate tests and devices, writing the results. + + for test in sorted(tests_dir.glob("*.amber")): + write_entry(test.name) + spirv_shaders = sorted( + tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv") + ) + for device in active_devices: + try: + if device.HasField("preprocess"): + # This just means spirv-op for now. + + assert spirv_opt_path # noqa + for spirv_shader in spirv_shaders: + spirv_opt_util.run_spirv_opt_on_spirv_shader( + spirv_shader, work_dir, ["-O"], spirv_opt_path + ) + elif device.HasField("shader_compiler"): + for spirv_shader in spirv_shaders: + shader_compiler_util.run_shader( + shader_compiler_device=device.shader_compiler, + shader_path=spirv_shader, + output_dir=work_dir, + timeout=DEFAULT_TIMEOUT, + ) + elif device.HasField("swift_shader"): + assert swift_shader_path # noqa + host_device_util.run_amber( + test, + work_dir, + dump_image=False, + dump_buffer=False, + icd=swift_shader_path, + ) + else: + raise AssertionError(f"Unsupported device {device.name}") + + write_entry("P") + except CalledProcessError: + write_entry("F") + except TimeoutExpired: + write_entry("T") + write_newline() + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/settings_util.py b/gfauto/gfauto/settings_util.py index 5ce0ee465..e63c603c7 100644 --- a/gfauto/gfauto/settings_util.py +++ b/gfauto/gfauto/settings_util.py @@ -30,6 +30,24 @@ DEFAULT_SETTINGS = Settings(maximum_duplicate_crashes=3) +class NoSettingsFile(Exception): + pass + + +def read_or_create(settings_path: Path) -> Settings: + try: + return read(settings_path) + except FileNotFoundError as exception: + message = f"Could not find settings file at: {settings_path}" + if not DEFAULT_SETTINGS_FILE_PATH.exists(): + write_default(DEFAULT_SETTINGS_FILE_PATH) + message += ( + f"; a default settings file has been created for you at {str(DEFAULT_SETTINGS_FILE_PATH)}. " + f"Please review it and then try again. " + ) + raise NoSettingsFile(message) from exception + + def read(settings_path: Path) -> Settings: result = proto_util.file_to_message(settings_path, Settings()) if not result.maximum_duplicate_crashes: diff --git a/gfauto/gfauto/shader_compiler_util.py b/gfauto/gfauto/shader_compiler_util.py index 9201434ca..18684959a 100644 --- a/gfauto/gfauto/shader_compiler_util.py +++ b/gfauto/gfauto/shader_compiler_util.py @@ -20,24 +20,43 @@ """ from pathlib import Path -from typing import List +from typing import List, Optional from gfauto import shader_job_util, subprocess_util, util from gfauto.device_pb2 import DeviceShaderCompiler from gfauto.gflogging import log +from gfauto.util import check_file_exists DEFAULT_TIMEOUT = 600 +def resolve_compiler_path(shader_compiler_device: DeviceShaderCompiler) -> Path: + try: + compiler_path: Path = util.tool_on_path(shader_compiler_device.binary) + except util.ToolNotOnPathError: + # If not found on PATH, then assume it is an absolute path. + compiler_path = Path(shader_compiler_device.binary) + check_file_exists(compiler_path) + return compiler_path + + def run_shader( - compiler_path: Path, shader_path: Path, args: List[str], output_dir: Path + shader_compiler_device: DeviceShaderCompiler, + shader_path: Path, + output_dir: Path, + compiler_path: Optional[Path] = None, + timeout: int = DEFAULT_TIMEOUT, ) -> Path: output_file_path = output_dir / (shader_path.name + ".out") + + if not compiler_path: + compiler_path = resolve_compiler_path(shader_compiler_device) + cmd = [str(compiler_path)] - cmd += args + cmd += list(shader_compiler_device.args) cmd += [str(shader_path), "-o", str(output_file_path)] cmd = util.prepend_catchsegv_if_available(cmd) - subprocess_util.run(cmd, verbose=True, timeout=DEFAULT_TIMEOUT) + subprocess_util.run(cmd, verbose=True, timeout=timeout) return output_file_path @@ -46,11 +65,7 @@ def run_shader_job( spirv_shader_job_path: Path, output_dir: Path, ) -> List[Path]: - try: - compiler_path: Path = util.tool_on_path(shader_compiler_device.binary) - except util.ToolNotOnPathError: - # If not found on PATH, then assume it is an absolute path. - compiler_path = Path(shader_compiler_device.binary) + compiler_path = resolve_compiler_path(shader_compiler_device) log(f"Running {str(compiler_path)} on shader job {str(spirv_shader_job_path)}") @@ -65,9 +80,9 @@ def run_shader_job( for shader_path in shader_paths: result.append( run_shader( + shader_compiler_device, compiler_path=compiler_path, shader_path=shader_path, - args=list(shader_compiler_device.args), output_dir=output_dir, ) ) diff --git a/gfauto/gfauto/subprocess_util.py b/gfauto/gfauto/subprocess_util.py index 741da3062..2f351d63c 100644 --- a/gfauto/gfauto/subprocess_util.py +++ b/gfauto/gfauto/subprocess_util.py @@ -25,6 +25,7 @@ import signal import subprocess import time +from pathlib import Path from typing import Dict, List, Optional, Union from gfauto.gflogging import log @@ -78,6 +79,7 @@ def run_helper( check_exit_code: bool = True, timeout: Optional[float] = None, env: Optional[Dict[str, str]] = None, + working_dir: Optional[Path] = None, ) -> subprocess.CompletedProcess: check( bool(cmd) and cmd[0] is not None and isinstance(cmd[0], str), @@ -98,6 +100,7 @@ def run_helper( env=env_child, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=working_dir, ) as process: try: stdout, stderr = process.communicate(input=None, timeout=timeout) @@ -128,10 +131,11 @@ def run( timeout: Optional[float] = None, verbose: bool = False, env: Optional[Dict[str, str]] = None, + working_dir: Optional[Path] = None, ) -> subprocess.CompletedProcess: log("Exec" + (" (verbose):" if verbose else ":") + str(cmd)) try: - result = run_helper(cmd, check_exit_code, timeout, env) + result = run_helper(cmd, check_exit_code, timeout, env, working_dir) except subprocess.TimeoutExpired as ex: log(LOG_COMMAND_TIMED_OUT_PREFIX + str(cmd)) # no return code to log in case of timeout diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py index 9d7fc8679..1c4e80eb1 100644 --- a/gfauto/gfauto/util.py +++ b/gfauto/gfauto/util.py @@ -39,6 +39,8 @@ def file_open_binary(file: pathlib.Path, mode: str) -> BinaryIO: # noqa VNE002 check("b" in mode, AssertionError(f"|mode|(=={mode}) should contain 'b'")) + if "w" in mode: + file_mkdirs_parent(file) # Type hint (no runtime check). result = cast(BinaryIO, open(str(file), mode)) return result @@ -70,10 +72,10 @@ def file_read_lines(file: pathlib.Path) -> List[str]: # noqa VNE002 return f.readlines() -def file_write_text(file: pathlib.Path, text: str) -> int: # noqa VNE002 - file_mkdirs_parent(file) +def file_write_text(file: pathlib.Path, text: str) -> Path: # noqa VNE002 with file_open_text(file, "w") as f: - return f.write(text) + f.write(text) + return file def mkdir_p_new(path: Path) -> Path: # noqa VNE002 @@ -266,3 +268,15 @@ def get_platform() -> str: if host == "Darwin": return "Mac" raise AssertionError("Unsupported platform: {}".format(host)) + + +def extract_archive(archive_file: Path, output_dir: Path) -> Path: + """ + Extract/unpack an archive. + + :return: output_dir + """ + gflogging.log(f"Extracting {str(archive_file)} to {str(output_dir)}") + shutil.unpack_archive(str(archive_file), extract_dir=str(output_dir)) + gflogging.log("Done") + return output_dir diff --git a/gfauto/gfautotests/test_gerrit_util.py b/gfauto/gfautotests/test_gerrit_util.py new file mode 100644 index 000000000..dfa81debf --- /dev/null +++ b/gfauto/gfautotests/test_gerrit_util.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gfauto import gerrit_util + + +def test_get_latest_change() -> None: + changes = [ + {"submitted": "2018-09-08 05:54:26.000000000"}, + {"submitted": "2019-06-10 11:54:26.000000000"}, + {"submitted": "2019-09-09 05:54:26.000000000"}, + {"submitted": "2019-09-10 05:54:26.000000000"}, # This one is the latest. + {"submitted": "2019-09-10 05:53:26.000000000"}, # This one is similar. + {"submitted": "2018-09-08 05:54:26.000000000"}, + ] + latest_change = gerrit_util.find_latest_change(changes) + assert latest_change == changes[3] + assert latest_change != changes[2] diff --git a/gfauto/setup.py b/gfauto/setup.py index 04bf6b02c..eb922d95a 100644 --- a/gfauto/setup.py +++ b/gfauto/setup.py @@ -31,7 +31,7 @@ license="Apache License 2.0", packages=["gfauto"], python_requires=">=3.6", - install_requires=["protobuf"], + install_requires=["protobuf", "requests", "python-dateutil"], package_data={"gfauto": ["*.proto", "*.pyi"]}, classifiers=[ "Environment :: Console", @@ -50,5 +50,7 @@ "add_amber_tests_to_cts = gfauto.add_amber_tests_to_cts:main", "gfauto_test_update_binaries = gfauto.test_update_binaries:main", "gfauto_test_create_readme = gfauto.test_create_readme:main", + "gfauto_download_cts_gf_tests = gfauto.download_cts_gf_tests:main", + "gfauto_run_cts_gf_tests = gfauto.run_cts_gf_tests:main", ]}, ) diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index 8d03230e6..96a1bcf31 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -179,3 +179,10 @@ mklink fullmatch winapi iter +Gerrit +khronos +cwd +params +deqp +tgz + From 74cbc9d11fd3cd89dd738280e87d153d4144596f Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 13 Sep 2019 17:31:31 +0100 Subject: [PATCH 130/180] Enhance reducer to include guard of compound statement when flattening it (#733) When simplifying a statement of the form: if(e) { s1; s2; ... } the reducer used to turn it into just: { s1; s2; ... } It now turns it into: { e; s1; s2; ... } For, while and do-while loops are similarly handled so that their guard (and the initializer and increment components, in the case of a for loop) are included in the flattened construct that the reducer produces. This is useful in cases where e.g. something appearing in the guard is needed to trigger a bug, but the control flow statement itself is not needed. Fixes #732. --- parent-all/pom.xml | 5 - .../graphicsfuzz/reducer/ReductionDriver.java | 2 +- .../CompoundToBlockReductionOpportunity.java | 67 -- ...lattenConditionalReductionOpportunity.java | 61 ++ ...tenControlFlowReductionOpportunities.java} | 76 +- ...lattenDoWhileLoopReductionOpportunity.java | 42 + .../FlattenForLoopReductionOpportunity.java | 49 + .../FlattenLoopReductionOpportunity.java | 58 ++ .../FlattenWhileLoopReductionOpportunity.java | 43 + .../IReductionOpportunityFinder.java | 12 +- .../ReductionOpportunities.java | 2 +- .../SwitchToLoopReductionOpportunity.java | 11 +- .../reducer/ReductionDriverTest.java | 128 +++ ...oundToBlockReductionOpportunitiesTest.java | 528 ---------- ...ControlFlowReductionOpportunitiesTest.java | 986 ++++++++++++++++++ ...witchToLoopReductionOpportunitiesTest.java | 5 + 16 files changed, 1431 insertions(+), 644 deletions(-) delete mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunity.java create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenConditionalReductionOpportunity.java rename reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/{CompoundToBlockReductionOpportunities.java => FlattenControlFlowReductionOpportunities.java} (65%) create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenDoWhileLoopReductionOpportunity.java create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenForLoopReductionOpportunity.java create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenLoopReductionOpportunity.java create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenWhileLoopReductionOpportunity.java delete mode 100755 reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java create mode 100755 reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunitiesTest.java diff --git a/parent-all/pom.xml b/parent-all/pom.xml index 24c8d771a..853b0261a 100644 --- a/parent-all/pom.xml +++ b/parent-all/pom.xml @@ -281,11 +281,6 @@ limitations under the License. ast 1.0 - - com.graphicsfuzz - astfuzzer - 1.0 - com.graphicsfuzz checkstyle-config diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java index a8275c5ca..4853dad44 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java @@ -112,7 +112,7 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.functionFinder(), IReductionOpportunityFinder.mutationFinder(), IReductionOpportunityFinder.loopMergeFinder(), - IReductionOpportunityFinder.compoundToBlockFinder(), + IReductionOpportunityFinder.flattenControlFlowFinder(), IReductionOpportunityFinder.switchToLoopFinder(), IReductionOpportunityFinder.outlinedStatementFinder(), IReductionOpportunityFinder.unwrapFinder(), diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunity.java deleted file mode 100644 index 42b2cc534..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunity.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.reductionopportunities; - -import com.graphicsfuzz.common.ast.IAstNode; -import com.graphicsfuzz.common.ast.stmt.BlockStmt; -import com.graphicsfuzz.common.ast.stmt.ForStmt; -import com.graphicsfuzz.common.ast.stmt.Stmt; -import com.graphicsfuzz.common.ast.visitors.VisitationDepth; -import java.util.ArrayList; -import java.util.List; - -public class CompoundToBlockReductionOpportunity extends AbstractReductionOpportunity { - - private final IAstNode parent; - private final Stmt compoundStmt; - private final Stmt childStmt; - - public CompoundToBlockReductionOpportunity(IAstNode parent, Stmt compoundStmt, Stmt childStmt, - VisitationDepth depth) { - super(depth); - this.parent = parent; - this.compoundStmt = compoundStmt; - this.childStmt = childStmt; - } - - @Override - public void applyReductionImpl() { - Stmt replacement; - if (compoundStmt instanceof ForStmt) { - final ForStmt forStmt = (ForStmt) compoundStmt; - assert childStmt == forStmt.getBody(); - List stmts = new ArrayList<>(); - stmts.add(forStmt.getInit()); - if (forStmt.getBody() instanceof BlockStmt) { - stmts.addAll(((BlockStmt) forStmt.getBody()).getStmts()); - } else { - stmts.add(forStmt.getBody()); - } - replacement = new BlockStmt(stmts, true); - } else { - replacement = childStmt; - } - parent.replaceChild(compoundStmt, replacement); - } - - @Override - public boolean preconditionHolds() { - return parent.hasChild(compoundStmt) - && compoundStmt.hasChild(childStmt); - } - -} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenConditionalReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenConditionalReductionOpportunity.java new file mode 100644 index 000000000..a9d4808aa --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenConditionalReductionOpportunity.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.IfStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.ArrayList; +import java.util.List; + +public class FlattenConditionalReductionOpportunity extends AbstractReductionOpportunity { + + private final IAstNode parent; + private final IfStmt conditional; + private final boolean replaceWithThenBranch; + + public FlattenConditionalReductionOpportunity(IAstNode parent, IfStmt conditional, + boolean replaceWithThenBranch, + VisitationDepth depth) { + super(depth); + this.parent = parent; + this.conditional = conditional; + this.replaceWithThenBranch = replaceWithThenBranch; + } + + @Override + void applyReductionImpl() { + final List newStmts = new ArrayList<>(); + newStmts.add(new ExprStmt(conditional.getCondition())); + Stmt branchToAdd = replaceWithThenBranch ? conditional.getThenStmt() : + conditional.getElseStmt(); + if (branchToAdd instanceof BlockStmt) { + newStmts.addAll(((BlockStmt) branchToAdd).getStmts()); + } else { + newStmts.add(branchToAdd); + } + parent.replaceChild(conditional, new BlockStmt(newStmts, true)); + } + + @Override + public boolean preconditionHolds() { + return parent.hasChild(conditional) && (replaceWithThenBranch || conditional.hasElseStmt()); + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java similarity index 65% rename from reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java rename to reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java index c8d81ceba..27e67d9de 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java @@ -27,21 +27,20 @@ import com.graphicsfuzz.common.util.ContainsTopLevelBreak; import com.graphicsfuzz.common.util.ContainsTopLevelContinue; import com.graphicsfuzz.common.util.ListConcat; -import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.SideEffectChecker; import java.util.Arrays; import java.util.List; -public class CompoundToBlockReductionOpportunities - extends ReductionOpportunitiesBase { +public class FlattenControlFlowReductionOpportunities + extends ReductionOpportunitiesBase { - private CompoundToBlockReductionOpportunities( + private FlattenControlFlowReductionOpportunities( TranslationUnit tu, ReducerContext context) { super(tu, context); } - static List findOpportunities( + static List findOpportunities( ShaderJob shaderJob, ReducerContext context) { return shaderJob.getShaders() @@ -50,11 +49,11 @@ static List findOpportunities( .reduce(Arrays.asList(), ListConcat::concatenate); } - private static List findOpportunitiesForShader( + private static List findOpportunitiesForShader( TranslationUnit tu, ReducerContext context) { - CompoundToBlockReductionOpportunities finder = - new CompoundToBlockReductionOpportunities(tu, context); + FlattenControlFlowReductionOpportunities finder = + new FlattenControlFlowReductionOpportunities(tu, context); finder.visit(tu); return finder.getOpportunities(); } @@ -62,51 +61,64 @@ private static List findOpportunitiesForSha @Override public void visitDoStmt(DoStmt doStmt) { super.visitDoStmt(doStmt); - handleLoopStmt(doStmt); + if (allowedToReduce(doStmt)) { + addOpportunity(new FlattenDoWhileLoopReductionOpportunity( + parentMap.getParent(doStmt), + doStmt, + getVistitationDepth())); + } } @Override public void visitForStmt(ForStmt forStmt) { super.visitForStmt(forStmt); - handleLoopStmt(forStmt); + if (allowedToReduce(forStmt)) { + addOpportunity(new FlattenForLoopReductionOpportunity( + parentMap.getParent(forStmt), + forStmt, + getVistitationDepth())); + } } @Override public void visitWhileStmt(WhileStmt whileStmt) { super.visitWhileStmt(whileStmt); - handleLoopStmt(whileStmt); - } - - private void handleLoopStmt(LoopStmt loopStmt) { - if (ContainsTopLevelBreak.check(loopStmt.getBody()) - || ContainsTopLevelContinue.check(loopStmt.getBody())) { - return; - } - if (MacroNames.isDeadByConstruction(loopStmt.getCondition())) { - return; + if (allowedToReduce(whileStmt)) { + addOpportunity(new FlattenWhileLoopReductionOpportunity( + parentMap.getParent(whileStmt), + whileStmt, + getVistitationDepth())); } - addOpportunity(loopStmt, loopStmt.getBody()); } @Override public void visitIfStmt(IfStmt ifStmt) { super.visitIfStmt(ifStmt); - addOpportunity(ifStmt, ifStmt.getThenStmt()); - if (ifStmt.hasElseStmt()) { - addOpportunity(ifStmt, ifStmt.getElseStmt()); - } - } - - private void addOpportunity(Stmt compoundStmt, - Stmt childStmt) { - if (allowedToReduce(compoundStmt)) { - addOpportunity(new CompoundToBlockReductionOpportunity( - parentMap.getParent(compoundStmt), compoundStmt, childStmt, + if (allowedToReduce(ifStmt)) { + addOpportunity(new FlattenConditionalReductionOpportunity( + parentMap.getParent(ifStmt), + ifStmt, + true, + getVistitationDepth())); + if (ifStmt.hasElseStmt()) { + addOpportunity(new FlattenConditionalReductionOpportunity( + parentMap.getParent(ifStmt), + ifStmt, + false, getVistitationDepth())); + } } } private boolean allowedToReduce(Stmt compoundStmt) { + if (compoundStmt instanceof LoopStmt) { + final LoopStmt loopStmt = (LoopStmt) compoundStmt; + if (ContainsTopLevelBreak.check(loopStmt.getBody()) + || ContainsTopLevelContinue.check(loopStmt.getBody())) { + return false; + } + } + return context.reduceEverywhere() || injectionTracker.enclosedByDeadCodeInjection() || injectionTracker.underUnreachableSwitchCase() diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenDoWhileLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenDoWhileLoopReductionOpportunity.java new file mode 100644 index 000000000..a2aadf569 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenDoWhileLoopReductionOpportunity.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.DoStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.ArrayList; +import java.util.List; + +public class FlattenDoWhileLoopReductionOpportunity extends FlattenLoopReductionOpportunity { + + public FlattenDoWhileLoopReductionOpportunity(IAstNode parent, DoStmt doStmt, + VisitationDepth depth) { + super(parent, doStmt, depth); + } + + @Override + void applyReductionImpl() { + final List newStmts = new ArrayList<>(); + addLoopBody(newStmts); + newStmts.add(new ExprStmt(getLoopStmt().getCondition())); + doReplacement(newStmts); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenForLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenForLoopReductionOpportunity.java new file mode 100644 index 000000000..143764d0d --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenForLoopReductionOpportunity.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.ArrayList; +import java.util.List; + +public class FlattenForLoopReductionOpportunity extends FlattenLoopReductionOpportunity { + + public FlattenForLoopReductionOpportunity(IAstNode parent, ForStmt forStmt, + VisitationDepth depth) { + super(parent, forStmt, depth); + } + + @Override + void applyReductionImpl() { + final List newStmts = new ArrayList<>(); + final ForStmt forStmt = (ForStmt) getLoopStmt(); + newStmts.add(forStmt.getInit()); + if (forStmt.hasCondition()) { + newStmts.add(new ExprStmt(forStmt.getCondition())); + } + addLoopBody(newStmts); + if (forStmt.hasIncrement()) { + newStmts.add(new ExprStmt(forStmt.getIncrement())); + } + doReplacement(newStmts); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenLoopReductionOpportunity.java new file mode 100644 index 000000000..2c6f7b3ed --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenLoopReductionOpportunity.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.LoopStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.List; + +public abstract class FlattenLoopReductionOpportunity extends AbstractReductionOpportunity { + + private final IAstNode parent; + private final LoopStmt loopStmt; + + FlattenLoopReductionOpportunity(IAstNode parent, LoopStmt loopStmt, + VisitationDepth depth) { + super(depth); + this.parent = parent; + this.loopStmt = loopStmt; + } + + LoopStmt getLoopStmt() { + return loopStmt; + } + + void doReplacement(List newStmts) { + parent.replaceChild(loopStmt, new BlockStmt(newStmts, true)); + } + + void addLoopBody(List newStmts) { + if (loopStmt.getBody() instanceof BlockStmt) { + newStmts.addAll(((BlockStmt) loopStmt.getBody()).getStmts()); + } else { + newStmts.add(loopStmt.getBody()); + } + } + + @Override + public final boolean preconditionHolds() { + return parent.hasChild(loopStmt); + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenWhileLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenWhileLoopReductionOpportunity.java new file mode 100644 index 000000000..98fde4ca8 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenWhileLoopReductionOpportunity.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.stmt.WhileStmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.ArrayList; +import java.util.List; + +public class FlattenWhileLoopReductionOpportunity extends FlattenLoopReductionOpportunity { + + public FlattenWhileLoopReductionOpportunity(IAstNode parent, WhileStmt whileStmt, + VisitationDepth depth) { + super(parent, whileStmt, depth); + } + + @Override + void applyReductionImpl() { + final List newStmts = new ArrayList<>(); + newStmts.add(new ExprStmt(getLoopStmt().getCondition())); + addLoopBody(newStmts); + doReplacement(newStmts); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java index 5f5f715f0..47f06b667 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java @@ -123,17 +123,17 @@ public String getName() { }; } - static IReductionOpportunityFinder compoundToBlockFinder() { - return new IReductionOpportunityFinder() { + static IReductionOpportunityFinder flattenControlFlowFinder() { + return new IReductionOpportunityFinder() { @Override - public List findOpportunities(ShaderJob shaderJob, - ReducerContext context) { - return CompoundToBlockReductionOpportunities.findOpportunities(shaderJob, context); + public List findOpportunities(ShaderJob shaderJob, + ReducerContext context) { + return FlattenControlFlowReductionOpportunities.findOpportunities(shaderJob, context); } @Override public String getName() { - return "compoundToBlock"; + return "flattenControlFlow"; } }; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java index fcc82af7b..bc16d6f4f 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java @@ -59,7 +59,7 @@ public static List getReductionOpportunities( IReductionOpportunityFinder.exprToConstantFinder(), IReductionOpportunityFinder.compoundExprToSubExprFinder(), IReductionOpportunityFinder.mutationFinder(), - IReductionOpportunityFinder.compoundToBlockFinder(), + IReductionOpportunityFinder.flattenControlFlowFinder(), IReductionOpportunityFinder.inlineInitializerFinder(), IReductionOpportunityFinder.inlineFunctionFinder(), IReductionOpportunityFinder.liveFragColorWriteFinder(), diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java index 7a0ddc616..ba0c089a5 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java @@ -21,8 +21,10 @@ import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.CaseLabel; import com.graphicsfuzz.common.ast.stmt.DoStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.Collections; import java.util.stream.Collectors; /** @@ -32,7 +34,7 @@ */ public class SwitchToLoopReductionOpportunity extends AbstractReductionOpportunity { - // The parent of the switch statemtnt + // The parent of the switch statement private final IAstNode parent; // The switch statement to be replaced @@ -46,9 +48,10 @@ public class SwitchToLoopReductionOpportunity extends AbstractReductionOpportuni @Override void applyReductionImpl() { - final BlockStmt loopBody = new BlockStmt( - switchStmt.getBody().getStmts().stream().filter(item -> !(item instanceof CaseLabel)) - .collect(Collectors.toList()), true); + final BlockStmt loopBody = new BlockStmt(Collections.emptyList(), true); + loopBody.addStmt(new ExprStmt(switchStmt.getExpr())); + switchStmt.getBody().getStmts().stream().filter(item -> !(item instanceof CaseLabel)) + .forEach(loopBody::addStmt); parent.replaceChild(switchStmt, new DoStmt(loopBody, new BoolConstantExpr(false))); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java index 3777bb9eb..398906236 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java @@ -23,8 +23,11 @@ import static org.junit.Assert.assertTrue; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; @@ -727,6 +730,131 @@ public void visitReturnStmt(ReturnStmt returnStmt) { } + @Test + public void testRemoveIfOrLoopButKeepGuard() throws Exception { + final String shaderIf = "#version 310 es\n" + + "void main() {\n" + + " if(1 > 0) {\n" + + " }\n" + + "}\n"; + + final String shaderWhile = "#version 310 es\n" + + "void main() {\n" + + " while(1 > 0) {\n" + + " }\n" + + "}\n"; + + final String shaderDoWhile = "#version 310 es\n" + + "void main() {\n" + + " do {\n" + + " } while(1 > 0);\n" + + "}\n"; + + final String shaderFor = "#version 310 es\n" + + "void main() {\n" + + " for (; 1 > 0; ) {\n" + + " }\n" + + "}\n"; + + final String expected = "#version 310 es\n" + + "void main() {\n" + + " 1 > 0;\n" + + "}\n"; + + final IFileJudge customFileJudge = + new CheckAstFeaturesFileJudge(Collections.singletonList( + () -> new CheckAstFeatureVisitor() { + @Override + public void visitBinaryExpr(BinaryExpr binaryExpr) { + super.visitBinaryExpr(binaryExpr); + if (binaryExpr.getOp() == BinOp.GT + && binaryExpr.getLhs() instanceof IntConstantExpr + && ((IntConstantExpr) binaryExpr.getLhs()).getNumericValue() == 1 + && binaryExpr.getRhs() instanceof IntConstantExpr + && ((IntConstantExpr) binaryExpr.getRhs()).getNumericValue() == 0) { + trigger(); + } + } + }), + ShaderKind.FRAGMENT, + fileOps); + + for (String shader : Arrays.asList(shaderIf, shaderWhile, shaderDoWhile, shaderFor)) { + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), + ParseHelper.parse(shader)); + + final File workDir = testFolder.getRoot(); + final File tempShaderJobFile = new File(workDir, "temp.json"); + fileOps.writeShaderJobFile(shaderJob, tempShaderJobFile); + + final String resultsPrefix = new ReductionDriver(new ReducerContext(true, + ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), + new IdGenerator()), + false, + fileOps, + customFileJudge, + workDir) + .doReduction(shaderJob, "temp", 0, 100); + + CompareAsts.assertEqualAsts(expected, ParseHelper.parse(new File(testFolder.getRoot(), resultsPrefix + ".frag"))); + } + + } + + @Test + public void testRemoveSwitchButKeepGuard() throws Exception { + + final String shaderSwitch = "#version 310 es\n" + + "void main() {\n" + + " switch(0) {\n" + + " case 1:\n" + + " break;\n" + + " }\n" + + "}\n"; + + final String expected = "#version 310 es\n" + + "void main() {\n" + + " 0;\n" + + "}\n"; + + final IFileJudge customFileJudge = + new CheckAstFeaturesFileJudge(Collections.singletonList( + () -> new CheckAstFeatureVisitor() { + @Override + public void visitIntConstantExpr(IntConstantExpr intConstantExpr) { + super.visitIntConstantExpr(intConstantExpr); + if (intConstantExpr.getNumericValue() == 0) { + trigger(); + } + } + }), + ShaderKind.FRAGMENT, + fileOps); + + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), + ParseHelper.parse(shaderSwitch)); + + final File workDir = testFolder.getRoot(); + final File tempShaderJobFile = new File(workDir, "temp.json"); + fileOps.writeShaderJobFile(shaderJob, tempShaderJobFile); + + final String resultsPrefix = new ReductionDriver(new ReducerContext(true, + ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), + new IdGenerator()), + false, + fileOps, + customFileJudge, + workDir) + .doReduction(shaderJob, "temp", 0, 100); + + CompareAsts.assertEqualAsts(expected, ParseHelper.parse(new File(testFolder.getRoot(), resultsPrefix + ".frag"))); + + } + private String getPrefix(File tempFile) { return FilenameUtils.removeExtension(tempFile.getName()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java deleted file mode 100755 index d3b7b8b43..000000000 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java +++ /dev/null @@ -1,528 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.reductionopportunities; - -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import com.graphicsfuzz.common.util.GlslParserException; -import com.graphicsfuzz.util.Constants; -import com.graphicsfuzz.common.util.IdGenerator; -import com.graphicsfuzz.common.util.ParseHelper; -import com.graphicsfuzz.common.util.ParseTimeoutException; -import com.graphicsfuzz.common.util.RandomWrapper; -import com.graphicsfuzz.common.util.ShaderJobFileOperations; -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class CompoundToBlockReductionOpportunitiesTest { - - private final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - - @Test - public void testUnderUnreachableSwitch() throws Exception { - final String original = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " if (true) {" - + " true;" - + " }" - + " case 2:" - + " if (true) {" - + " true;" - + " } else false;" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " while(true) { 1; }" - + " }" - + "}"; - final String expected1 = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " { true; }" - + " case 2:" - + " if (true) {" - + " true;" - + " } else false;" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " while(true) { 1; }" - + " }" - + "}"; - final String expected2 = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " if (true) {" - + " true;" - + " }" - + " case 2:" - + " { true; }" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " while(true) { 1; }" - + " }" - + "}"; - final String expected3 = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " if (true) {" - + " true;" - + " }" - + " case 2:" - + " false;" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " while(true) { 1; }" - + " }" - + "}"; - final String expected4 = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " if (true) {" - + " true;" - + " }" - + " case 2:" - + " if (true) {" - + " true;" - + " } else false;" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " { 1; }" - + " }" - + "}"; - check(false, original, expected1, expected2, expected3, expected4); - } - - @Test - public void testDoNotRemoveDeadIf() throws Exception { - final String original = "" - + "void main() {" - + " int a = 2;" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " a = a + 1;" - + " }" - + "}"; - final List opportunities = - CompoundToBlockReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(ParseHelper.parse(original)), - new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - new IdGenerator())); - assertEquals(0, opportunities.size()); - } - - @Test - public void testInDeadCode() throws Exception { - final String original = "" - + "void main() {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (false) {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - final String expected = "" - + "void main() {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - check(false, original, expected); - } - - @Test - public void testDeadIfReduceEverywhere() throws Exception { - final String original = "" - + "void main() {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - final String expected1 = "" - + "void main() {" - + " {" // first if changed to block - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - final String expected2 = "" - + "void main() {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " {" // second if changed to block - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - check(true, original, expected1, expected2); - } - - @Test - public void testInDeadCode2() throws Exception { - final String original = "" - + "void main() {" - + " int a = 4;" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (a) {" - + " if (a > 0) {" - + " int a = 2;" - + " a = a + 1;" - + " } else" - + " a++;" - + " }" - + " }" - + "}"; - final String expected1 = "" - + "void main() {" - + " int a = 4;" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " {" - + " if (a > 0) {" - + " int a = 2;" - + " a = a + 1;" - + " } else" - + " a++;" - + " }" - + " }" - + "}"; - final String expected2 = "" - + "void main() {" - + " int a = 4;" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (a) {" - + " {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + " }" - + "}"; - final String expected3 = "" - + "void main() {" - + " int a = 4;" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (a) {" - + " a++;" - + " }" - + " }" - + "}"; - check(false, original, expected1, expected2, expected3); - } - - @Test - public void testInLiveCode() throws Exception { - final String original = "void main() {" - + " int x;" - + " {" - + " int GLF_live1hello = 4;" - + " if (GLF_live1hello < 46)" - + " GLF_live1hello++;" - + " }" - + " if (true) {" - + " x++;" - + " }" - + "}"; - final String expected = "void main() {" - + " int x;" - + " {" - + " int GLF_live1hello = 4;" - + " GLF_live1hello++;" - + " }" - + " if (true) {" - + " x++;" - + " }" - + "}"; - check(false, original, expected); - } - - @Test - public void testInDeadFunction() throws Exception { - final String original = "" - + "int foo(int x) {" - + " do {" - + " x++;" - + " } while(x < 100);" - + "}" - + "int bar(int x) {" - + " do {" - + " x++;" - + " break;" - + " } while(x < 100);" - + "}" - + "int baz(int x) {" - + " do " - + " break;" - + " while(x < 100);" - + "}" - + "void main() {" - + "}"; - final String expected = "" - + "int foo(int x) {" - + " {" - + " x++;" - + " }" - + "}" - + "int bar(int x) {" - + " do {" - + " x++;" - + " break;" - + " } while(x < 100);" - + "}" - + "int baz(int x) {" - + " do " - + " break;" - + " while(x < 100);" - + "}" - + "void main() {" - + "}"; - check(false, original, expected); - } - - @Test - public void testReduceEverywhere() throws Exception { - final String original = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " if (i > j) {" - + " } else if(j > 5) {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - - final String expected1 = "" - + "void main() {" - + " {" - + " int i = 0;" - + " for (int j = 0; j < 200; j++) {" - + " if (i > j) {" - + " } else if(j > 5) {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - - final String expected2 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " {" - + " int j = 0;" - + " if (i > j) {" - + " } else if(j > 5) {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - final String expected3 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " {" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - final String expected4 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " if(j > 5) {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - final String expected5 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " if (i > j) {" - + " } else if(j > 5) {" - + " j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - final String expected6 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " if (i > j) {" - + " } else {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - - check(true, original, expected1, expected2, expected3, expected4, expected5, expected6); - } - - @Test - public void reduceEverywhereSimpleFor() throws Exception { - final String original = "void main() {" - + " for (int i = 0; i < 10; i++) ;" - + "}"; - final String expected = "void main() {" - + " { int i = 0; ; }" - + "}"; - check(true, original, expected); - } - - @Test - public void reduceEverywhereSimpleFor2() throws Exception { - final String original = "void main() {" - + " for (int i = 0; i < 10; i++) { }" - + "}"; - final String expected = "void main() {" - + " { int i = 0; }" - + "}"; - check(true, original, expected); - } - - private void check(boolean reduceEverywhere, String original, String... expected) - throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - final TranslationUnit tu = ParseHelper.parse(original); - List ops = - getOps(tu, reduceEverywhere); - final Set actualSet = new HashSet<>(); - for (int i = 0; i < ops.size(); i++) { - final TranslationUnit clonedTu = tu.clone(); - List clonedOps = - getOps(clonedTu, reduceEverywhere); - assertEquals(ops.size(), clonedOps.size()); - clonedOps.get(i).applyReduction(); - actualSet.add(PrettyPrinterVisitor.prettyPrintAsString(clonedTu)); - } - final Set expectedSet = new HashSet<>(); - for (String anExpected : expected) { - expectedSet.add(PrettyPrinterVisitor - .prettyPrintAsString(ParseHelper.parse(anExpected))); - } - assertEquals(expectedSet, actualSet); - } - - private List getOps(TranslationUnit tu, - boolean reduceEverywhere) { - return ReductionOpportunities. - getReductionOpportunities( - MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext( - reduceEverywhere, - ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), - new IdGenerator()), - fileOps - ).stream() - .filter(item -> item instanceof CompoundToBlockReductionOpportunity) - .map(item -> (CompoundToBlockReductionOpportunity) item) - .collect(Collectors.toList()); - } - - @Test - public void testDoNotTurnForLoopIntoBlock() throws Exception { - final String shader = "#version 310 es\n" - + "void GLF_live4doConvert()\n" - + "{\n" - + "}\n" - + "void main()\n" - + "{\n" - + " vec4 c = vec4(0.0, 0.0, 0.0, 1.0);\n" - + " for(\n" - + " int i = 0;\n" - + " i < 3;\n" - + " i ++\n" - + " )\n" - + " {\n" - + " c[i] = c[i] * c[i];\n" - + " }\n" - + " " + Constants.LIVE_PREFIX + "4doConvert();\n" - + "}"; - final TranslationUnit tu = ParseHelper.parse(shader); - final List ops = - CompoundToBlockReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_310, - new RandomWrapper(0), new IdGenerator())); - assertEquals(0, ops.size()); - } - - @Test - public void doNotReplaceDeadCodeInjectionWithBody() throws Exception { - final String original = "void main() {" - + " if (" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false))) {" - + " discard;" - + " }" - + "}"; - final List opportunities = - CompoundToBlockReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(ParseHelper.parse(original)), - new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - new IdGenerator())); - assertEquals(0, opportunities.size()); - } - -} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunitiesTest.java new file mode 100755 index 000000000..a83b1e562 --- /dev/null +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunitiesTest.java @@ -0,0 +1,986 @@ +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; +import com.graphicsfuzz.util.Constants; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class FlattenControlFlowReductionOpportunitiesTest { + + private final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + @Test + public void testUnderUnreachableSwitch() throws Exception { + final String original = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " if (true) {" + + " true;" + + " }" + + " case 2:" + + " if (true) {" + + " true;" + + " } else false;" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " while(true) { 1; }" + + " }" + + "}"; + final String expected1 = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " { true; true; }" + + " case 2:" + + " if (true) {" + + " true;" + + " } else false;" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " while(true) { 1; }" + + " }" + + "}"; + final String expected2 = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " if (true) {" + + " true;" + + " }" + + " case 2:" + + " { true; true; }" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " while(true) { 1; }" + + " }" + + "}"; + final String expected3 = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " if (true) {" + + " true;" + + " }" + + " case 2:" + + " { true; false; }" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " while(true) { 1; }" + + " }" + + "}"; + final String expected4 = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " if (true) {" + + " true;" + + " }" + + " case 2:" + + " if (true) {" + + " true;" + + " } else false;" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " { true; 1; }" + + " }" + + "}"; + check(false, original, expected1, expected2, expected3, expected4); + } + + @Test + public void testDoNotRemoveDeadIf() throws Exception { + final String original = "" + + "void main() {" + + " int a = 2;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " a = a + 1;" + + " }" + + "}"; + final List opportunities = + getOps(ParseHelper.parse(original), false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testInDeadCode() throws Exception { + final String original = "" + + "void main() {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (false) {" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + final String expected = "" + + "void main() {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " {" + + " false;" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + check(false, original, expected); + } + + @Test + public void testDeadIfReduceEverywhere() throws Exception { + final String original = "" + + "void main() {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + final String expected1 = "" + + "void main() {" + + " {" // first if changed to block + + " " + Constants.GLF_DEAD + "(false);" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + final String expected2 = "" + + "void main() {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " {" // second if changed to block + + " " + Constants.GLF_DEAD + "(false);" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + check(true, original, expected1, expected2); + } + + @Test + public void testInDeadCode2() throws Exception { + final String original = "" + + "void main() {" + + " int a = 4;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (a) {" + + " if (a > 0) {" + + " int a = 2;" + + " a = a + 1;" + + " } else" + + " a++;" + + " }" + + " }" + + "}"; + final String expected1 = "" + + "void main() {" + + " int a = 4;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " {" + + " a;" + + " if (a > 0) {" + + " int a = 2;" + + " a = a + 1;" + + " } else" + + " a++;" + + " }" + + " }" + + "}"; + final String expected2 = "" + + "void main() {" + + " int a = 4;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (a) {" + + " {" + + " a > 0;" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + " }" + + "}"; + final String expected3 = "" + + "void main() {" + + " int a = 4;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (a) {" + + " {" + + " a > 0;" + + " a++;" + + " }" + + " }" + + " }" + + "}"; + check(false, original, expected1, expected2, expected3); + } + + @Test + public void testInLiveCode() throws Exception { + final String original = "void main() {" + + " int x;" + + " {" + + " int GLF_live1hello = 4;" + + " if (GLF_live1hello < 46)" + + " GLF_live1hello++;" + + " }" + + " if (true) {" + + " x++;" + + " }" + + "}"; + final String expected = "void main() {" + + " int x;" + + " {" + + " int GLF_live1hello = 4;" + + " {" + + " GLF_live1hello < 46;" + + " GLF_live1hello++;" + + " }" + + " }" + + " if (true) {" + + " x++;" + + " }" + + "}"; + check(false, original, expected); + } + + @Test + public void testInDeadFunction() throws Exception { + final String original = "" + + "int foo(int x) {" + + " do {" + + " x++;" + + " } while(x < 100);" + + "}" + + "int bar(int x) {" + + " do {" + + " x++;" + + " break;" + + " } while(x < 100);" + + "}" + + "int baz(int x) {" + + " do " + + " break;" + + " while(x < 100);" + + "}" + + "void main() {" + + "}"; + final String expected = "" + + "int foo(int x) {" + + " {" + + " x++;" + + " x < 100;" + + " }" + + "}" + + "int bar(int x) {" + + " do {" + + " x++;" + + " break;" + + " } while(x < 100);" + + "}" + + "int baz(int x) {" + + " do " + + " break;" + + " while(x < 100);" + + "}" + + "void main() {" + + "}"; + check(false, original, expected); + } + + @Test + public void testReduceEverywhere() throws Exception { + final String original = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " if (i > j) {" + + " } else if(j > 5) {" + + " while(j > 5) j--;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + final String expected1 = "" + + "void main() {" + + " {" + + " int i = 0;" + + " i < 100;" + + " for (int j = 0; j < 200; j++) {" + + " if (i > j) {" + + " } else if(j > 5) {" + + " while(j > 5) j--;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " i++;" + + " }" + + "}"; + + final String expected2 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " {" + + " int j = 0;" + + " j < 200;" + + " if (i > j) {" + + " } else if(j > 5) {" + + " while(j > 5) j--;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " j++;" + + " }" + + " }" + + "}"; + + final String expected3 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " {" + + " i > j;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + final String expected4 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " {" + + " i > j;" + + " if(j > 5) {" + + " while(j > 5) j--;" + + " }" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + final String expected5 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " if (i > j) {" + + " } else {" + + " j > 5;" + + " while(j > 5) j--;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + final String expected6 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " if (i > j) {" + + " } else if(j > 5) {" + + " {" + + " j > 5;" + + " j--;" + + " }" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + check(true, original, expected1, expected2, expected3, expected4, expected5, expected6); + } + + @Test + public void reduceEverywhereSimpleFor() throws Exception { + final String original = "void main() {" + + " for (int i = 0; i < 10; i++) ;" + + "}"; + final String expected = "void main() {" + + " { int i = 0; i < 10; ; i++; }" + + "}"; + check(true, original, expected); + } + + @Test + public void reduceEverywhereSimpleFor2() throws Exception { + final String original = "void main() {" + + " for (int i = 0; i < 10; i++) { }" + + "}"; + final String expected = "void main() {" + + " { int i = 0; i < 10; i++; }" + + "}"; + check(true, original, expected); + } + + private void check(boolean reduceEverywhere, String original, String... expected) + throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { + final TranslationUnit tu = ParseHelper.parse(original); + List ops = + getOps(tu, reduceEverywhere); + final Set actualSet = new HashSet<>(); + for (int i = 0; i < ops.size(); i++) { + final TranslationUnit clonedTu = tu.clone(); + List clonedOps = + getOps(clonedTu, reduceEverywhere); + assertEquals(ops.size(), clonedOps.size()); + clonedOps.get(i).applyReduction(); + actualSet.add(PrettyPrinterVisitor.prettyPrintAsString(clonedTu)); + } + final Set expectedSet = new HashSet<>(); + for (String anExpected : expected) { + expectedSet.add(PrettyPrinterVisitor + .prettyPrintAsString(ParseHelper.parse(anExpected))); + } + assertEquals(expectedSet, actualSet); + } + + @Test + public void testDoNotTurnForLoopIntoBlock() throws Exception { + final String shader = "#version 310 es\n" + + "void GLF_live4doConvert()\n" + + "{\n" + + "}\n" + + "void main()\n" + + "{\n" + + " vec4 c = vec4(0.0, 0.0, 0.0, 1.0);\n" + + " for(\n" + + " int i = 0;\n" + + " i < 3;\n" + + " i ++\n" + + " )\n" + + " {\n" + + " c[i] = c[i] * c[i];\n" + + " }\n" + + " " + Constants.LIVE_PREFIX + "4doConvert();\n" + + "}"; + final TranslationUnit tu = ParseHelper.parse(shader); + final List ops = + getOps(tu, false); + assertEquals(0, ops.size()); + } + + @Test + public void doNotReplaceDeadCodeInjectionWithBody() throws Exception { + final String original = "void main() {" + + " if (" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false))) {" + + " discard;" + + " }" + + "}"; + final List opportunities = + getOps(ParseHelper.parse(original), false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testIfWithReduceEverywhere() throws Exception { + // Fine to reduce this conditional to its guard as we are not preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " if (x > 0) {\n" + + " ++x;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " {\n" + + " x > 0;\n" + + " ++x;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, true); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testForWithReduceEverywhere() throws Exception { + // Fine to reduce this loop to its guard as we are not preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " for (i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " {\n" + + " i = 0;\n" + + " i < 100;\n" + + " i++;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, true); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testWhileWithReduceEverywhere() throws Exception { + // Fine to reduce this loop to its guard as we are not preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " while (x > 0) {\n" + + " x--;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " {\n" + + " x > 0;" + + " x--;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, true); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoWhileWithReduceEverywhere() throws Exception { + // Fine to reduce this loop to its guard as we are not preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " do {\n" + + " x++;\n" + + " } while (x > 0);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " {\n" + + " x++;\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, true); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testNoIfWithPreserveSemantics() throws Exception { + // No reduction opportunities here if preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " if (x > 0) {\n" + + " ++x;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " x > 0;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testNoForWithPreserveSemantics() throws Exception { + // No reduction opportunities here if preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " for (i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " i < 100;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testNoWhileWithPreserveSemantics() throws Exception { + // No reduction opportunities here if preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " while (x > 0) {\n" + + " x--;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " x > 0;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testNoDoWhileWithPreserveSemantics() throws Exception { + // No reduction opportunities here if preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " do {\n" + + " x++;\n" + + " } while (x > 0);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " x > 0;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testNoSwitchWithPreserveSemantics() throws Exception { + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " switch (x + 3) {\n" + + " default:\n" + + " x++;\n" + + " break;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " x + 3;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testIfWithPreserveSemanticsSideEffectFree() throws Exception { + // Despite preserving semantics, fine to reduce this conditional to its guard as the + // conditional is side effect-free. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " if (x > 0) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " {\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testForWithPreserveSemanticsSideEffectFree() throws Exception { + // Despite preserving semantics, fine to reduce this loop to its guard as the loop is side + // effect-free. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " i = 0;\n" + + " for (; i < 100;) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " i = 0;\n" + + " {\n" + + " ;\n" + + " i < 100;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testWhileWithPreserveSemanticsSideEffectFree() throws Exception { + // Despite preserving semantics, fine to reduce this loop to its guard as the loop is side + // effect-free. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " while (x > 0) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " {\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoWhileWithPreserveSemanticsSideEffectFree() throws Exception { + // Despite preserving semantics, fine to reduce this loop to its guard as the loop is side + // effect-free. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " do {\n" + + " } while (x > 0);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " {\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testIfWithPreserveSemanticsUnreachableSwitch() throws Exception { + // Fine to reduce two of these loops to their guards despite preserving semantics as they are + // under unreachable switch cases. + final String original = "#version 310 es\n" + + "void main() {\n" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " {\n" + + " int x = 2;\n" + + " if (x > 0) {\n" + + " ++x;\n" + + " }\n" + + " }\n" + + " case 0:" + + " {\n" + + " int y = 2;\n" + + " if (y > 0) {\n" + + " ++y;\n" + + " }\n" + + " }\n" + + " break;\n" + + " default:\n" + + " {\n" + + " int z = 2;\n" + + " if (z > 0) {\n" + + " ++z;\n" + + " }\n" + + " }\n" + + " }" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " {\n" + + " int x = 2;\n" + + " {\n" + + " x > 0;\n" + + " ++x;\n" + + " }\n" + + " }\n" + + " case 0:" + + " {\n" + + " int y = 2;\n" + + " if (y > 0) {\n" + + " ++y;\n" + + " }\n" + + " }\n" + + " break;\n" + + " default:\n" + + " {\n" + + " int z = 2;\n" + + " {\n" + + " z > 0;\n" + + " ++z;\n" + + " }\n" + + " }\n" + + " }" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(2, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + opportunities.get(1).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testForWithPreserveSemanticsDeadCode() throws Exception { + // Fine to reduce this loop to its guard despite preserving semantics as we are in dead code. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " if (" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false))) {" + + " for (i = 0; i < 100; i++) {\n" + + " }\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " if (" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false))) {" + + " {\n" + + " i = 0;\n" + + " i < 100;\n" + + " i++;\n" + + " }\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testWhileWithPreserveSemanticsLiveCode() throws Exception { + // Fine to reduce this loop to its guard despite preserving semantics as we are in injected + // live code. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int " + Constants.LIVE_PREFIX + "x = 100;\n" + + " while (" + Constants.LIVE_PREFIX + "x > 0) {\n" + + " " + Constants.LIVE_PREFIX + "x--;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int " + Constants.LIVE_PREFIX + "x = 100;\n" + + " {\n" + + " " + Constants.LIVE_PREFIX + "x > 0;\n" + + " " + Constants.LIVE_PREFIX + "x--;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoWhileWithPreserveSemanticsDeadFunction() throws Exception { + // Fine to reduce this loop to its guard despite preserving semantics as we are in a dead + // function. + final String original = "#version 310 es\n" + + "void " + Constants.GLF_DEAD + "_foo() {\n" + + " int x = 100;\n" + + " do {\n" + + " x++;\n" + + " } while (x > 0);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void " + Constants.GLF_DEAD + "_foo() {\n" + + " int x = 100;\n" + + " {\n" + + " x++;\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + private List getOps(TranslationUnit tu, + boolean reduceEverywhere) { + return FlattenControlFlowReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(reduceEverywhere, + tu.getShadingLanguageVersion(), new RandomWrapper(0), + new IdGenerator())); + } + +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java index 90b2b0f37..592c4c36b 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java @@ -64,6 +64,7 @@ public void testDoReduceWhenPreservingSemanticsIfInDeadCode() throws Exception { final String expected = "void main() {\n" + " if (" + Constants.GLF_DEAD + "(false)) {\n" + " do {\n" + + " 0;" + " return;\n" + " } while (false);\n" + " }\n" @@ -102,6 +103,7 @@ public void testNoBreaks() throws Exception { + " int a;\n" + " int x = 3;\n" + " do {\n" + + " x;\n" + " {\n" + " x = 1;\n" + " }\n" @@ -146,6 +148,7 @@ public void testWithBreaks() throws Exception { + " int a;\n" + " int x = 3;\n" + " do {\n" + + " x;\n" + " {\n" + " x = 1;\n" + " }\n" @@ -184,9 +187,11 @@ public void testTwoSwitches() throws Exception { final String expected = "void main() {\n" + " int x;\n" + " do {\n" + + " 0;\n" + " x = 1;\n" + " } while (false);\n" + " do {\n" + + " 1;\n" + " x = 2;\n" + " } while (false);\n" + "}\n"; From 98e6cdd1f9423aefbd7db7de4d7403a4d897be13 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Fri, 13 Sep 2019 18:37:25 +0100 Subject: [PATCH 131/180] gfauto: improvements and fixes to *_cts_gf_tests (#740) * Downloaded .amber tests have their shaders extracted, and now also are zipped for easier bug reporting. * Create work directory in run_cts_gf_tests.py. * Util function for creating zips. --- gfauto/gfauto/download_cts_gf_tests.py | 7 +++++++ gfauto/gfauto/run_cts_gf_tests.py | 4 +++- gfauto/gfauto/util.py | 22 +++++++++++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/gfauto/gfauto/download_cts_gf_tests.py b/gfauto/gfauto/download_cts_gf_tests.py index 85d5f22c6..520801dbc 100644 --- a/gfauto/gfauto/download_cts_gf_tests.py +++ b/gfauto/gfauto/download_cts_gf_tests.py @@ -114,6 +114,13 @@ def download_cts_graphicsfuzz_tests( # pylint: disable=too-many-locals; amber_file, output_dir=amber_file.parent, binaries=binaries ) + zip_files = [ + util.ZipEntry(f, Path(f.name)) + for f in sorted(shader_dir.glob(f"{amber_file.stem}.*")) + ] + + util.create_zip(amber_file.with_suffix(".zip"), zip_files) + GERRIT_COOKIE_ARGUMENT_DESCRIPTION = ( "The Gerrit cookie used for authentication. To get this, log in to the Khronos Gerrit page in your " diff --git a/gfauto/gfauto/run_cts_gf_tests.py b/gfauto/gfauto/run_cts_gf_tests.py index 933948841..b9dc06467 100644 --- a/gfauto/gfauto/run_cts_gf_tests.py +++ b/gfauto/gfauto/run_cts_gf_tests.py @@ -37,7 +37,7 @@ DEFAULT_TIMEOUT = 30 -def main() -> None: # pylint: disable=too-many-locals,too-many-branches; +def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; parser = argparse.ArgumentParser( description="Runs GraphicsFuzz AmberScript tests on the active devices listed in " "the settings.json file." @@ -71,6 +71,8 @@ def main() -> None: # pylint: disable=too-many-locals,too-many-branches; work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" + util.mkdirs_p(work_dir) + with util.file_open_text(Path("results.txt"), "w") as log_handle: def write_entry(entry: str) -> None: diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py index 1c4e80eb1..21275c9a1 100644 --- a/gfauto/gfauto/util.py +++ b/gfauto/gfauto/util.py @@ -25,9 +25,12 @@ import pathlib import platform import shutil +import zipfile from contextlib import contextmanager from pathlib import Path -from typing import Any, BinaryIO, Iterator, List, TextIO, cast +from typing import Any, BinaryIO, Iterator, List, Optional, TextIO, cast + +import attr from gfauto import gflogging @@ -280,3 +283,20 @@ def extract_archive(archive_file: Path, output_dir: Path) -> Path: shutil.unpack_archive(str(archive_file), extract_dir=str(output_dir)) gflogging.log("Done") return output_dir + + +@attr.dataclass +class ZipEntry: + path: Path + path_in_archive: Optional[Path] = None + + +def create_zip(output_file_path: Path, entries: List[ZipEntry]) -> Path: + gflogging.log(f"Creating zip {str(output_file_path)}:") + with zipfile.ZipFile( + output_file_path, "w", compression=zipfile.ZIP_DEFLATED + ) as file_handle: + for entry in entries: + file_handle.write(entry.path, entry.path_in_archive) + gflogging.log(f"Adding: {entry.path} {entry.path_in_archive or ''}") + return output_file_path From 938b8921fd26e03a0f538b4e785c3ac2e59a9a4f Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 17 Sep 2019 14:10:22 +0100 Subject: [PATCH 132/180] gfauto: save spirv-fuzz failures (#742) --- gfauto/gfauto/fuzz.py | 4 ++++ gfauto/gfauto/fuzz_spirv_test.py | 20 +++++++++++++------- gfauto/gfauto/settings.proto | 4 ++++ gfauto/gfauto/settings_pb2.py | 13 ++++++++++--- gfauto/gfauto/settings_pb2.pyi | 6 ++++-- gfauto/gfauto/settings_util.py | 2 +- 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index 162f49c25..16f31e44e 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -118,6 +118,8 @@ SPIRV_FUZZ = False +FUZZ_FAILURES_DIR_NAME = "fuzz_failures" + def get_random_name() -> str: # TODO: could change to human-readable random name or the date. @@ -172,6 +174,7 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many active_devices = devices_util.get_active_devices(settings.device_list) reports_dir = Path() / "reports" + fuzz_failures_dir = reports_dir / FUZZ_FAILURES_DIR_NAME temp_dir = Path() / "temp" references_dir = Path() / "references" donors_dir = Path() / "donors" @@ -256,6 +259,7 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many fuzz_spirv_test.fuzz_spirv( staging_dir, reports_dir, + fuzz_failures_dir, active_devices, spirv_fuzz_shaders, settings, diff --git a/gfauto/gfauto/fuzz_spirv_test.py b/gfauto/gfauto/fuzz_spirv_test.py index 28373fac8..cd7730534 100644 --- a/gfauto/gfauto/fuzz_spirv_test.py +++ b/gfauto/gfauto/fuzz_spirv_test.py @@ -20,6 +20,7 @@ """ import random +import subprocess from pathlib import Path from typing import List, Optional @@ -126,6 +127,7 @@ def handle_test( def fuzz_spirv( staging_dir: Path, reports_dir: Path, + fuzz_failures_dir: Path, active_devices: List[Device], spirv_fuzz_shaders: List[Path], settings: Settings, @@ -143,13 +145,17 @@ def fuzz_spirv( ) # TODO: Allow using downloaded spirv-fuzz. - - spirv_fuzz_util.run_generate_on_shader_job( - util.tool_on_path("spirv-fuzz"), - reference_spirv_shader_job, - template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, - seed=str(random.getrandbits(spirv_fuzz_util.GENERATE_SEED_BITS)), - ) + try: + spirv_fuzz_util.run_generate_on_shader_job( + util.tool_on_path("spirv-fuzz"), + reference_spirv_shader_job, + template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, + seed=str(random.getrandbits(spirv_fuzz_util.GENERATE_SEED_BITS)), + ) + except subprocess.CalledProcessError: + if len(list(fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures: + util.copy_dir(staging_dir, fuzz_failures_dir / staging_dir.name) + return test_dirs = [ make_test( diff --git a/gfauto/gfauto/settings.proto b/gfauto/gfauto/settings.proto index 67b2ccb72..a95596a53 100644 --- a/gfauto/gfauto/settings.proto +++ b/gfauto/gfauto/settings.proto @@ -21,6 +21,7 @@ import "gfauto/device.proto"; // A fuzzing session requires a Settings proto stored in "settings.json". +// Default values are defined in settings_util.py. message Settings { // A list of devices, including the list of active devices names that will actually be used in this session. DeviceList device_list = 1; @@ -33,4 +34,7 @@ message Settings { // Allow at most this many duplicate crashes per device. // If 0 (the protobuf default value), a different default value will actually be used, as determined by gfauto. uint32 maximum_duplicate_crashes = 3; + + // Store at most this many fuzzing failure files. + uint32 maximum_fuzz_failures = 4; } diff --git a/gfauto/gfauto/settings_pb2.py b/gfauto/gfauto/settings_pb2.py index 5b7f9c340..0e277b787 100644 --- a/gfauto/gfauto/settings_pb2.py +++ b/gfauto/gfauto/settings_pb2.py @@ -22,7 +22,7 @@ package='gfauto', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\x15gfauto/settings.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\x7f\n\x08Settings\x12\'\n\x0b\x64\x65vice_list\x18\x01 \x01(\x0b\x32\x12.gfauto.DeviceList\x12\'\n\x0f\x63ustom_binaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binary\x12!\n\x19maximum_duplicate_crashes\x18\x03 \x01(\rb\x06proto3') + serialized_pb=_b('\n\x15gfauto/settings.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\x9e\x01\n\x08Settings\x12\'\n\x0b\x64\x65vice_list\x18\x01 \x01(\x0b\x32\x12.gfauto.DeviceList\x12\'\n\x0f\x63ustom_binaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binary\x12!\n\x19maximum_duplicate_crashes\x18\x03 \x01(\r\x12\x1d\n\x15maximum_fuzz_failures\x18\x04 \x01(\rb\x06proto3') , dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,gfauto_dot_device__pb2.DESCRIPTOR,]) @@ -57,6 +57,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='maximum_fuzz_failures', full_name='gfauto.Settings.maximum_fuzz_failures', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -69,8 +76,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=75, - serialized_end=202, + serialized_start=76, + serialized_end=234, ) _SETTINGS.fields_by_name['device_list'].message_type = gfauto_dot_device__pb2._DEVICELIST diff --git a/gfauto/gfauto/settings_pb2.pyi b/gfauto/gfauto/settings_pb2.pyi index 72b5f1291..4b92adecb 100644 --- a/gfauto/gfauto/settings_pb2.pyi +++ b/gfauto/gfauto/settings_pb2.pyi @@ -33,6 +33,7 @@ from typing_extensions import ( class Settings(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... maximum_duplicate_crashes = ... # type: int + maximum_fuzz_failures = ... # type: int @property def device_list(self) -> gfauto___device_pb2___DeviceList: ... @@ -45,6 +46,7 @@ class Settings(google___protobuf___message___Message): device_list : typing___Optional[gfauto___device_pb2___DeviceList] = None, custom_binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, maximum_duplicate_crashes : typing___Optional[int] = None, + maximum_fuzz_failures : typing___Optional[int] = None, ) -> None: ... @classmethod def FromString(cls, s: bytes) -> Settings: ... @@ -52,7 +54,7 @@ class Settings(google___protobuf___message___Message): def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): def HasField(self, field_name: typing_extensions___Literal[u"device_list"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",u"device_list",u"maximum_duplicate_crashes"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",u"device_list",u"maximum_duplicate_crashes",u"maximum_fuzz_failures"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"device_list",b"device_list"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",b"custom_binaries",u"device_list",b"device_list",u"maximum_duplicate_crashes",b"maximum_duplicate_crashes"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",b"custom_binaries",u"device_list",b"device_list",u"maximum_duplicate_crashes",b"maximum_duplicate_crashes",u"maximum_fuzz_failures",b"maximum_fuzz_failures"]) -> None: ... diff --git a/gfauto/gfauto/settings_util.py b/gfauto/gfauto/settings_util.py index e63c603c7..3589e82f2 100644 --- a/gfauto/gfauto/settings_util.py +++ b/gfauto/gfauto/settings_util.py @@ -27,7 +27,7 @@ DEFAULT_SETTINGS_FILE_PATH = Path("settings.json") -DEFAULT_SETTINGS = Settings(maximum_duplicate_crashes=3) +DEFAULT_SETTINGS = Settings(maximum_duplicate_crashes=3, maximum_fuzz_failures=10) class NoSettingsFile(Exception): From 7e231fff5aa3a2c29b530aad93179ff09dabf7e8 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 17 Sep 2019 15:03:52 +0100 Subject: [PATCH 133/180] Update binaries (#743) --- assembly-binaries/pom.xml | 4 ++-- .../generator/tool/Fragment2ComputeTest.java | 9 +++++++-- gfauto/gfauto/binaries_util.py | 12 ++++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/assembly-binaries/pom.xml b/assembly-binaries/pom.xml index 2fc61903b..5d0d93dc1 100644 --- a/assembly-binaries/pom.xml +++ b/assembly-binaries/pom.xml @@ -58,10 +58,10 @@ limitations under the License. - 04b2b8e2543b4643c533b20ca1a9d88c72fea370 + b97215064186d731eac68adcc5ade4c7b96b265b 9d3202492d78aae5ced08ff14679b81c98d71c15 0565d5aee279a4782689297ada0aa2489e24ad3e - 1586e566f4949b1957e7c32454cbf27e501ed632 + 0f167ce7125795df62ae5893f553e5608c9652f4 d987dc0e5f8aeae0dc2f5aac2013e9a3edd54465 diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/Fragment2ComputeTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/Fragment2ComputeTest.java index dc7411564..b89203ffe 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/Fragment2ComputeTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/Fragment2ComputeTest.java @@ -21,11 +21,12 @@ import com.graphicsfuzz.util.ToolPaths; import java.io.File; import java.nio.file.Paths; +import java.util.Arrays; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; public class Fragment2ComputeTest { @@ -40,7 +41,11 @@ public void testTranslationToCompute() throws Exception { final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - for (File reference : referencesDir.listFiles((dir, name) -> name.endsWith(".json"))) { + String[] blacklist = {"trigonometric_strip", "mergesort_mosaic"}; + + for (File reference : + referencesDir.listFiles((dir, name) -> name.endsWith(".json") + && Arrays.stream(blacklist).noneMatch(s -> name.contains(s)))) { File outputShaderJob = temporaryFolder.newFile(reference.getName()); Fragment2Compute.mainHelper(reference.getAbsolutePath(), outputShaderJob.getAbsolutePath()); assertTrue(fileOps.getUnderlyingShaderFile(outputShaderJob, ShaderKind.COMPUTE) diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py index 2af6ef9c9..ee76a8844 100644 --- a/gfauto/gfauto/binaries_util.py +++ b/gfauto/gfauto/binaries_util.py @@ -56,13 +56,13 @@ "Mac_x64_RelWithDebInfo", ] -DEFAULT_SPIRV_TOOLS_VERSION = "06407250a169c6a03b3765e86619075af1a8c187" +DEFAULT_SPIRV_TOOLS_VERSION = "ad7f2c5c4c7f51360e9e079109a9217aa5ba5cc0" DEFAULT_BINARIES = [ Binary( name="glslangValidator", tags=["Debug"], - version="f04f1f93a70f4608ffa9903b20bfb95f20a063f5", + version="fe0b2bd694bb07004a2db859c5714c321c26b751", ), Binary(name="spirv-opt", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), Binary(name="spirv-dis", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), @@ -519,4 +519,12 @@ def get_graphics_fuzz_121() -> List[recipe_wrap.RecipeWrap]: version_hash="06407250a169c6a03b3765e86619075af1a8c187", build_version_hash="04b2b8e2543b4643c533b20ca1a9d88c72fea370", ) + + _get_built_in_glslang_version( + version_hash="fe0b2bd694bb07004a2db859c5714c321c26b751", + build_version_hash="0f167ce7125795df62ae5893f553e5608c9652f4", + ) + + _get_built_in_spirv_tools_version( + version_hash="ad7f2c5c4c7f51360e9e079109a9217aa5ba5cc0", + build_version_hash="b97215064186d731eac68adcc5ade4c7b96b265b", + ) ) From ad957eddbccfe4b5280955efd2a7571df3b7ccf7 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Wed, 18 Sep 2019 11:41:01 +0100 Subject: [PATCH 134/180] Fix scripts to make shaders spirv-fuzz-ready (#741) Minor edits to scripts that take a GraphicsFuzz shader and turn it into a shader suitable for use with spirv-fuzz. --- .../convert-shaders-to-spirv-fuzz-form.py | 12 ++++++------ ...shader_job_uniforms_to_spirv_fuzz_facts.py | 19 +++++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py b/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py index 2efccfbe8..4aa05e335 100644 --- a/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py +++ b/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py @@ -58,21 +58,21 @@ def main_helper(args): shader_job_uniforms_to_spirv_fuzz_facts.main_helper([json_in_output_dir, os.path.join(args.output_dir, shader_job_prefix - + ".facts")]) + + ".frag.facts")]) runspv.convert_glsl_to_spv(os.path.join(args.output_dir, shader_job_prefix + ".frag"), - os.path.join(args.output_dir, shader_job_prefix + ".spv")) + os.path.join(args.output_dir, shader_job_prefix + ".frag.spv")) for opt_settings in ["-O", "-Os"]: shutil.copyfile(os.path.join(args.output_dir, shader_job_prefix + ".json"), os.path.join(args.output_dir, shader_job_prefix + opt_settings + ".json")) - shutil.copyfile(os.path.join(args.output_dir, shader_job_prefix + ".facts"), + shutil.copyfile(os.path.join(args.output_dir, shader_job_prefix + ".frag.facts"), os.path.join(args.output_dir, shader_job_prefix + opt_settings - + ".facts")) - runspv.run_spirv_opt(os.path.join(args.output_dir, shader_job_prefix + ".spv"), + + ".frag.facts")) + runspv.run_spirv_opt(os.path.join(args.output_dir, shader_job_prefix + ".frag.spv"), [opt_settings], os.path.join(args.output_dir, shader_job_prefix + opt_settings - + ".spv")) + + ".frag.spv")) if __name__ == '__main__': diff --git a/python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py b/python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py index d2e2eb61a..67a43c49c 100644 --- a/python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py +++ b/python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py @@ -57,23 +57,26 @@ def main_helper(args) -> None: if name == "$compute": continue + scalar_uniform_functions = ['glUniform1i', 'glUniform1f'] + vector_uniform_functions = ['glUniform2i', 'glUniform3i', 'glUniform4i', 'glUniform2f', + 'glUniform3f', 'glUniform4f'] + + if not (entry["func"] in scalar_uniform_functions or entry["func"] in vector_uniform_functions): + print("Ignoring unsupported uniform function " + entry["func"]) + continue + # Make a separate fact for every component value of each uniform. for i in range(0, len(entry["args"])): # Every uniform is in its own struct, so we index into the first element of that struct # using index 0. If the uniform is a vector, we need to further index into the vector. - if entry["func"] in ['glUniform1i', 'glUniform1f']: + if entry["func"] in scalar_uniform_functions: # The uniform is a scalar. assert i == 0 index_list = [0] - elif entry["func"] in ['glUniform2i', 'glUniform3i', 'glUniform4i', 'glUniform2f', - 'glUniform3f', 'glUniform4f']: + else: + assert entry["func"] in vector_uniform_functions # The uniform is a vector, so we have two levels of indexing. index_list = [0, i] - else: - # We can deal with other uniforms in due course. - index_list = [] - print("Unsupported uniform function " + entry["func"]) - exit(1) # We need to pass the component value as an integer. If it is a float, we need to # reinterpret its bits as an integer. From d12af6f5ef26bb800d195d57c0edd587cadb2b19 Mon Sep 17 00:00:00 2001 From: Ilkka Saarelainen Date: Wed, 18 Sep 2019 14:41:35 +0300 Subject: [PATCH 135/180] Update generate_cts_test_template.py (#745) Changes the paths to be relative to the python script itself instead of the working directory. --- gfauto/gfauto/generate_cts_test_template.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gfauto/gfauto/generate_cts_test_template.py b/gfauto/gfauto/generate_cts_test_template.py index 9561f345b..1e82a1a68 100644 --- a/gfauto/gfauto/generate_cts_test_template.py +++ b/gfauto/gfauto/generate_cts_test_template.py @@ -25,7 +25,7 @@ import sys from pathlib import Path -from gfauto import artifact_util, tool +from gfauto import artifact_util, tool, util def main() -> None: @@ -39,10 +39,12 @@ def main() -> None: artifact_util.recipes_write_built_in() + bug_dir = util.norm_path(Path(__file__).absolute()).parent + tool.glsl_shader_job_crash_to_amber_script_for_google_cts( - source_dir=Path() / "reduced_glsl_manual", - output_amber=Path() / "name-of-test-TODO.amber", - work_dir=Path() / "work", + source_dir=bug_dir / "reduced_glsl_manual", + output_amber=bug_dir / "name-of-test-TODO.amber", + work_dir=bug_dir / "work", # One sentence, 58 characters max., no period, no line breaks. short_description="A fragment shader with TODO", comment_text="""The test passes because TODO""", From e7bddd56e74c1fe3afd57ee8f0062c42fe870f6c Mon Sep 17 00:00:00 2001 From: jarisiru <53901722+jarisiru@users.noreply.github.com> Date: Wed, 18 Sep 2019 15:13:35 +0300 Subject: [PATCH 136/180] Fix binarytree shader (#746) The sinh() function caused the shader to be numerically unstable (differing result image with 3 and 64 bit floats). Since this was only used for making the output image look nice, replacing it with tan() still creates a relatively nice image while leaving the actual binary tree functionality alone. --- shaders/src/main/glsl/samples/300es/binarysearch_tree.frag | 2 +- shaders/src/main/glsl/samples/310es/binarysearch_tree.frag | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag b/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag index ad843ed32..944fb705c 100644 --- a/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag +++ b/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag @@ -170,7 +170,7 @@ void main() { } } } - float a = sinh(x + y * float(sum)); + float a = tan(x + y * float(sum)); _GLF_color = vec4(hueColor(a), 1.); } diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag b/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag index 273458383..3810b48b6 100644 --- a/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag +++ b/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag @@ -170,7 +170,7 @@ void main() { } } } - float a = sinh(x + y * float(sum)); + float a = tan(x + y * float(sum)); _GLF_color = vec4(hueColor(a), 1.); } From 86367a09ec30fb3007f9d436dc8d7a33c5a13f55 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Wed, 18 Sep 2019 13:13:48 +0100 Subject: [PATCH 137/180] gfauto: fix type warnings (#744) --- gfauto/gfauto/subprocess_util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gfauto/gfauto/subprocess_util.py b/gfauto/gfauto/subprocess_util.py index 2f351d63c..28d8b6eda 100644 --- a/gfauto/gfauto/subprocess_util.py +++ b/gfauto/gfauto/subprocess_util.py @@ -26,7 +26,7 @@ import subprocess import time from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union from gfauto.gflogging import log from gfauto.util import check @@ -69,9 +69,11 @@ def log_returncode( def posix_kill_group(process: subprocess.Popen) -> None: - os.killpg(process.pid, signal.SIGTERM) + # Work around type warnings that will only show up on Windows: + os_alias: Any = os + os_alias.killpg(process.pid, signal.SIGTERM) time.sleep(1) - os.killpg(process.pid, signal.SIGKILL) + os_alias.killpg(process.pid, signal.SIGKILL) def run_helper( From 4bde68171e7c44c40f6b027d3cd1dcede26a3570 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Wed, 18 Sep 2019 13:15:09 +0100 Subject: [PATCH 138/180] gfauto: handle reboots (#747) Add a new status UNRESPONSIVE. Also, check and record the status in the test.json and use this in gfauto_interestingness_test (i.e. use it during reductions). Fix #696 Fix #748 --- gfauto/gfauto/android_device.py | 69 +++++++++++++------- gfauto/gfauto/fuzz.py | 1 + gfauto/gfauto/fuzz_glsl_test.py | 9 ++- gfauto/gfauto/fuzz_spirv_test.py | 6 +- gfauto/gfauto/gfauto_interestingness_test.py | 33 +++++++--- gfauto/gfauto/test.proto | 2 + gfauto/gfauto/test_pb2.py | 19 ++++-- gfauto/gfauto/test_pb2.pyi | 6 +- 8 files changed, 103 insertions(+), 42 deletions(-) diff --git a/gfauto/gfauto/android_device.py b/gfauto/gfauto/android_device.py index 1933ec0d6..10320f091 100644 --- a/gfauto/gfauto/android_device.py +++ b/gfauto/gfauto/android_device.py @@ -42,6 +42,10 @@ BUSY_WAIT_SLEEP_SLOW = 1.0 +ADB_DEFAULT_TIME_LIMIT = 30 + +ADB_SHORT_LOGCAT_TIME_LIMIT = 3 + def adb_path() -> Path: if "ANDROID_HOME" in os.environ: @@ -57,6 +61,7 @@ def adb_helper( adb_args: List[str], check_exit_code: bool, verbose: bool = False, + timeout: Optional[int] = ADB_DEFAULT_TIME_LIMIT, ) -> subprocess.CompletedProcess: adb_cmd = [str(adb_path())] @@ -67,25 +72,32 @@ def adb_helper( adb_cmd.extend(adb_args) return subprocess_util.run( - adb_cmd, - check_exit_code=check_exit_code, - timeout=fuzz.AMBER_RUN_TIME_LIMIT, - verbose=verbose, + adb_cmd, check_exit_code=check_exit_code, timeout=timeout, verbose=verbose ) def adb_check( - serial: Optional[str], adb_args: List[str], verbose: bool = False + serial: Optional[str], + adb_args: List[str], + verbose: bool = False, + timeout: Optional[int] = ADB_DEFAULT_TIME_LIMIT, ) -> subprocess.CompletedProcess: - return adb_helper(serial, adb_args, check_exit_code=True, verbose=verbose) + return adb_helper( + serial, adb_args, check_exit_code=True, verbose=verbose, timeout=timeout + ) def adb_can_fail( - serial: Optional[str], adb_args: List[str], verbose: bool = False + serial: Optional[str], + adb_args: List[str], + verbose: bool = False, + timeout: Optional[int] = ADB_DEFAULT_TIME_LIMIT, ) -> subprocess.CompletedProcess: - return adb_helper(serial, adb_args, check_exit_code=False, verbose=verbose) + return adb_helper( + serial, adb_args, check_exit_code=False, verbose=verbose, timeout=timeout + ) def stay_awake_warning(serial: Optional[str] = None) -> None: @@ -150,6 +162,7 @@ def get_all_android_devices() -> List[Device]: def prepare_device(wait_for_screen: bool, serial: Optional[str] = None) -> None: + adb_check(serial, ["wait-for-device"], timeout=None) adb_check(serial, ["logcat", "-c"]) res = adb_can_fail(serial, ["shell", "test -e " + ANDROID_DEVICE_AMBER]) check( @@ -250,25 +263,33 @@ def run_amber_on_device_helper( result: Optional[CompletedProcess] = None try: - result = adb_can_fail(serial, cmd, verbose=True) + result = adb_can_fail( + serial, cmd, verbose=True, timeout=fuzz.AMBER_RUN_TIME_LIMIT + ) except subprocess.TimeoutExpired: status = fuzz.STATUS_TIMEOUT - if result: - if result.returncode != 0: - status = fuzz.STATUS_CRASH - else: - status = fuzz.STATUS_SUCCESS - - if not skip_render: - adb_check( - serial, - # The /. syntax means the contents of the results directory will be copied into output_dir. - ["pull", ANDROID_DEVICE_RESULT_DIR + "/.", str(output_dir)], - ) - - # Grab log: - adb_check(serial, ["logcat", "-d"], verbose=True) + try: + if result: + if result.returncode != 0: + status = fuzz.STATUS_CRASH + else: + status = fuzz.STATUS_SUCCESS + + if not skip_render: + adb_check( + serial, + # The /. syntax means the contents of the results directory will be copied into output_dir. + ["pull", ANDROID_DEVICE_RESULT_DIR + "/.", str(output_dir)], + ) + + # Grab log. Use a short time limit to increase the chance of detecting a device reboot. + adb_check( + serial, ["logcat", "-d"], verbose=True, timeout=ADB_SHORT_LOGCAT_TIME_LIMIT + ) + except subprocess.SubprocessError: + # If we fail in getting the results directory or log, assume the device has rebooted. + status = fuzz.STATUS_UNRESPONSIVE log("\nSTATUS " + status + "\n") diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index 16f31e44e..b4598fa55 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -108,6 +108,7 @@ STATUS_TOOL_CRASH = "TOOL_CRASH" STATUS_CRASH = "CRASH" +STATUS_UNRESPONSIVE = "UNRESPONSIVE" STATUS_TOOL_TIMEOUT = "TOOL_TIMEOUT" STATUS_TIMEOUT = "TIMEOUT" STATUS_SUCCESS = "SUCCESS" diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py index 4f0a59fc8..fcc9befe3 100644 --- a/gfauto/gfauto/fuzz_glsl_test.py +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -207,6 +207,8 @@ def maybe_add_report( report_subdirectory_name = "crashes" elif status == fuzz.STATUS_TOOL_CRASH: report_subdirectory_name = "tool_crashes" + elif status == fuzz.STATUS_UNRESPONSIVE: + report_subdirectory_name = "unresponsive" if not report_subdirectory_name: return None @@ -243,6 +245,7 @@ def maybe_add_report( test_metadata = test_util.metadata_read(test_dir_in_reports) test_metadata.crash_signature = signature test_metadata.device.CopyFrom(device) + test_metadata.expected_status = status test_util.metadata_write(test_metadata, test_dir_in_reports) return test_dir_in_reports @@ -301,7 +304,11 @@ def handle_test( # Run on all devices. for device in active_devices: status = run(test_dir, binary_manager, device) - if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): + if status in ( + fuzz.STATUS_CRASH, + fuzz.STATUS_TOOL_CRASH, + fuzz.STATUS_UNRESPONSIVE, + ): issue_found = True if status == fuzz.STATUS_TOOL_CRASH: # No need to run further on real devices if the pre-processing step failed. diff --git a/gfauto/gfauto/fuzz_spirv_test.py b/gfauto/gfauto/fuzz_spirv_test.py index cd7730534..cf00be9d8 100644 --- a/gfauto/gfauto/fuzz_spirv_test.py +++ b/gfauto/gfauto/fuzz_spirv_test.py @@ -102,7 +102,11 @@ def handle_test( # Run on all devices. for device in active_devices: status = run(test_dir, binary_manager, device) - if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): + if status in ( + fuzz.STATUS_CRASH, + fuzz.STATUS_TOOL_CRASH, + fuzz.STATUS_UNRESPONSIVE, + ): issue_found = True if status == fuzz.STATUS_TOOL_CRASH: # No need to run further on real devices if the pre-processing step failed. diff --git a/gfauto/gfauto/gfauto_interestingness_test.py b/gfauto/gfauto/gfauto_interestingness_test.py index cd0bef984..605a45d7e 100644 --- a/gfauto/gfauto/gfauto_interestingness_test.py +++ b/gfauto/gfauto/gfauto_interestingness_test.py @@ -123,22 +123,39 @@ def main() -> None: ) status = result_util.get_status(output_dir) - if status not in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): - log("Shader run did not crash; not interesting.") - sys.exit(1) + if test.expected_status: + log("") + log(f"Expected status: {test.expected_status}") + log(f"Actual status: {status}") log_contents = util.file_read_text(result_util.get_log_path(output_dir)) signature = signature_util.get_signature_from_log_contents(log_contents) + log("") log(f"Expected signature: {test.crash_signature}") log(f"Actual signature: {signature}") - if signature == test.crash_signature: - log("Interesting!") - return + log("") + + if test.expected_status: + if status != test.expected_status: + log("status != expected_status; not interesting") + sys.exit(1) + else: + # There is no expected status, so just assume it needs to be one of the "bad" statuses: + if status not in ( + fuzz.STATUS_CRASH, + fuzz.STATUS_TOOL_CRASH, + fuzz.STATUS_UNRESPONSIVE, + ): + log("shader run did not fail; not interesting.") + sys.exit(1) + + if signature != test.crash_signature: + log("signature != crash_signature; not interesting") + sys.exit(1) - log("Not interesting") - sys.exit(1) + log("Interesting!") if __name__ == "__main__": diff --git a/gfauto/gfauto/test.proto b/gfauto/gfauto/test.proto index 5079496ba..7c7885cf9 100644 --- a/gfauto/gfauto/test.proto +++ b/gfauto/gfauto/test.proto @@ -30,6 +30,8 @@ message Test { string crash_signature = 2; Device device = 3; repeated Binary binaries = 4; + // The expected status when running the test. E.g. CRASH, TOOL_CRASH, TIMEOUT. See fuzz.py. + string expected_status = 6; } message TestGlsl { diff --git a/gfauto/gfauto/test_pb2.py b/gfauto/gfauto/test_pb2.py index 4e3093d80..3faede5aa 100644 --- a/gfauto/gfauto/test_pb2.py +++ b/gfauto/gfauto/test_pb2.py @@ -22,7 +22,7 @@ package='gfauto', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\x11gfauto/test.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\xb8\x01\n\x04Test\x12 \n\x04glsl\x18\x01 \x01(\x0b\x32\x10.gfauto.TestGlslH\x00\x12+\n\nspirv_fuzz\x18\x05 \x01(\x0b\x32\x15.gfauto.TestSpirvFuzzH\x00\x12\x17\n\x0f\x63rash_signature\x18\x02 \x01(\t\x12\x1e\n\x06\x64\x65vice\x18\x03 \x01(\x0b\x32\x0e.gfauto.Device\x12 \n\x08\x62inaries\x18\x04 \x03(\x0b\x32\x0e.gfauto.BinaryB\x06\n\x04test\"\"\n\x08TestGlsl\x12\x16\n\x0espirv_opt_args\x18\x01 \x03(\t\"\x0f\n\rTestSpirvFuzzb\x06proto3') + serialized_pb=_b('\n\x11gfauto/test.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\xd1\x01\n\x04Test\x12 \n\x04glsl\x18\x01 \x01(\x0b\x32\x10.gfauto.TestGlslH\x00\x12+\n\nspirv_fuzz\x18\x05 \x01(\x0b\x32\x15.gfauto.TestSpirvFuzzH\x00\x12\x17\n\x0f\x63rash_signature\x18\x02 \x01(\t\x12\x1e\n\x06\x64\x65vice\x18\x03 \x01(\x0b\x32\x0e.gfauto.Device\x12 \n\x08\x62inaries\x18\x04 \x03(\x0b\x32\x0e.gfauto.Binary\x12\x17\n\x0f\x65xpected_status\x18\x06 \x01(\tB\x06\n\x04test\"\"\n\x08TestGlsl\x12\x16\n\x0espirv_opt_args\x18\x01 \x03(\t\"\x0f\n\rTestSpirvFuzzb\x06proto3') , dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,gfauto_dot_device__pb2.DESCRIPTOR,]) @@ -71,6 +71,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='expected_status', full_name='gfauto.Test.expected_status', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -87,7 +94,7 @@ index=0, containing_type=None, fields=[]), ], serialized_start=72, - serialized_end=256, + serialized_end=281, ) @@ -117,8 +124,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=258, - serialized_end=292, + serialized_start=283, + serialized_end=317, ) @@ -141,8 +148,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=294, - serialized_end=309, + serialized_start=319, + serialized_end=334, ) _TEST.fields_by_name['glsl'].message_type = _TESTGLSL diff --git a/gfauto/gfauto/test_pb2.pyi b/gfauto/gfauto/test_pb2.pyi index 6338db327..3daf356d9 100644 --- a/gfauto/gfauto/test_pb2.pyi +++ b/gfauto/gfauto/test_pb2.pyi @@ -35,6 +35,7 @@ from typing_extensions import ( class Test(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... crash_signature = ... # type: typing___Text + expected_status = ... # type: typing___Text @property def glsl(self) -> TestGlsl: ... @@ -55,6 +56,7 @@ class Test(google___protobuf___message___Message): crash_signature : typing___Optional[typing___Text] = None, device : typing___Optional[gfauto___device_pb2___Device] = None, binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, + expected_status : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod def FromString(cls, s: bytes) -> Test: ... @@ -62,10 +64,10 @@ class Test(google___protobuf___message___Message): def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): def HasField(self, field_name: typing_extensions___Literal[u"device",u"glsl",u"spirv_fuzz",u"test"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"binaries",u"crash_signature",u"device",u"glsl",u"spirv_fuzz",u"test"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"binaries",u"crash_signature",u"device",u"expected_status",u"glsl",u"spirv_fuzz",u"test"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"device",b"device",u"glsl",b"glsl",u"spirv_fuzz",b"spirv_fuzz",u"test",b"test"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"binaries",b"binaries",u"crash_signature",b"crash_signature",u"device",b"device",u"glsl",b"glsl",u"spirv_fuzz",b"spirv_fuzz",u"test",b"test"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"binaries",b"binaries",u"crash_signature",b"crash_signature",u"device",b"device",u"expected_status",b"expected_status",u"glsl",b"glsl",u"spirv_fuzz",b"spirv_fuzz",u"test",b"test"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"test",b"test"]) -> typing_extensions___Literal["glsl","spirv_fuzz"]: ... class TestGlsl(google___protobuf___message___Message): From 1794a1be9305f4220079d3e8b7c5c70166d78435 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 19 Sep 2019 12:40:49 +0100 Subject: [PATCH 139/180] Use clamp instead of ternary to make array access in bounds (#752) Live code injection has a feature that attempts to force all array accesses to be in bounds. This was being achieved by using a ternary expression to clamp an array index expression into an acceptable range, turning: A[e] into: A[e >= 0 && e < len(A) ? e : 0] A problem with this is that when e has side effects, evaluating e multiple times can make the access end up being out of bounds anyway. For example in: int A[10]; A[++i >= 0 && ++i < 10 ? ++i : 0] = 123; if i is 8 then ++i will be less than 10, so ++i will be used to index the array ... but will evaluate to 10. This change addresses this problem for recent shading language versions by using the clamp built-in instead of a ternary. For older shading language versions we still use a ternary - living with the problem as we do not deem it worth fixing at present. The clamping is expressed in a macro, _GLF_MAKE_IN_BOUNDS, to pave the way for avoiding losing in-bounds clamping during test case reduction. The reducer has been updated so that in its final simplification pass it can get rid of these new macros. Fixes #679. --- .../common/tool/PrettyPrinterVisitor.java | 54 ++++-- .../graphicsfuzz/common/util/MacroNames.java | 8 + .../common/util/ParseHelperTest.java | 4 +- .../DonateCodeTransformation.java | 2 +- .../donation/MakeArrayAccessesInBounds.java | 56 +++--- .../MakeArrayAccessesInBoundsTest.java | 33 ++-- .../EliminateGraphicsFuzzDefines.java | 183 ++++++++++++++++++ .../EliminateInjectionMacrosVisitor.java | 93 --------- .../graphicsfuzz/reducer/util/Simplify.java | 8 +- .../reducer/util/SimplifyTest.java | 76 ++++++++ .../java/com/graphicsfuzz/util/Constants.java | 4 + 11 files changed, 362 insertions(+), 159 deletions(-) create mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateGraphicsFuzzDefines.java delete mode 100644 reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java diff --git a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java index 0e68bb368..2a26dd91e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java +++ b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java @@ -72,6 +72,7 @@ import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.util.Constants; @@ -639,7 +640,7 @@ public void visitTranslationUnit(TranslationUnit translationUnit) { } if (emitGraphicsFuzzDefines) { - emitGraphicsFuzzDefines(out); + emitGraphicsFuzzDefines(out, translationUnit.getShadingLanguageVersion()); } super.visitTranslationUnit(translationUnit); @@ -695,20 +696,47 @@ public static String defaultIndent(int level) { return result.toString(); } - public static void emitGraphicsFuzzDefines(PrintStream out) { + public static void emitGraphicsFuzzDefines(PrintStream out, + ShadingLanguageVersion shadingLanguageVersion) { out.append("\n"); out.append("#ifndef REDUCER\n"); - out.append("#define " + Constants.GLF_ZERO + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_ONE + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_FALSE + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_TRUE + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_IDENTITY + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_DEAD + "(X) (X)\n"); - out.append("#define " + Constants.GLF_FUZZED + "(X) (X)\n"); - out.append("#define " + Constants.GLF_WRAPPED_LOOP + "(X) X\n"); - out.append("#define " + Constants.GLF_WRAPPED_IF_TRUE + "(X) X\n"); - out.append("#define " + Constants.GLF_WRAPPED_IF_FALSE + "(X) X\n"); - out.append("#define " + Constants.GLF_SWITCH + "(X) X\n"); + out.append("#define " + Constants.GLF_ZERO + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_ONE + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_FALSE + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_TRUE + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_IDENTITY + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_DEAD + "(X) (X)\n"); + out.append("#define " + Constants.GLF_FUZZED + "(X) (X)\n"); + out.append("#define " + Constants.GLF_WRAPPED_LOOP + "(X) X\n"); + out.append("#define " + Constants.GLF_WRAPPED_IF_TRUE + "(X) X\n"); + out.append("#define " + Constants.GLF_WRAPPED_IF_FALSE + "(X) X\n"); + out.append("#define " + Constants.GLF_SWITCH + "(X) X\n"); + + // The preferred way to make array accesses in bounds is to use 'clamp'. However, 'clamp' + // on integer types is not available in all shading language versions. When it is not + // available, we resort to a solution based on the ternary operator. This has the disadvantage + // of evaluating the array index multiple times, so that if it has side effects these will + // occur multiple times (potentially leading to the access ending up being out of bounds if the + // side effects change the index). If deemed worth prioritizing, this could be worked around + // by declaring appropriate 'clamp' functions (but with appropriately obfuscated names to avoid + // collisions with user-defined integer clamp functions). + + if (shadingLanguageVersion.supportedClampInt()) { + out.append("#define " + Constants.GLF_MAKE_IN_BOUNDS_INT + "(IDX, SZ) clamp(IDX, 0, SZ - 1)" + + "\n"); + } else { + out.append("#define " + Constants.GLF_MAKE_IN_BOUNDS_INT + "(IDX, SZ) ((IDX) < 0 ? 0 : (" + + "(IDX) >= SZ ? SZ - 1 : (IDX)))\n"); + } + + if (shadingLanguageVersion.supportedClampUint()) { + out.append("#define " + Constants.GLF_MAKE_IN_BOUNDS_UINT + "(IDX, SZ) clamp(IDX, 0u, SZ - " + + "1u)\n"); + } else { + out.append("#define " + Constants.GLF_MAKE_IN_BOUNDS_UINT + "(IDX, SZ) ((IDX) >= SZ ? SZ - " + + "1u : (IDX))\n"); + } + out.append("#endif\n"); out.append("\n"); out.append(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES + "\n"); diff --git a/ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java b/ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java index 0fc4f9f51..3ec58adc1 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java +++ b/ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java @@ -85,6 +85,14 @@ public static boolean isSwitch(Expr expr) { return isCallToNamedFunction(expr, Constants.GLF_SWITCH); } + public static boolean isMakeInBoundsInt(Expr expr) { + return isCallToNamedFunction(expr, Constants.GLF_MAKE_IN_BOUNDS_INT); + } + + public static boolean isMakeInBoundsUint(Expr expr) { + return isCallToNamedFunction(expr, Constants.GLF_MAKE_IN_BOUNDS_UINT); + } + private static boolean isCallToNamedFunction(Expr expr, String functionName) { return expr instanceof FunctionCallExpr && ((FunctionCallExpr) expr).getCallee() .equals(functionName); diff --git a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java index cc7338b51..2ae7cd2ac 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java @@ -105,7 +105,7 @@ private void checkTranslationUnit(TranslationUnit tu) { public void testParseCurrentHeaderNoMacros() throws Exception { File tempFile = testFolder.newFile("shader.frag"); PrintStream ps = new PrintStream(new FileOutputStream(tempFile)); - PrettyPrinterVisitor.emitGraphicsFuzzDefines(ps); + PrettyPrinterVisitor.emitGraphicsFuzzDefines(ps, ShadingLanguageVersion.ESSL_310); ps.println(TEST_PROGRAM); ps.close(); TranslationUnit tu = ParseHelper.parse(tempFile); @@ -116,7 +116,7 @@ public void testParseCurrentHeaderNoMacros() throws Exception { public void testParseCurrentHeaderWithMacros() throws Exception { File tempFile = testFolder.newFile("shader.frag"); PrintStream ps = new PrintStream(new FileOutputStream(tempFile)); - PrettyPrinterVisitor.emitGraphicsFuzzDefines(ps); + PrettyPrinterVisitor.emitGraphicsFuzzDefines(ps, ShadingLanguageVersion.ESSL_310); ps.println(TEST_PROGRAM); ps.close(); TranslationUnit tu = ParseHelper.parse(tempFile); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java index 8cb945990..a5f6e24f2 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java @@ -296,7 +296,7 @@ private void makeInjectedArrayAccessesInBounds(TranslationUnit tu, List injectedStmts) { Typer typer = new Typer(tu); for (Stmt stmt : injectedStmts) { - MakeArrayAccessesInBounds.makeInBounds(stmt, typer); + MakeArrayAccessesInBounds.makeInBounds(stmt, typer, tu); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java index 8e114a39a..f8a161ca3 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java @@ -17,10 +17,13 @@ package com.graphicsfuzz.generator.transformation.donation; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.TernaryExpr; @@ -29,9 +32,11 @@ import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.Typer; +import com.graphicsfuzz.util.Constants; -public class MakeArrayAccessesInBounds extends StandardVisitor { +public class MakeArrayAccessesInBounds extends ScopeTrackingVisitor { private Typer typer; @@ -39,12 +44,19 @@ private MakeArrayAccessesInBounds(Typer typer) { this.typer = typer; } - public static void makeInBounds(IAstNode node, Typer typer) { + /** + * Clamp array / vector / matrix accesses to ensure they will be in-bounds. + * @param node AST node under which accesses should be made in bound. + * @param typer Type info for the AST. + * @param tu The translation unit in which node is contained. + */ + public static void makeInBounds(IAstNode node, Typer typer, TranslationUnit tu) { new MakeArrayAccessesInBounds(typer).visit(node); } @Override public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { + super.visitArrayIndexExpr(arrayIndexExpr); Type type = typer.lookupType(arrayIndexExpr.getArray()); if (type == null) { return; @@ -57,34 +69,20 @@ public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { return; } indexType = indexType.getWithoutQualifiers(); - if (indexType == BasicType.UINT) { - arrayIndexExpr.setIndex(new TernaryExpr( - new BinaryExpr( - new ParenExpr(arrayIndexExpr.getIndex().clone()), - new UIntConstantExpr(getSize(type).toString() + "u"), - BinOp.LT), - arrayIndexExpr.getIndex(), - new UIntConstantExpr("0u")) - ); - } else { - assert indexType == BasicType.INT; - arrayIndexExpr.setIndex(new TernaryExpr( - new BinaryExpr( - new BinaryExpr( - new ParenExpr(arrayIndexExpr.getIndex().clone()), - new IntConstantExpr("0"), - BinOp.GE), - new BinaryExpr( - new ParenExpr(arrayIndexExpr.getIndex().clone()), - new IntConstantExpr(getSize(type).toString()), - BinOp.LT), - BinOp.LAND), - arrayIndexExpr.getIndex(), - new IntConstantExpr("0")) - ); - } + assert indexType == BasicType.INT || indexType == BasicType.UINT; + + final Expr arraySize = indexType == BasicType.INT + ? new IntConstantExpr(getSize(type).toString()) + : new UIntConstantExpr(getSize(type).toString() + "u"); + + final Expr clampedIndexExpr = + new FunctionCallExpr(indexType == BasicType.INT ? Constants.GLF_MAKE_IN_BOUNDS_INT : + Constants.GLF_MAKE_IN_BOUNDS_UINT, + arrayIndexExpr.getIndex(), + arraySize); + arrayIndexExpr.setIndex(clampedIndexExpr); + } - super.visitArrayIndexExpr(arrayIndexExpr); } private static Integer getSize(Type type) { diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java index fc239df28..074cc2e03 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.util.Constants; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -30,10 +31,11 @@ public class MakeArrayAccessesInBoundsTest { @Test public void testBasic() throws Exception { final String shader = "#version 300 es\nvoid main() { int A[5]; int x = 17; A[x] = 2; }"; - final String expected = "#version 300 es\nvoid main() { int A[5]; int x = 17; A[(x) >= 0 && (x) < 5 ? x : 0] = 2; }"; + final String expected = "#version 300 es\nvoid main() { int A[5]; int x = 17; A[" + + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 5)] = 2; }"; final TranslationUnit tu = ParseHelper.parse(shader); final Typer typer = new Typer(tu); - MakeArrayAccessesInBounds.makeInBounds(tu, typer); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -42,12 +44,12 @@ public void testBasic() throws Exception { public void testMatrixVector() throws Exception { final String shader = "#version 300 es\nvoid main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100; As[x][y][z] = 2.0; }"; final String expected = "#version 300 es\nvoid main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100;" - + "As[(x) >= 0 && (x) < 5 ? x : 0]" - + " /* column */ [(y) >= 0 && (y) < 4 ? y : 0]" - + " /* row */ [(z) >= 0 && (z) < 2 ? z : 0] = 2.0; }"; + + "As[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 5)]" + + " /* column */ [" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(y, 4)]" + + " /* row */ [" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(z, 2)] = 2.0; }"; final TranslationUnit tu = ParseHelper.parse(shader); final Typer typer = new Typer(tu); - MakeArrayAccessesInBounds.makeInBounds(tu, typer); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -70,15 +72,15 @@ public void testMatrixVector2() throws Exception { + " int x = 17;" + " int y = -22;" + " int z = 100;" - + " mat3x4 A = As[(x) >= 0 && (x) < 5 ? x : 0];" + + " mat3x4 A = As[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 5)];" + " vec4 v;" - + " v = A[(y) >= 0 && (y) < 3 ? y : 0];" + + " v = A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(y, 3)];" + " float f;" - + " f = v[(z) >= 0 && (z) < 4 ? z : 0];" + + " f = v[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(z, 4)];" + "}"; final TranslationUnit tu = ParseHelper.parse(shader); final Typer typer = new Typer(tu); - MakeArrayAccessesInBounds.makeInBounds(tu, typer); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -93,11 +95,11 @@ public void testUIntConstantExprIndex() throws Exception { final String expected = "#version 300 es\n" + "void main() { vec3 stuff[16];" + " uint x = 19u;" - + " vec3 f = stuff[(x) < 16u ? x : 0u];" + + " vec3 f = stuff[" + Constants.GLF_MAKE_IN_BOUNDS_UINT + "(x, 16u)];" + "}"; final TranslationUnit tu = ParseHelper.parse(shader); final Typer typer = new Typer(tu); - MakeArrayAccessesInBounds.makeInBounds(tu, typer); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -112,11 +114,12 @@ public void testUIntFunctionCallReturnIndex() throws Exception { final String expected = "#version 310 es\n" + "void main() { vec3 stuff[16];" + " uint uselessOut;" - + " vec3 f = stuff[(uaddCarry(19u, 15u, uselessOut)) < 16u ? uaddCarry(19u, 15u, uselessOut) : 0u];" + + " vec3 f = stuff[" + Constants.GLF_MAKE_IN_BOUNDS_UINT + "(uaddCarry(19u, 15u, " + + "uselessOut), 16u)];" + "}"; final TranslationUnit tu = ParseHelper.parse(shader); final Typer typer = new Typer(tu); - MakeArrayAccessesInBounds.makeInBounds(tu, typer); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -129,7 +132,7 @@ public void testUIntStaticallyInBounds() throws Exception { + "}"; final TranslationUnit tu = ParseHelper.parse(shader); final Typer typer = new Typer(tu); - MakeArrayAccessesInBounds.makeInBounds(tu, typer); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(shader)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateGraphicsFuzzDefines.java b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateGraphicsFuzzDefines.java new file mode 100644 index 000000000..1c91bbcd5 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateGraphicsFuzzDefines.java @@ -0,0 +1,183 @@ +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.glslreducers; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.ConstantExpr; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; +import com.graphicsfuzz.common.ast.expr.ParenExpr; +import com.graphicsfuzz.common.ast.expr.TernaryExpr; +import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.MacroNames; + +public class EliminateGraphicsFuzzDefines extends StandardVisitor { + + private final ShadingLanguageVersion shadingLanguageVersion; + + private EliminateGraphicsFuzzDefines(ShadingLanguageVersion shadingLanguageVersion) { + this.shadingLanguageVersion = shadingLanguageVersion; + } + + /** + * Eliminates all GraphicsFuzz macros from the given translation unit, by inlining them. + * @param tu The translation unit to be processed. + * @return A clone of the given translation unit, with all GraphicsFuzz macros inlined. + */ + public static TranslationUnit transform(TranslationUnit tu) { + final TranslationUnit result = tu.clone(); + new EliminateGraphicsFuzzDefines(tu.getShadingLanguageVersion()).visit(result); + return result; + } + + @Override + protected void visitChildFromParent(IAstNode child, IAstNode parent) { + super.visitChildFromParent(child, parent); + if (child instanceof FunctionCallExpr) { + final FunctionCallExpr functionCallExpr = (FunctionCallExpr) child; + if (MacroNames.isIdentity(functionCallExpr) + || MacroNames.isZero(functionCallExpr) + || MacroNames.isOne(functionCallExpr) + || MacroNames.isFalse(functionCallExpr) + || MacroNames.isTrue(functionCallExpr)) { + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, functionCallExpr.getChild(1))); + } else if (MacroNames.isFuzzed(functionCallExpr) + || MacroNames.isDeadByConstruction(functionCallExpr) + || MacroNames.isSwitch(functionCallExpr) + || MacroNames.isLoopWrapper(functionCallExpr) + || MacroNames.isIfWrapperFalse(functionCallExpr) + || MacroNames.isIfWrapperTrue(functionCallExpr)) { + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, functionCallExpr.getChild(0))); + } else if (MacroNames.isMakeInBoundsInt(functionCallExpr)) { + expandMakeInBoundsInt(parent, functionCallExpr); + } else if (MacroNames.isMakeInBoundsUint(functionCallExpr)) { + expandMakeInBoundsUint(parent, functionCallExpr); + } + } + } + + private void expandMakeInBoundsInt(IAstNode parent, FunctionCallExpr functionCallExpr) { + + // This replaces a call to _GLF_MAKE_IN_BOUNDS_INT with either: + // - a call to clamp, if the shading language version supports integer clamping + // - a ternary expression otherwise + // The expression that is generated matches the definition of the macro, which can be found in + // PrettyPrinterVisitor.emitGraphicsFuzzDefines. + + final ConstantExpr one = new IntConstantExpr("1"); + final ConstantExpr zero = new IntConstantExpr("0"); + if (shadingLanguageVersion.supportedClampInt()) { + expandMakeInBoundsMacroToClamp(parent, functionCallExpr, one, + zero); + } else { + final Expr indexLessThanZero = + new BinaryExpr(new ParenExpr(functionCallExpr.getChild(0).clone()), + zero, BinOp.LT); + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, + new TernaryExpr(indexLessThanZero, zero.clone(), + new ParenExpr(getTernaryUpperBoundCheckForMakeInBounds(functionCallExpr, one))))); + } + } + + private void expandMakeInBoundsUint(IAstNode parent, FunctionCallExpr functionCallExpr) { + + // Similar to expandMakeInBoundsInt, but for the 'uint' case. + + final ConstantExpr one = new UIntConstantExpr("1u"); + if (shadingLanguageVersion.supportedClampUint()) { + expandMakeInBoundsMacroToClamp(parent, functionCallExpr, one, + new UIntConstantExpr("0u")); + } else { + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, + getTernaryUpperBoundCheckForMakeInBounds(functionCallExpr, + one))); + + } + } + + private Expr getTernaryUpperBoundCheckForMakeInBounds(FunctionCallExpr functionCallExpr, + ConstantExpr one) { + return new TernaryExpr(new BinaryExpr(new ParenExpr(functionCallExpr.getChild(0).clone()), + functionCallExpr.getChild(1).clone(), + BinOp.GE), + new BinaryExpr(functionCallExpr.getChild(1).clone(), one, BinOp.SUB), + new ParenExpr(functionCallExpr.getChild(0).clone())); + } + + private void expandMakeInBoundsMacroToClamp(IAstNode parent, FunctionCallExpr functionCallExpr, + ConstantExpr one, ConstantExpr zero) { + final Expr sizeMinusOne = new BinaryExpr(functionCallExpr.getChild(1), + one, + BinOp.SUB); + final Expr clamp = new FunctionCallExpr("clamp", functionCallExpr.getChild(0), + zero, + sizeMinusOne); + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, + clamp)); + } + + private IAstNode addParenthesesIfNecessary(IAstNode parent, Expr child) { + if (child instanceof ConstantExpr + || child instanceof ParenExpr + || child instanceof VariableIdentifierExpr + || child instanceof FunctionCallExpr) { + // Parentheses is unnecessary in cases such as _GLF_FUNCTION(1), + // _GLF_FUNCTION((1)), _GLF_FUNCTION(a), _GLF_FUNCTION(sin(a)). + return child; + } + + if (!(parent instanceof Expr)) { + // No parentheses needed if the parent is not an expression, + // for example, int x = _GLF_FUNCTION(a + b). + return child; + } + + if (parent instanceof ParenExpr) { + // If parent is a parentheses expression, adding more parentheses would be + // redundant, + // e.g. (_GLF_FUNCTION(a + b)). + return child; + } + + if (parent instanceof FunctionCallExpr) { + // Parentheses is unnecessary if parent is a function call expression. + // For example, foo(_GLF_FUNCTION(a)). + + // This asserts that the binary expression inside the function call is not a comma operator + // as it is invalid to have a comma appear directly here, e.g. _GLF_IDENTITY(expr, a, b) is + // not valid since a and b are treated as function arguments instead. + assert (!(child instanceof BinaryExpr) || ((BinaryExpr) child).getOp() != BinOp.COMMA); + return child; + } + + return new ParenExpr(child); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java deleted file mode 100644 index a32478d5a..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.glslreducers; - -import com.graphicsfuzz.common.ast.IAstNode; -import com.graphicsfuzz.common.ast.expr.BinOp; -import com.graphicsfuzz.common.ast.expr.BinaryExpr; -import com.graphicsfuzz.common.ast.expr.ConstantExpr; -import com.graphicsfuzz.common.ast.expr.Expr; -import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; -import com.graphicsfuzz.common.ast.expr.ParenExpr; -import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; -import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import com.graphicsfuzz.common.util.MacroNames; - -public class EliminateInjectionMacrosVisitor extends StandardVisitor { - - @Override - protected void visitChildFromParent(IAstNode child, IAstNode parent) { - super.visitChildFromParent(child, parent); - if (child instanceof FunctionCallExpr) { - final FunctionCallExpr functionCallExpr = (FunctionCallExpr) child; - if (MacroNames.isIdentity(functionCallExpr) - || MacroNames.isZero(functionCallExpr) - || MacroNames.isOne(functionCallExpr) - || MacroNames.isFalse(functionCallExpr) - || MacroNames.isTrue(functionCallExpr)) { - parent.replaceChild(child, - addParenthesesIfNecessary(parent, functionCallExpr.getChild(1))); - } else if (MacroNames.isFuzzed(functionCallExpr) - || MacroNames.isDeadByConstruction(functionCallExpr) - || MacroNames.isSwitch(functionCallExpr) - || MacroNames.isLoopWrapper(functionCallExpr) - || MacroNames.isIfWrapperFalse(functionCallExpr) - || MacroNames.isIfWrapperTrue(functionCallExpr) - ) { - parent.replaceChild(child, - addParenthesesIfNecessary(parent, functionCallExpr.getChild(0))); - } - } - } - - private IAstNode addParenthesesIfNecessary(IAstNode parent, Expr child) { - if (child instanceof ConstantExpr - || child instanceof ParenExpr - || child instanceof VariableIdentifierExpr - || child instanceof FunctionCallExpr) { - // Parentheses is unnecessary in cases such as _GLF_FUNCTION(1), - // _GLF_FUNCTION((1)), _GLF_FUNCTION(a), _GLF_FUNCTION(sin(a)). - return child; - } - - if (!(parent instanceof Expr)) { - // No parentheses needed if the parent is not an expression, - // for example, int x = _GLF_FUNCTION(a + b). - return child; - } - - if (parent instanceof ParenExpr) { - // If parent is parentheses, adding a new parentheses would be redundant, - // e.g. (_GLF_FUNCTION(a + b)). - return child; - } - - if (parent instanceof FunctionCallExpr) { - // Parentheses is unnecessary if parent is a function call expression. - // For example, foo(_GLF_FUNCTION(a)). - - // This asserts that the binary expression inside the function call is not a comma operator - // as it is invalid to have a comma appear directly here, e.g. _GLF_IDENTITY(expr, a, b) is - // not valid since a and b are treated as function arguments instead. - assert (!(child instanceof BinaryExpr) || ((BinaryExpr) child).getOp() != BinOp.COMMA); - return child; - } - - return new ParenExpr(child); - } - -} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/util/Simplify.java b/reducer/src/main/java/com/graphicsfuzz/reducer/util/Simplify.java index 4cf7f3b3c..598d33435 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/util/Simplify.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/util/Simplify.java @@ -18,16 +18,12 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.util.AddBraces; -import com.graphicsfuzz.reducer.glslreducers.EliminateInjectionMacrosVisitor; -import java.util.Optional; +import com.graphicsfuzz.reducer.glslreducers.EliminateGraphicsFuzzDefines; public class Simplify { public static TranslationUnit simplify(TranslationUnit tu) { - TranslationUnit temp = tu.clone(); - new EliminateInjectionMacrosVisitor().visit(temp); - temp = AddBraces.transform(temp); - return temp; + return AddBraces.transform(EliminateGraphicsFuzzDefines.transform(tu)); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java index 1af838a27..96636f151 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java @@ -145,4 +145,80 @@ public void testFunctionCallParenthesesRemoved() throws Exception { final TranslationUnit simplifiedTu = Simplify.simplify(tu); CompareAsts.assertEqualAsts(expected, simplifiedTu); } + + @Test + public void testMakeInBoundsIntClampRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " float A[4];\n" + + " A[_GLF_MAKE_IN_BOUNDS_INT(A[_GLF_MAKE_IN_BOUNDS_INT(3, 4)], 4)] =\n" + + " A[_GLF_MAKE_IN_BOUNDS_INT(1 + 2, 4)];\n" + + "}\n" + ); + final String expected = "#version 310 es\n" + + "void main() {\n" + + " float A[4];\n" + + " A[clamp(A[clamp(3, 0, 4 - 1)], 0, 4 - 1)] =\n" + + " A[clamp(1 + 2, 0, 4 - 1)];\n" + + "}\n"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + + @Test + public void testMakeInBoundsUintClampRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " float A[4];\n" + + " A[_GLF_MAKE_IN_BOUNDS_UINT(A[_GLF_MAKE_IN_BOUNDS_UINT(3u, 4u)], 4u)] =\n" + + " A[_GLF_MAKE_IN_BOUNDS_UINT(uint(1 + 2), 4u)];\n" + + "}\n" + ); + final String expected = "#version 310 es\n" + + "void main() {\n" + + " float A[4];\n" + + " A[clamp(A[clamp(3u, 0u, 4u - 1u)], 0u, 4u - 1u)] =\n" + + " A[clamp(uint(1 + 2), 0u, 4u - 1u)];\n" + + "}\n"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + + @Test + public void testMakeInBoundsIntTernaryRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " float A[4];\n" + + " A[_GLF_MAKE_IN_BOUNDS_INT(A[_GLF_MAKE_IN_BOUNDS_INT(3, 4)], 4)] = 1.0;\n" + + "}\n" + ); + final String inner = "A[((3) < 0 ? 0 : ((3) >= 4 ? 4 - 1 : (3)))]"; + final String expected = "#version 100\n" + + "void main() {\n" + + " float A[4];\n" + + " A[((" + inner + ") < 0 ? 0 : ((" + inner + ") >= 4 ? 4 - 1 : (" + inner + ")))] = 1" + + ".0;\n" + + "}\n"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + + @Test + public void testMakeInBoundsUintTernaryRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " float A[4];\n" + + " A[_GLF_MAKE_IN_BOUNDS_UINT(A[_GLF_MAKE_IN_BOUNDS_UINT(3u, 4u)], 4u)] = 1.0;\n" + + "}\n" + ); + final String inner = "A[((3u) >= 4u ? 4u - 1u : (3u))]"; + final String expected = "#version 100\n" + + "void main() {\n" + + " float A[4];\n" + + " A[((" + inner + ") >= 4u ? 4u - 1u : (" + inner + "))] = 1.0;\n" + + "}\n"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + } diff --git a/util/src/main/java/com/graphicsfuzz/util/Constants.java b/util/src/main/java/com/graphicsfuzz/util/Constants.java index 31c76384f..e8987d4bc 100755 --- a/util/src/main/java/com/graphicsfuzz/util/Constants.java +++ b/util/src/main/java/com/graphicsfuzz/util/Constants.java @@ -45,6 +45,10 @@ private Constants() { public static final String GLF_IDENTITY = "_GLF_IDENTITY"; public static final String GLF_SWITCH = "_GLF_SWITCH"; + // Macro names used to make array accesses in bounds + public static final String GLF_MAKE_IN_BOUNDS_INT = "_GLF_MAKE_IN_BOUNDS_INT"; + public static final String GLF_MAKE_IN_BOUNDS_UINT = "_GLF_MAKE_IN_BOUNDS_UINT"; + public static final String GLF_PRIMITIVE = "_GLF_PRIMITIVE"; public static final String GLF_PRIMITIVE_GLOBAL = "_GLF_PRIMITIVE_GLOBAL"; public static final String GLF_COMPUTE = "_GLF_COMPUTE"; From cb0a9e37ab4b3c10debacd43425f57055faebba0 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 19 Sep 2019 16:25:49 +0100 Subject: [PATCH 140/180] gfauto: fix .idea/inspectionProfiles/profiles_settings.xml (#755) --- gfauto/.idea/inspectionProfiles/profiles_settings.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gfauto/.idea/inspectionProfiles/profiles_settings.xml b/gfauto/.idea/inspectionProfiles/profiles_settings.xml index 333a4e80e..4e0cac616 100644 --- a/gfauto/.idea/inspectionProfiles/profiles_settings.xml +++ b/gfauto/.idea/inspectionProfiles/profiles_settings.xml @@ -1,7 +1,6 @@ - + \ No newline at end of file From dbd1ecc4508d546f93795f4a17aea645c1168f94 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 19 Sep 2019 23:24:01 +0100 Subject: [PATCH 141/180] Remove VkScript support from runspv (#754) AmberScript support is now working well, so we no longer need VkScript support to cross-check against. Fixes #751. --- .../main/python/drivers/glsl-to-spv-worker.py | 7 +- python/src/main/python/drivers/runspv.py | 276 ++---------------- .../test_scripts/test_runspv/runspv_tests.py | 208 ------------- 3 files changed, 24 insertions(+), 467 deletions(-) diff --git a/python/src/main/python/drivers/glsl-to-spv-worker.py b/python/src/main/python/drivers/glsl-to-spv-worker.py index a5fae4011..d3a5c13a7 100755 --- a/python/src/main/python/drivers/glsl-to-spv-worker.py +++ b/python/src/main/python/drivers/glsl-to-spv-worker.py @@ -17,7 +17,6 @@ import argparse import os import random -import shutil import subprocess import sys import time @@ -209,8 +208,7 @@ def do_image_job( force=args.force, is_android=(args.target == 'android'), skip_render=skip_render, - spirv_opt_args=resolved_spirvopt_args, - use_amberscript=True + spirv_opt_args=resolved_spirvopt_args ) except Exception as ex: runspv.log('Exception: ' + str(ex)) @@ -305,8 +303,7 @@ def do_compute_job( force=args.force, is_android=(args.target == 'android'), skip_render=comp_job.skipRender, - spirv_opt_args=spirv_opt_args, - use_amberscript=True + spirv_opt_args=spirv_opt_args ) except Exception as ex: runspv.log('Exception: ' + str(ex)) diff --git a/python/src/main/python/drivers/runspv.py b/python/src/main/python/drivers/runspv.py index 2682f2ec2..fe1f82ca8 100755 --- a/python/src/main/python/drivers/runspv.py +++ b/python/src/main/python/drivers/runspv.py @@ -53,8 +53,6 @@ TARGET_HELP = 'One of \'host\' (run on host machine) or \'android\' (run on Android device).' -USE_AMBERSCRIPT_OPTION_HELP = 'Use AmberScript (not VkScript) test file format.' - ################################################################################ # Constants @@ -826,65 +824,6 @@ def spv_get_disassembly(shader_filename): return subprocess_helper(cmd).stdout -def uniform_json_to_vkscript(uniform_json): - """ - Returns the string representing VkScript version of uniform declarations. - Skips the special '$compute' key, if present. - - { - "myuniform": { - "func": "glUniform1f", - "args": [ 42.0 ], - "binding": 3 - }, - "$compute": { ... will be ignored ... } - } - - becomes: - - # myuniform - uniform ubo 0:3 float 0 42.0 - - """ - - uniform_types = { - 'glUniform1i': 'int', - 'glUniform2i': 'ivec2', - 'glUniform3i': 'ivec3', - 'glUniform4i': 'ivec4', - 'glUniform1f': 'float', - 'glUniform2f': 'vec2', - 'glUniform3f': 'vec3', - 'glUniform4f': 'vec4', - } - - descriptor_set = 0 # always 0 in our tests - offset = 0 # We never have uniform offset in our tests - - result = '' - with open_helper(uniform_json, 'r') as f: - j = json.load(f) - for name, entry in j.items(): - - if name == '$compute': - continue - - func = entry['func'] - if func not in uniform_types.keys(): - raise AssertionError('Error: unknown uniform type for function: ' + func) - uniform_type = uniform_types[func] - - result += '# ' + name + '\n' - result += 'uniform ubo {}:{}'.format(descriptor_set, entry['binding']) - result += ' ' + uniform_type - result += ' {}'.format(offset) - for arg in entry['args']: - result += ' {}'.format(arg) - result += '\n' - - return result - - def get_shader_as_comment(shader: str) -> str: with open_helper(shader, 'r') as f: lines = f.readlines() @@ -951,50 +890,6 @@ def get_header_comment_original_source_comp( return get_header_comment_original_source(None, None, comp_original, spirv_args) -def vkscriptify_image( - vert, - frag, - uniform_json, - vert_original, - frag_original, - spirv_args -): - """ - Generates VkScript representation of an image test - """ - - result = '# Generated\n\n' - - result += '# A test for a bug found by GraphicsFuzz.\n\n' - - result += get_header_comment_original_source_image(vert_original, frag_original, spirv_args) - - result += '[require]\n' - result += 'fbsize 256 256\n\n' - - result += '[require]\n' - result += 'fence_timeout ' + str(AMBER_FENCE_TIMEOUT_MS) + '\n\n' - - if vert: - result += '[vertex shader spirv]\n' - result += spv_get_disassembly(vert) - else: - result += '[vertex shader passthrough]\n' - result += '\n\n' - - result += '[fragment shader spirv]\n' - result += spv_get_disassembly(frag) - result += '\n\n' - - result += '[test]\n' - result += '## Uniforms\n' - result += uniform_json_to_vkscript(uniform_json) - result += '\n' - result += 'draw rect -1 -1 2 2\n' - - return result - - def amberscript_uniform_buffer_decl(uniform_json): ''' Returns the string representing AmberScript version of uniform declarations. @@ -1157,8 +1052,7 @@ def run_image_amber( force: bool, is_android: bool, skip_render: bool, - spirv_opt_args: Optional[List[str]], - use_amberscript: bool, + spirv_opt_args: Optional[List[str]] ): # The vertex shader is optional; passthrough will be used if it is not present assert not vert_original or os.path.isfile(vert_original) @@ -1173,28 +1067,16 @@ def run_image_amber( device_image = ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/image_0.png' - if use_amberscript: - shader_test_file = os.path.join(output_dir, 'tmp_shader_test.amber') - with open_helper(shader_test_file, 'w') as f: - f.write(amberscriptify_image( - vert, - frag, - json_file, - vert_original, - frag_original, - spirv_opt_args, - )) - else: - shader_test_file = os.path.join(output_dir, 'tmp_shader_test.vkscript') - with open_helper(shader_test_file, 'w') as f: - f.write(vkscriptify_image( - vert, - frag, - json_file, - vert_original, - frag_original, - spirv_opt_args, - )) + shader_test_file = os.path.join(output_dir, 'tmp_shader_test.amber') + with open_helper(shader_test_file, 'w') as f: + f.write(amberscriptify_image( + vert, + frag, + json_file, + vert_original, + frag_original, + spirv_opt_args, + )) if is_android: prepare_device(force, using_legacy_worker=False) @@ -1309,106 +1191,6 @@ def amber_check_buffer_single_type(json_filename): raise ValueError('Amber only supports one type per buffer') -def comp_json_to_vkscript(comp_json): - """ - Returns the string representing VkScript version of compute shader setup, - found under the special "$compute" key in JSON - - { - "my_uniform_name": { ... ignored by this function ... }, - - "$compute": { - "num_groups": [12, 13, 14]; - "buffer": { - "binding": 123, - "input": [42, 43, 44, 45] - } - } - - } - - becomes: - - ssbo 123 subdata int 0 42 43 44 45 - - compute 12 13 14 - - """ - - with open_helper(comp_json, 'r') as f: - j = json.load(f) - - assert '$compute' in j.keys(), 'Cannot find "$compute" key in JSON file' - j = j['$compute'] - - result = "" - - binding = j['buffer']['binding'] - offset = 0 - - for field_info in j['buffer']['fields']: - result += ( - 'ssbo ' - + str(binding) - + ' subdata ' - + field_info['type'] - + ' ' - + str(offset) - ) - for datum in field_info['data']: - result += ' ' + str(datum) - offset += 4 - result += '\n' - result += '\n\n' - - result += 'compute' - result += ' ' + str(j['num_groups'][0]) - result += ' ' + str(j['num_groups'][1]) - result += ' ' + str(j['num_groups'][2]) - result += '\n' - - return result - - -def vkscriptify_comp( - comp_spv: str, - comp_json: str, - comp_original: Optional[str], - spirv_args: Optional[List[str]] -): - """ - Generates an Vkscript representation of a compute test - """ - - result = '# Generated\n\n' - - result += '# A test for a bug found by GraphicsFuzz.\n\n' - - result += get_spirv_opt_args_comment(spirv_args) - - if comp_original and filename_extension_suggests_glsl(comp_original): - result += '# Derived from the following GLSL.\n\n' - result += '# Compute shader GLSL:\n' - result += get_shader_as_comment(comp_original) - result += '\n\n' - - result += '[require]\n' - result += 'fence_timeout ' + str(AMBER_FENCE_TIMEOUT_MS) + '\n\n' - - result += '[compute shader spirv]\n' - result += spv_get_disassembly(comp_spv) - result += '\n\n' - - result += '[test]\n' - result += '## Uniforms\n' - result += uniform_json_to_vkscript(comp_json) - result += '## SSBO\n' - result += comp_json_to_vkscript(comp_json) - result += '\n' - - return result - - def amberscript_comp_buff_decl(comp_json): """ Returns the string representing AmberScript declaration of buffers for a @@ -1621,8 +1403,7 @@ def run_compute_amber( force: bool, is_android: bool, skip_render: bool, - spirv_opt_args: Optional[List[str]], - use_amberscript: bool, + spirv_opt_args: Optional[List[str]] ) -> None: assert os.path.isfile(comp_original) @@ -1635,24 +1416,14 @@ def run_compute_amber( comp = prepare_shader(output_dir, comp_original, spirv_opt_args) - if use_amberscript: - shader_test_file = os.path.join(output_dir, 'tmp_shader_test.amber') - with open_helper(shader_test_file, 'w') as f: - f.write(amberscriptify_comp( - comp, - json_file, - comp_original, - spirv_opt_args, - )) - else: - shader_test_file = os.path.join(output_dir, 'tmp_shader_test.vkscript') - with open_helper(shader_test_file, 'w') as f: - f.write(vkscriptify_comp( - comp, - json_file, - comp_original, - spirv_opt_args, - )) + shader_test_file = os.path.join(output_dir, 'tmp_shader_test.amber') + with open_helper(shader_test_file, 'w') as f: + f.write(amberscriptify_comp( + comp, + json_file, + comp_original, + spirv_opt_args, + )) # FIXME: in case of multiple SSBOs, we should pass the binding of the ones to be dumped ssbo_binding = str(get_ssbo_binding(json_file)) @@ -1819,7 +1590,6 @@ def main_helper(args): # Optional arguments parser.add_argument('--serial', help=SERIAL_OPTION_HELP) parser.add_argument('--legacy-worker', action='store_true', help=LEGACY_OPTION_HELP) - parser.add_argument('--use-amberscript', action='store_true', help=USE_AMBERSCRIPT_OPTION_HELP) parser.add_argument('--skip-render', action='store_true', help=SKIP_RENDER_OPTION_HELP) parser.add_argument('--spirvopt', help=SPIRV_OPT_OPTION_HELP) parser.add_argument('--force', action='store_true', help=FORCE_OPTION_HELP) @@ -1910,8 +1680,7 @@ def main_helper(args): force=args.force, is_android=(args.target == 'android'), skip_render=args.skip_render, - spirv_opt_args=spirv_args, - use_amberscript=args.use_amberscript, + spirv_opt_args=spirv_args ) return @@ -1935,8 +1704,7 @@ def main_helper(args): force=args.force, is_android=(args.target == 'android'), skip_render=args.skip_render, - spirv_opt_args=spirv_args, - use_amberscript=args.use_amberscript, + spirv_opt_args=spirv_args ) finally: log_to_file = None diff --git a/python/src/main/python/test_scripts/test_runspv/runspv_tests.py b/python/src/main/python/test_scripts/test_runspv/runspv_tests.py index 3dfd7ce2a..479282dcc 100755 --- a/python/src/main/python/test_scripts/test_runspv/runspv_tests.py +++ b/python/src/main/python/test_scripts/test_runspv/runspv_tests.py @@ -118,8 +118,6 @@ def check_images_match(tmp_path: pathlib2.Path, is_amber_1: bool, is_amber_2: bool, fuzzy_image_comparison: bool, - use_amberscript_1: bool=None, - use_amberscript_2: bool=None, spirvopt_options_1: str=None, spirvopt_options_2: str=None): out_dir_1 = tmp_path / 'out_1' @@ -128,16 +126,12 @@ def check_images_match(tmp_path: pathlib2.Path, args_1 = ['android' if is_android_1 else 'host', get_image_test(json_filename), str(out_dir_1)] if not is_amber_1: args_1.append('--legacy-worker') - elif use_amberscript_1: - args_1.append('--use-amberscript') if spirvopt_options_1: args_1.append('--spirvopt=' + spirvopt_options_1) args_2 = ['android' if is_android_2 else 'host', get_image_test(json_filename), str(out_dir_2)] if not is_amber_2: args_2.append('--legacy-worker') - elif use_amberscript_2: - args_2.append('--use-amberscript') if spirvopt_options_2: args_2.append('--spirvopt=' + spirvopt_options_2) @@ -181,26 +175,9 @@ def check_images_match_spirvopt(tmp_path: pathlib2.Path, json_filename: str, opt spirvopt_options_2=None) -def check_images_match_vkscript_amberscript(tmp_path: pathlib2.Path, json_filename: str, - is_android: bool) -> None: - check_images_match(tmp_path, - json_filename=json_filename, - is_android_1=is_android, - is_android_2=is_android, - is_amber_1=True, - is_amber_2=True, - fuzzy_image_comparison=False, - use_amberscript_1=False, - use_amberscript_2=True, - spirvopt_options_1=None, - spirvopt_options_2=None) - - def check_compute_matches(tmp_path: pathlib2.Path, json_filename: str, is_android_1: bool, is_android_2: bool, - use_amberscript_1: bool=None, - use_amberscript_2: bool=None, spirvopt_options_1: str=None, spirvopt_options_2: str=None): out_dir_1 = tmp_path / 'out_1' @@ -208,10 +185,6 @@ def check_compute_matches(tmp_path: pathlib2.Path, json_filename: str, args_1 = ['android' if is_android_1 else 'host', get_compute_samples_dir() + os.sep + json_filename, str(out_dir_1)] args_2 = ['android' if is_android_2 else 'host', get_compute_samples_dir() + os.sep + json_filename, str(out_dir_2)] - if use_amberscript_1: - args_1.append('--use-amberscript') - if use_amberscript_2: - args_2.append('--use-amberscript') if spirvopt_options_1: args_1.append('--spirvopt=' + spirvopt_options_1) if spirvopt_options_2: @@ -243,17 +216,6 @@ def check_host_and_android_match_compute(tmp_path: pathlib2.Path, json_filename: is_android_2=True) -def check_compute_matches_vkscript_amberscript(tmp_path: pathlib2.Path, json_filename: str, is_android: bool): - check_compute_matches(tmp_path, - json_filename=json_filename, - is_android_1=is_android, - is_android_2=is_android, - use_amberscript_1=False, - use_amberscript_2=True, - spirvopt_options_1=None, - spirvopt_options_2=None) - - def check_no_image_skip_render(tmp_path: pathlib2.Path, is_android: bool, is_legacy_worker: bool, json_filename: str): out_dir = tmp_path / 'out' @@ -809,173 +771,3 @@ def test_image_0006_spirvopt_android_legacy(tmp_path: pathlib2.Path): options='-Os', is_android=True, is_amber=False) - -################################# -# VkScript vs AmberScript - -def test_image_0000_vksript_amberscript_host(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0000.json', - is_android=False) - - -def test_image_0000_vksript_amberscript_android(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0000.json', - is_android=True) - - -def test_image_0001_vksript_amberscript_host(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0001.json', - is_android=False) - - -def test_image_0001_vksript_amberscript_android(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0001.json', - is_android=True) - - -def test_image_0002_vksript_amberscript_host(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0002.json', - is_android=False) - - -def test_image_0002_vksript_amberscript_android(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0002.json', - is_android=True) - - -def test_image_0003_vksript_amberscript_host(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0003.json', - is_android=False) - - -def test_image_0003_vksript_amberscript_android(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0003.json', - is_android=True) - - -def test_image_0004_vksript_amberscript_host(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0004.json', - is_android=False) - - -def test_image_0004_vksript_amberscript_android(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0004.json', - is_android=True) - - -def test_image_0005_vksript_amberscript_host(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0005.json', - is_android=False) - - -def test_image_0005_vksript_amberscript_android(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0005.json', - is_android=True) - - -def test_image_0006_vksript_amberscript_host(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0006.json', - is_android=False) - - -def test_image_0006_vksript_amberscript_android(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0006.json', - is_android=True) - - -def test_image_0007_vksript_amberscript_host(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0007.json', - is_android=False) - - -def test_image_0007_vksript_amberscript_android(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0007.json', - is_android=True) - - -def test_image_0008_vksript_amberscript_host(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0008.json', - is_android=False) - - -def test_image_0008_vksript_amberscript_android(tmp_path: pathlib2.Path): - check_images_match_vkscript_amberscript(tmp_path, - 'image_test_0008.json', - is_android=True) - - -def test_compute_0001_vkscript_amberscript_host(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0001-findmax.json', - is_android=False) - - -def test_compute_0001_vkscript_amberscript_android(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0001-findmax.json', - is_android=True) - - -def test_compute_0002_vkscript_amberscript_host(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0002-smooth-mean.json', - is_android=False) - - -def test_compute_0002_vkscript_amberscript_android(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0002-smooth-mean.json', - is_android=True) - - -def test_compute_0003_vkscript_amberscript_host(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0003-random-middle-square.json', - is_android=False) - - -def test_compute_0003_vkscript_amberscript_android(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0003-random-middle-square.json', - is_android=True) - - -def test_compute_0004_vkscript_amberscript_host(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0004-koggestone.json', - is_android=False) - - -def test_compute_0004_vkscript_amberscript_android(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0004-koggestone.json', - is_android=True) - - -def test_compute_0005_vkscript_amberscript_host(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0005-sklansky.json', - is_android=False) - - -def test_compute_0005_vkscript_amberscript_android(tmp_path: pathlib2.Path): - check_compute_matches_vkscript_amberscript(tmp_path, - 'comp-0005-sklansky.json', - is_android=True) From 0d0d4716e952ee27d521b5faac360ae605a85c4a Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 24 Sep 2019 10:49:28 +0100 Subject: [PATCH 142/180] gfauto: better handle Android device reboot (#757) Fixes #696 --- gfauto/gfauto/android_device.py | 96 ++++++++++++++++++++++++++------- gfauto/whitelist.dic | 1 + 2 files changed, 79 insertions(+), 18 deletions(-) diff --git a/gfauto/gfauto/android_device.py b/gfauto/gfauto/android_device.py index 10320f091..521fba084 100644 --- a/gfauto/gfauto/android_device.py +++ b/gfauto/gfauto/android_device.py @@ -42,6 +42,10 @@ BUSY_WAIT_SLEEP_SLOW = 1.0 +WAIT_AFTER_BOOT_ANIMATION = 10 + +WAIT_AFTER_BOOT_AND_UNLOCK = 20 + ADB_DEFAULT_TIME_LIMIT = 30 ADB_SHORT_LOGCAT_TIME_LIMIT = 3 @@ -101,15 +105,23 @@ def adb_can_fail( def stay_awake_warning(serial: Optional[str] = None) -> None: - res = adb_check(serial, ["shell", "settings get global stay_on_while_plugged_in"]) - if str(res.stdout).strip() == "0": - log('\nWARNING: please enable "Stay Awake" from developer settings\n') + try: + res = adb_check( + serial, ["shell", "settings get global stay_on_while_plugged_in"] + ) + if str(res.stdout).strip() == "0": + log('\nWARNING: please enable "Stay Awake" from developer settings\n') + except subprocess.CalledProcessError: + log( + "Failed to check Stay Awake setting. This can happen if the device has just booted." + ) def is_screen_off_or_locked(serial: Optional[str] = None) -> bool: """:return: True: the screen is off or locked. False: unknown.""" res = adb_can_fail(serial, ["shell", "dumpsys nfc"]) if res.returncode != 0: + log("Failed to run dumpsys.") return False stdout = str(res.stdout) @@ -162,8 +174,71 @@ def get_all_android_devices() -> List[Device]: def prepare_device(wait_for_screen: bool, serial: Optional[str] = None) -> None: + device_was_booting_or_locked = False + + res = adb_can_fail(serial, ["get-state"]) + if res.returncode != 0 or res.stdout.strip() != "device": + device_was_booting_or_locked = True + adb_check(serial, ["wait-for-device"], timeout=None) + + # Conservatively check that booting has finished. + while True: + + log("Checking if boot animation has finished.") + + res_bootanim = adb_can_fail(serial, ["shell", "getprop init.svc.bootanim"]) + res_bootanim_exit = adb_can_fail( + serial, ["shell", "getprop service.bootanim.exit"] + ) + if res_bootanim.returncode != 0 and res_bootanim_exit != 0: + # Both commands failed so there is no point in trying to use either result. + log("Could not check boot animation; continuing.") + break + if ( + res_bootanim.stdout.strip() != "running" + and res_bootanim_exit.stdout.strip() != "0" + ): + # Both commands suggest the boot animation is NOT running. + # This may include one or both commands returning nothing because the property doesn't exist on this device. + log("Boot animation is not running.") + break + + # If either command suggests the boot animation is running, we assume it is accurate, so we wait. + device_was_booting_or_locked = True + time.sleep(BUSY_WAIT_SLEEP_SLOW) + + if device_was_booting_or_locked: + log( + f"Device appeared to be booting previously, so waiting a further {WAIT_AFTER_BOOT_ANIMATION} seconds." + ) + time.sleep(WAIT_AFTER_BOOT_ANIMATION) + + if wait_for_screen: + stay_awake_warning(serial) + # We cannot reliably know if the screen is on, but this function definitely knows if it is + # off or locked. So we wait here while we definitely know there is an issue. + count = 0 + while is_screen_off_or_locked(serial): + log( + "\nWARNING: The screen appears to be off or locked. Please unlock the device and ensure 'Stay Awake' is enabled in developer settings.\n" + ) + device_was_booting_or_locked = True + time.sleep(BUSY_WAIT_SLEEP_SLOW) + count += 1 + if count > 1 and (count % 3) == 0: + log("Pressing the menu key.") + adb_can_fail(serial, ["shell", "input keyevent 82"], verbose=True) + + if device_was_booting_or_locked: + log( + f"Device appeared to be booting previously, so waiting a further {WAIT_AFTER_BOOT_AND_UNLOCK} seconds." + ) + time.sleep(WAIT_AFTER_BOOT_AND_UNLOCK) + + log("Clearing logcat.") adb_check(serial, ["logcat", "-c"]) + res = adb_can_fail(serial, ["shell", "test -e " + ANDROID_DEVICE_AMBER]) check( res.returncode == 0, @@ -182,21 +257,6 @@ def prepare_device(wait_for_screen: bool, serial: Optional[str] = None) -> None: ], ) - if wait_for_screen: - stay_awake_warning(serial) - # We cannot reliably know if the screen is on, but this function definitely knows if it is - # off or locked. So we wait here while we definitely know there is an issue. - count = 0 - while is_screen_off_or_locked(serial): - log( - "\nWARNING: The screen appears to be off or locked. Please unlock the device and ensure 'Stay Awake' is enabled in developer settings.\n" - ) - time.sleep(BUSY_WAIT_SLEEP_SLOW) - count += 1 - if count > 1 and (count % 10) == 0: - log("Pressing the menu key.") - adb_can_fail(serial, ["shell", "input keyevent 82"], verbose=True) - def run_amber_on_device( amber_script_file: Path, diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index 96a1bcf31..fd295f7d1 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -185,4 +185,5 @@ cwd params deqp tgz +bootanim From c2fe5c21c1d8fac1952553ef2ff35f06a6a50213 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Fri, 27 Sep 2019 13:28:56 +0100 Subject: [PATCH 143/180] gfauto: add GLSL wrong image support (#761) --- gfauto/README.md | 8 +- gfauto/gfauto/binaries_util.py | 12 +- gfauto/gfauto/fuzz_glsl_test.py | 365 ++++++++++++------- gfauto/gfauto/fuzz_spirv_test.py | 13 +- gfauto/gfauto/gfauto_interestingness_test.py | 82 +++-- gfauto/gfauto/glsl_generate_util.py | 21 ++ gfauto/gfauto/result_util.py | 20 + gfauto/gfauto/signature_util.py | 11 + gfauto/gfauto/spirv_fuzz_util.py | 1 + gfauto/gfauto/test_util.py | 27 +- gfauto/gfauto/tool.py | 221 ++++------- gfauto/whitelist.dic | 4 +- 12 files changed, 447 insertions(+), 338 deletions(-) diff --git a/gfauto/README.md b/gfauto/README.md index 91dd13c33..fd1c5d93a 100644 --- a/gfauto/README.md +++ b/gfauto/README.md @@ -159,7 +159,13 @@ import secrets Assuming you saved those to `../seeds.txt`, you can run parallel instances of `gfauto_fuzz` using: ```sh -parallel -j 64 gfauto_fuzz --skip_writing_binary_recipes --iteration_seed -- $(cat ../seeds.txt) +parallel -j 32 gfauto_fuzz --skip_writing_binary_recipes --iteration_seed -- $(cat ../seeds.txt) ``` This is probably only suitable for testing the `host_preprocessor` and `swift_shader` virtual devices; running parallel tests on actual hardware is likely to give unreliable results. + +You can run parallel instances of gfauto (just for increased throughput, not with fixed seeds) using: + +```sh +parallel -j 32 -i gfauto_fuzz --skip_writing_binary_recipes -- $(seq 100) +``` diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py index ee76a8844..73bfd80a2 100644 --- a/gfauto/gfauto/binaries_util.py +++ b/gfauto/gfauto/binaries_util.py @@ -56,7 +56,7 @@ "Mac_x64_RelWithDebInfo", ] -DEFAULT_SPIRV_TOOLS_VERSION = "ad7f2c5c4c7f51360e9e079109a9217aa5ba5cc0" +DEFAULT_SPIRV_TOOLS_VERSION = "6b072126595dd8c2448eb1fda616251c5e6d7079" DEFAULT_BINARIES = [ Binary( @@ -72,7 +72,7 @@ Binary( name="swift_shader_icd", tags=["Debug"], - version="f25a1c68473b868ce61e97fe5b830c0cdd7e8181", + version="b6fa949c45397bd1fbfda769a104b9e8884f343e", ), ] @@ -527,4 +527,12 @@ def get_graphics_fuzz_121() -> List[recipe_wrap.RecipeWrap]: version_hash="ad7f2c5c4c7f51360e9e079109a9217aa5ba5cc0", build_version_hash="b97215064186d731eac68adcc5ade4c7b96b265b", ) + + _get_built_in_spirv_tools_version( + version_hash="6b072126595dd8c2448eb1fda616251c5e6d7079", + build_version_hash="74886e02e26453ee1dcba4290157e9c8a5e8d07e", + ) + + _get_built_in_swift_shader_version( + version_hash="b6fa949c45397bd1fbfda769a104b9e8884f343e", + build_version_hash="70e8d53b94227fed094975771d96f240f7d00911", + ) ) diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py index fcc9befe3..7d4311f39 100644 --- a/gfauto/gfauto/fuzz_glsl_test.py +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -22,7 +22,7 @@ import random import subprocess from pathlib import Path -from typing import List, Optional +from typing import Iterable, List, Optional from gfauto import ( amber_converter, @@ -108,17 +108,22 @@ def create_staging_tests( staging_name = staging_dir.name template_source_dir = staging_dir / "source_template" - # Copy in a randomly chosen reference. - reference_glsl_shader_job = shader_job_util.copy( - random.choice(references), - template_source_dir / test_util.REFERENCE_DIR / test_util.SHADER_JOB, - ) + # Pick a randomly chosen reference. + unprepared_reference_shader_job = random.choice(references) # TODO: Allow GraphicsFuzz to be downloaded. + # Create the prepared (for Vulkan GLSL) reference. + glsl_generate_util.run_prepare_reference( + util.tool_on_path("graphicsfuzz-tool"), + unprepared_reference_shader_job, + template_source_dir / test_util.REFERENCE_DIR / test_util.SHADER_JOB, + ) + + # Generate the variant (GraphicsFuzz requires the unprepared reference as input). glsl_generate_util.run_generate( util.tool_on_path("graphicsfuzz-tool"), - reference_glsl_shader_job, + unprepared_reference_shader_job, donors_dir, template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, seed=str(random.getrandbits(glsl_generate_util.GENERATE_SEED_BITS)), @@ -176,28 +181,21 @@ def run( if not device: device = test.device - log(f"Running test on device:\n{device.name}") - result_output_dir = run_shader_job( - shader_job=test_util.get_shader_job_path(test_dir, is_variant=True), - output_dir=test_util.get_results_directory( - test_dir, device.name, is_variant=True - ), - test=test, - device=device, + source_dir=test_util.get_source_dir(test_dir), + output_dir=test_util.get_results_directory(test_dir, device.name), binary_manager=binary_manager, + device=device, ) return result_util.get_status(result_output_dir) -def maybe_add_report( +def maybe_add_report( # pylint: disable=too-many-locals; test_dir: Path, reports_dir: Path, device: Device, settings: Settings ) -> Optional[Path]: - result_output_dir = test_util.get_results_directory( - test_dir, device.name, is_variant=True - ) + result_output_dir = test_util.get_results_directory(test_dir, device.name) status = result_util.get_status(result_output_dir) @@ -225,14 +223,15 @@ def maybe_add_report( if (signature_dir / "NOT_INTERESTING").exists(): return None - # If we have reached the maximum number of crashes per signature for this device, don't create a report. - num_duplicates = [ - report_dir - for report_dir in signature_dir.iterdir() - if report_dir.is_dir() and report_dir.name.endswith(f"_{device.name}") - ] - if len(num_duplicates) >= settings.maximum_duplicate_crashes: - return None + if signature != signature_util.BAD_IMAGE_SIGNATURE: + # If we have reached the maximum number of crashes per signature for this device, don't create a report. + num_duplicates = [ + report_dir + for report_dir in signature_dir.iterdir() + if report_dir.is_dir() and report_dir.name.endswith(f"_{device.name}") + ] + if len(num_duplicates) >= settings.maximum_duplicate_crashes: + return None # We include the device name in the directory name because it is possible that this test crashes on two # different devices but gives the same crash signature in both cases (e.g. for generic signatures @@ -242,6 +241,39 @@ def maybe_add_report( util.copy_dir(test_dir, test_dir_in_reports) + if signature != signature_util.BAD_IMAGE_SIGNATURE: + + # If we found a crash, rename the directories for all shaders other than the variant. Thus, only the variant + # shader will run. + + bad_shader_name = result_util.get_status_bad_shader_name( + test_util.get_results_directory(test_dir_in_reports, device.name) + ) + + # TODO: Could possibly improve this. Could try scanning the Amber log to figure out which shader failed? + + if not bad_shader_name: + log("WARNING: assuming that the bad shader is the variant") + bad_shader_name = test_util.VARIANT_DIR + + shader_jobs = tool.get_shader_jobs( + test_util.get_source_dir(test_dir_in_reports) + ) + found_bad_shader = False + for shader_job in shader_jobs: + if shader_job.name == bad_shader_name: + found_bad_shader = True + else: + shader_job.shader_job.parent.rename( + shader_job.shader_job.parent.parent / f"_{shader_job.name}" + ) + check( + found_bad_shader, + AssertionError( + f"Could not find bad shader at: {test_util.get_source_dir(test_dir_in_reports) / bad_shader_name}" + ), + ) + test_metadata = test_util.metadata_read(test_dir_in_reports) test_metadata.crash_signature = signature test_metadata.device.CopyFrom(device) @@ -255,19 +287,21 @@ def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: test = test_util.metadata_read(test_dir) try: - part_1_reduced_test = run_reduction( + reduced_test = test_dir + reduced_test = run_reduction( test_dir_reduction_output=test_dir, - test_dir_to_reduce=test_dir, + test_dir_to_reduce=reduced_test, preserve_semantics=True, reduction_name="1", ) - part_2_reduced_test = run_reduction( - test_dir_reduction_output=test_dir, - test_dir_to_reduce=part_1_reduced_test, - preserve_semantics=False, - reduction_name="2", - ) + if test.crash_signature != signature_util.BAD_IMAGE_SIGNATURE: + reduced_test = run_reduction( + test_dir_reduction_output=test_dir, + test_dir_to_reduce=reduced_test, + preserve_semantics=False, + reduction_name="2", + ) device_name = test.device.name @@ -276,8 +310,7 @@ def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: test_dir, device_name, fuzz.BEST_REDUCTION_NAME ) util.make_directory_symlink( - new_symlink_file_path=best_reduced_test_link, - existing_dir=part_2_reduced_test, + new_symlink_file_path=best_reduced_test_link, existing_dir=reduced_test ) except ReductionFailedError as ex: # Create a symlink to the failed reduction so it is easy to investigate failed reductions. @@ -332,111 +365,160 @@ def handle_test( return issue_found -def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branches; - shader_job: Path, +def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branches, too-many-locals, too-many-statements; + source_dir: Path, output_dir: Path, - test: Test, - device: Device, binary_manager: binaries_util.BinaryManager, - use_default_binaries: bool = False, + test: Optional[Test] = None, + device: Optional[Device] = None, + ignore_test_and_device_binaries: bool = False, + shader_overrides: Iterable[tool.NameAndShaderJob] = (), ) -> Path: - with util.file_open_text(output_dir / "log.txt", "w") as log_file: try: gflogging.push_stream_for_logging(log_file) - child_binary_manager = binary_manager - if not use_default_binaries: - child_binary_manager = child_binary_manager.get_child_binary_manager( - list(device.binaries) + list(test.binaries) - ) - # TODO: Find amber path. NDK or host. # TODO: If Amber is going to be used, check if Amber can use Vulkan debug layers now, and if not, pass that # info down via a bool. + if not test: + test = test_util.metadata_read_from_path( + source_dir / test_util.TEST_METADATA + ) + + if not device: + device = test.device + + log(f"Running test on device:\n{device.name}") + + if not ignore_test_and_device_binaries: + binary_manager = binary_manager.get_child_binary_manager( + list(device.binaries) + list(test.binaries) + ) + spirv_opt_hash: Optional[str] = None if test.glsl.spirv_opt_args: - spirv_opt_hash = child_binary_manager.get_binary_by_name( + spirv_opt_hash = binary_manager.get_binary_by_name( binaries_util.SPIRV_OPT_NAME ).version - try: - spirv_asm_shader_job_path, spirv_shader_job_path = tool.compile_shader_job( - shader_job, - output_dir, - child_binary_manager, - list(test.glsl.spirv_opt_args), - ) - except subprocess.CalledProcessError: - util.file_write_text( - result_util.get_status_path(output_dir), fuzz.STATUS_TOOL_CRASH - ) - return output_dir - except subprocess.TimeoutExpired: - util.file_write_text( - result_util.get_status_path(output_dir), fuzz.STATUS_TOOL_TIMEOUT - ) - return output_dir - - is_compute = bool( - shader_job_util.get_related_files( - shader_job, [shader_job_util.EXT_COMP] - ) - ) - - # Device types: |preprocess| and |shader_compiler| don't need an AmberScript file. + shader_jobs = tool.get_shader_jobs(source_dir, overrides=shader_overrides) - if device.HasField("preprocess"): - # The "preprocess" device type just needs to get this far, so this is a success. - util.file_write_text( - result_util.get_status_path(output_dir), fuzz.STATUS_SUCCESS - ) - return output_dir + combined_spirv_shader_jobs: List[tool.SpirvCombinedShaderJob] = [] - if device.HasField("shader_compiler"): + for shader_job in shader_jobs: try: - shader_compiler_util.run_shader_job( - device.shader_compiler, spirv_shader_job_path, output_dir - ) - # The shader compiler succeeded; this is a success. - util.file_write_text( - result_util.get_status_path(output_dir), fuzz.STATUS_SUCCESS + combined_spirv_shader_jobs.append( + tool.compile_shader_job( + name=shader_job.name, + input_json=shader_job.shader_job, + work_dir=output_dir / shader_job.name, + binary_paths=binary_manager, + spirv_opt_args=list(test.glsl.spirv_opt_args), + ) ) - return output_dir except subprocess.CalledProcessError: - util.file_write_text( - result_util.get_status_path(output_dir), fuzz.STATUS_CRASH + result_util.write_status( + output_dir, fuzz.STATUS_TOOL_CRASH, shader_job.name ) return output_dir except subprocess.TimeoutExpired: - util.file_write_text( - result_util.get_status_path(output_dir), fuzz.STATUS_TIMEOUT + result_util.write_status( + output_dir, fuzz.STATUS_TOOL_TIMEOUT, shader_job.name ) return output_dir + # Device types: |preprocess| and |shader_compiler| don't need an AmberScript file. + + if device.HasField("preprocess"): + # The "preprocess" device type just needs to get this far, so this is a success. + result_util.write_status(output_dir, fuzz.STATUS_SUCCESS) + return output_dir + + if device.HasField("shader_compiler"): + for combined_spirv_shader_job in combined_spirv_shader_jobs: + try: + shader_compiler_util.run_shader_job( + device.shader_compiler, + combined_spirv_shader_job.spirv_shader_job, + output_dir, + ) + except subprocess.CalledProcessError: + result_util.write_status( + output_dir, + fuzz.STATUS_CRASH, + combined_spirv_shader_job.name, + ) + return output_dir + except subprocess.TimeoutExpired: + result_util.write_status( + output_dir, + fuzz.STATUS_TIMEOUT, + combined_spirv_shader_job.name, + ) + return output_dir + + # The shader compiler succeeded on all files; this is a success. + result_util.write_status(output_dir, fuzz.STATUS_SUCCESS) + return output_dir + # Other device types need an AmberScript file. - amber_script_file = tool.amberfy( - spirv_asm_shader_job_path, - output_dir / "test.amber", - amber_converter.AmberfySettings( + amber_converter_shader_job_files = [ + amber_converter.ShaderJobFile( + name_prefix=combined_spirv_shader_job.name, + asm_spirv_shader_job_json=combined_spirv_shader_job.spirv_asm_shader_job, + glsl_source_json=combined_spirv_shader_job.glsl_source_shader_job, + processing_info="", + ) + for combined_spirv_shader_job in combined_spirv_shader_jobs + ] + + # Check if the first is the reference shader; if so, pull it out into its own variable. + + reference: Optional[amber_converter.ShaderJobFile] = None + variants = amber_converter_shader_job_files + + if ( + amber_converter_shader_job_files[0].name_prefix + == test_util.REFERENCE_DIR + ): + reference = amber_converter_shader_job_files[0] + variants = variants[1:] + elif len(variants) > 1: + raise AssertionError( + "More than one variant, but no reference. This is unexpected." + ) + + amber_script_file = amber_converter.spirv_asm_shader_job_to_amber_script( + shader_job_file_amber_test=amber_converter.ShaderJobFileBasedAmberTest( + reference_asm_spirv_job=reference, variants_asm_spirv_job=variants + ), + output_amber_script_file_path=output_dir / "test.amber", + amberfy_settings=amber_converter.AmberfySettings( spirv_opt_args=list(test.glsl.spirv_opt_args), spirv_opt_hash=spirv_opt_hash, ), - shader_job, + ) + + is_compute = bool( + shader_job_util.get_related_files( + combined_spirv_shader_jobs[0].spirv_shader_job, + [shader_job_util.EXT_COMP], + ) ) if device.HasField("host") or device.HasField("swift_shader"): icd: Optional[Path] = None if device.HasField("swift_shader"): - icd = child_binary_manager.get_binary_path_by_name( + icd = binary_manager.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME ).path - # Run the shader on the host using Amber. + # Run the test on the host using Amber. host_device_util.run_amber( amber_script_file, output_dir, @@ -475,17 +557,22 @@ def run_reduction( test_dir_to_reduce: Path, preserve_semantics: bool, reduction_name: str = "reduction1", - device_name: Optional[str] = None, ) -> Path: test = test_util.metadata_read(test_dir_to_reduce) - if not device_name and not test.device: - raise AssertionError( + check( + bool(test.device), + AssertionError( f"Cannot reduce {str(test_dir_to_reduce)}; device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}" - ) + ), + ) - if not device_name: - device_name = test.device.name + check( + bool(test.device.name), + AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)}; device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}" + ), + ) if not test.crash_signature: raise AssertionError( @@ -493,17 +580,33 @@ def run_reduction( f"for now, only crash reductions are supported" ) - reduced_test_dir_1 = test_util.get_reduced_test_dir( - test_dir_reduction_output, device_name, reduction_name + # E.g. reports/crashes/no_signature/d50c96e8_opt_rand2_test_phone_ABC/results/phone_ABC/reductions/1 + # Will contain work/ and source/ + reduced_test_dir = test_util.get_reduced_test_dir( + test_dir_reduction_output, test.device.name, reduction_name ) + source_dir = test_util.get_source_dir(test_dir_to_reduce) + + shader_jobs = tool.get_shader_jobs(source_dir) + + # TODO: if needed, this could become a parameter to this function. + name_of_shader_to_reduce = shader_jobs[0].name + + if len(shader_jobs) > 1: + check( + len(shader_jobs) == 2 and shader_jobs[1].name == test_util.VARIANT_DIR, + AssertionError( + "Can only reduce tests with shader jobs reference and variant, or just variant." + ), + ) + name_of_shader_to_reduce = shader_jobs[1].name + reduction_work_variant_dir = run_glsl_reduce( - input_shader_job=test_util.get_shader_job_path( - test_dir_to_reduce, is_variant=True - ), - test_metadata_path=test_util.get_metadata_path(test_dir_to_reduce), + source_dir=source_dir, + name_of_shader_to_reduce=name_of_shader_to_reduce, output_dir=test_util.get_reduction_work_directory( - reduced_test_dir_1, is_variant=True + reduced_test_dir, name_of_shader_to_reduce ), preserve_semantics=preserve_semantics, ) @@ -521,33 +624,39 @@ def run_reduction( ), ) - # Finally, write the test metadata and shader job, so the returned directory can be used as a test_dir. + # Finally, create the source_dir so the returned directory can be used as a test_dir. - test_util.metadata_write(test, reduced_test_dir_1) + util.copy_dir(source_dir, test_util.get_source_dir(reduced_test_dir)) shader_job_util.copy( final_reduced_shader_job_path, - test_util.get_shader_job_path(reduced_test_dir_1, is_variant=True), + test_util.get_shader_job_path(reduced_test_dir, name_of_shader_to_reduce), ) - return reduced_test_dir_1 + return reduced_test_dir def run_glsl_reduce( - input_shader_job: Path, - test_metadata_path: Path, + source_dir: Path, + name_of_shader_to_reduce: str, output_dir: Path, preserve_semantics: bool = False, ) -> Path: + input_shader_job = source_dir / name_of_shader_to_reduce / test_util.SHADER_JOB + cmd = [ str(tool_on_path("glsl-reduce")), str(input_shader_job), "--output", str(output_dir), + # This ensures the arguments that follow are all positional arguments. "--", "gfauto_interestingness_test", - str(test_metadata_path), + str(source_dir), + # --override_shader requires two parameters to follow; the second will be added by glsl-reduce (the shader.json file). + "--override_shader", + str(name_of_shader_to_reduce), ] if preserve_semantics: @@ -589,21 +698,17 @@ def create_summary_and_reproduce_glsl( reduced_glsl = util.copy_dir(reduced_source_dir, summary_dir / "reduced_glsl") run_shader_job( - unreduced_glsl / test_util.VARIANT_DIR / test_util.SHADER_JOB, - summary_dir / "unreduced_glsl_result" / test_util.VARIANT_DIR, - test_metadata, - device, - binary_manager, + source_dir=unreduced_glsl, + output_dir=(summary_dir / "unreduced_glsl_result"), + binary_manager=binary_manager, ) variant_reduced_glsl_result: Optional[Path] = None if reduced_glsl: variant_reduced_glsl_result = run_shader_job( - reduced_glsl / test_util.VARIANT_DIR / test_util.SHADER_JOB, - summary_dir / "reduced_glsl_result" / test_util.VARIANT_DIR, - test_metadata, - device, - binary_manager, + source_dir=reduced_glsl, + output_dir=(summary_dir / "reduced_glsl_result"), + binary_manager=binary_manager, ) # Some post-processing for common error types. diff --git a/gfauto/gfauto/fuzz_spirv_test.py b/gfauto/gfauto/fuzz_spirv_test.py index cf00be9d8..821f8c2de 100644 --- a/gfauto/gfauto/fuzz_spirv_test.py +++ b/gfauto/gfauto/fuzz_spirv_test.py @@ -35,7 +35,6 @@ util, ) from gfauto.device_pb2 import Device -from gfauto.gflogging import log from gfauto.settings_pb2 import Settings from gfauto.test_pb2 import Test, TestSpirvFuzz @@ -74,16 +73,11 @@ def run( if not device: device = test.device - log(f"Running test on device:\n{device.name}") - result_output_dir = fuzz_glsl_test.run_shader_job( - shader_job=test_util.get_shader_job_path(test_dir, is_variant=True), - output_dir=test_util.get_results_directory( - test_dir, device.name, is_variant=True - ), - test=test, - device=device, + source_dir=test_util.get_source_dir(test_dir), + output_dir=test_util.get_results_directory(test_dir, device.name), binary_manager=binary_manager, + device=device, ) return result_util.get_status(result_output_dir) @@ -157,6 +151,7 @@ def fuzz_spirv( seed=str(random.getrandbits(spirv_fuzz_util.GENERATE_SEED_BITS)), ) except subprocess.CalledProcessError: + util.mkdirs_p(fuzz_failures_dir) if len(list(fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures: util.copy_dir(staging_dir, fuzz_failures_dir / staging_dir.name) return diff --git a/gfauto/gfauto/gfauto_interestingness_test.py b/gfauto/gfauto/gfauto_interestingness_test.py index 605a45d7e..e665d9c47 100644 --- a/gfauto/gfauto/gfauto_interestingness_test.py +++ b/gfauto/gfauto/gfauto_interestingness_test.py @@ -26,6 +26,7 @@ import argparse import sys from pathlib import Path +from typing import List, Optional, Tuple from gfauto import ( artifact_util, @@ -35,10 +36,10 @@ result_util, signature_util, test_util, + tool, util, ) from gfauto.gflogging import log -from gfauto.util import check, check_file_exists # TODO: Maybe add helper method and throw exceptions instead of calling sys.exit. @@ -48,21 +49,25 @@ # A device (or test?) could then even specify a custom interestingness command, although the default one would probably # be the same for all devices and it would look at the device info in the test_json? -# TODO: Support more than just GLSL crash tests. - -def main() -> None: +def main() -> None: # pylint: disable=too-many-statements; parser = argparse.ArgumentParser( - description="Interestingness test that runs a shader using Amber, " + description="Interestingness test that runs a test using Amber, " "calculates the crash signature based on the result, and returns 0 " "if the signature matches the expected crash signature." ) parser.add_argument( - "test_json", - help="The .json test metadata file path that describes how to run the test.", + "source_dir", + help="The source directory containing the shaders and the test.json file that describes how to run the test.", + ) + parser.add_argument( + "--override_shader", + nargs=2, + metavar=("shader_name", "shader_job_json"), + help='Override one of the shader jobs. E.g.: "--override_shader variant temp/variant.json". Note that ' + "the output directory will be set to shader_job_json/ (with the .json extension removed) by default in this case.", ) - parser.add_argument("shader_job_json", help="The .json shader job file path.") parser.add_argument( "--use_default_binaries", @@ -77,51 +82,64 @@ def main() -> None: action="store_true", ) - parser.add_argument("--output", help="Output directory.", default=None) + parser.add_argument( + "--output", + help="Output directory. Required unless --override_shader is used; in this case " + "the output will be shader_job_json/ (a directory alongside the shader json with the same name).", + default=None, + ) parsed_args = parser.parse_args(sys.argv[1:]) - test_json: Path = Path(parsed_args.test_json) - shader_job_json: Path = Path(parsed_args.shader_job_json) + source_dir: Path = Path(parsed_args.source_dir) + override_shader: Optional[Tuple[str, str]] = parsed_args.override_shader + use_default_binaries: bool = parsed_args.use_default_binaries fallback_binaries: bool = parsed_args.fallback_binaries or use_default_binaries - output: Path = Path( - parsed_args.output - ) if parsed_args.output else shader_job_json.with_suffix("") - - check_file_exists(test_json) - check_file_exists(shader_job_json) + output: Path + if parsed_args.output: + output = Path(parsed_args.output) + elif override_shader: + output = Path(override_shader[1]).with_suffix("") + else: + raise AssertionError("Need --output or --override_shader parameter.") artifact_util.recipes_write_built_in() - test = test_util.metadata_read_from_path(test_json) - - check( - test.HasField("glsl") and bool(test.device) and bool(test.crash_signature), - AssertionError( - f"Provided test json {str(test_json)} does not have entries: glsl, device, crash_signature" - ), - ) - binary_manager = binaries_util.BinaryManager( binaries_util.DEFAULT_BINARIES if fallback_binaries else [], util.get_platform(), binaries_util.BUILT_IN_BINARY_RECIPES_PATH_PREFIX, ) + shader_overrides: List[tool.NameAndShaderJob] = [] + + if override_shader: + shader_overrides.append( + tool.NameAndShaderJob( + name=override_shader[0], shader_job=Path(override_shader[1]) + ) + ) + + # We don't need to read this to run the shader, but we need it afterwards anyway. + test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) + output_dir = fuzz_glsl_test.run_shader_job( - shader_job_json, + source_dir=source_dir, output_dir=output, - test=test, - device=test.device, binary_manager=binary_manager, - use_default_binaries=use_default_binaries, + test=test, + ignore_test_and_device_binaries=use_default_binaries, + shader_overrides=shader_overrides, ) log( - f"gfauto_interestingness_test: finished running {str(shader_job_json)} in {str(output_dir)}." + f"gfauto_interestingness_test: finished running {str(source_dir)} in {str(output_dir)}." ) + if override_shader: + log(f"The {override_shader[0]} shader was overridden with {override_shader[1]}") + status = result_util.get_status(output_dir) if test.expected_status: log("") @@ -142,7 +160,7 @@ def main() -> None: log("status != expected_status; not interesting") sys.exit(1) else: - # There is no expected status, so just assume it needs to be one of the "bad" statuses: + # There is no expected status given, so just assume it needs to be one of the "bad" statuses: if status not in ( fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH, diff --git a/gfauto/gfauto/glsl_generate_util.py b/gfauto/gfauto/glsl_generate_util.py index 9220fdeff..5153be344 100644 --- a/gfauto/gfauto/glsl_generate_util.py +++ b/gfauto/gfauto/glsl_generate_util.py @@ -28,6 +28,27 @@ GENERATE_SEED_BITS = 64 +def run_prepare_reference( + graphicsfuzz_tool_path: Path, + input_reference_shader_json: Path, + output_reference_shader_json: Path, +) -> Path: + util.file_mkdirs_parent(output_reference_shader_json) + cmd = [ + str(graphicsfuzz_tool_path), + "com.graphicsfuzz.generator.tool.PrepareReference", + "--generate-uniform-bindings", + "--max-uniforms", + "10", + str(input_reference_shader_json), + str(output_reference_shader_json), + ] + + subprocess_util.run(cmd) + + return output_reference_shader_json + + def run_generate( graphicsfuzz_tool_path: Path, reference_shader_json: pathlib.Path, diff --git a/gfauto/gfauto/result_util.py b/gfauto/gfauto/result_util.py index 07e2cb5fe..ff2fecbb3 100644 --- a/gfauto/gfauto/result_util.py +++ b/gfauto/gfauto/result_util.py @@ -21,14 +21,34 @@ """ from pathlib import Path +from typing import Optional from gfauto import util +def write_status( + result_output_dir: Path, status: str, bad_shader_name: Optional[str] = None +) -> None: + util.file_write_text(get_status_path(result_output_dir), status) + if bad_shader_name: + util.file_write_text( + get_status_bad_shader_name_path(result_output_dir), bad_shader_name + ) + + def get_status_path(result_output_dir: Path) -> Path: return result_output_dir / "STATUS" +def get_status_bad_shader_name(result_output_dir: Path) -> str: + bad_shader_name_path = get_status_bad_shader_name_path(result_output_dir) + return util.file_read_text_or_else(bad_shader_name_path, "") + + +def get_status_bad_shader_name_path(result_output_dir: Path) -> Path: + return result_output_dir / "BAD_SHADER" + + def get_status(result_output_dir: Path) -> str: status_file = get_status_path(result_output_dir) return util.file_read_text_or_else(status_file, "UNEXPECTED_ERROR") diff --git a/gfauto/gfauto/signature_util.py b/gfauto/gfauto/signature_util.py index cc226ecc7..6cfa0e956 100644 --- a/gfauto/gfauto/signature_util.py +++ b/gfauto/gfauto/signature_util.py @@ -118,6 +118,13 @@ PATTERN_LLVM_ERROR_DIAGNOSIS = re.compile(r"ERROR: LLVM DIAGNOSIS INFO: (.*)") +PATTERN_AMBER_TOLERANCE_ERROR = re.compile( + r"is greater th[ae]n tolerance|Buffers have different values" +) + +BAD_IMAGE_SIGNATURE = "bad_image" + + def remove_hex_like(string: str) -> str: temp = string # Remove hex like chunks of 4 or more. @@ -308,6 +315,10 @@ def get_signature_from_log_contents( # pylint: disable=too-many-return-statemen if "Resource deadlock would occur" in log_contents: return "Resource_deadlock_would_occur" + match = re.search(PATTERN_AMBER_TOLERANCE_ERROR, log_contents) + if match: + return BAD_IMAGE_SIGNATURE + return "no_signature" diff --git a/gfauto/gfauto/spirv_fuzz_util.py b/gfauto/gfauto/spirv_fuzz_util.py index 8b9b41d33..b12fea034 100644 --- a/gfauto/gfauto/spirv_fuzz_util.py +++ b/gfauto/gfauto/spirv_fuzz_util.py @@ -42,6 +42,7 @@ def run_generate( AssertionError(f"Expected {str(output_shader_spv)} to end with .spv"), ) + util.file_mkdirs_parent(output_shader_spv) cmd = [ str(spirv_fuzz_path), str(reference_shader_spv), diff --git a/gfauto/gfauto/test_util.py b/gfauto/gfauto/test_util.py index 497ec3d67..275ce5aca 100644 --- a/gfauto/gfauto/test_util.py +++ b/gfauto/gfauto/test_util.py @@ -63,36 +63,21 @@ def metadata_write_to_path(metadata: Test, test_metadata_path: Path) -> Path: return test_metadata_path -def get_shader_job_path(test_dir: Path, is_variant: bool = True) -> Path: - return ( - test_dir - / "source" - / (VARIANT_DIR if is_variant else REFERENCE_DIR) - / SHADER_JOB - ) +def get_shader_job_path(test_dir: Path, shader_name: str) -> Path: + return test_dir / "source" / shader_name / SHADER_JOB def get_device_directory(test_dir: Path, device_name: str) -> Path: return test_dir / "results" / device_name -def get_results_directory( - test_dir: Path, device_name: str, is_variant: bool = True -) -> Path: - return get_device_directory(test_dir, device_name) / ( - VARIANT_DIR if is_variant else REFERENCE_DIR - ) +def get_results_directory(test_dir: Path, device_name: str) -> Path: + return get_device_directory(test_dir, device_name) / "result" def get_reduced_test_dir(test_dir: Path, device_name: str, reduction_name: str) -> Path: return get_device_directory(test_dir, device_name) / "reductions" / reduction_name -def get_reduction_work_directory( - reduced_test_dir: Path, is_variant: bool = True -) -> Path: - return ( - reduced_test_dir - / "reduction_work" - / (VARIANT_DIR if is_variant else REFERENCE_DIR) - ) +def get_reduction_work_directory(reduced_test_dir: Path, name_of_shader: str) -> Path: + return reduced_test_dir / "reduction_work" / name_of_shader diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index f5e688146..33f5fe83b 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -20,7 +20,7 @@ """ from pathlib import Path -from typing import Iterator, List, Optional, Tuple +from typing import Dict, Iterable, Iterator, List, Optional from attr import dataclass @@ -36,7 +36,6 @@ test_util, util, ) -from gfauto.util import check AMBER_COMMAND_PROBE_TOP_LEFT_RED = "probe rgba (0, 0) (1, 0, 0, 1)\n" @@ -162,25 +161,33 @@ def validate_spirv_shader_job( @dataclass -class SpirvAndSpirvAsmShaderJob: +class SpirvCombinedShaderJob: + name: str spirv_asm_shader_job: Path spirv_shader_job: Path + glsl_source_shader_job: Optional[Path] def __iter__(self) -> Iterator[Path]: return iter((self.spirv_asm_shader_job, self.spirv_shader_job)) def compile_shader_job( + name: str, input_json: Path, work_dir: Path, binary_paths: binaries_util.BinaryGetter, spirv_opt_args: Optional[List[str]] = None, -) -> SpirvAndSpirvAsmShaderJob: +) -> SpirvCombinedShaderJob: result = input_json + glsl_source_shader_job: Optional[Path] = None + # If GLSL: - if shader_job_util.get_related_suffixes_that_exist(input_json): + if shader_job_util.get_related_suffixes_that_exist( + result, language_suffix=(shader_job_util.SUFFIX_GLSL,) + ): + glsl_source_shader_job = result result = shader_job_util.copy(result, work_dir / "0_glsl" / result.name) @@ -221,126 +228,70 @@ def compile_shader_job( validate_spirv_shader_job(result_spirv, binary_paths) - return SpirvAndSpirvAsmShaderJob(result, result_spirv) - - -def glsl_shader_job_to_amber_script( - input_json: Path, - output_amber: Path, - work_dir: Path, - binary_paths: binaries_util.BinaryGetter, - amberfy_settings: amber_converter.AmberfySettings, - spirv_opt_args: Optional[List[str]] = None, -) -> Path: - - spirv_asm_shader_job = compile_shader_job( - input_json, work_dir, binary_paths, spirv_opt_args - ).spirv_asm_shader_job - - amber_test_path = amberfy( - spirv_asm_shader_job, output_amber, amberfy_settings, input_json + return SpirvCombinedShaderJob( + name=name, + spirv_asm_shader_job=result, + spirv_shader_job=result_spirv, + glsl_source_shader_job=glsl_source_shader_job, ) - return amber_test_path - - -def get_compilation_settings( - binary_paths: Optional[binaries_util.BinaryGetter] = None, - spirv_opt_args: Optional[List[str]] = None, - spirv_opt_hash: Optional[str] = None, - test_metadata_path: Optional[Path] = None, -) -> Tuple[binaries_util.BinaryGetter, Optional[List[str]], Optional[str]]: - - if not binary_paths: - check( - bool(test_metadata_path), - AssertionError("Must have test_metadata_path or binary_paths"), - ) - assert test_metadata_path # noqa - binary_paths = binaries_util.BinaryManager( - binaries_util.BinaryManager.get_binary_list_from_test_metadata( - test_metadata_path - ) - ) - - if test_metadata_path: - check( - bool(not spirv_opt_args), - AssertionError("Cannot have spirv_opt_args AND test_metadata_path"), - ) - spirv_opt_args = list( - test_util.metadata_read_from_path(test_metadata_path).glsl.spirv_opt_args - ) - - if spirv_opt_args and not spirv_opt_hash: - spirv_opt_hash = binary_paths.get_binary_path_by_name( - binaries_util.SPIRV_OPT_NAME - ).binary.version - - return (binary_paths, spirv_opt_args, spirv_opt_hash) - def glsl_shader_job_crash_to_amber_script_for_google_cts( + source_dir: Path, output_amber: Path, work_dir: Path, short_description: str, comment_text: str, copyright_year: str, extra_commands: str, - binary_paths: Optional[binaries_util.BinaryGetter] = None, - spirv_opt_args: Optional[List[str]] = None, - spirv_opt_hash: Optional[str] = None, - test_metadata_path: Optional[Path] = None, - input_json: Optional[Path] = None, - source_dir: Optional[Path] = None, ) -> Path: - """ - Converts a GLSL shader job to an Amber script suitable for adding to the CTS. - - :param input_json: - :param output_amber: - :param work_dir: - :param short_description: One sentence, 58 characters max., no period, no line breaks. - :param comment_text: Why the test should pass. Can have line breaks. Ideally make sure lines are not too long. - :param copyright_year: - :param extra_commands: - :param binary_paths: - :param spirv_opt_args: - :param spirv_opt_hash: - :param test_metadata_path: - :param source_dir: - :return: - """ - if source_dir: - input_json = source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB - test_metadata_path = source_dir / test_util.TEST_METADATA - else: - check(bool(input_json), AssertionError("Need input_json or source_dir")) - assert input_json # noqa - - # Get compilation settings - (binary_paths, spirv_opt_args, spirv_opt_hash) = get_compilation_settings( - binary_paths, spirv_opt_args, spirv_opt_hash, test_metadata_path + """Converts a GLSL shader job to an Amber script suitable for adding to the CTS.""" + return glsl_shader_job_wrong_image_to_amber_script_for_google_cts( + source_dir=source_dir, + output_amber=output_amber, + work_dir=work_dir, + short_description=short_description, + comment_text=comment_text, + copyright_year=copyright_year, + extra_commands=extra_commands, ) - return glsl_shader_job_to_amber_script( - input_json, - output_amber, - work_dir, - binary_paths, - amber_converter.AmberfySettings( - copyright_header_text=get_copyright_header_google(copyright_year), - add_graphics_fuzz_comment=True, - short_description=short_description, - comment_text=comment_text, - use_default_fence_timeout=True, - extra_commands=extra_commands, - spirv_opt_args=spirv_opt_args, - spirv_opt_hash=spirv_opt_hash, + +@dataclass +class NameAndShaderJob: + name: str + shader_job: Path + + +def get_shader_jobs( + source_dir: Path, overrides: Iterable[NameAndShaderJob] = () +) -> List[NameAndShaderJob]: + shader_job_dir_names = [test_util.REFERENCE_DIR] + [ + shader_directory.name + for shader_directory in source_dir.glob(test_util.VARIANT_DIR + "*") + if shader_directory.is_dir() + ] + + # name -> Path + shader_jobs: Dict[str, Path] = { + shader_job_dir_name: source_dir / shader_job_dir_name / test_util.SHADER_JOB + for shader_job_dir_name in shader_job_dir_names + if (source_dir / shader_job_dir_name / test_util.SHADER_JOB).is_file() + } + + for override in overrides: + shader_jobs[override.name] = override.shader_job + + shader_jobs_list = sorted( + ( + NameAndShaderJob(name, Path(shader_job)) + for (name, shader_job) in shader_jobs.items() ), - spirv_opt_args=spirv_opt_args, + key=lambda x: x.name, ) + return shader_jobs_list + def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( source_dir: Path, @@ -349,49 +300,34 @@ def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( short_description: str, comment_text: str, copyright_year: str, - binary_paths: Optional[binaries_util.BinaryGetter] = None, - spirv_opt_args: Optional[List[str]] = None, - spirv_opt_hash: Optional[str] = None, + extra_commands: str, ) -> Path: - """ - Converts a GLSL shader job of a wrong image case to an Amber script suitable for adding to the CTS. - - :param source_dir: - :param output_amber: - :param work_dir: - :param short_description: One sentence, 58 characters max., no period, no line breaks. - :param comment_text: Why the test should pass. Can have line breaks. Ideally make sure lines are not too long. - :param copyright_year: - :param binary_paths: - :param spirv_opt_args: - :param spirv_opt_hash: - :return - """ - test_metadata_path = source_dir / test_util.TEST_METADATA - - # Populate shader jobs with reference and all available variants - shader_jobs = [test_util.REFERENCE_DIR] + sorted( - variant.name - for variant in source_dir.glob(test_util.VARIANT_DIR + "*") - if variant.is_dir() - ) + """Converts a GLSL shader job of a wrong image case to an Amber script suitable for adding to the CTS.""" + shader_jobs = get_shader_jobs(source_dir) - # Get compilation settings - (binary_paths, spirv_opt_args, spirv_opt_hash) = get_compilation_settings( - binary_paths, spirv_opt_args, spirv_opt_hash, test_metadata_path + test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) + binary_manager = binaries_util.BinaryManager() + binary_manager = binary_manager.get_child_binary_manager( + list(test.device.binaries) + list(test.binaries) ) + spirv_opt_args = list(test.glsl.spirv_opt_args) + spirv_opt_hash = binary_manager.get_binary_by_name( + binaries_util.SPIRV_OPT_NAME + ).version + # Compile all shader jobs shader_job_files = [ amber_converter.ShaderJobFile( - shader_job, + shader_job.name, compile_shader_job( - source_dir / shader_job / test_util.SHADER_JOB, - work_dir / shader_job, - binary_paths, + shader_job.name, + shader_job.shader_job, + work_dir / shader_job.name, + binary_manager, spirv_opt_args=spirv_opt_args, ).spirv_asm_shader_job, - source_dir / shader_job / test_util.SHADER_JOB, + shader_job.shader_job, "", ) for shader_job in shader_jobs @@ -411,5 +347,6 @@ def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( use_default_fence_timeout=True, spirv_opt_args=spirv_opt_args, spirv_opt_hash=spirv_opt_hash, + extra_commands=extra_commands, ), ) diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index fd295f7d1..2eeb3f560 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -186,4 +186,6 @@ params deqp tgz bootanim - +metavar +d50c96e8 +rand2 From 042b713eb1530f9fb7879ad49597388fdfd8e4aa Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 1 Oct 2019 09:59:44 +0100 Subject: [PATCH 144/180] gfauto: add spirv-fuzz reduction and other fixes (#763) * Add support for spirv-fuzz shrinking. * Add flag for spirv-fuzz fuzzing. * spirv-fuzz: support spirv-opt. * Fix CTS test generation. --- gfauto/gfauto/fuzz.py | 34 ++- gfauto/gfauto/fuzz_glsl_test.py | 116 ++++---- gfauto/gfauto/fuzz_spirv_test.py | 266 ++++++++++++++++++- gfauto/gfauto/generate_cts_test_template.py | 2 +- gfauto/gfauto/gfauto_interestingness_test.py | 64 +++-- gfauto/gfauto/test.proto | 2 +- gfauto/gfauto/test_pb2.py | 11 +- gfauto/gfauto/test_pb2.pyi | 7 + gfauto/gfauto/tool.py | 113 +++++++- gfauto/gfauto/util.py | 7 + 10 files changed, 522 insertions(+), 100 deletions(-) diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index b4598fa55..74b9eb113 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -40,7 +40,6 @@ test_util, util, ) -from gfauto.device_pb2 import Device from gfauto.gflogging import log # Root: @@ -117,8 +116,6 @@ # Python normally uses 256 bits internally when seeding its RNG, hence this choice. ITERATION_SEED_BITS = 256 -SPIRV_FUZZ = False - FUZZ_FAILURES_DIR_NAME = "fuzz_failures" @@ -148,6 +145,12 @@ def main() -> None: action="store_true", ) + parser.add_argument( + "--use_spirv_fuzz", + help="Do fuzzing using spirv-fuzz, which must be on your PATH.", + action="store_true", + ) + parsed_args = parser.parse_args(sys.argv[1:]) settings_path = Path(parsed_args.settings) @@ -155,11 +158,17 @@ def main() -> None: parsed_args.iteration_seed ) skip_writing_binary_recipes: bool = parsed_args.skip_writing_binary_recipes + use_spirv_fuzz: bool = parsed_args.use_spirv_fuzz with util.file_open_text(Path(f"log_{get_random_name()}.txt"), "w") as log_file: gflogging.push_stream_for_logging(log_file) try: - main_helper(settings_path, iteration_seed, skip_writing_binary_recipes) + main_helper( + settings_path, + iteration_seed, + skip_writing_binary_recipes, + use_spirv_fuzz, + ) finally: gflogging.pop_stream_for_logging() @@ -168,6 +177,7 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many settings_path: Path, iteration_seed_override: Optional[int], skip_writing_binary_recipes: bool, + use_spirv_fuzz: bool, ) -> None: settings = settings_util.read_or_create(settings_path) @@ -215,6 +225,8 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many # so that we don't need to specify it and update it in the device list (on disk). # Thus, when we save the test, the device will contain the version of SwiftShader we used. for device in active_devices: + + # noinspection PyTypeChecker if device.HasField("swift_shader"): swift_binaries = [ binary @@ -256,7 +268,7 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many # - Reduce each report (on the given device). # - Produce a summary for each report. - if SPIRV_FUZZ: + if use_spirv_fuzz: fuzz_spirv_test.fuzz_spirv( staging_dir, reports_dir, @@ -285,16 +297,14 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many def create_summary_and_reproduce( - test_dir: Path, - binary_manager: binaries_util.BinaryManager, - device: Optional[Device] = None, + test_dir: Path, binary_manager: binaries_util.BinaryManager ) -> None: util.mkdirs_p(test_dir / "summary") test_metadata = test_util.metadata_read(test_dir) - if test_metadata.HasField("glsl"): - fuzz_glsl_test.create_summary_and_reproduce_glsl( - test_dir, binary_manager, device - ) + + # noinspection PyTypeChecker + if test_metadata.HasField("glsl") or test_metadata.HasField("spirv_fuzz"): + fuzz_glsl_test.create_summary_and_reproduce(test_dir, binary_manager) else: raise AssertionError("Unrecognized test type") diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py index 7d4311f39..1b43177da 100644 --- a/gfauto/gfauto/fuzz_glsl_test.py +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -372,8 +372,15 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc test: Optional[Test] = None, device: Optional[Device] = None, ignore_test_and_device_binaries: bool = False, - shader_overrides: Iterable[tool.NameAndShaderJob] = (), + shader_job_overrides: Iterable[tool.NameAndShaderJob] = (), + shader_job_shader_overrides: Optional[ + tool.ShaderJobNameToShaderOverridesMap + ] = None, ) -> Path: + + if not shader_job_shader_overrides: + shader_job_shader_overrides = {} + with util.file_open_text(output_dir / "log.txt", "w") as log_file: try: gflogging.push_stream_for_logging(log_file) @@ -399,24 +406,36 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc ) spirv_opt_hash: Optional[str] = None - if test.glsl.spirv_opt_args: + spirv_opt_args: Optional[List[str]] = None + if test.glsl.spirv_opt_args or test.spirv_fuzz.spirv_opt_args: spirv_opt_hash = binary_manager.get_binary_by_name( binaries_util.SPIRV_OPT_NAME ).version + spirv_opt_args = ( + list(test.glsl.spirv_opt_args) + if test.glsl.spirv_opt_args + else list(test.spirv_fuzz.spirv_opt_args) + ) - shader_jobs = tool.get_shader_jobs(source_dir, overrides=shader_overrides) + shader_jobs = tool.get_shader_jobs( + source_dir, overrides=shader_job_overrides + ) combined_spirv_shader_jobs: List[tool.SpirvCombinedShaderJob] = [] for shader_job in shader_jobs: try: + shader_overrides = shader_job_shader_overrides.get( + shader_job.name, None + ) combined_spirv_shader_jobs.append( tool.compile_shader_job( name=shader_job.name, input_json=shader_job.shader_job, work_dir=output_dir / shader_job.name, binary_paths=binary_manager, - spirv_opt_args=list(test.glsl.spirv_opt_args), + spirv_opt_args=spirv_opt_args, + shader_overrides=shader_overrides, ) ) except subprocess.CalledProcessError: @@ -432,11 +451,13 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc # Device types: |preprocess| and |shader_compiler| don't need an AmberScript file. + # noinspection PyTypeChecker if device.HasField("preprocess"): # The "preprocess" device type just needs to get this far, so this is a success. result_util.write_status(output_dir, fuzz.STATUS_SUCCESS) return output_dir + # noinspection PyTypeChecker if device.HasField("shader_compiler"): for combined_spirv_shader_job in combined_spirv_shader_jobs: try: @@ -498,8 +519,7 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc ), output_amber_script_file_path=output_dir / "test.amber", amberfy_settings=amber_converter.AmberfySettings( - spirv_opt_args=list(test.glsl.spirv_opt_args), - spirv_opt_hash=spirv_opt_hash, + spirv_opt_args=spirv_opt_args, spirv_opt_hash=spirv_opt_hash ), ) @@ -510,9 +530,11 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc ) ) + # noinspection PyTypeChecker if device.HasField("host") or device.HasField("swift_shader"): icd: Optional[Path] = None + # noinspection PyTypeChecker if device.HasField("swift_shader"): icd = binary_manager.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME @@ -528,6 +550,7 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc ) return output_dir + # noinspection PyTypeChecker if device.HasField("android"): android_device.run_amber_on_device( @@ -560,24 +583,15 @@ def run_reduction( ) -> Path: test = test_util.metadata_read(test_dir_to_reduce) - check( - bool(test.device), - AssertionError( - f"Cannot reduce {str(test_dir_to_reduce)}; device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}" - ), - ) - - check( - bool(test.device.name), - AssertionError( - f"Cannot reduce {str(test_dir_to_reduce)}; device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}" - ), - ) + if not test.device or not test.device.name: + raise AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)}; " + f"device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}" + ) if not test.crash_signature: raise AssertionError( - f"Cannot reduce {str(test_dir_to_reduce)} because there is no crash string specified; " - f"for now, only crash reductions are supported" + f"Cannot reduce {str(test_dir_to_reduce)} because there is no crash string specified." ) # E.g. reports/crashes/no_signature/d50c96e8_opt_rand2_test_phone_ABC/results/phone_ABC/reductions/1 @@ -618,9 +632,7 @@ def run_reduction( check( final_reduced_shader_job_path.exists(), ReductionFailedError( - "Reduction failed; not yet handled", - reduction_name, - reduction_work_variant_dir, + "Reduction failed.", reduction_name, reduction_work_variant_dir ), ) @@ -654,8 +666,8 @@ def run_glsl_reduce( "--", "gfauto_interestingness_test", str(source_dir), - # --override_shader requires two parameters to follow; the second will be added by glsl-reduce (the shader.json file). - "--override_shader", + # --override_shader_job requires two parameters to follow; the second will be added by glsl-reduce (the shader.json file). + "--override_shader_job", str(name_of_shader_to_reduce), ] @@ -674,40 +686,36 @@ def run_glsl_reduce( return output_dir -def create_summary_and_reproduce_glsl( - test_dir: Path, - binary_manager: binaries_util.BinaryManager, - device: Optional[Device] = None, +def create_summary_and_reproduce( + test_dir: Path, binary_manager: binaries_util.BinaryManager ) -> None: test_metadata = test_util.metadata_read(test_dir) - if not device: - device = test_metadata.device summary_dir = test_dir / "summary" - unreduced_glsl = util.copy_dir( - test_util.get_source_dir(test_dir), summary_dir / "unreduced_glsl" + unreduced = util.copy_dir( + test_util.get_source_dir(test_dir), summary_dir / "unreduced" ) reduced_test_dir = test_util.get_reduced_test_dir( test_dir, test_metadata.device.name, fuzz.BEST_REDUCTION_NAME ) reduced_source_dir = test_util.get_source_dir(reduced_test_dir) - reduced_glsl: Optional[Path] = None + reduced: Optional[Path] = None if reduced_source_dir.exists(): - reduced_glsl = util.copy_dir(reduced_source_dir, summary_dir / "reduced_glsl") + reduced = util.copy_dir(reduced_source_dir, summary_dir / "reduced") run_shader_job( - source_dir=unreduced_glsl, - output_dir=(summary_dir / "unreduced_glsl_result"), + source_dir=unreduced, + output_dir=(summary_dir / "unreduced_result"), binary_manager=binary_manager, ) variant_reduced_glsl_result: Optional[Path] = None - if reduced_glsl: + if reduced: variant_reduced_glsl_result = run_shader_job( - source_dir=reduced_glsl, - output_dir=(summary_dir / "reduced_glsl_result"), + source_dir=reduced, + output_dir=(summary_dir / "reduced_result"), binary_manager=binary_manager, ) @@ -729,17 +737,22 @@ def tool_crash_summary_bug_report_dir( # pylint: disable=too-many-locals; variant_reduced_glsl_result_dir: Path, output_dir: Path, binary_manager: binaries_util.BinaryManager, -) -> Path: +) -> Optional[Path]: # Create a simple script and README. shader_job = reduced_glsl_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB + if not shader_job.is_file(): + return None + test_metadata: Test = test_util.metadata_read_from_path( reduced_glsl_source_dir / test_util.TEST_METADATA ) shader_files = shader_job_util.get_related_files( - shader_job, shader_job_util.EXT_ALL + shader_job, + shader_job_util.EXT_ALL, + (shader_job_util.SUFFIX_GLSL, shader_job_util.SUFFIX_SPIRV), ) check( len(shader_files) > 0, @@ -779,19 +792,26 @@ def tool_crash_summary_bug_report_dir( # pylint: disable=too-many-locals; "Issue found using [GraphicsFuzz](https://github.com/google/graphicsfuzz).\n\n" ) readme += "Tool versions:\n\n" - readme += f"* glslangValidator commit hash: {binary_manager.get_binary_by_name(binaries_util.GLSLANG_VALIDATOR_NAME).version}\n" - if test_metadata.glsl.spirv_opt_args: + # noinspection PyTypeChecker + if test_metadata.HasField("glsl"): + readme += f"* glslangValidator commit hash: {binary_manager.get_binary_by_name(binaries_util.GLSLANG_VALIDATOR_NAME).version}\n" + + if test_metadata.glsl.spirv_opt_args or test_metadata.spirv_fuzz.spirv_opt_args: readme += f"* spirv-opt commit hash: {binary_manager.get_binary_by_name(binaries_util.SPIRV_OPT_NAME).version}\n" readme += "\nTo reproduce:\n\n" readme += f"`glslangValidator -V shader{shader_extension} -o shader{shader_extension}.spv`\n\n" - if spv_files and not test_metadata.glsl.spirv_opt_args: - # There was an .spv file and no spirv-opt, so validate the SPIR-V. + if ( + test_metadata.HasField("glsl") + and spv_files + and not test_metadata.glsl.spirv_opt_args + ): + # GLSL was converted to SPIR-V, and spirv-opt was not run, so indicate that we should validate the SPIR-V. readme += f"`spirv-val shader{shader_extension}.spv`\n\n" - if test_metadata.glsl.spirv_opt_args: + if test_metadata.glsl.spirv_opt_args or test_metadata.spirv_fuzz.spirv_opt_args: readme += f"`spirv-opt shader{shader_extension}.spv -o temp.spv --validate-after-all {' '.join(test_metadata.glsl.spirv_opt_args)}`\n\n" files_to_list = glsl_files + spv_files + asm_files diff --git a/gfauto/gfauto/fuzz_spirv_test.py b/gfauto/gfauto/fuzz_spirv_test.py index 821f8c2de..fec600dcb 100644 --- a/gfauto/gfauto/fuzz_spirv_test.py +++ b/gfauto/gfauto/fuzz_spirv_test.py @@ -28,15 +28,21 @@ binaries_util, fuzz, fuzz_glsl_test, + gflogging, result_util, shader_job_util, spirv_fuzz_util, + spirv_opt_util, + subprocess_util, test_util, + tool, util, ) from gfauto.device_pb2 import Device +from gfauto.fuzz_glsl_test import ReductionFailedError from gfauto.settings_pb2 import Settings from gfauto.test_pb2 import Test, TestSpirvFuzz +from gfauto.util import check def make_test( @@ -48,9 +54,7 @@ def make_test( # Create the subtest by copying the base source. util.copy_dir(base_source_dir, test_util.get_source_dir(subtest_dir)) - test = Test(spirv_fuzz=TestSpirvFuzz()) - - # TODO: Handle spirv_opt_args. + test = Test(spirv_fuzz=TestSpirvFuzz(spirv_opt_args=spirv_opt_args)) test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-dis")]) test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-val")]) @@ -83,6 +87,222 @@ def run( return result_util.get_status(result_output_dir) +def run_spirv_fuzz_shrink( + source_dir: Path, + name_of_shader_job_to_reduce: str, + transformation_suffix_to_reduce: str, + output_dir: Path, +) -> Path: + input_shader_job = source_dir / name_of_shader_job_to_reduce / test_util.SHADER_JOB + + original_spirv_file = input_shader_job.with_suffix( + transformation_suffix_to_reduce + ).with_suffix(shader_job_util.SUFFIX_SPIRV_ORIG) + transformations_file = input_shader_job.with_suffix(transformation_suffix_to_reduce) + + util.mkdirs_p(output_dir) + + final_shader = output_dir / "final.spv" + + # E.g. transformation_suffix_to_reduce == ".frag.transformations" + + # E.g. ".frag.spv" + shader_suffix_to_override = ( + util.remove_end( + transformation_suffix_to_reduce, shader_job_util.SUFFIX_TRANSFORMATIONS + ) + + shader_job_util.SUFFIX_SPIRV + ) + + cmd = [ + str(util.tool_on_path("spirv-fuzz")), + str(original_spirv_file), + "-o", + str(final_shader), + f"--shrink={str(transformations_file)}", + f"--shrinker-temp-file-prefix={str(output_dir / 'temp_')}", + # This ensures the arguments that follow are all positional arguments. + "--", + "gfauto_interestingness_test", + str(source_dir), + # --override_shader requires three parameters to follow; the third will be added by spirv-fuzz (the shader.spv file). + "--override_shader", + name_of_shader_job_to_reduce, + shader_suffix_to_override, + ] + + # Log the reduction. + with util.file_open_text(output_dir / "command.log", "w") as f: + gflogging.push_stream_for_logging(f) + try: + # The reducer can fail, but it will typically output an exception file, so we can ignore the exit code. + subprocess_util.run(cmd, verbose=True, check_exit_code=False) + finally: + gflogging.pop_stream_for_logging() + + return final_shader + + +def run_reduction( + test_dir_reduction_output: Path, + test_dir_to_reduce: Path, + shader_job_name_to_reduce: str, + transformation_suffix_to_reduce: str, + preserve_semantics: bool, + reduction_name: str = "reduction1", +) -> Path: + test = test_util.metadata_read(test_dir_to_reduce) + + check( + bool(test.device and test.device.name), + AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)}; " + f"device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}" + ), + ) + + check( + bool(test.crash_signature), + AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)} because there is no crash string specified." + ), + ) + + check( + preserve_semantics, + AssertionError( + "preserve_semantics must be true for spirv reductions (for now)" + ), + ) + + # E.g. reports/crashes/no_signature/d50c96e8_opt_rand2_test_phone_ABC/results/phone_ABC/reductions/1 + # Will contain work/ and source/ + reduced_test_dir = test_util.get_reduced_test_dir( + test_dir_reduction_output, test.device.name, reduction_name + ) + + source_dir = test_util.get_source_dir(test_dir_to_reduce) + output_dir = test_util.get_reduction_work_directory( + reduced_test_dir, shader_job_name_to_reduce + ) + final_shader_path = run_spirv_fuzz_shrink( + source_dir=source_dir, + name_of_shader_job_to_reduce=shader_job_name_to_reduce, + transformation_suffix_to_reduce=transformation_suffix_to_reduce, + output_dir=output_dir, + ) + + check( + final_shader_path.exists(), + ReductionFailedError("Reduction failed.", reduction_name, output_dir), + ) + + # Finally, create the source_dir so the returned directory can be used as a test_dir. + + # Copy the original source directory. + util.copy_dir(source_dir, test_util.get_source_dir(reduced_test_dir)) + + # And then replace the shader. + + final_shader_prefix = final_shader_path.with_suffix("") + + output_shader_prefix = ( + test_util.get_source_dir(reduced_test_dir) + / shader_job_name_to_reduce + / test_util.SHADER_JOB + ).with_suffix(transformation_suffix_to_reduce) + + util.copy_file( + final_shader_prefix.with_suffix(shader_job_util.SUFFIX_SPIRV), + output_shader_prefix.with_suffix(shader_job_util.SUFFIX_SPIRV), + ) + + util.copy_file( + final_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), + output_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), + ) + + util.copy_file( + final_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS_JSON), + output_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS_JSON), + ) + + return reduced_test_dir + + +def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: + test = test_util.metadata_read(test_dir) + + check( + bool(test.device and test.device.name), + AssertionError( + f"Cannot reduce {str(test_dir)}; " + f"device must be specified in {str(test_util.get_metadata_path(test_dir))}" + ), + ) + + check( + bool(test.crash_signature), + AssertionError( + f"Cannot reduce {str(test_dir)} because there is no crash string specified." + ), + ) + + source_dir = test_util.get_source_dir(test_dir) + + shader_jobs = tool.get_shader_jobs(source_dir) + + # TODO: if needed, this could become a parameter to this function. + shader_job_to_reduce = shader_jobs[0] + + if len(shader_jobs) > 1: + check( + len(shader_jobs) == 2 and shader_jobs[1].name == test_util.VARIANT_DIR, + AssertionError( + "Can only reduce tests with shader jobs reference and variant, or just variant." + ), + ) + shader_job_to_reduce = shader_jobs[1] + + shader_transformation_suffixes = shader_job_util.get_related_suffixes_that_exist( + shader_job_to_reduce.shader_job, + language_suffix=(shader_job_util.SUFFIX_TRANSFORMATIONS,), + ) + + try: + reduced_test = test_dir + + for index, suffix in enumerate(shader_transformation_suffixes): + reduced_test = run_reduction( + test_dir_reduction_output=test_dir, + test_dir_to_reduce=reduced_test, + shader_job_name_to_reduce=shader_job_to_reduce.name, + transformation_suffix_to_reduce=suffix, + preserve_semantics=True, + reduction_name=f"{index}_{suffix.split('.')[1]}", + ) + # TODO: reduce without preserving semantics if crash. + + device_name = test.device.name + + # Create a symlink to the "best" reduction. + best_reduced_test_link = test_util.get_reduced_test_dir( + test_dir, device_name, fuzz.BEST_REDUCTION_NAME + ) + util.make_directory_symlink( + new_symlink_file_path=best_reduced_test_link, existing_dir=reduced_test + ) + except ReductionFailedError as ex: + # Create a symlink to the failed reduction so it is easy to investigate failed reductions. + link_to_failed_reduction_path = ( + reports_dir / "failed_reductions" / f"{test_dir.name}_{ex.reduction_name}" + ) + util.make_directory_symlink( + new_symlink_file_path=link_to_failed_reduction_path, + existing_dir=ex.reduction_work_dir, + ) + + def handle_test( test_dir: Path, reports_dir: Path, @@ -115,9 +335,13 @@ def handle_test( if report_dir: report_paths.append(report_dir) - # TODO: reductions. + # For each report, run a reduction on the target device with the device-specific crash signature. + for test_dir_in_reports in report_paths: + run_reduction_on_report(test_dir_in_reports, reports_dir) - # TODO: summaries. + # For each report, create a summary and reproduce the bug. + for test_dir_in_reports in report_paths: + fuzz.create_summary_and_reproduce(test_dir_in_reports, binary_manager) return issue_found @@ -162,7 +386,37 @@ def fuzz_spirv( staging_dir / f"{staging_name}_no_opt_test", spirv_opt_args=None, binary_manager=binary_manager, - ) + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_O_test", + spirv_opt_args=["-O"], + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_Os_test", + spirv_opt_args=["-Os"], + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand1_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand2_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand3_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), ] for test_dir in test_dirs: diff --git a/gfauto/gfauto/generate_cts_test_template.py b/gfauto/gfauto/generate_cts_test_template.py index 1e82a1a68..9bae73c84 100644 --- a/gfauto/gfauto/generate_cts_test_template.py +++ b/gfauto/gfauto/generate_cts_test_template.py @@ -42,7 +42,7 @@ def main() -> None: bug_dir = util.norm_path(Path(__file__).absolute()).parent tool.glsl_shader_job_crash_to_amber_script_for_google_cts( - source_dir=bug_dir / "reduced_glsl_manual", + source_dir=bug_dir / "reduced_manual", output_amber=bug_dir / "name-of-test-TODO.amber", work_dir=bug_dir / "work", # One sentence, 58 characters max., no period, no line breaks. diff --git a/gfauto/gfauto/gfauto_interestingness_test.py b/gfauto/gfauto/gfauto_interestingness_test.py index e665d9c47..7b7c06a54 100644 --- a/gfauto/gfauto/gfauto_interestingness_test.py +++ b/gfauto/gfauto/gfauto_interestingness_test.py @@ -50,7 +50,7 @@ # be the same for all devices and it would look at the device info in the test_json? -def main() -> None: # pylint: disable=too-many-statements; +def main() -> None: # pylint: disable=too-many-statements, too-many-locals; parser = argparse.ArgumentParser( description="Interestingness test that runs a test using Amber, " "calculates the crash signature based on the result, and returns 0 " @@ -62,11 +62,19 @@ def main() -> None: # pylint: disable=too-many-statements; help="The source directory containing the shaders and the test.json file that describes how to run the test.", ) parser.add_argument( - "--override_shader", + "--override_shader_job", nargs=2, - metavar=("shader_name", "shader_job_json"), - help='Override one of the shader jobs. E.g.: "--override_shader variant temp/variant.json". Note that ' - "the output directory will be set to shader_job_json/ (with the .json extension removed) by default in this case.", + metavar=("shader_job_name", "shader_job_json"), + help='Override one of the shader jobs. E.g.: "--override_shader_job variant temp/variant.json". Note that ' + "the output directory will be set to shader_job_json/ (with the .json extension removed) by default in this case. ", + ) + + parser.add_argument( + "--override_shader", + nargs=3, + metavar=("shader_name", "suffix", "shader_path"), + help='Override one of the shaders. E.g.: "--override_shader variant .frag.spv temp/my_shader.spv". Note that ' + "the output directory will be set to shader_path/ (with the .spv extension removed) by default in this case. ", ) parser.add_argument( @@ -84,25 +92,27 @@ def main() -> None: # pylint: disable=too-many-statements; parser.add_argument( "--output", - help="Output directory. Required unless --override_shader is used; in this case " - "the output will be shader_job_json/ (a directory alongside the shader json with the same name).", + help="Output directory. Required unless --override_shader[_job] is used; see --override_shader[_job] for details.", default=None, ) parsed_args = parser.parse_args(sys.argv[1:]) source_dir: Path = Path(parsed_args.source_dir) - override_shader: Optional[Tuple[str, str]] = parsed_args.override_shader + override_shader_job: Optional[Tuple[str, str]] = parsed_args.override_shader_job + override_shader: Optional[Tuple[str, str, str]] = parsed_args.override_shader use_default_binaries: bool = parsed_args.use_default_binaries fallback_binaries: bool = parsed_args.fallback_binaries or use_default_binaries output: Path if parsed_args.output: output = Path(parsed_args.output) + elif override_shader_job: + output = Path(override_shader_job[1]).with_suffix("") elif override_shader: - output = Path(override_shader[1]).with_suffix("") + output = Path(override_shader[2]).with_suffix("") else: - raise AssertionError("Need --output or --override_shader parameter.") + raise AssertionError("Need --output or --override_shader[_job] parameter.") artifact_util.recipes_write_built_in() @@ -112,15 +122,32 @@ def main() -> None: # pylint: disable=too-many-statements; binaries_util.BUILT_IN_BINARY_RECIPES_PATH_PREFIX, ) - shader_overrides: List[tool.NameAndShaderJob] = [] + shader_job_overrides: List[tool.NameAndShaderJob] = [] - if override_shader: - shader_overrides.append( + if override_shader_job: + shader_job_overrides.append( tool.NameAndShaderJob( - name=override_shader[0], shader_job=Path(override_shader[1]) + name=override_shader_job[0], shader_job=Path(override_shader_job[1]) ) ) + shader_overrides: tool.ShaderJobNameToShaderOverridesMap = {} + + if override_shader: + override = tool.ShaderPathWithNameAndSuffix( + name=override_shader[0], + suffix=override_shader[1], + path=Path(override_shader[2]), + ) + shader_overrides[override.name] = {override.suffix: override} + + # E.g. shader_overrides == + # { + # "variant": { + # ".frag.spv": ShaderPathWithNameAndSuffix("variant", ".frag.spv", Path("path/to/shader.frag.spv")) + # } + # } + # We don't need to read this to run the shader, but we need it afterwards anyway. test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) @@ -130,15 +157,18 @@ def main() -> None: # pylint: disable=too-many-statements; binary_manager=binary_manager, test=test, ignore_test_and_device_binaries=use_default_binaries, - shader_overrides=shader_overrides, + shader_job_overrides=shader_job_overrides, + shader_job_shader_overrides=shader_overrides, ) log( f"gfauto_interestingness_test: finished running {str(source_dir)} in {str(output_dir)}." ) - if override_shader: - log(f"The {override_shader[0]} shader was overridden with {override_shader[1]}") + if override_shader_job: + log( + f"The {override_shader_job[0]} shader was overridden with {override_shader_job[1]}" + ) status = result_util.get_status(output_dir) if test.expected_status: diff --git a/gfauto/gfauto/test.proto b/gfauto/gfauto/test.proto index 7c7885cf9..5c1dce29e 100644 --- a/gfauto/gfauto/test.proto +++ b/gfauto/gfauto/test.proto @@ -40,5 +40,5 @@ message TestGlsl { // Spirv-fuzz generated spirv test. message TestSpirvFuzz { - + repeated string spirv_opt_args = 1; } diff --git a/gfauto/gfauto/test_pb2.py b/gfauto/gfauto/test_pb2.py index 3faede5aa..bbec8bdbd 100644 --- a/gfauto/gfauto/test_pb2.py +++ b/gfauto/gfauto/test_pb2.py @@ -22,7 +22,7 @@ package='gfauto', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\x11gfauto/test.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\xd1\x01\n\x04Test\x12 \n\x04glsl\x18\x01 \x01(\x0b\x32\x10.gfauto.TestGlslH\x00\x12+\n\nspirv_fuzz\x18\x05 \x01(\x0b\x32\x15.gfauto.TestSpirvFuzzH\x00\x12\x17\n\x0f\x63rash_signature\x18\x02 \x01(\t\x12\x1e\n\x06\x64\x65vice\x18\x03 \x01(\x0b\x32\x0e.gfauto.Device\x12 \n\x08\x62inaries\x18\x04 \x03(\x0b\x32\x0e.gfauto.Binary\x12\x17\n\x0f\x65xpected_status\x18\x06 \x01(\tB\x06\n\x04test\"\"\n\x08TestGlsl\x12\x16\n\x0espirv_opt_args\x18\x01 \x03(\t\"\x0f\n\rTestSpirvFuzzb\x06proto3') + serialized_pb=_b('\n\x11gfauto/test.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\xd1\x01\n\x04Test\x12 \n\x04glsl\x18\x01 \x01(\x0b\x32\x10.gfauto.TestGlslH\x00\x12+\n\nspirv_fuzz\x18\x05 \x01(\x0b\x32\x15.gfauto.TestSpirvFuzzH\x00\x12\x17\n\x0f\x63rash_signature\x18\x02 \x01(\t\x12\x1e\n\x06\x64\x65vice\x18\x03 \x01(\x0b\x32\x0e.gfauto.Device\x12 \n\x08\x62inaries\x18\x04 \x03(\x0b\x32\x0e.gfauto.Binary\x12\x17\n\x0f\x65xpected_status\x18\x06 \x01(\tB\x06\n\x04test\"\"\n\x08TestGlsl\x12\x16\n\x0espirv_opt_args\x18\x01 \x03(\t\"\'\n\rTestSpirvFuzz\x12\x16\n\x0espirv_opt_args\x18\x01 \x03(\tb\x06proto3') , dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,gfauto_dot_device__pb2.DESCRIPTOR,]) @@ -136,6 +136,13 @@ file=DESCRIPTOR, containing_type=None, fields=[ + _descriptor.FieldDescriptor( + name='spirv_opt_args', full_name='gfauto.TestSpirvFuzz.spirv_opt_args', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -149,7 +156,7 @@ oneofs=[ ], serialized_start=319, - serialized_end=334, + serialized_end=358, ) _TEST.fields_by_name['glsl'].message_type = _TESTGLSL diff --git a/gfauto/gfauto/test_pb2.pyi b/gfauto/gfauto/test_pb2.pyi index 3daf356d9..a2065bf09 100644 --- a/gfauto/gfauto/test_pb2.pyi +++ b/gfauto/gfauto/test_pb2.pyi @@ -89,10 +89,17 @@ class TestGlsl(google___protobuf___message___Message): class TestSpirvFuzz(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + spirv_opt_args = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] def __init__(self, + *, + spirv_opt_args : typing___Optional[typing___Iterable[typing___Text]] = None, ) -> None: ... @classmethod def FromString(cls, s: bytes) -> TestSpirvFuzz: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"spirv_opt_args"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"spirv_opt_args",b"spirv_opt_args"]) -> None: ... diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index 33f5fe83b..4f6238ee7 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -36,6 +36,7 @@ test_util, util, ) +from gfauto.util import check AMBER_COMMAND_PROBE_TOP_LEFT_RED = "probe rgba (0, 0) (1, 0, 0, 1)\n" @@ -46,6 +47,24 @@ ) +@dataclass +class NameAndShaderJob: + name: str + shader_job: Path + + +@dataclass +class ShaderPathWithNameAndSuffix: + name: str + suffix: str + path: Path + + +ShaderSuffixToShaderOverride = Dict[str, ShaderPathWithNameAndSuffix] + +ShaderJobNameToShaderOverridesMap = Dict[str, ShaderSuffixToShaderOverride] + + def get_copyright_header_google(year: str) -> str: return f"""Copyright {year} Google LLC @@ -177,35 +196,83 @@ def compile_shader_job( work_dir: Path, binary_paths: binaries_util.BinaryGetter, spirv_opt_args: Optional[List[str]] = None, + shader_overrides: Optional[ShaderSuffixToShaderOverride] = None, ) -> SpirvCombinedShaderJob: result = input_json glsl_source_shader_job: Optional[Path] = None - # If GLSL: - if shader_job_util.get_related_suffixes_that_exist( + glsl_suffixes = shader_job_util.get_related_suffixes_that_exist( result, language_suffix=(shader_job_util.SUFFIX_GLSL,) - ): + ) + + spirv_suffixes = shader_job_util.get_related_suffixes_that_exist( + result, language_suffix=[shader_job_util.SUFFIX_SPIRV] + ) + + # If GLSL: + if glsl_suffixes: glsl_source_shader_job = result result = shader_job_util.copy(result, work_dir / "0_glsl" / result.name) + if shader_overrides: + raise AssertionError("Shader overrides are not supported for GLSL") + result = glslang_glsl_shader_job_to_spirv( result, work_dir / "1_spirv" / result.name, binary_paths ) - # If SPIR-V: - elif shader_job_util.get_related_suffixes_that_exist( - input_json, language_suffix=[shader_job_util.SUFFIX_SPIRV] - ): + elif spirv_suffixes: + result = shader_job_util.copy( result, work_dir / "1_spirv" / result.name, # Copy all spirv-fuzz related files too: language_suffix=shader_job_util.SUFFIXES_SPIRV_FUZZ, ) + + if shader_overrides: + for suffix in spirv_suffixes: + shader_override = shader_overrides.get(suffix) + if shader_override: + check( + name == shader_override.name, + AssertionError( + f"shader job name {name} does not match shader override job name {shader_override.name}" + ), + ) + check( + shader_override.suffix == suffix, + AssertionError( + f"shader suffix {suffix} does not match shader override suffix {shader_override.suffix}" + ), + ) + + # These will be used as prefixes via .with_suffix(). + # E.g. path/to/temp.spv + source_prefix = shader_override.path + # E.g. path/to/shader.json -> path/to/shader.frag.spv + dest_prefix = result.with_suffix(suffix) + + util.copy_file_if_exists(source_prefix, dest_prefix) + util.copy_file_if_exists( + source_prefix.with_suffix( + shader_job_util.SUFFIX_TRANSFORMATIONS + ), + dest_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), + ) + util.copy_file_if_exists( + source_prefix.with_suffix( + shader_job_util.SUFFIX_TRANSFORMATIONS_JSON + ), + dest_prefix.with_suffix( + shader_job_util.SUFFIX_TRANSFORMATIONS_JSON + ), + ) else: + # result has not changed, which means nothing was executed above. raise AssertionError(f"Unrecognized shader job type: {str(input_json)}") result_spirv = result @@ -257,10 +324,24 @@ def glsl_shader_job_crash_to_amber_script_for_google_cts( ) -@dataclass -class NameAndShaderJob: - name: str - shader_job: Path +# +# @dataclass +# class Shader: +# suffix: str +# path: Path +# +# +# @dataclass +# class ShaderJob: +# name: str +# path: Path +# shader_files: Dict[str, Shader] +# +# +# @dataclass +# class SourceDirFiles: +# test_metadata: Path +# shader_jobs: Dict[str, ShaderJob] def get_shader_jobs( @@ -333,10 +414,16 @@ def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( for shader_job in shader_jobs ] + reference_asm_spirv_job: Optional[amber_converter.ShaderJobFile] = None + + if shader_job_files[0].name_prefix == test_util.REFERENCE_DIR: + reference_asm_spirv_job = shader_job_files[0] + del shader_job_files[0] + return amber_converter.spirv_asm_shader_job_to_amber_script( amber_converter.ShaderJobFileBasedAmberTest( - reference_asm_spirv_job=shader_job_files[0], - variants_asm_spirv_job=shader_job_files[1:], + reference_asm_spirv_job=reference_asm_spirv_job, + variants_asm_spirv_job=shader_job_files, ), output_amber, amber_converter.AmberfySettings( diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py index 21275c9a1..481a2bd01 100644 --- a/gfauto/gfauto/util.py +++ b/gfauto/gfauto/util.py @@ -149,10 +149,17 @@ def prepend_catchsegv_if_available( return cmd +def copy_file_if_exists(source_file_path: Path, dest_file_path: Path) -> Optional[Path]: + if not source_file_path.is_file(): + return None + return copy_file(source_file_path, dest_file_path) + + def copy_file( source_file_path: pathlib.Path, dest_file_path: pathlib.Path ) -> pathlib.Path: file_mkdirs_parent(dest_file_path) + gflogging.log(f"Copying {str(source_file_path)} to {str(dest_file_path)}") shutil.copy(str(source_file_path), str(dest_file_path)) return dest_file_path From 194e8e9006b16315e26622f45bbb577fdab3ad7a Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 1 Oct 2019 17:03:26 +0100 Subject: [PATCH 145/180] gfauto: remove writing of default recipes (#764) And add safe concurrent recipe execution. --- .../inspectionProfiles/gfauto_inspections.xml | 1 + gfauto/README.md | 6 +- gfauto/gfauto/artifact_util.py | 155 +++++++---- gfauto/gfauto/binaries_util.py | 241 ++++++++++-------- gfauto/gfauto/download_cts_gf_tests.py | 5 +- gfauto/gfauto/fuzz.py | 32 +-- gfauto/gfauto/generate_cts_test_template.py | 4 +- gfauto/gfauto/gfauto_interestingness_test.py | 12 +- gfauto/gfauto/run_cts_gf_tests.py | 4 +- gfauto/gfauto/test_create_readme.py | 11 +- gfauto/gfauto/tool.py | 14 +- gfauto/gfauto/util.py | 32 ++- gfauto/whitelist.dic | 4 + 13 files changed, 311 insertions(+), 210 deletions(-) diff --git a/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml b/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml index eec3863dc..35f2f7b1d 100644 --- a/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml +++ b/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml @@ -12,5 +12,6 @@ + \ No newline at end of file diff --git a/gfauto/README.md b/gfauto/README.md index fd1c5d93a..7e5e7bc3a 100644 --- a/gfauto/README.md +++ b/gfauto/README.md @@ -146,7 +146,7 @@ The `active_device_names` list should be modified to include only the unique dev * Including multiple, identical devices (with the same GPU and drivers) is not recommended, as it will currently result in redundant testing. * `host_preprocessor` should always be included first, as this virtual device detects failures in tools such as `glslangValidator` and `spirv-opt`, and will ensure such failures will not be logged for real devices. -* You can use `--settings SETTINGS.json` to use a settings file other than `settings.json` (the default). In this way, you can run multiple instances of `gfauto_fuzz` in parallel to test *different* devices. When starting, `gfauto_fuzz` writes the built-in recipes to `//binaries/`, and this code is not safe to run concurrently; run `gfauto_fuzz` at least once to ensure the recipes are written and then use the `--skip_writing_binary_recipes` option to prevent other instances of `gfauto_fuzz` from writing them on start-up. The code that downloads binaries is not safe to run concurrently at the time of writing; running one instance for a few minutes to download the latest binaries is usually sufficient to then allow additional instances to be started without conflicts. +* You can use `--settings SETTINGS.json` to use a settings file other than `settings.json` (the default). In this way, you can run multiple instances of `gfauto_fuzz` in parallel to test *different* devices. You can generate a space-separated list of seeds as follows: @@ -159,7 +159,7 @@ import secrets Assuming you saved those to `../seeds.txt`, you can run parallel instances of `gfauto_fuzz` using: ```sh -parallel -j 32 gfauto_fuzz --skip_writing_binary_recipes --iteration_seed -- $(cat ../seeds.txt) +parallel -j 32 gfauto_fuzz --iteration_seed -- $(cat ../seeds.txt) ``` This is probably only suitable for testing the `host_preprocessor` and `swift_shader` virtual devices; running parallel tests on actual hardware is likely to give unreliable results. @@ -167,5 +167,5 @@ This is probably only suitable for testing the `host_preprocessor` and `swift_sh You can run parallel instances of gfauto (just for increased throughput, not with fixed seeds) using: ```sh -parallel -j 32 -i gfauto_fuzz --skip_writing_binary_recipes -- $(seq 100) +parallel -j 32 -i gfauto_fuzz -- $(seq 100) ``` diff --git a/gfauto/gfauto/artifact_util.py b/gfauto/gfauto/artifact_util.py index f5b9348e1..9159811be 100644 --- a/gfauto/gfauto/artifact_util.py +++ b/gfauto/gfauto/artifact_util.py @@ -20,15 +20,10 @@ """ import pathlib -from typing import List, Optional, Tuple - -from gfauto import ( - binaries_util, - gflogging, - proto_util, - recipe_download_and_extract_archive_set, - util, -) +import time +from typing import Dict, List, Optional, Tuple + +from gfauto import gflogging, proto_util, recipe_download_and_extract_archive_set, util from gfauto.artifact_pb2 import ArtifactMetadata from gfauto.common_pb2 import ArchiveSet from gfauto.gflogging import log @@ -39,6 +34,11 @@ ARTIFACT_RECIPE_FILE_NAME = "recipe.json" ARTIFACT_RECIPE_LOG_FILE_NAME = "recipe.log" ARTIFACT_ROOT_FILE_NAME = "ROOT" +ARTIFACT_EXECUTING_LOCK_FILE_NAME = "EXECUTING_LOCK" + +BUSY_WAIT_IN_SECONDS = 1 + +RecipeMap = Dict[str, Recipe] class ArtifactWrap: @@ -49,15 +49,6 @@ def __init__(self, path: str, metadata: Optional[ArtifactMetadata] = None): ) -def recipes_write_built_in() -> None: - log( - "Writing built-in binary recipes to //binaries/ (where // is the ROOT directory)" - ) - for recipe_wrap in binaries_util.BUILT_IN_BINARY_RECIPES: - if not artifact_get_metadata_file_path(recipe_wrap.path).exists(): - recipe_wrap.write() - - def binary_artifacts_find(artifact_path_prefix: str) -> List[Tuple[ArchiveSet, str]]: artifact_paths = artifacts_find(artifact_path_prefix) result: List[Tuple[ArchiveSet, str]] = [] @@ -174,7 +165,7 @@ def artifact_write_recipe( json_text = proto_util.message_to_json(recipe, including_default_value_fields=True) json_file_path = artifact_get_recipe_file_path(artifact_path) - util.file_write_text(json_file_path, json_text) + util.file_write_text_atomic(json_file_path, json_text) return artifact_path @@ -195,7 +186,7 @@ def artifact_write_metadata( artifact_metadata, including_default_value_fields=True ) json_file_path = artifact_get_metadata_file_path(artifact_path) - util.file_write_text(json_file_path, json_text) + util.file_write_text_atomic(json_file_path, json_text) return artifact_path @@ -207,41 +198,113 @@ def artifact_read_metadata(artifact_path: str = "") -> ArtifactMetadata: return artifact_metadata -def artifact_execute_recipe_if_needed(artifact_path: str = "") -> None: - artifact_execute_recipe(artifact_path, only_if_artifact_json_missing=True) +def artifact_execute_recipe_if_needed( + artifact_path: str = "", built_in_recipes: Optional[RecipeMap] = None +) -> None: + artifact_execute_recipe( + artifact_path, + only_if_artifact_json_missing=True, + built_in_recipes=built_in_recipes, + ) -def artifact_execute_recipe( - artifact_path: str = "", only_if_artifact_json_missing: bool = False +def artifact_execute_recipe( # pylint: disable=too-many-branches; + artifact_path: str = "", + only_if_artifact_json_missing: bool = False, + built_in_recipes: Optional[RecipeMap] = None, ) -> None: - artifact_path = artifact_path_absolute(artifact_path) + executing_lock_file_path = artifact_get_inner_file_path( + ARTIFACT_EXECUTING_LOCK_FILE_NAME, artifact_path + ) - if ( - only_if_artifact_json_missing - and artifact_get_metadata_file_path(artifact_path).exists() - ): - return - - recipe = artifact_read_recipe(artifact_path) + busy_waiting = False + first_wait = True - with util.file_open_text( - artifact_get_recipe_log_file_path(artifact_path), "w" - ) as f: - gflogging.push_stream_for_logging(f) - try: - if recipe.HasField("download_and_extract_archive_set"): - recipe_download_and_extract_archive_set.recipe_download_and_extract_archive_set( - recipe.download_and_extract_archive_set, artifact_path + # We may have to retry if another process appears to be executing this recipe. + while True: + if busy_waiting: + time.sleep(BUSY_WAIT_IN_SECONDS) + if first_wait: + log( + f"Waiting for {artifact_path} due to lock file {executing_lock_file_path}" ) - else: - raise NotImplementedError( - "Artifact {} has recipe type {} and this is not implemented".format( - artifact_path, recipe.WhichOneof("recipe") - ) + first_wait = False + + if executing_lock_file_path.exists(): + # Retry. + busy_waiting = True + continue + + # Several processes can still execute here concurrently; the above check is just an optimization. + + # The metadata file should be written atomically once the artifact is ready for use, so if it exists, we can + # just return. + if ( + only_if_artifact_json_missing + and artifact_get_metadata_file_path(artifact_path).exists() + ): + return + + # The recipe file should be written atomically. If it exists, we are fine to continue. If not and if we have + # the recipe in |built_in_recipes|, more than one process might write it, but the final rename from TEMP_FILE -> + # RECIPE.json is atomic, so *some* process will succeed and the contents will be valid. Thus, we should be fine + # to continue once we have written the recipe. + if ( + not artifact_get_recipe_file_path(artifact_path).exists() + and built_in_recipes + ): + built_in_recipe = built_in_recipes[artifact_path] + if not built_in_recipe: + raise FileNotFoundError( + str(artifact_get_recipe_file_path(artifact_path)) ) + # This is atomic; should not fail. + artifact_write_recipe(built_in_recipe, artifact_path) + + recipe = artifact_read_recipe(artifact_path) + + # Create EXECUTING_LOCK file. The "x" means exclusive creation. This will fail if the file already exists; + # i.e. another process won the race and is executing the recipe; if so, we retry from the beginning of this + # function (and will return early). Otherwise, we can continue. We don't need to keep the file open; the file + # is not opened with exclusive access, just created exclusively. + try: + with util.file_open_text(executing_lock_file_path, "x") as lock_file: + lock_file.write("locked") + except FileExistsError: + # Retry. + busy_waiting = True + continue + + # If we fail here (e.g. KeyboardInterrupt), we won't remove the lock file. But any alternative will either have + # the same problem (interrupts can happen at almost any time) or could end up accidentally removing the lock + # file made by another process, so this is the safest approach. Users can manually delete lock files if needed; + # the log output indicates the file on which we are blocked. + + try: + with util.file_open_text( + artifact_get_recipe_log_file_path(artifact_path), "w" + ) as f: + gflogging.push_stream_for_logging(f) + try: + if recipe.HasField("download_and_extract_archive_set"): + recipe_download_and_extract_archive_set.recipe_download_and_extract_archive_set( + recipe.download_and_extract_archive_set, artifact_path + ) + else: + raise NotImplementedError( + "Artifact {} has recipe type {} and this is not implemented".format( + artifact_path, recipe.WhichOneof("recipe") + ) + ) + finally: + gflogging.pop_stream_for_logging() finally: - gflogging.pop_stream_for_logging() + # Delete the lock file when we have finished. Ignore errors. + try: + executing_lock_file_path.unlink() + except OSError: + log(f"WARNING: failed to delete: {str(executing_lock_file_path)}") def artifact_get_inner_file_path(inner_file: str, artifact_path: str) -> pathlib.Path: diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py index 73bfd80a2..3e515d946 100644 --- a/gfauto/gfauto/binaries_util.py +++ b/gfauto/gfauto/binaries_util.py @@ -31,8 +31,11 @@ from gfauto.common_pb2 import Archive, ArchiveSet, Binary from gfauto.gflogging import log from gfauto.recipe_pb2 import Recipe, RecipeDownloadAndExtractArchiveSet +from gfauto.util import check -LATEST_GRAPHICSFUZZ_ARTIFACT = "//binaries/graphicsfuzz_v1.2.1" +BINARY_RECIPES_PREFIX = "//binaries" + +LATEST_GRAPHICSFUZZ_ARTIFACT = f"{BINARY_RECIPES_PREFIX}/graphicsfuzz_v1.2.1" GLSLANG_VALIDATOR_NAME = "glslangValidator" SPIRV_OPT_NAME = "spirv-opt" @@ -42,7 +45,9 @@ SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG = "no-validate-after-all" -BUILT_IN_BINARY_RECIPES_PATH_PREFIX = "//binaries" +BUILT_IN_BINARY_RECIPES_PATH_PREFIX = f"{BINARY_RECIPES_PREFIX}/built_in" + +CUSTOM_BINARY_RECIPES_PATH_PREFIX = f"{BINARY_RECIPES_PREFIX}/custom" PLATFORM_SUFFIXES_DEBUG = ["Linux_x64_Debug", "Windows_x64_Debug", "Mac_x64_Debug"] PLATFORM_SUFFIXES_RELEASE = [ @@ -97,103 +102,6 @@ def __init__(self, binary: Binary): super().__init__(f"Could not find binary path for binary: \n{binary}") -class BinaryManager(BinaryGetter): - """ - Implements BinaryGetter. - - An instance of BinaryManager is the main way that code accesses binaries. BinaryManger allows certain tests and/or - devices to override binaries by passing a list of binary versions that take priority, so the correct versions are - always used. Plus, the current platform will be used when deciding which binary to download and return. - - See the Binary proto. - - _binary_list: A list of Binary with name, version, configuration. This is used to map a binary name to a Binary. - _resolved_paths: A map containing: Binary (serialized bytes) -> Path - _binary_artifacts: A list of all available binary artifacts/recipes stored as tuples: (ArchiveSet, artifact_path). - """ - - _binary_list: List[Binary] - _resolved_paths: Dict[bytes, Path] - _binary_artifacts: List[Tuple[ArchiveSet, str]] - - def __init__( - self, - binary_list: Optional[List[Binary]] = None, - platform: Optional[str] = None, - binary_artifacts_prefix: Optional[str] = BUILT_IN_BINARY_RECIPES_PATH_PREFIX, - ): - self._binary_list = binary_list or DEFAULT_BINARIES - self._resolved_paths = {} - self._platform = platform or util.get_platform() - self._binary_artifacts = [] - - if binary_artifacts_prefix: - self._binary_artifacts.extend( - artifact_util.binary_artifacts_find(binary_artifacts_prefix) - ) - - @staticmethod - def get_binary_list_from_test_metadata(test_json_path: Path) -> List[Binary]: - test_metadata = test_util.metadata_read_from_path(test_json_path) - result: List[Binary] = [] - if test_metadata.device: - result.extend(test_metadata.device.binaries) - result.extend(test_metadata.binaries) - return result - - def get_binary_path(self, binary: Binary) -> Path: - result = self._resolved_paths.get(binary.SerializePartialToString()) - if result: - return result - log(f"Finding path of binary:\n{binary}") - binary_tags = set(binary.tags) - binary_tags.add(self._platform) - for (archive_set, artifact_path) in self._binary_artifacts: - for artifact_binary in archive_set.binaries: # type: Binary - if artifact_binary.name != binary.name: - continue - if artifact_binary.version != binary.version: - continue - recipe_binary_tags = set(artifact_binary.tags) - if not binary_tags.issubset(recipe_binary_tags): - continue - artifact_util.artifact_execute_recipe_if_needed(artifact_path) - result = artifact_util.artifact_get_inner_file_path( - artifact_binary.path, artifact_path - ) - self._resolved_paths[binary.SerializePartialToString()] = result - return result - raise BinaryPathNotFound(binary) - - @staticmethod - def get_binary_by_name_from_list(name: str, binary_list: List[Binary]) -> Binary: - for binary in binary_list: - if binary.name == name: - return binary - raise BinaryNotFound( - f"Could not find binary named {name} in list:\n{binary_list}" - ) - - def get_binary_path_by_name(self, name: str) -> BinaryPathAndInfo: - binary = self.get_binary_by_name(name) - return BinaryPathAndInfo(self.get_binary_path(binary), binary) - - def get_binary_by_name(self, name: str) -> Binary: - return self.get_binary_by_name_from_list(name, self._binary_list) - - def get_child_binary_manager(self, binary_list: List[Binary]) -> "BinaryManager": - result = BinaryManager( - binary_list + self._binary_list, - self._platform, - binary_artifacts_prefix=None, - ) - # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. - result._resolved_paths = self._resolved_paths - # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. - result._binary_artifacts = self._binary_artifacts - return result - - @attr.dataclass class ToolNameAndPath: name: str @@ -245,7 +153,7 @@ def _get_built_in_binary_recipe_from_build_github_repo( result.append( recipe_wrap.RecipeWrap( - f"//binaries/{project_name}_{version_hash}_{platform_suffix}", + f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/{project_name}_{version_hash}_{platform_suffix}", Recipe( download_and_extract_archive_set=RecipeDownloadAndExtractArchiveSet( archive_set=ArchiveSet( @@ -325,7 +233,7 @@ def _get_built_in_glslang_version( def get_graphics_fuzz_121() -> List[recipe_wrap.RecipeWrap]: return [ recipe_wrap.RecipeWrap( - "//binaries/graphicsfuzz_v1.2.1", + f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/graphicsfuzz_v1.2.1", Recipe( download_and_extract_archive_set=RecipeDownloadAndExtractArchiveSet( archive_set=ArchiveSet( @@ -536,3 +444,134 @@ def get_graphics_fuzz_121() -> List[recipe_wrap.RecipeWrap]: build_version_hash="70e8d53b94227fed094975771d96f240f7d00911", ) ) + +BUILT_IN_BINARY_RECIPES_MAP: Dict[str, Recipe] = { + recipe_wrap.path: recipe_wrap.recipe for recipe_wrap in BUILT_IN_BINARY_RECIPES +} + + +class BinaryManager(BinaryGetter): + """ + Implements BinaryGetter. + + An instance of BinaryManager is the main way that code accesses binaries. BinaryManger allows certain tests and/or + devices to override binaries by passing a list of binary versions that take priority, so the correct versions are + always used. Plus, the current platform will be used when deciding which binary to download and return. + + See the Binary proto. + + _binary_list: A list of Binary with name, version, configuration. This is used to map a binary name to a Binary. + _resolved_paths: A map containing: Binary (serialized bytes) -> Path + _binary_artifacts: A list of all available binary artifacts/recipes stored as tuples: (ArchiveSet, artifact_path). + _built_in_binary_recipes: This is needed to pass to artifact_util.artifact_execute_recipe_if_needed() so that the + recipe file can be written on-demand from our in-memory list of built-in recipes. + """ + + _binary_list: List[Binary] + _resolved_paths: Dict[bytes, Path] + _binary_artifacts: List[Tuple[ArchiveSet, str]] + _built_in_binary_recipes: artifact_util.RecipeMap + + def __init__( + self, + binary_list: Optional[List[Binary]] = None, + platform: Optional[str] = None, + built_in_binary_recipes: Optional[Dict[str, Recipe]] = None, + custom_binary_artifacts_prefix: Optional[str] = None, + ): + self._binary_list = binary_list or DEFAULT_BINARIES + self._resolved_paths = {} + self._platform = platform or util.get_platform() + self._binary_artifacts = [] + self._built_in_binary_recipes = {} + + # When changing this constructor, check self.get_child_binary_manager(). + + if built_in_binary_recipes: + self._built_in_binary_recipes = built_in_binary_recipes + # For each recipe, add a tuple (ArchiveSet, artifact_path) to self._binary_artifacts. + for (artifact_path, recipe) in self._built_in_binary_recipes.items(): + check( + recipe.HasField("download_and_extract_archive_set"), + AssertionError(f"Bad built-in recipe: {recipe}"), + ) + archive_set: RecipeDownloadAndExtractArchiveSet = recipe.download_and_extract_archive_set + self._binary_artifacts.append((archive_set.archive_set, artifact_path)) + + if custom_binary_artifacts_prefix: + self._binary_artifacts.extend( + artifact_util.binary_artifacts_find(custom_binary_artifacts_prefix) + ) + + @staticmethod + def get_binary_list_from_test_metadata(test_json_path: Path) -> List[Binary]: + test_metadata = test_util.metadata_read_from_path(test_json_path) + result: List[Binary] = [] + if test_metadata.device: + result.extend(test_metadata.device.binaries) + result.extend(test_metadata.binaries) + return result + + def get_binary_path(self, binary: Binary) -> Path: + result = self._resolved_paths.get(binary.SerializePartialToString()) + if result: + return result + log(f"Finding path of binary:\n{binary}") + binary_tags = set(binary.tags) + binary_tags.add(self._platform) + for (archive_set, artifact_path) in self._binary_artifacts: + for artifact_binary in archive_set.binaries: # type: Binary + if artifact_binary.name != binary.name: + continue + if artifact_binary.version != binary.version: + continue + recipe_binary_tags = set(artifact_binary.tags) + if not binary_tags.issubset(recipe_binary_tags): + continue + artifact_util.artifact_execute_recipe_if_needed( + artifact_path, self._built_in_binary_recipes + ) + result = artifact_util.artifact_get_inner_file_path( + artifact_binary.path, artifact_path + ) + self._resolved_paths[binary.SerializePartialToString()] = result + return result + raise BinaryPathNotFound(binary) + + @staticmethod + def get_binary_by_name_from_list(name: str, binary_list: List[Binary]) -> Binary: + for binary in binary_list: + if binary.name == name: + return binary + raise BinaryNotFound( + f"Could not find binary named {name} in list:\n{binary_list}" + ) + + def get_binary_path_by_name(self, name: str) -> BinaryPathAndInfo: + binary = self.get_binary_by_name(name) + return BinaryPathAndInfo(self.get_binary_path(binary), binary) + + def get_binary_by_name(self, name: str) -> Binary: + return self.get_binary_by_name_from_list(name, self._binary_list) + + def get_child_binary_manager( + self, binary_list: List[Binary], prepend: bool = False + ) -> "BinaryManager": + child_binary_list = binary_list + if prepend: + child_binary_list += self._binary_list + result = BinaryManager(child_binary_list, self._platform) + # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. + result._resolved_paths = self._resolved_paths + # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. + result._binary_artifacts = self._binary_artifacts + # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. + result._built_in_binary_recipes = self._built_in_binary_recipes + return result + + +def get_default_binary_manager() -> BinaryManager: + return BinaryManager( + built_in_binary_recipes=BUILT_IN_BINARY_RECIPES_MAP, + custom_binary_artifacts_prefix=CUSTOM_BINARY_RECIPES_PATH_PREFIX, + ) diff --git a/gfauto/gfauto/download_cts_gf_tests.py b/gfauto/gfauto/download_cts_gf_tests.py index 520801dbc..e5265e6e2 100644 --- a/gfauto/gfauto/download_cts_gf_tests.py +++ b/gfauto/gfauto/download_cts_gf_tests.py @@ -25,7 +25,6 @@ from gfauto import ( amber_converter, - artifact_util, binaries_util, fuzz, gerrit_util, @@ -145,9 +144,7 @@ def main() -> None: # Need git. git_tool = util.tool_on_path("git") - artifact_util.recipes_write_built_in() - - binaries = binaries_util.BinaryManager() + binaries = binaries_util.get_default_binary_manager() download_cts_graphicsfuzz_tests(git_tool, cookie, binaries) diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index 74b9eb113..d3cd84571 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -24,7 +24,6 @@ import secrets import shutil import sys -import uuid from pathlib import Path from typing import Optional @@ -121,7 +120,7 @@ def get_random_name() -> str: # TODO: could change to human-readable random name or the date. - return uuid.uuid4().hex + return util.get_random_name() def main() -> None: @@ -138,13 +137,6 @@ def main() -> None: help="The seed to use for one fuzzing iteration (useful for reproducing an issue).", ) - parser.add_argument( - "--skip_writing_binary_recipes", - help="Don't write the default binary recipes to ROOT/binaries. " - "This is useful when running multiple, parallel instances of gfauto because otherwise the writes will race.", - action="store_true", - ) - parser.add_argument( "--use_spirv_fuzz", help="Do fuzzing using spirv-fuzz, which must be on your PATH.", @@ -157,27 +149,18 @@ def main() -> None: iteration_seed: Optional[int] = None if parsed_args.iteration_seed is None else int( parsed_args.iteration_seed ) - skip_writing_binary_recipes: bool = parsed_args.skip_writing_binary_recipes use_spirv_fuzz: bool = parsed_args.use_spirv_fuzz with util.file_open_text(Path(f"log_{get_random_name()}.txt"), "w") as log_file: gflogging.push_stream_for_logging(log_file) try: - main_helper( - settings_path, - iteration_seed, - skip_writing_binary_recipes, - use_spirv_fuzz, - ) + main_helper(settings_path, iteration_seed, use_spirv_fuzz) finally: gflogging.pop_stream_for_logging() def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements; - settings_path: Path, - iteration_seed_override: Optional[int], - skip_writing_binary_recipes: bool, - use_spirv_fuzz: bool, + settings_path: Path, iteration_seed_override: Optional[int], use_spirv_fuzz: bool ) -> None: settings = settings_util.read_or_create(settings_path) @@ -201,9 +184,6 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many ) util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") - if not skip_writing_binary_recipes: - artifact_util.recipes_write_built_in() - # Log a warning if there is no tool on the PATH for printing stack traces. util.prepend_catchsegv_if_available([], log_warning=True) @@ -215,10 +195,8 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json")) - binary_manager = binaries_util.BinaryManager( - list(settings.custom_binaries) + binaries_util.DEFAULT_BINARIES, - util.get_platform(), - binary_artifacts_prefix=binaries_util.BUILT_IN_BINARY_RECIPES_PATH_PREFIX, + binary_manager = binaries_util.get_default_binary_manager().get_child_binary_manager( + list(settings.custom_binaries), prepend=True ) # For convenience, we add the default (i.e. newest) SwiftShader ICD (binary) to any swift_shader devices diff --git a/gfauto/gfauto/generate_cts_test_template.py b/gfauto/gfauto/generate_cts_test_template.py index 9bae73c84..59c56d488 100644 --- a/gfauto/gfauto/generate_cts_test_template.py +++ b/gfauto/gfauto/generate_cts_test_template.py @@ -25,7 +25,7 @@ import sys from pathlib import Path -from gfauto import artifact_util, tool, util +from gfauto import tool, util def main() -> None: @@ -37,8 +37,6 @@ def main() -> None: # - check copyright_year # - check extra_commands - artifact_util.recipes_write_built_in() - bug_dir = util.norm_path(Path(__file__).absolute()).parent tool.glsl_shader_job_crash_to_amber_script_for_google_cts( diff --git a/gfauto/gfauto/gfauto_interestingness_test.py b/gfauto/gfauto/gfauto_interestingness_test.py index 7b7c06a54..8af729862 100644 --- a/gfauto/gfauto/gfauto_interestingness_test.py +++ b/gfauto/gfauto/gfauto_interestingness_test.py @@ -29,7 +29,6 @@ from typing import List, Optional, Tuple from gfauto import ( - artifact_util, binaries_util, fuzz, fuzz_glsl_test, @@ -50,7 +49,7 @@ # be the same for all devices and it would look at the device info in the test_json? -def main() -> None: # pylint: disable=too-many-statements, too-many-locals; +def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too-many-branches; parser = argparse.ArgumentParser( description="Interestingness test that runs a test using Amber, " "calculates the crash signature based on the result, and returns 0 " @@ -114,13 +113,10 @@ def main() -> None: # pylint: disable=too-many-statements, too-many-locals; else: raise AssertionError("Need --output or --override_shader[_job] parameter.") - artifact_util.recipes_write_built_in() + binary_manager = binaries_util.get_default_binary_manager() - binary_manager = binaries_util.BinaryManager( - binaries_util.DEFAULT_BINARIES if fallback_binaries else [], - util.get_platform(), - binaries_util.BUILT_IN_BINARY_RECIPES_PATH_PREFIX, - ) + if not fallback_binaries: + binary_manager = binary_manager.get_child_binary_manager(binary_list=[]) shader_job_overrides: List[tool.NameAndShaderJob] = [] diff --git a/gfauto/gfauto/run_cts_gf_tests.py b/gfauto/gfauto/run_cts_gf_tests.py index b9dc06467..6c4a98ba3 100644 --- a/gfauto/gfauto/run_cts_gf_tests.py +++ b/gfauto/gfauto/run_cts_gf_tests.py @@ -23,7 +23,6 @@ from typing import Optional from gfauto import ( - artifact_util, binaries_util, devices_util, fuzz, @@ -66,8 +65,7 @@ def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-man active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. - artifact_util.recipes_write_built_in() - binaries = binaries_util.BinaryManager() + binaries = binaries_util.get_default_binary_manager() work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" diff --git a/gfauto/gfauto/test_create_readme.py b/gfauto/gfauto/test_create_readme.py index bd4f9b4f7..860ee55e3 100644 --- a/gfauto/gfauto/test_create_readme.py +++ b/gfauto/gfauto/test_create_readme.py @@ -23,8 +23,7 @@ import sys from pathlib import Path -from gfauto import artifact_util, binaries_util, fuzz_glsl_test, test_util -from gfauto.binaries_util import BinaryManager +from gfauto import binaries_util, fuzz_glsl_test, test_util from gfauto.util import check, check_dir_exists @@ -57,12 +56,12 @@ def main() -> None: check_dir_exists(source_dir) check_dir_exists(result_dir) - artifact_util.recipes_write_built_in() - - binary_manager = BinaryManager(binaries_util.DEFAULT_BINARIES) - test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) + binary_manager = binaries_util.get_default_binary_manager().get_child_binary_manager( + binary_list=list(test.binaries) + list(test.device.binaries) + ) + check(test.HasField("glsl"), AssertionError("Only glsl tests currently supported")) check( diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index 4f6238ee7..4a9179e46 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -88,12 +88,15 @@ def get_binary_paths_using_artifact_system( # Deprecated. - artifact_util.recipes_write_built_in() - artifact_util.artifact_execute_recipe_if_needed(artifact_path) + artifact_util.artifact_execute_recipe_if_needed( + artifact_path, binaries_util.BUILT_IN_BINARY_RECIPES_MAP + ) artifact_metadata = artifact_util.artifact_read_metadata(artifact_path) return binaries_util.BinaryManager( - list(artifact_metadata.data.extracted_archive_set.archive_set.binaries) + list(artifact_metadata.data.extracted_archive_set.archive_set.binaries), + built_in_binary_recipes=binaries_util.BUILT_IN_BINARY_RECIPES_MAP, + custom_binary_artifacts_prefix=artifact_path, ) @@ -387,9 +390,8 @@ def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( shader_jobs = get_shader_jobs(source_dir) test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) - binary_manager = binaries_util.BinaryManager() - binary_manager = binary_manager.get_child_binary_manager( - list(test.device.binaries) + list(test.binaries) + binary_manager = binaries_util.get_default_binary_manager().get_child_binary_manager( + binary_list=list(test.device.binaries) + list(test.binaries) ) spirv_opt_args = list(test.glsl.spirv_opt_args) diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py index 481a2bd01..59deb728c 100644 --- a/gfauto/gfauto/util.py +++ b/gfauto/gfauto/util.py @@ -25,6 +25,7 @@ import pathlib import platform import shutil +import uuid import zipfile from contextlib import contextmanager from pathlib import Path @@ -42,7 +43,7 @@ def file_open_binary(file: pathlib.Path, mode: str) -> BinaryIO: # noqa VNE002 check("b" in mode, AssertionError(f"|mode|(=={mode}) should contain 'b'")) - if "w" in mode: + if "w" in mode or "x" in mode: file_mkdirs_parent(file) # Type hint (no runtime check). result = cast(BinaryIO, open(str(file), mode)) @@ -51,7 +52,7 @@ def file_open_binary(file: pathlib.Path, mode: str) -> BinaryIO: # noqa VNE002 def file_open_text(file: pathlib.Path, mode: str) -> TextIO: # noqa VNE002 check("b" not in mode, AssertionError(f"|mode|(=={mode}) should not contain 'b'")) - if "w" in mode: + if "w" in mode or "x" in mode: file_mkdirs_parent(file) # Type hint (no runtime check). result = cast(TextIO, open(str(file), mode, encoding="utf-8", errors="ignore")) @@ -75,6 +76,26 @@ def file_read_lines(file: pathlib.Path) -> List[str]: # noqa VNE002 return f.readlines() +def file_write_text_atomic(file: Path, text: str) -> Path: # noqa VNE002 + """ + Writes |text| to |file| atomically by first writing to a temporary file alongside |file| then renaming the + temporary file to |file|. + + :param file: + :param text: + :return: + """ + temp_file: Path = file.parent / get_random_name() + with file_open_text(temp_file, "x") as f: + f.write(text) + f.flush() + os.fsync(f.fileno()) + # Will not fail if dest already exists; will just silently replace dest. + os.replace(str(temp_file), str(file)) + + return file + + def file_write_text(file: pathlib.Path, text: str) -> Path: # noqa VNE002 with file_open_text(file, "w") as f: f.write(text) @@ -95,7 +116,8 @@ def mkdir_p_new(path: Path) -> Path: # noqa VNE002 def mkdirs_p(path: pathlib.Path) -> Path: # noqa VNE002 - path.mkdir(parents=True, exist_ok=True) + # Use os.makedirs, as this is more likely to be atomic. + os.makedirs(str(path), exist_ok=True) return path @@ -307,3 +329,7 @@ def create_zip(output_file_path: Path, entries: List[ZipEntry]) -> Path: file_handle.write(entry.path, entry.path_in_archive) gflogging.log(f"Adding: {entry.path} {entry.path_in_archive or ''}") return output_file_path + + +def get_random_name() -> str: + return uuid.uuid4().hex diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index 2eeb3f560..cc5c82ab2 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -189,3 +189,7 @@ bootanim metavar d50c96e8 rand2 +makedirs +unlink +fsync +fileno From 11117821c11c3a264708bcc904c9c7fed6c0b4c9 Mon Sep 17 00:00:00 2001 From: jarisiru <53901722+jarisiru@users.noreply.github.com> Date: Thu, 3 Oct 2019 14:24:22 +0300 Subject: [PATCH 146/180] Added fudge factor to calculation to make it more stable (#758) Multiplying floating point 1.0 with a large number followed by fract() may not result in expected result. Adding a fudge factor before fract stabilizes this, making 64bit and 32bit float return the same result even when asking more results than the default grid in the shader does. With the default grid, I could mask out 3 of the bits from the 32bit float before I got a different result compared to 64bit floats. Fixes #750. --- shaders/src/main/glsl/samples/300es/quicksort_palette.frag | 2 +- shaders/src/main/glsl/samples/310es/quicksort_palette.frag | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shaders/src/main/glsl/samples/300es/quicksort_palette.frag b/shaders/src/main/glsl/samples/300es/quicksort_palette.frag index c41018986..a33448293 100644 --- a/shaders/src/main/glsl/samples/300es/quicksort_palette.frag +++ b/shaders/src/main/glsl/samples/300es/quicksort_palette.frag @@ -82,7 +82,7 @@ vec3 palette(vec3 a, vec3 b, vec3 c, vec3 d) { float randomize(vec2 co) { - return float(floor(fract(sin(dot(co.xy ,vec2(12.5, 3.))) * 4250.) + 0.5)); + return float(floor(fract(sin(dot(co.xy ,vec2(12.5, 3.))) * 4250. + 0.02) + 0.5)); } bool puzzlelize(vec2 pos) diff --git a/shaders/src/main/glsl/samples/310es/quicksort_palette.frag b/shaders/src/main/glsl/samples/310es/quicksort_palette.frag index af897b24c..95ebbad29 100644 --- a/shaders/src/main/glsl/samples/310es/quicksort_palette.frag +++ b/shaders/src/main/glsl/samples/310es/quicksort_palette.frag @@ -82,7 +82,7 @@ vec3 palette(vec3 a, vec3 b, vec3 c, vec3 d) { float randomize(vec2 co) { - return float(floor(fract(sin(dot(co.xy ,vec2(12.5, 3.))) * 4250.) + 0.5)); + return float(floor(fract(sin(dot(co.xy ,vec2(12.5, 3.))) * 4250. + 0.02) + 0.5)); } bool puzzlelize(vec2 pos) From d8456fb7e410ebf76197a286ae23735fed4a0bb7 Mon Sep 17 00:00:00 2001 From: Jiradet Ounjai Date: Thu, 3 Oct 2019 14:45:51 +0200 Subject: [PATCH 147/180] KnownValue Generator: add support for UINT type (#716) Adds uint type support during expression generation, and updates LiteralFuzzer to provide fuzzed expressions of uint type. Fixes #661 --- .../ExpressionGenerator.java | 38 ++++++++++++------- .../semanticschanging/LiteralFuzzer.java | 5 +++ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java index db244c0a7..846cc2e38 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java @@ -254,9 +254,7 @@ private Expr generateForLoopValue(Value value, FactManager factManager, FunctionDefinition currentFunction, Stmt stmtToInsertBefore) { - // TODO(https://github.com/google/graphicsfuzz/issues/661): we can also generate for loop - // value of UINT type. - if (value.getType() != BasicType.INT) { + if (value.getType() != BasicType.INT && value.getType() != BasicType.UINT) { return null; } // If the given fact manager is at global scope, the caller expects a expression generated by @@ -275,7 +273,7 @@ private Expr generateForLoopValue(Value value, generateExpr(factManager, currentFunction, stmtToInsertBefore, - new NumericValue(BasicType.INT, Optional.of(0))) + new NumericValue((BasicType) value.getType(), Optional.of(0))) )); final VariablesDeclaration variablesDecl = new VariablesDeclaration(value.getType(), variableDeclInfo); @@ -305,7 +303,7 @@ private Expr generateForLoopValue(Value value, final int remainder = original % divisor; // Values of numbers which will be used for the for loop. - final Value divisorValue = new NumericValue(BasicType.INT, Optional.of(divisor)); + final Value divisorValue = new NumericValue((BasicType) value.getType(), Optional.of(divisor)); final Value iterationValue = new NumericValue(BasicType.INT, Optional.of(isIncrement ? 0 : iterations)); final Value conditionValue = new NumericValue(BasicType.INT, @@ -378,9 +376,8 @@ private Expr generateAdditionValue(Value value, if (!(value instanceof NumericValue)) { return null; } - // TODO(https://github.com/google/graphicsfuzz/issues/661): we can also generate addition - // value of UINT type. - if (value.getType() != BasicType.INT && value.getType() != BasicType.FLOAT) { + if (value.getType() != BasicType.INT && value.getType() != BasicType.FLOAT + && value.getType() != BasicType.UINT) { return null; } // If the given value is unknown, we are free to choose any arbitrary numbers for addition. @@ -393,7 +390,8 @@ private Expr generateAdditionValue(Value value, BinOp.ADD); } - Number summandA; + final Number expected = ((NumericValue) value).getValue().get(); + Number summandA = null; // Given the expected type, we have to retrieve all values from the fact manager and filter only // the facts that are known to compute particular values. final List knownValues = factManager.getValuesFromType(value.getType()) @@ -411,14 +409,26 @@ private Expr generateAdditionValue(Value value, // TODO(https://github.com/google/graphicsfuzz/issues/688): range of numbers [-10, 10] is // temporarily used here, we have to change how the summand is generated. summandA = (float) generator.nextInt(21) - 10; - } else { + } else if (value.getType() == BasicType.UINT) { + // We pick a random number in a range of numbers [1- expectedValue] so that when + // subtracting the random summand with the expected value we would get the non-negative + // result. + summandA = generator.nextInt(Math.max(1, expected.intValue())); + } else if (value.getType() == BasicType.INT) { summandA = generator.nextInt(INT_MAX); } } - final Number expected = ((NumericValue) value).getValue().get(); + assert summandA != null; // To get the second summand, we subtract original value by the the first summand. final Number summandB = subtractNumbers(expected, summandA); + // We have a chance to have a summandA based on known fact that is greater than the expected + // value making the summandB obtained by subtraction negative. We thus have to ensure that + // the result of subtraction is non-negative number as since it is not legal to have singed + // number for uint type. + if (value.getType() == BasicType.UINT && String.valueOf(summandB).contains("-")) { + return null; + } // Randomly decide whether summandA or summandB should be the first summand. final boolean summandAFirst = generator.nextBoolean(); final Number firstSummand = summandAFirst ? summandA : summandB; @@ -561,7 +571,7 @@ private String parseNameFromValue(Value value) { if (value instanceof NumericValue) { final NumericValue numericValue = (NumericValue) value; float floatValue = numericValue.getValue().get().floatValue(); - if (floatValue < 0.0) { + if (String.valueOf(floatValue).contains("-")) { name.append(NEGATIVE); floatValue = Math.abs(floatValue); } @@ -735,8 +745,8 @@ private List getAvailableTypes() { // We will have to consider supporting more types but at the moment as we are using // LiteralFuzzer to generate literal expressions so we are unable to cover all basic type now. // The following are types currently supported by LiteralFuzzer. - return Arrays.asList(BasicType.BOOL, BasicType.INT, BasicType.FLOAT, BasicType.VEC2, - BasicType.VEC3, BasicType.VEC4); + return Arrays.asList(BasicType.BOOL, BasicType.INT, BasicType.UINT, BasicType.FLOAT, + BasicType.VEC2, BasicType.VEC3, BasicType.VEC4); } private Value fuzzValue(Type type, boolean forceGenerateKnownValue) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java index e678835e1..c97bbeb11 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.ast.expr.FloatConstantExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; +import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.util.IRandom; @@ -49,6 +50,10 @@ public Optional fuzz(Type type) { return Optional.of(new IntConstantExpr( String.valueOf(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN))); } + if (type == BasicType.UINT) { + return Optional.of(new UIntConstantExpr( + String.valueOf(Math.abs(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN) + "u"))); + } if (type == BasicType.FLOAT) { return Optional.of(new FloatConstantExpr(LiteralFuzzer.randomFloatString(generator))); } From 64ba9d4c8189c40c6c1da9240628972cabcfa779 Mon Sep 17 00:00:00 2001 From: jarisiru <53901722+jarisiru@users.noreply.github.com> Date: Thu, 3 Oct 2019 16:20:23 +0300 Subject: [PATCH 148/180] Added fixed point variants of mandelbrot shader (#770) Added variants of the fixed point shaders where the actual fractal calculation is done via fixed point math to avoid floating point issues. Otherwise the shaders are as before. For fixed point, 8.8 was chosen so that 32 bit integers are enough in all situations (fixed point multiplications and divisions require double the bits of the operands). --- .../samples/300es/mandelbrot_fixed_point.frag | 76 +++++++++++++++++++ .../samples/300es/mandelbrot_fixed_point.json | 9 +++ .../samples/310es/mandelbrot_fixed_point.frag | 76 +++++++++++++++++++ .../samples/310es/mandelbrot_fixed_point.json | 9 +++ 4 files changed, 170 insertions(+) create mode 100644 shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.frag create mode 100644 shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.json create mode 100644 shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.frag create mode 100644 shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.json diff --git a/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.frag b/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.frag new file mode 100644 index 000000000..ee2237e0d --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.frag @@ -0,0 +1,76 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +vec3 pickColor(int i) { + return vec3(float(i) / 50.0, float(i) / 120.0, float(i) / 140.0); +} + +vec3 mand(float xCoord, float yCoord) +{ + int xpos = int(xCoord) * 256; + int ypos = int(yCoord) * 256; + int height = int(resolution.y) * 256; + int width = int(resolution.x) * 256; + + int c_re = ((xpos - width / 2) * 819) / width - 102; + int c_im = ((ypos - height / 2) * 819) / width; + + int x = 0, y = 0; + int iteration = 0; + for (int k = 0; k < 1000; k++) + { + if (x * x + y * y > 262144) + { + break; + } + int x_new = ((x * x - y * y) / 256 + c_re); + y = (2 * x * y / 256 + c_im); + x = x_new; + iteration++; + } + if (iteration < 1000) + { + return pickColor(iteration); + } + else + { + return vec3(0.0, 0.0, 0.5); + } +} + +void main() { + vec3 data[16]; + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + data[4*j + i] = mand(gl_FragCoord.x + float(i - 1), gl_FragCoord.y + float(j - 1)); + } + } + vec3 sum = vec3(0.0); + for (int i = 0; i < 16; i++) { + sum += data[i]; + } + sum /= vec3(16.0); + _GLF_color = vec4(sum, 1.0); +} + diff --git a/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.json b/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.json new file mode 100644 index 000000000..28c3c3e61 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.frag b/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.frag new file mode 100644 index 000000000..636df8f2e --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.frag @@ -0,0 +1,76 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +vec3 pickColor(int i) { + return vec3(float(i) / 50.0, float(i) / 120.0, float(i) / 140.0); +} + +vec3 mand(float xCoord, float yCoord) +{ + int xpos = int(xCoord) * 256; + int ypos = int(yCoord) * 256; + int height = int(resolution.y) * 256; + int width = int(resolution.x) * 256; + + int c_re = ((xpos - width / 2) * 819) / width - 102; + int c_im = ((ypos - height / 2) * 819) / width; + + int x = 0, y = 0; + int iteration = 0; + for (int k = 0; k < 1000; k++) + { + if (x * x + y * y > 262144) + { + break; + } + int x_new = ((x * x - y * y) / 256 + c_re); + y = (2 * x * y / 256 + c_im); + x = x_new; + iteration++; + } + if (iteration < 1000) + { + return pickColor(iteration); + } + else + { + return vec3(0.0, 0.0, 0.5); + } +} + +void main() { + vec3 data[16]; + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + data[4*j + i] = mand(gl_FragCoord.x + float(i - 1), gl_FragCoord.y + float(j - 1)); + } + } + vec3 sum = vec3(0.0); + for (int i = 0; i < 16; i++) { + sum += data[i]; + } + sum /= vec3(16.0); + _GLF_color = vec4(sum, 1.0); +} + diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.json b/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.json new file mode 100644 index 000000000..28c3c3e61 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From 6b4cdaefa066fcc859dffd25786fb6df4204e2be Mon Sep 17 00:00:00 2001 From: jarisiru <53901722+jarisiru@users.noreply.github.com> Date: Thu, 3 Oct 2019 18:24:08 +0300 Subject: [PATCH 149/180] Fix acosh() input range (#753) acosh() is only defined for input values >=1, but the shader used values that are less than that (including negative values). This leads to undefined behavior as per SPIR-V spec. In C++ the result may have bee NaN, which is optional in Vulkan. This change fixes the shader so that all acosh() input values are >=1. --- shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag | 2 +- shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag b/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag index d215742eb..8afcebd6c 100644 --- a/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag +++ b/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag @@ -138,7 +138,7 @@ void main() { color.x += atanh(color.x) * cosh(injectionSwitch.y) * smoothstep(color, injectionSwitch, gl_FragCoord.yy).x; break; } else if (int(gl_FragCoord[1]) < 120) { - color = fract(acosh(vecCoor.yx - trunc(float(data[3])))); + color = fract(acosh(clamp(vecCoor.yx - trunc(float(data[3])), 1.0, 1000.0))); color.x += (isnan(gl_FragCoord.x) ? log2(gl_FragCoord.x) : log2(gl_FragCoord.y)); break; } else if (int(gl_FragCoord[1]) < 150) { diff --git a/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag index a57cc536e..ba5da010a 100644 --- a/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag +++ b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag @@ -136,7 +136,7 @@ void main() { color.x += atanh(color.x) * cosh(injectionSwitch.y) * smoothstep(color, injectionSwitch, gl_FragCoord.yy).x; break; } else if (int(gl_FragCoord[1]) < 120) { - color = fract(acosh(vecCoor.yx - trunc(float(data[3])))); + color = fract(acosh(clamp(vecCoor.yx - trunc(float(data[3])), 1.0, 1000.0))); color.x += (isnan(gl_FragCoord.x) ? log2(gl_FragCoord.x) : log2(gl_FragCoord.y)); break; } else if (int(gl_FragCoord[1]) < 150) { From bff37c9bf040439804dfed487298c18ede8a969f Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Fri, 4 Oct 2019 04:05:39 -0500 Subject: [PATCH 150/180] Add 130es Muller's Method shader (#698) - Given a uniform vec3 polynomial representing the constants of a quadratic function, and a uniform vec3 initial_xvalues that represents the xvalues to begin the algorithm with (with initial_xvalues.y being the midpoint of [initial_xvalues.x, initialxvalues.z]), approximates a root of polynomial nearest to initial_xvalues by approximating using quadratic functions. - Uses a while loop - Math-heavy - Renders red if the test passes (the method finds the correct root, given a small margin of error), otherwise green - From some fuzz testing, this shouldn't be vulnerable to floating point issues and should give consistent results. Related issue: #545. --- .../src/main/glsl/samples/310es/muller.frag | 76 +++++++++++++++++++ .../src/main/glsl/samples/310es/muller.json | 18 +++++ 2 files changed, 94 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/muller.frag create mode 100644 shaders/src/main/glsl/samples/310es/muller.json diff --git a/shaders/src/main/glsl/samples/310es/muller.frag b/shaders/src/main/glsl/samples/310es/muller.frag new file mode 100644 index 000000000..72c48900c --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/muller.frag @@ -0,0 +1,76 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec3 polynomial; + +uniform vec3 initial_xvalues; + +float fx(float x) +{ + return polynomial.x * pow(x, 2.0) + polynomial.y * x + polynomial.z; +} + +void main() +{ + // n + float x2 = initial_xvalues.x; + // n - 1 + float x1 = initial_xvalues.y; + // n - 2 + float x0 = initial_xvalues.z; + // temp storage + float temp = 0.0; + // constants in the quadratic formula + // calculated with derived formulas for Muller's method + float A = 0.0; + float B = 0.0; + float C = 0.0; + + while(abs(x2 - x1) >= .000000000000001) + { + float h0 = x0 - x2; + float h1 = x1 - x2; + float k0 = fx(x0) - fx(x2); + float k1 = fx(x1) - fx(x2); + temp = x2; + A = (((h1) * (k0)) - ((h0) * (k1))) / + ((pow((h0), 2.0) * (h1)) - (pow((h1), 2.0) * (h0))); + B = (((k0) - (A * (pow((h0), 2.0)))) / (h0)); + C = fx(x2); + + // get the new root + x2 = x2 - ((2.0 * C) / (B + sign(B) * sqrt(pow(B, 2.0) - (4.0 * A * C)))); + + // move up in the sequence. + x0 = x1; + x1 = temp; // x1 = original x2; + } + if(x2 <= -0.9 && x2 >= -1.1) + { + _GLF_color = vec4(1.0, 0.0, 0.0, 1.0); + } + else + { + _GLF_color = vec4(0.0, 1.0, 0.0, 1.0); + } +} diff --git a/shaders/src/main/glsl/samples/310es/muller.json b/shaders/src/main/glsl/samples/310es/muller.json new file mode 100644 index 000000000..1bf27217a --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/muller.json @@ -0,0 +1,18 @@ +{ + "polynomial": { + "func": "glUniform3f", + "args": [ + 1.0, + 3.0, + 2.0 + ] + }, + "initial_xvalues": { + "func": "glUniform3f", + "args": [ + 5.0, + 3.0, + 1.0 + ] + } +} From 9a9303ad90bba47774a4eef756639d91b2b2caff Mon Sep 17 00:00:00 2001 From: jarisiru <53901722+jarisiru@users.noreply.github.com> Date: Fri, 4 Oct 2019 12:21:51 +0300 Subject: [PATCH 151/180] Zoom into mandelbrot lake to avoid float issues (#767) Due to the iterative nature of mandelbrot fractal calculations, even minute differences of float accuracy will make the images incomparable. Zooming into the lake will make the images equal while retaining the complexity of the functions. --- .../generator/tool/GenerateShaderFamilyTest.java | 2 +- .../graphicsfuzz/generator/tool/GlslGenerateTest.java | 2 +- .../100/{mandelbrot_blurry.frag => mandelbrot_zoom.frag} | 9 ++++++--- .../100/{mandelbrot_blurry.json => mandelbrot_zoom.json} | 0 .../{mandelbrot_blurry.frag => mandelbrot_zoom.frag} | 9 ++++++--- .../{mandelbrot_blurry.json => mandelbrot_zoom.json} | 0 .../{mandelbrot_blurry.frag => mandelbrot_zoom.frag} | 9 ++++++--- .../{mandelbrot_blurry.json => mandelbrot_zoom.json} | 0 ...tfunctions.frag => mandelbrot_zoom_intfunctions.frag} | 9 ++++++--- ...tfunctions.json => mandelbrot_zoom_intfunctions.json} | 0 .../{mandelbrot_blurry.frag => mandelbrot_zoom.frag} | 9 ++++++--- .../{mandelbrot_blurry.json => mandelbrot_zoom.json} | 0 .../{mandelbrot_blurry.frag => mandelbrot_zoom.frag} | 9 ++++++--- .../{mandelbrot_blurry.json => mandelbrot_zoom.json} | 0 .../{mandelbrot_blurry.frag => mandelbrot_zoom.frag} | 9 ++++++--- .../{mandelbrot_blurry.json => mandelbrot_zoom.json} | 0 .../java/com/graphicsfuzz/tester/GeneratorUnitTest.java | 4 ++-- 17 files changed, 46 insertions(+), 25 deletions(-) rename shaders/src/main/glsl/samples/100/{mandelbrot_blurry.frag => mandelbrot_zoom.frag} (81%) rename shaders/src/main/glsl/samples/100/{mandelbrot_blurry.json => mandelbrot_zoom.json} (100%) rename shaders/src/main/glsl/samples/300es/{mandelbrot_blurry.frag => mandelbrot_zoom.frag} (81%) rename shaders/src/main/glsl/samples/300es/{mandelbrot_blurry.json => mandelbrot_zoom.json} (100%) rename shaders/src/main/glsl/samples/310es/{mandelbrot_blurry.frag => mandelbrot_zoom.frag} (81%) rename shaders/src/main/glsl/samples/310es/{mandelbrot_blurry.json => mandelbrot_zoom.json} (100%) rename shaders/src/main/glsl/samples/310es/{mandelbrot_blurry_intfunctions.frag => mandelbrot_zoom_intfunctions.frag} (85%) rename shaders/src/main/glsl/samples/310es/{mandelbrot_blurry_intfunctions.json => mandelbrot_zoom_intfunctions.json} (100%) rename shaders/src/main/glsl/samples/donors/{mandelbrot_blurry.frag => mandelbrot_zoom.frag} (81%) rename shaders/src/main/glsl/samples/donors/{mandelbrot_blurry.json => mandelbrot_zoom.json} (100%) rename shaders/src/main/glsl/samples/webgl1/{mandelbrot_blurry.frag => mandelbrot_zoom.frag} (81%) rename shaders/src/main/glsl/samples/webgl1/{mandelbrot_blurry.json => mandelbrot_zoom.json} (100%) rename shaders/src/main/glsl/samples/webgl2/{mandelbrot_blurry.frag => mandelbrot_zoom.frag} (81%) rename shaders/src/main/glsl/samples/webgl2/{mandelbrot_blurry.json => mandelbrot_zoom.json} (100%) diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java index 5b28930a4..32929461c 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java @@ -62,7 +62,7 @@ public void testGenerateSmallWebGL1ShaderFamily() throws Exception { @Test public void testGenerateSmall300esShaderFamily() throws Exception { final String samplesSubdir = "300es"; - final String referenceShaderName = "mandelbrot_blurry"; + final String referenceShaderName = "mandelbrot_zoom"; final int numVariants = 3; int seed = 2; checkShaderFamilyGeneration(samplesSubdir, referenceShaderName, numVariants, diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java index a3a176165..3986984fa 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java @@ -151,7 +151,7 @@ private void checkFragmentShaderFamilyGeneration(String references, generateShaderFamily(references, donors, numVariants, prefix, outputDir, seed, extraArgs, true); for (String reference : Arrays.asList("bubblesort_flag", "colorgrid_modulo", - "mandelbrot_blurry", "prefix_sum", "squares")) { + "mandelbrot_zoom", "prefix_sum", "squares")) { final File expectedOutputDirectory = new File(temporaryFolder.getRoot(), prefix + "_" + reference); assertTrue(expectedOutputDirectory.isDirectory()); diff --git a/shaders/src/main/glsl/samples/100/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/100/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/100/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/100/mandelbrot_zoom.frag index 72f105246..4023d98a2 100644 --- a/shaders/src/main/glsl/samples/100/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/100/mandelbrot_zoom.frag @@ -28,8 +28,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -44,7 +47,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/100/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/100/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/100/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/100/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/300es/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/300es/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/300es/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/300es/mandelbrot_zoom.frag index 99cc3c753..477a673c3 100644 --- a/shaders/src/main/glsl/samples/300es/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/300es/mandelbrot_zoom.frag @@ -30,8 +30,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -46,7 +49,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/300es/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/300es/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/300es/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/300es/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/310es/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/310es/mandelbrot_zoom.frag index 00329aeb0..e1e10ab7a 100644 --- a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom.frag @@ -30,8 +30,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -46,7 +49,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/310es/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/310es/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.frag b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.frag similarity index 85% rename from shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.frag rename to shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.frag index 87e3a039e..2f4464d08 100644 --- a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.frag +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.frag @@ -31,8 +31,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; if (0.0 > resolution.x) { @@ -59,7 +62,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < bitfieldInsert(iterationCap, 0, 0, 0)) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.json b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.json similarity index 100% rename from shaders/src/main/glsl/samples/310es/mandelbrot_blurry_intfunctions.json rename to shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.json diff --git a/shaders/src/main/glsl/samples/donors/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/donors/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/donors/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/donors/mandelbrot_zoom.frag index b52569b8d..846ba7cf4 100644 --- a/shaders/src/main/glsl/samples/donors/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/donors/mandelbrot_zoom.frag @@ -24,8 +24,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -40,7 +43,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/donors/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/donors/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/donors/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/donors/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.frag index 6a65e258c..1f1b8926d 100644 --- a/shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.frag @@ -29,8 +29,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -45,7 +48,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.frag index 90303fb31..cfb7f97e4 100644 --- a/shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.frag @@ -31,8 +31,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -47,7 +50,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.json diff --git a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java index 1bad33a66..17b982e61 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java @@ -142,8 +142,8 @@ public void testDonateDeadCode() throws Exception { Util.getDonorsFolder(), GenerationParams.normal(ShaderKind.FRAGMENT, true)), TransformationProbabilities.likelyDonateDeadCode(), "donatedead", - Arrays.asList("bubblesort_flag.json", "squares.json", "mandelbrot_blurry.json"), - Arrays.asList("bubblesort_flag.json", "squares.json", "mandelbrot_blurry.json")); + Arrays.asList("bubblesort_flag.json", "squares.json", "mandelbrot_zoom.json"), + Arrays.asList("bubblesort_flag.json", "squares.json", "mandelbrot_zoom.json")); // Reason for blacklisting^: slow. } From 0683f734d1e49212bdea32261c04d839b116a028 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Fri, 4 Oct 2019 13:15:07 +0100 Subject: [PATCH 152/180] gfauto: add spirv-reduce support (#771) ...for much better crash reductions. --- gfauto/gfauto/fuzz_spirv_test.py | 163 ++++++++++++++++++++----------- 1 file changed, 104 insertions(+), 59 deletions(-) diff --git a/gfauto/gfauto/fuzz_spirv_test.py b/gfauto/gfauto/fuzz_spirv_test.py index fec600dcb..0c4360656 100644 --- a/gfauto/gfauto/fuzz_spirv_test.py +++ b/gfauto/gfauto/fuzz_spirv_test.py @@ -31,6 +31,7 @@ gflogging, result_util, shader_job_util, + signature_util, spirv_fuzz_util, spirv_opt_util, subprocess_util, @@ -87,18 +88,25 @@ def run( return result_util.get_status(result_output_dir) -def run_spirv_fuzz_shrink( +def run_spirv_reduce_or_shrink( source_dir: Path, name_of_shader_job_to_reduce: str, - transformation_suffix_to_reduce: str, + extension_to_reduce: str, output_dir: Path, + preserve_semantics: bool, ) -> Path: input_shader_job = source_dir / name_of_shader_job_to_reduce / test_util.SHADER_JOB original_spirv_file = input_shader_job.with_suffix( - transformation_suffix_to_reduce - ).with_suffix(shader_job_util.SUFFIX_SPIRV_ORIG) - transformations_file = input_shader_job.with_suffix(transformation_suffix_to_reduce) + extension_to_reduce + shader_job_util.SUFFIX_SPIRV_ORIG + ) + + transformed_spirv_file = input_shader_job.with_suffix( + extension_to_reduce + shader_job_util.SUFFIX_SPIRV + ) + transformations_file = input_shader_job.with_suffix( + extension_to_reduce + shader_job_util.SUFFIX_TRANSFORMATIONS + ) util.mkdirs_p(output_dir) @@ -106,30 +114,42 @@ def run_spirv_fuzz_shrink( # E.g. transformation_suffix_to_reduce == ".frag.transformations" - # E.g. ".frag.spv" - shader_suffix_to_override = ( - util.remove_end( - transformation_suffix_to_reduce, shader_job_util.SUFFIX_TRANSFORMATIONS - ) - + shader_job_util.SUFFIX_SPIRV - ) - - cmd = [ - str(util.tool_on_path("spirv-fuzz")), - str(original_spirv_file), - "-o", - str(final_shader), - f"--shrink={str(transformations_file)}", - f"--shrinker-temp-file-prefix={str(output_dir / 'temp_')}", - # This ensures the arguments that follow are all positional arguments. - "--", - "gfauto_interestingness_test", - str(source_dir), - # --override_shader requires three parameters to follow; the third will be added by spirv-fuzz (the shader.spv file). - "--override_shader", - name_of_shader_job_to_reduce, - shader_suffix_to_override, - ] + # E.g. ".frag.??" -> ".frag.spv" + shader_suffix_to_override = extension_to_reduce + shader_job_util.SUFFIX_SPIRV + + if preserve_semantics: + cmd = [ + str(util.tool_on_path("spirv-fuzz")), + str(original_spirv_file), + "-o", + str(final_shader), + f"--shrink={str(transformations_file)}", + f"--shrinker-temp-file-prefix={str(output_dir / 'temp_')}", + # This ensures the arguments that follow are all positional arguments. + "--", + "gfauto_interestingness_test", + str(source_dir), + # --override_shader requires three parameters to follow; the third will be added by spirv-fuzz (the shader.spv file). + "--override_shader", + name_of_shader_job_to_reduce, + shader_suffix_to_override, + ] + else: + cmd = [ + str(util.tool_on_path("spirv-reduce")), + str(transformed_spirv_file), + "-o", + str(final_shader), + f"--temp-file-prefix={str(output_dir / 'temp_')}", + # This ensures the arguments that follow are all positional arguments. + "--", + "gfauto_interestingness_test", + str(source_dir), + # --override_shader requires three parameters to follow; the third will be added by spirv-reduce (the shader.spv file). + "--override_shader", + name_of_shader_job_to_reduce, + shader_suffix_to_override, + ] # Log the reduction. with util.file_open_text(output_dir / "command.log", "w") as f: @@ -147,7 +167,7 @@ def run_reduction( test_dir_reduction_output: Path, test_dir_to_reduce: Path, shader_job_name_to_reduce: str, - transformation_suffix_to_reduce: str, + extension_to_reduce: str, preserve_semantics: bool, reduction_name: str = "reduction1", ) -> Path: @@ -168,13 +188,6 @@ def run_reduction( ), ) - check( - preserve_semantics, - AssertionError( - "preserve_semantics must be true for spirv reductions (for now)" - ), - ) - # E.g. reports/crashes/no_signature/d50c96e8_opt_rand2_test_phone_ABC/results/phone_ABC/reductions/1 # Will contain work/ and source/ reduced_test_dir = test_util.get_reduced_test_dir( @@ -185,12 +198,22 @@ def run_reduction( output_dir = test_util.get_reduction_work_directory( reduced_test_dir, shader_job_name_to_reduce ) - final_shader_path = run_spirv_fuzz_shrink( - source_dir=source_dir, - name_of_shader_job_to_reduce=shader_job_name_to_reduce, - transformation_suffix_to_reduce=transformation_suffix_to_reduce, - output_dir=output_dir, - ) + if preserve_semantics: + final_shader_path = run_spirv_reduce_or_shrink( + source_dir=source_dir, + name_of_shader_job_to_reduce=shader_job_name_to_reduce, + extension_to_reduce=extension_to_reduce, + output_dir=output_dir, + preserve_semantics=preserve_semantics, + ) + else: + final_shader_path = run_spirv_reduce_or_shrink( + source_dir=source_dir, + name_of_shader_job_to_reduce=shader_job_name_to_reduce, + extension_to_reduce=extension_to_reduce, + output_dir=output_dir, + preserve_semantics=preserve_semantics, + ) check( final_shader_path.exists(), @@ -204,33 +227,37 @@ def run_reduction( # And then replace the shader. - final_shader_prefix = final_shader_path.with_suffix("") - + # Destination file. E.g. reductions/source/variant/shader.frag.spv output_shader_prefix = ( test_util.get_source_dir(reduced_test_dir) / shader_job_name_to_reduce / test_util.SHADER_JOB - ).with_suffix(transformation_suffix_to_reduce) + ).with_suffix(extension_to_reduce + shader_job_util.SUFFIX_SPIRV) util.copy_file( - final_shader_prefix.with_suffix(shader_job_util.SUFFIX_SPIRV), + final_shader_path.with_suffix(shader_job_util.SUFFIX_SPIRV), output_shader_prefix.with_suffix(shader_job_util.SUFFIX_SPIRV), ) - util.copy_file( - final_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), - output_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), - ) + if preserve_semantics: + util.copy_file( + final_shader_path.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), + output_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), + ) - util.copy_file( - final_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS_JSON), - output_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS_JSON), - ) + util.copy_file( + final_shader_path.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS_JSON), + output_shader_prefix.with_suffix( + shader_job_util.SUFFIX_TRANSFORMATIONS_JSON + ), + ) return reduced_test_dir -def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: +def run_reduction_on_report( # pylint: disable=too-many-locals; + test_dir: Path, reports_dir: Path +) -> None: test = test_util.metadata_read(test_dir) check( @@ -269,19 +296,37 @@ def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: language_suffix=(shader_job_util.SUFFIX_TRANSFORMATIONS,), ) + shader_spv_suffixes = shader_job_util.get_related_suffixes_that_exist( + shader_job_to_reduce.shader_job, language_suffix=(shader_job_util.SUFFIX_SPIRV,) + ) + try: reduced_test = test_dir for index, suffix in enumerate(shader_transformation_suffixes): + # E.g. .frag.transformations -> .frag + extension_to_reduce = str(Path(suffix).with_suffix("")) reduced_test = run_reduction( test_dir_reduction_output=test_dir, test_dir_to_reduce=reduced_test, shader_job_name_to_reduce=shader_job_to_reduce.name, - transformation_suffix_to_reduce=suffix, + extension_to_reduce=extension_to_reduce, preserve_semantics=True, - reduction_name=f"{index}_{suffix.split('.')[1]}", + reduction_name=f"0_{index}_{suffix.split('.')[1]}", ) - # TODO: reduce without preserving semantics if crash. + + if test.crash_signature != signature_util.BAD_IMAGE_SIGNATURE: + for index, suffix in enumerate(shader_spv_suffixes): + # E.g. .frag.spv -> .frag + extension_to_reduce = str(Path(suffix).with_suffix("")) + reduced_test = run_reduction( + test_dir_reduction_output=test_dir, + test_dir_to_reduce=reduced_test, + shader_job_name_to_reduce=shader_job_to_reduce.name, + extension_to_reduce=extension_to_reduce, + preserve_semantics=False, + reduction_name=f"1_{index}_{suffix.split('.')[1]}", + ) device_name = test.device.name From 331d7c1695cf8118ee2626eebf6c7ff50e714204 Mon Sep 17 00:00:00 2001 From: Abel Briggs Date: Tue, 8 Oct 2019 09:40:51 -0500 Subject: [PATCH 153/180] Add identity to put data in a larger type and pull it back out (#704) Fixes #675. --- .../fuzzer/OpaqueExpressionGenerator.java | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 4d2e0625d..614f45a20 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -89,9 +89,11 @@ public OpaqueExpressionGenerator(IRandom generator, GenerationParams generationP expressionIdentities.add(new IdentityClamp()); expressionIdentities.add(new IdentityRewriteComposite()); + expressionIdentities.add(new IdentityCompositeConstructorExpansion()); if (shadingLanguageVersion.supportedMixNonfloatBool()) { expressionIdentities.add(new IdentityMixBvec()); } + if (shadingLanguageVersion.supportedTranspose()) { expressionIdentities.add(new IdentityDoubleTranspose()); } @@ -1418,6 +1420,102 @@ public boolean preconditionHolds(Expr expr, BasicType basicType, boolean constCo } } + /** + * Identity transformation to insert an expression into a wider vector or matrix using a + * type constructor, then extract it back out again using another type constructor. When + * performed, transforms an expression e of type t to identity(t(identity(m(identity(e), ..)))), + * where m is a type of equal or greater width than t. + * The rules for this smaller -> larger -> smaller transformation are as follows: + * If the given type is a vector or scalar, the inner constructor type can be a vector of the + * same element type with equal/greater width than the given type, or (provided that the + * given type is a floating point genType) it can be a matrix with equal/greater column + * width than the given type. + * If the given type is a matrix, the inner constructor type can be a matrix with + * equal/greater column/row width than the given type. + */ + private class IdentityCompositeConstructorExpansion extends AbstractIdentityTransformation { + private IdentityCompositeConstructorExpansion() { + super(BasicType.allBasicTypes().stream() + .filter(item -> !Arrays.asList(BasicType.BVEC4, BasicType.IVEC4, BasicType.UVEC4, + BasicType.MAT4X4) + .contains(item)) + .collect(Collectors.toList()), true); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert !Arrays.asList(BasicType.BVEC4, BasicType.IVEC4, BasicType.UVEC4, BasicType.MAT4X4) + .contains(type); + final List innerConstructorTypes = new ArrayList(); + // Our inner constructor type will be equal or larger in width than our given type. The goal + // of the next set of conditionals is to populate this list with all types that can fit our + // given type. + final int maxVectorSize = 4; + if (!type.isMatrix()) { + for (int i = type.getNumElements(); i <= maxVectorSize; i++) { + innerConstructorTypes.add(BasicType.makeVectorType(type.getElementType(), i)); + } + if (type.getElementType() == BasicType.FLOAT) { + innerConstructorTypes.addAll(BasicType.allSquareMatrixTypes()); + if (shadingLanguageVersion.supportedNonSquareMatrices()) { + innerConstructorTypes.addAll(BasicType.allNonSquareMatrixTypes()); + } + } + } else { + for (BasicType constructorType : BasicType.allMatrixTypes()) { + if (type.getNumRows() <= constructorType.getNumRows() + && type.getNumColumns() <= constructorType.getNumColumns()) { + if (BasicType.allSquareMatrixTypes().contains(constructorType)) { + innerConstructorTypes.add(constructorType); + } else if (shadingLanguageVersion.supportedNonSquareMatrices()) { + innerConstructorTypes.add(constructorType); + } + } + } + } + assert !innerConstructorTypes.isEmpty(); + assert innerConstructorTypes.contains(type); + final BasicType randomInnerConstructorType = + innerConstructorTypes.get(generator.nextInt(innerConstructorTypes.size())); + final List innerConstructorArgs = new ArrayList(); + innerConstructorArgs.add( + applyIdentityFunction(expr.clone(), type, constContext, depth, fuzzer)); + // GLSL won't fill in the blanks of the inner constructor unless a matrix is being constructed + // from another matrix. + if (!type.isMatrix()) { + final int numExcessConstructorArgs = + randomInnerConstructorType.getNumElements() - type.getNumElements(); + for (int i = 0; i < numExcessConstructorArgs; i++) { + // We add a boolean if the element type is boolean, and a zero/one otherwise. + innerConstructorArgs.add( + type.getElementType() == BasicType.BOOL + ? makeOpaqueBoolean(generator.nextBoolean(), + BasicType.BOOL, + constContext, + depth, + fuzzer) + : makeOpaqueZeroOrOne(generator.nextBoolean(), + type.getElementType(), + constContext, + depth, + fuzzer)); + } + } + return identityConstructor( + expr, + applyIdentityFunction( + new TypeConstructorExpr( + type.toString(), + applyIdentityFunction( + new TypeConstructorExpr( + randomInnerConstructorType.toString(), + innerConstructorArgs), + randomInnerConstructorType, constContext, depth, fuzzer)), + type, constContext, depth, fuzzer)); + } + } + /** * Identity transformation to transpose a matrix twice. When performed, transforms an expression * of a matrix m -> transpose(identity(transpose(identity(m)))). From 8c86ceb516f03749add87f2a8dfab5310635ab76 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Fri, 11 Oct 2019 10:57:08 +0100 Subject: [PATCH 154/180] Update and improve licenses.py (#773) --- build/travis/licenses.py | 198 ++++++++++++++---- .../licenses/android-support-libraries.txt | 0 .../licenses/elg-headers.txt | 0 {build => third_party}/licenses/javacpp.txt | 0 .../licenses/libpng-LICENSE.txt | 0 .../licenses/max-sills-shaders.txt | 0 {build => third_party}/licenses/minigbm.txt | 0 .../licenses/opengl-headers.txt | 0 .../licenses/valters-mednis-shaders.txt | 0 .../licenses/zt-process-killer.txt | 0 10 files changed, 163 insertions(+), 35 deletions(-) rename {build => third_party}/licenses/android-support-libraries.txt (100%) rename {build => third_party}/licenses/elg-headers.txt (100%) rename {build => third_party}/licenses/javacpp.txt (100%) rename {build => third_party}/licenses/libpng-LICENSE.txt (100%) rename {build => third_party}/licenses/max-sills-shaders.txt (100%) rename {build => third_party}/licenses/minigbm.txt (100%) rename {build => third_party}/licenses/opengl-headers.txt (100%) rename {build => third_party}/licenses/valters-mednis-shaders.txt (100%) rename {build => third_party}/licenses/zt-process-killer.txt (100%) diff --git a/build/travis/licenses.py b/build/travis/licenses.py index 808071719..032ba6701 100644 --- a/build/travis/licenses.py +++ b/build/travis/licenses.py @@ -48,7 +48,7 @@ def get_extras(): 'comment': 'Used by angle', 'name': 'libpng', 'url': 'http://libpng.org/pub/png/libpng.html', - 'license_file': 'build/licenses/libpng-LICENSE.txt', + 'license_file': path('third_party', 'licenses', 'libpng-LICENSE.txt'), 'license_url': '', 'skipped': '', }, @@ -58,7 +58,16 @@ def get_extras(): 'name': 'minigbm', 'url': 'https://chromium.googlesource.com/chromiumos/platform/minigbm/+/master', 'license_url': '', - 'license_file': 'build/licenses/minigbm.txt', + 'license_file': path('third_party', 'licenses', 'minigbm.txt'), + 'skipped': '', + }, + + 'rapidjson': { + 'comment': 'Used by angle', + 'name': 'RapidJSON', + 'url': 'https://github.com/TenCent/rapidjson', + 'license_url': 'https://raw.githubusercontent.com/Tencent/rapidjson/6a6bed2759d42891f9e29a37b21315d3192890ed/license.txt', + 'license_file': '', 'skipped': '', }, @@ -107,14 +116,11 @@ def get_extras(): 'skipped': '', }, - 'vulkan-validationLayers': { + 'vulkan-validation-layers': { 'comment': 'Used by angle', 'name': 'Vulkan Validation Layers - Vulkan Ecosystem Components', 'url': 'https://github.com/KhronosGroup/Vulkan-ValidationLayers', - 'license_url': [ - 'http://www.apache.org/licenses/LICENSE-2.0.txt', - 'https://raw.githubusercontent.com/KhronosGroup/Vulkan-ValidationLayers/44b7b80e5e52f2962077346caa4ed175798d16df/COPYRIGHT.txt' - ], + 'license_url': 'https://raw.githubusercontent.com/KhronosGroup/Vulkan-ValidationLayers/b8d149f81be4496ef8df11097e17365268ea832f/LICENSE.txt', 'license_file': '', 'skipped': '', }, @@ -166,29 +172,38 @@ def get_extras(): 'skipped': '', }, - 'llvm': { + 'libbacktrace': { + 'comment': 'Used by swiftshader', + 'name': 'libbacktrace', + 'url': 'https://github.com/ianlancetaylor/libbacktrace', + 'license_url': 'https://raw.githubusercontent.com/ianlancetaylor/libbacktrace/5a99ff7fed66b8ea8f09c9805c138524a7035ece/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + 'llvm-7.0': { 'comment': '', - 'name': 'LLVM', + 'name': 'LLVM 7.0', 'url': 'https://llvm.org/', - 'license_url': 'https://raw.githubusercontent.com/google/swiftshader/720aec1cd6ebf4c4d74326c5faaddd57ee351609/third_party/LLVM/LICENSE.TXT', + 'license_url': 'https://raw.githubusercontent.com/google/swiftshader/ae022faf53b9f648324874063d7147ba7b555417/third_party/llvm-7.0/llvm/LICENSE.TXT', 'license_file': '', 'skipped': '', }, - 'stlport-cpp11-extension': { + 'marl': { 'comment': '', - 'name': 'stlport-cpp11-extension', - 'url': 'https://github.com/google/swiftshader/tree/720aec1cd6ebf4c4d74326c5faaddd57ee351609/third_party/stlport-cpp11-extension', - 'license_url': 'http://www.apache.org/licenses/LICENSE-2.0.txt', + 'name': 'Marl', + 'url': 'https://github.com/google/marl', + 'license_url': 'https://raw.githubusercontent.com/google/swiftshader/ae022faf53b9f648324874063d7147ba7b555417/third_party/marl/LICENSE', 'license_file': '', 'skipped': '', }, - 'subzero': { + 'stlport-cpp11-extension': { 'comment': '', - 'name': 'Subzero from Chromium', - 'url': 'https://github.com/google/swiftshader/tree/720aec1cd6ebf4c4d74326c5faaddd57ee351609/third_party/subzero', - 'license_url': 'https://raw.githubusercontent.com/google/swiftshader/720aec1cd6ebf4c4d74326c5faaddd57ee351609/third_party/subzero/LICENSE.TXT', + 'name': 'stlport-cpp11-extension', + 'url': 'https://github.com/google/swiftshader/tree/ae022faf53b9f648324874063d7147ba7b555417/third_party/stlport-cpp11-extension', + 'license_url': 'http://www.apache.org/licenses/LICENSE-2.0.txt', 'license_file': '', 'skipped': '', }, @@ -199,7 +214,7 @@ def get_extras(): 'name': 'OpenGL headers', 'url': 'https://github.com/KhronosGroup/OpenGL-Registry', 'license_url': '', - 'license_file': 'build/licenses/opengl-headers.txt', + 'license_file': path('third_party', 'licenses', 'opengl-headers.txt'), 'skipped': '', }, @@ -208,7 +223,7 @@ def get_extras(): 'name': 'EGL headers', 'url': 'https://github.com/KhronosGroup/EGL-Registry', 'license_url': '', - 'license_file': 'build/licenses/opengl-headers.txt', + 'license_file': path('third_party', 'licenses', 'opengl-headers.txt'), 'skipped': '', }, @@ -240,7 +255,7 @@ def get_extras(): }, 'lodepng': { - 'comment': '', + 'comment': 'Used by Vulkan worker and Amber', 'name': 'LodePNG', 'url': 'https://github.com/lvandeve/lodepng', 'license_url': 'https://raw.githubusercontent.com/lvandeve/lodepng/941de186edfc68bca5ba1043423d0937b4baf3c6/LICENSE', @@ -269,7 +284,7 @@ def get_extras(): 'https://www.shadertoy.com/view/lsKXDW', ], 'license_url': '', - 'license_file': 'build/licenses/max-sills-shaders.txt', + 'license_file': path('third_party', 'licenses', 'max-sills-shaders.txt'), 'skipped': '', }, @@ -278,7 +293,7 @@ def get_extras(): 'name': 'Valters Mednis\' GLSL shaders', 'url':'https://www.shadertoy.com/view/ltVGWG', 'license_url': '', - 'license_file': 'build/licenses/valters-mednis-shaders.txt', + 'license_file': path('third_party', 'licenses', 'valters-mednis-shaders.txt'), 'skipped': '', }, @@ -307,7 +322,7 @@ def get_extras(): 'name': 'Android support libraries', 'url': 'https://source.android.com/', 'license_url': '', - 'license_file': 'build/licenses/android-support-libraries.txt', + 'license_file': path('third_party', 'licenses', 'android-support-libraries.txt'), 'skipped': '', }, @@ -360,7 +375,7 @@ def get_extras(): 'name': 'zt-process-killer', 'url': 'https://github.com/zeroturnaround/zt-process-killer', 'license_url': '', - 'license_file': 'build/licenses/zt-process-killer.txt', + 'license_file': path('third_party', 'licenses', 'zt-process-killer.txt'), 'skipped': '', }, @@ -385,6 +400,117 @@ def get_extras(): }, + 'amber': { + 'comment': '', + 'name': 'Amber', + 'url': 'https://github.com/google/amber', + 'license_url': 'https://raw.githubusercontent.com/google/amber/ab41eacc3fba9ea3365a57b1712837cb4ed6d2c4/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + + + 'amdvlk': { + 'comment': '', + 'name': 'AMD Open Source Driver for Vulkan (AMDVLK)', + 'url': 'https://github.com/GPUOpen-Drivers/AMDVLK', + 'license_url': 'https://raw.githubusercontent.com/GPUOpen-Drivers/AMDVLK/808b6d5603653e6efa7a0da46b4f519f0c105b27/LICENSE.txt', + 'license_file': '', + 'skipped': '', + }, + + 'cwpack': { + 'comment': '', + 'name': 'CWPack', + 'url': 'https://github.com/clwi/CWPack', + 'license_url': 'https://raw.githubusercontent.com/clwi/CWPack/43583ff9abe6f5e68602eccb24d5f5c3aceac51c/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + 'metrohash': { + 'comment': '', + 'name': 'MetroHash', + 'url': 'https://github.com/jandrewrogers/MetroHash', + 'license_url': 'https://raw.githubusercontent.com/jandrewrogers/MetroHash/690a521d9beb2e1050cc8f273fdabc13b31bf8f6/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + 'llvm': { + 'comment': 'Used by AMDVLK', + 'name': 'LLVM', + 'url': 'https://llvm.org/', + 'license_url': 'https://raw.githubusercontent.com/GPUOpen-Drivers/llvm/21b028dc97ee9b0c6629ff83c2924b3b80c5d6e7/LICENSE.TXT', + 'license_file': '', + 'skipped': '', + }, + + 'llpc': { + 'comment': '', + 'name': 'LLVM-Based Pipeline Compiler (LLPC)', + 'url': 'https://github.com/GPUOpen-Drivers/llpc', + 'license_url': [ + 'https://raw.githubusercontent.com/GPUOpen-Drivers/llpc/8b8fc751408274f1f96064e183956d6fab1d54d1/LICENSE', + 'https://raw.githubusercontent.com/GPUOpen-Drivers/AMDVLK/808b6d5603653e6efa7a0da46b4f519f0c105b27/LICENSE.txt', + ], + 'license_file': '', + 'skipped': '', + }, + + 'pal': { + 'comment': '', + 'name': 'Platform Abstraction Library (PAL)', + 'url': 'https://github.com/GPUOpen-Drivers/pal', + 'license_url': 'https://raw.githubusercontent.com/GPUOpen-Drivers/pal/3e28ea331cf8e7b0e5733e5bd7f7d16582c603c5/LICENSE.txt', + 'license_file': '', + 'skipped': '', + }, + + 'spvgen': { + 'comment': '', + 'name': 'SPVGEN', + 'url': 'https://github.com/GPUOpen-Drivers/spvgen', + 'license_url': [ + 'https://raw.githubusercontent.com/GPUOpen-Drivers/spvgen/2f31d1170e8a12a66168b23235638c4bbc43ecdc/LICENSE', + 'https://raw.githubusercontent.com/GPUOpen-Drivers/AMDVLK/808b6d5603653e6efa7a0da46b4f519f0c105b27/LICENSE.txt', + ], + 'license_file': '', + 'skipped': '', + }, + + 'xgl': { + 'comment': '', + 'name': 'Vulkan API Layer (XGL)', + 'url': 'https://github.com/GPUOpen-Drivers/xgl', + 'license_url': 'https://raw.githubusercontent.com/GPUOpen-Drivers/xgl/3252b6223947f9fc67399e0798b1062983925fce/LICENSE.txt', + 'license_file': '', + 'skipped': '', + }, + + + 'gfauto': { + 'comment': '', + 'name': 'GraphicsFuzz auto (gfauto)', + 'url': 'https://github.com/google/graphicsfuzz/tree/dev/gfauto', + 'license_url': 'https://raw.githubusercontent.com/google/graphicsfuzz/143f46a1298a2a1e012b8b3ab31fc87c2075e82f/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + + # Android stuff: + + 'bionic': { + 'comment': '', + 'name': 'Android bionic libc', + 'url': 'https://android.googlesource.com/platform/bionic/', + 'license_url': 'https://raw.githubusercontent.com/aosp-mirror/platform_bionic/48da33389b086d1a52524feee049d8219dc4a190/libc/NOTICE', + 'license_file': '', + 'skipped': '', + }, + } @@ -414,7 +540,7 @@ def get_maven_dependencies_populated(): 'name': 'The Alphanum Algorithm', 'url': 'http://www.davekoelle.com/alphanum.html', 'license_url': '', - 'license_file': 'third_party/alphanum-comparator/LICENSE', + 'license_file': path('third_party', 'alphanum-comparator', 'LICENSE'), 'skipped': '', }, 'com.graphicsfuzz:assembly-binaries': { @@ -470,7 +596,7 @@ def get_maven_dependencies_populated(): 'name': 'Animated GIF Writer', 'url': 'http://elliot.kroo.net/software/java/GifSequenceWriter/', 'license_url': '', - 'license_file': 'third_party/gif-sequence-writer/LICENSE', + 'license_file': path('third_party', 'gif-sequence-writer', 'LICENSE'), 'skipped': '', }, 'com.graphicsfuzz.thirdparty:jquery-js': { @@ -479,8 +605,8 @@ def get_maven_dependencies_populated(): 'url': 'https://jquery.org/', 'license_url': '', 'license_file': [ - 'third_party/jquery-js/LICENSE.txt', - 'third_party/jquery-js/AUTHORS.txt', + path('third_party', 'jquery-js', 'LICENSE.txt'), + path('third_party', 'jquery-js', 'AUTHORS.txt'), ], 'skipped': '', }, @@ -489,7 +615,7 @@ def get_maven_dependencies_populated(): 'name': 'six', 'url': 'https://pypi.org/project/six/', 'license_url': '', - 'license_file': 'third_party/python-six/LICENSE', + 'license_file': path('third_party', 'python-six', 'LICENSE'), 'skipped': '', }, 'com.graphicsfuzz:python': { @@ -513,7 +639,7 @@ def get_maven_dependencies_populated(): 'name': 'Semantic UI', 'url': 'https://github.com/semantic-org/semantic-ui/', 'license_url': '', - 'license_file': 'third_party/semantic-ui/LICENSE', + 'license_file': path('third_party', 'semantic-ui', 'LICENSE'), 'skipped': '', }, 'com.graphicsfuzz:server-static-public': { @@ -561,7 +687,7 @@ def get_maven_dependencies_populated(): 'name': 'Apache Thrift', 'url': 'https://thrift.apache.org/', 'license_url': '', - 'license_file': ['third_party/thrift-js/NOTICE', 'third_party/thrift-js/LICENSE'], + 'license_file': [path('third_party', 'thrift-js', 'NOTICE'), path('third_party', 'thrift-js', 'LICENSE')], 'skipped': '', }, 'com.graphicsfuzz.thirdparty:thrift-py': { @@ -780,7 +906,7 @@ def get_maven_dependencies_populated(): 'name': 'JavaCPP', 'url': 'https://github.com/bytedeco/javacpp', 'license_url': '', - 'license_file': 'build/licenses/javacpp.txt', + 'license_file': path('third_party', 'licenses', 'javacpp.txt'), 'skipped': '', }, 'org.bytedeco.javacpp-presets:opencv': { @@ -788,7 +914,7 @@ def get_maven_dependencies_populated(): 'name': 'JavaCPP Presets (OpenCV)', 'url': 'https://github.com/bytedeco/javacpp-presets', 'license_url': '', - 'license_file': 'build/licenses/javacpp.txt', + 'license_file': path('third_party', 'licenses', 'javacpp.txt'), 'skipped': '', }, 'org.eclipse.jetty:jetty-http': { @@ -954,9 +1080,11 @@ def go(): errors='ignore') as fout: fout.write('\n') + fout.write('Open source licenses for the GraphicsFuzz project.\n') + fout.write('https://github.com/google/graphicsfuzz\n\n') fout.write('Summary of projects:\n\n') - for (dep, details) in dependencies_populated.items(): + for (dep, details) in sorted(dependencies_populated.items(), key=lambda x: x[1]['name'].lower()): print('Dependency: ' + dep) if len(details['skipped']) > 0: print('Skipping (' + details['skipped'] + ')') diff --git a/build/licenses/android-support-libraries.txt b/third_party/licenses/android-support-libraries.txt similarity index 100% rename from build/licenses/android-support-libraries.txt rename to third_party/licenses/android-support-libraries.txt diff --git a/build/licenses/elg-headers.txt b/third_party/licenses/elg-headers.txt similarity index 100% rename from build/licenses/elg-headers.txt rename to third_party/licenses/elg-headers.txt diff --git a/build/licenses/javacpp.txt b/third_party/licenses/javacpp.txt similarity index 100% rename from build/licenses/javacpp.txt rename to third_party/licenses/javacpp.txt diff --git a/build/licenses/libpng-LICENSE.txt b/third_party/licenses/libpng-LICENSE.txt similarity index 100% rename from build/licenses/libpng-LICENSE.txt rename to third_party/licenses/libpng-LICENSE.txt diff --git a/build/licenses/max-sills-shaders.txt b/third_party/licenses/max-sills-shaders.txt similarity index 100% rename from build/licenses/max-sills-shaders.txt rename to third_party/licenses/max-sills-shaders.txt diff --git a/build/licenses/minigbm.txt b/third_party/licenses/minigbm.txt similarity index 100% rename from build/licenses/minigbm.txt rename to third_party/licenses/minigbm.txt diff --git a/build/licenses/opengl-headers.txt b/third_party/licenses/opengl-headers.txt similarity index 100% rename from build/licenses/opengl-headers.txt rename to third_party/licenses/opengl-headers.txt diff --git a/build/licenses/valters-mednis-shaders.txt b/third_party/licenses/valters-mednis-shaders.txt similarity index 100% rename from build/licenses/valters-mednis-shaders.txt rename to third_party/licenses/valters-mednis-shaders.txt diff --git a/build/licenses/zt-process-killer.txt b/third_party/licenses/zt-process-killer.txt similarity index 100% rename from build/licenses/zt-process-killer.txt rename to third_party/licenses/zt-process-killer.txt From 4f08872ca1dbd635fa1ef948b453e559d9dccf3e Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Wed, 16 Oct 2019 10:01:17 +0100 Subject: [PATCH 155/180] licenses.py: sort properly (#774) --- build/travis/licenses.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build/travis/licenses.py b/build/travis/licenses.py index 032ba6701..01cf8a911 100644 --- a/build/travis/licenses.py +++ b/build/travis/licenses.py @@ -1071,6 +1071,8 @@ def go(): print('Missing dependency license information ' + dep) sys.exit(1) + dependencies_populated_list = sorted(dependencies_populated.items(), key=lambda x: x[1]['name'].lower()) + # Write an OPEN_SOURCE_LICENSES.TXT file. with io.open( 'OPEN_SOURCE_LICENSES.TXT', @@ -1084,7 +1086,7 @@ def go(): fout.write('https://github.com/google/graphicsfuzz\n\n') fout.write('Summary of projects:\n\n') - for (dep, details) in sorted(dependencies_populated.items(), key=lambda x: x[1]['name'].lower()): + for (dep, details) in dependencies_populated_list: print('Dependency: ' + dep) if len(details['skipped']) > 0: print('Skipping (' + details['skipped'] + ')') @@ -1103,7 +1105,7 @@ def go(): fout.write('\n') fout.write('All projects and licenses:\n') - for (dep, details) in dependencies_populated.items(): + for (dep, details) in dependencies_populated_list: print('Dependency: ' + dep) if len(details['skipped']) > 0: print('Skipping (' + details['skipped'] + ')') From b91d34e5c39441fc4a27f08d47b47608d45f9ddd Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Wed, 16 Oct 2019 10:01:37 +0100 Subject: [PATCH 156/180] Add "lite" build profile (#775) --- graphicsfuzz/pom.xml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/graphicsfuzz/pom.xml b/graphicsfuzz/pom.xml index 06b1bf236..655c114a9 100644 --- a/graphicsfuzz/pom.xml +++ b/graphicsfuzz/pom.xml @@ -29,6 +29,23 @@ limitations under the License. ../parent-all/pom.xml + + **/*.pdb + + false + + + + + lite + + **/*.pdb,**/swiftshader/**,**/angle/** + **/opencv-4* + true + + + + @@ -210,6 +227,10 @@ limitations under the License. + + + + @@ -246,8 +267,8 @@ limitations under the License. - - + + @@ -260,7 +281,7 @@ limitations under the License. - + From b82cf495af1dea454218a332b88d2d309657594d Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 17 Oct 2019 10:50:13 +0100 Subject: [PATCH 157/180] Remove unused uniforms from sample shaders (#776) --- shaders/src/main/glsl/samples/100/colorgrid_modulo.frag | 1 - shaders/src/main/glsl/samples/300es/colorgrid_modulo.frag | 1 - .../src/main/glsl/samples/300es/selection_sort_struct.frag | 2 -- .../src/main/glsl/samples/300es/selection_sort_struct.json | 7 ------- shaders/src/main/glsl/samples/300es/squares.json | 7 ------- shaders/src/main/glsl/samples/310es/colorgrid_modulo.frag | 1 - .../src/main/glsl/samples/310es/selection_sort_struct.frag | 2 -- .../src/main/glsl/samples/310es/selection_sort_struct.json | 7 ------- shaders/src/main/glsl/samples/310es/squares.json | 7 ------- .../src/main/glsl/samples/310es/squares_intfunctions.json | 7 ------- shaders/src/main/glsl/samples/webgl1/colorgrid_modulo.frag | 1 - shaders/src/main/glsl/samples/webgl2/colorgrid_modulo.frag | 1 - 12 files changed, 44 deletions(-) diff --git a/shaders/src/main/glsl/samples/100/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/100/colorgrid_modulo.frag index 642e99d5e..10978c024 100644 --- a/shaders/src/main/glsl/samples/100/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/100/colorgrid_modulo.frag @@ -18,7 +18,6 @@ precision mediump float; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { diff --git a/shaders/src/main/glsl/samples/300es/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/300es/colorgrid_modulo.frag index 5567f1af9..f2487d549 100644 --- a/shaders/src/main/glsl/samples/300es/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/300es/colorgrid_modulo.frag @@ -20,7 +20,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { diff --git a/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag b/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag index ad151bb49..b833868b1 100644 --- a/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag +++ b/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag @@ -20,8 +20,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; - uniform vec2 resolution; struct Obj { diff --git a/shaders/src/main/glsl/samples/300es/selection_sort_struct.json b/shaders/src/main/glsl/samples/300es/selection_sort_struct.json index d7b58d068..269b946d9 100644 --- a/shaders/src/main/glsl/samples/300es/selection_sort_struct.json +++ b/shaders/src/main/glsl/samples/300es/selection_sort_struct.json @@ -1,11 +1,4 @@ { - "injectionSwitch": { - "func": "glUniform2f", - "args": [ - 0.0, - 1.0 - ] - }, "resolution": { "func": "glUniform2f", "args": [ diff --git a/shaders/src/main/glsl/samples/300es/squares.json b/shaders/src/main/glsl/samples/300es/squares.json index bc2d1fb1b..fdc0723fa 100644 --- a/shaders/src/main/glsl/samples/300es/squares.json +++ b/shaders/src/main/glsl/samples/300es/squares.json @@ -5,13 +5,6 @@ 0.0 ] }, - "mouse": { - "func": "glUniform2f", - "args": [ - 0.0, - 0.0 - ] - }, "resolution": { "func": "glUniform2f", "args": [ diff --git a/shaders/src/main/glsl/samples/310es/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/310es/colorgrid_modulo.frag index 8416a648b..4961cb25c 100644 --- a/shaders/src/main/glsl/samples/310es/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/310es/colorgrid_modulo.frag @@ -20,7 +20,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { diff --git a/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag b/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag index 44f9d0adf..827b02271 100644 --- a/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag +++ b/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag @@ -20,8 +20,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; - uniform vec2 resolution; struct Obj { diff --git a/shaders/src/main/glsl/samples/310es/selection_sort_struct.json b/shaders/src/main/glsl/samples/310es/selection_sort_struct.json index d7b58d068..269b946d9 100644 --- a/shaders/src/main/glsl/samples/310es/selection_sort_struct.json +++ b/shaders/src/main/glsl/samples/310es/selection_sort_struct.json @@ -1,11 +1,4 @@ { - "injectionSwitch": { - "func": "glUniform2f", - "args": [ - 0.0, - 1.0 - ] - }, "resolution": { "func": "glUniform2f", "args": [ diff --git a/shaders/src/main/glsl/samples/310es/squares.json b/shaders/src/main/glsl/samples/310es/squares.json index bc2d1fb1b..fdc0723fa 100644 --- a/shaders/src/main/glsl/samples/310es/squares.json +++ b/shaders/src/main/glsl/samples/310es/squares.json @@ -5,13 +5,6 @@ 0.0 ] }, - "mouse": { - "func": "glUniform2f", - "args": [ - 0.0, - 0.0 - ] - }, "resolution": { "func": "glUniform2f", "args": [ diff --git a/shaders/src/main/glsl/samples/310es/squares_intfunctions.json b/shaders/src/main/glsl/samples/310es/squares_intfunctions.json index 14446b79a..a12f190d9 100644 --- a/shaders/src/main/glsl/samples/310es/squares_intfunctions.json +++ b/shaders/src/main/glsl/samples/310es/squares_intfunctions.json @@ -5,13 +5,6 @@ 0.0 ] }, - "mouse": { - "func": "glUniform2f", - "args": [ - 0.0, - 0.0 - ] - }, "resolution": { "func": "glUniform2f", "args": [ diff --git a/shaders/src/main/glsl/samples/webgl1/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/webgl1/colorgrid_modulo.frag index 093653d47..4f0c4a2e4 100644 --- a/shaders/src/main/glsl/samples/webgl1/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/webgl1/colorgrid_modulo.frag @@ -19,7 +19,6 @@ precision mediump float; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { diff --git a/shaders/src/main/glsl/samples/webgl2/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/webgl2/colorgrid_modulo.frag index 4aa24d598..71033c147 100644 --- a/shaders/src/main/glsl/samples/webgl2/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/webgl2/colorgrid_modulo.frag @@ -21,7 +21,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { From f661316522b9d4183103f7acc8751696ab77f433 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 29 Oct 2019 15:59:28 +0000 Subject: [PATCH 158/180] gfauto: various fixes (#785) * gfauto: stop if we cannot get stack traces * gfauto: add instructions for Python issue * Add log file to spirv fuzz failures * Handle glsl-fuzz failures * gfauto: update whitelist.dic * gfauto: clearer error for missing directories * gfauto: better no settings file error message * gfauto: use address as signature as last resort * gfauto: use histogram comparison * gfauto: dump all buffers via Amber * gfauto: allow disabling reductions in settings * gfauto: tool.py: don't require spirv-opt --- gfauto/README.md | 41 ++++++++- gfauto/gfauto/amber_converter.py | 2 +- gfauto/gfauto/android_device.py | 2 +- gfauto/gfauto/fuzz.py | 67 +++++++++----- gfauto/gfauto/fuzz_glsl_test.py | 139 ++++++++++++++++++------------ gfauto/gfauto/fuzz_spirv_test.py | 23 +++-- gfauto/gfauto/host_device_util.py | 11 ++- gfauto/gfauto/settings.proto | 10 +++ gfauto/gfauto/settings_pb2.py | 25 +++++- gfauto/gfauto/settings_pb2.pyi | 10 ++- gfauto/gfauto/settings_util.py | 16 ++-- gfauto/gfauto/signature_util.py | 10 ++- gfauto/gfauto/tool.py | 10 ++- gfauto/gfauto/util.py | 4 +- gfauto/whitelist.dic | 2 + 15 files changed, 265 insertions(+), 107 deletions(-) diff --git a/gfauto/README.md b/gfauto/README.md index 7e5e7bc3a..18f926047 100644 --- a/gfauto/README.md +++ b/gfauto/README.md @@ -15,7 +15,12 @@ GraphicsFuzz auto (this project) provides scripts for running these tools with m > On Windows, you can use the Git Bash shell, or adapt the commands (including those inside `dev_shell.sh.template`) to the Windows command prompt. -Execute `./dev_shell.sh.template`. If the default settings don't work, make a copy of the file called `dev_shell.sh` and modify according to the comments before executing. `pip` must be installed for the version of Python you wish to use. +Execute `./dev_shell.sh.template`. If the default settings don't work, make a copy of the file called `dev_shell.sh` and modify according to the comments before executing. `pip` must be installed for the version of Python you wish to use. Note that you can do e.g. `export PYTHON=python3.6.8` to set your preferred Python binary. We currently target Python 3.6. + +> Pip for Python 3.6 may be broken on certain Debian distributions. +> You can just use the newer Python 3.7+ version provided by your +> distribution. +> See "Installing Python" below if you want to use Python 3.6. The script generates and activates a Python virtual environment (located at `.venv/`) with all dependencies installed. @@ -169,3 +174,37 @@ You can run parallel instances of gfauto (just for increased throughput, not wit ```sh parallel -j 32 -i gfauto_fuzz -- $(seq 100) ``` + +# Installing Python + +To manually install Python on your Linux distribution, you can use `pyenv`. + +https://github.com/pyenv/pyenv#basic-github-checkout + +In summary: + +* Install the required packages recommended [here](https://github.com/pyenv/pyenv/wiki/Common-build-problems). + +* Then: + +```sh +git clone https://github.com/pyenv/pyenv.git ~/.pyenv + +# Add the following two lines to your ~/.bashrc file. +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH + +# In a new terminal: +eval "$(pyenv init -) +pyenv install 3.6.9 +pyenv global 3.6.9 + +# Now execute the development shell script, as usual. +export PYTHON="python" +./dev_shell.sh.template +``` + +You can reactivate the development shell later, +as explained above, using +`source .venv/bin/activate`, +without having to re-execute the above `pyenv` commands. diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py index 994034c0d..d8241db3e 100644 --- a/gfauto/gfauto/amber_converter.py +++ b/gfauto/gfauto/amber_converter.py @@ -571,7 +571,7 @@ def graphics_shader_job_amber_test_to_amber_script( for pipeline_index in range(1, len(jobs)): prefix_0 = jobs[0].name_prefix prefix_1 = jobs[pipeline_index].name_prefix - result += f"EXPECT {prefix_0}_framebuffer RMSE_BUFFER {prefix_1}_framebuffer TOLERANCE 7" + result += f"EXPECT {prefix_0}_framebuffer EQ_HISTOGRAM_EMD_BUFFER {prefix_1}_framebuffer TOLERANCE 0.005" result += "\n" if amberfy_settings.extra_commands: diff --git a/gfauto/gfauto/android_device.py b/gfauto/gfauto/android_device.py index 521fba084..ec351c6b8 100644 --- a/gfauto/gfauto/android_device.py +++ b/gfauto/gfauto/android_device.py @@ -306,7 +306,7 @@ def run_amber_on_device_helper( amber_flags += " -ps" else: if dump_image: - amber_flags += f" -i {fuzz.IMAGE_FILE_NAME} -I variant_framebuffer" + amber_flags += f" -I variant_framebuffer -i {fuzz.VARIANT_IMAGE_FILE_NAME} -I reference_framebuffer -i {fuzz.REFERENCE_IMAGE_FILE_NAME}" if dump_buffer: amber_flags += f" -b {fuzz.BUFFER_FILE_NAME} -B 0" diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index d3cd84571..067aa7838 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -25,7 +25,7 @@ import shutil import sys from pathlib import Path -from typing import Optional +from typing import List, Optional from gfauto import ( artifact_util, @@ -40,6 +40,7 @@ util, ) from gfauto.gflogging import log +from gfauto.util import check_dir_exists # Root: # - donors/ (contains GLSL shader jobs) @@ -97,7 +98,8 @@ # -IMAGE_FILE_NAME = "image.png" +REFERENCE_IMAGE_FILE_NAME = "reference.png" +VARIANT_IMAGE_FILE_NAME = "variant.png" BUFFER_FILE_NAME = "buffer.bin" BEST_REDUCTION_NAME = "best" @@ -143,6 +145,12 @@ def main() -> None: action="store_true", ) + parser.add_argument( + "--force_no_stack_traces", + help="Continue even if we cannot get stack traces (using catchsegv or cdb).", + action="store_true", + ) + parsed_args = parser.parse_args(sys.argv[1:]) settings_path = Path(parsed_args.settings) @@ -150,17 +158,25 @@ def main() -> None: parsed_args.iteration_seed ) use_spirv_fuzz: bool = parsed_args.use_spirv_fuzz + force_no_stack_traces: bool = parsed_args.force_no_stack_traces with util.file_open_text(Path(f"log_{get_random_name()}.txt"), "w") as log_file: gflogging.push_stream_for_logging(log_file) try: - main_helper(settings_path, iteration_seed, use_spirv_fuzz) + main_helper( + settings_path, iteration_seed, use_spirv_fuzz, force_no_stack_traces + ) + except settings_util.NoSettingsFile as exception: + log(str(exception)) finally: gflogging.pop_stream_for_logging() def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements; - settings_path: Path, iteration_seed_override: Optional[int], use_spirv_fuzz: bool + settings_path: Path, + iteration_seed_override: Optional[int], + use_spirv_fuzz: bool, + force_no_stack_traces: bool, ) -> None: settings = settings_util.read_or_create(settings_path) @@ -174,26 +190,35 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many donors_dir = Path() / "donors" spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders" - if donors_dir.exists(): - try: - artifact_util.artifact_path_get_root() - except FileNotFoundError: - log( - "Could not find ROOT file (in the current directory or above) to mark where binaries should be stored. " - "Creating a ROOT file in the current directory." - ) - util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") + try: + artifact_util.artifact_path_get_root() + except FileNotFoundError: + log( + "Could not find ROOT file (in the current directory or above) to mark where binaries should be stored. " + "Creating a ROOT file in the current directory." + ) + util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") # Log a warning if there is no tool on the PATH for printing stack traces. - util.prepend_catchsegv_if_available([], log_warning=True) - - # TODO: make GraphicsFuzz find donors recursively. - references = sorted(references_dir.rglob("*.json")) + prepended = util.prepend_catchsegv_if_available([], log_warning=True) + if not force_no_stack_traces and not prepended: + raise AssertionError("Stopping because we cannot get stack traces.") - # Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file. - references = [ref for ref in references if shader_job_util.get_related_files(ref)] + spirv_fuzz_shaders: List[Path] = [] + references: List[Path] = [] - spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json")) + if use_spirv_fuzz: + check_dir_exists(spirv_fuzz_shaders_dir) + spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json")) + else: + check_dir_exists(references_dir) + check_dir_exists(donors_dir) + # TODO: make GraphicsFuzz find donors recursively. + references = sorted(references_dir.rglob("*.json")) + # Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file. + references = [ + ref for ref in references if shader_job_util.get_related_files(ref) + ] binary_manager = binaries_util.get_default_binary_manager().get_child_binary_manager( list(settings.custom_binaries), prepend=True @@ -260,6 +285,7 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many fuzz_glsl_test.fuzz_glsl( staging_dir, reports_dir, + fuzz_failures_dir, active_devices, references, donors_dir, @@ -289,4 +315,3 @@ def create_summary_and_reproduce( if __name__ == "__main__": main() - sys.exit(0) diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py index 1b43177da..5520c2e7f 100644 --- a/gfauto/gfauto/fuzz_glsl_test.py +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -59,52 +59,13 @@ def __init__(self, message: str, reduction_name: str, reduction_work_dir: Path): def fuzz_glsl( staging_dir: Path, reports_dir: Path, + fuzz_failures_dir: Path, active_devices: List[Device], references: List[Path], donors_dir: Path, settings: Settings, binary_manager: binaries_util.BinaryManager, ) -> None: - test_dirs = create_staging_tests( - staging_dir, references, donors_dir, binary_manager - ) - - for test_dir in test_dirs: - if handle_test(test_dir, reports_dir, active_devices, binary_manager, settings): - # If we generated a report, don't bother trying other optimization combinations. - break - - -def make_test( - base_source_dir: Path, - subtest_dir: Path, - spirv_opt_args: Optional[List[str]], - binary_manager: binaries_util.BinaryManager, -) -> Path: - # Create the subtest by copying the base source. - util.copy_dir(base_source_dir, test_util.get_source_dir(subtest_dir)) - - test = Test(glsl=TestGlsl(spirv_opt_args=spirv_opt_args)) - - test.binaries.extend([binary_manager.get_binary_by_name(name="glslangValidator")]) - test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-dis")]) - test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-val")]) - if spirv_opt_args: - test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-opt")]) - - # Write the test metadata. - test_util.metadata_write(test, subtest_dir) - - return subtest_dir - - -def create_staging_tests( - staging_dir: Path, - references: List[Path], - donors_dir: Path, - binary_manager: binaries_util.BinaryManager, -) -> List[Path]: - staging_name = staging_dir.name template_source_dir = staging_dir / "source_template" @@ -113,21 +74,35 @@ def create_staging_tests( # TODO: Allow GraphicsFuzz to be downloaded. - # Create the prepared (for Vulkan GLSL) reference. - glsl_generate_util.run_prepare_reference( - util.tool_on_path("graphicsfuzz-tool"), - unprepared_reference_shader_job, - template_source_dir / test_util.REFERENCE_DIR / test_util.SHADER_JOB, - ) + try: + with util.file_open_text(staging_dir / "log.txt", "w") as log_file: + try: + gflogging.push_stream_for_logging(log_file) + + # Create the prepared (for Vulkan GLSL) reference. + glsl_generate_util.run_prepare_reference( + util.tool_on_path("graphicsfuzz-tool"), + unprepared_reference_shader_job, + template_source_dir + / test_util.REFERENCE_DIR + / test_util.SHADER_JOB, + ) - # Generate the variant (GraphicsFuzz requires the unprepared reference as input). - glsl_generate_util.run_generate( - util.tool_on_path("graphicsfuzz-tool"), - unprepared_reference_shader_job, - donors_dir, - template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, - seed=str(random.getrandbits(glsl_generate_util.GENERATE_SEED_BITS)), - ) + # Generate the variant (GraphicsFuzz requires the unprepared reference as input). + glsl_generate_util.run_generate( + util.tool_on_path("graphicsfuzz-tool"), + unprepared_reference_shader_job, + donors_dir, + template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, + seed=str(random.getrandbits(glsl_generate_util.GENERATE_SEED_BITS)), + ) + finally: + gflogging.pop_stream_for_logging() + except subprocess.CalledProcessError: + util.mkdirs_p(fuzz_failures_dir) + if len(list(fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures: + util.copy_dir(staging_dir, fuzz_failures_dir / staging_dir.name) + return test_dirs = [ make_test( @@ -168,7 +143,33 @@ def create_staging_tests( ), ] - return test_dirs + for test_dir in test_dirs: + if handle_test(test_dir, reports_dir, active_devices, binary_manager, settings): + # If we generated a report, don't bother trying other optimization combinations. + break + + +def make_test( + base_source_dir: Path, + subtest_dir: Path, + spirv_opt_args: Optional[List[str]], + binary_manager: binaries_util.BinaryManager, +) -> Path: + # Create the subtest by copying the base source. + util.copy_dir(base_source_dir, test_util.get_source_dir(subtest_dir)) + + test = Test(glsl=TestGlsl(spirv_opt_args=spirv_opt_args)) + + test.binaries.extend([binary_manager.get_binary_by_name(name="glslangValidator")]) + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-dis")]) + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-val")]) + if spirv_opt_args: + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-opt")]) + + # Write the test metadata. + test_util.metadata_write(test, subtest_dir) + + return subtest_dir def run( @@ -283,6 +284,29 @@ def maybe_add_report( # pylint: disable=too-many-locals; return test_dir_in_reports +def should_reduce_report(settings: Settings, test_dir: Path) -> bool: + test = test_util.metadata_read(test_dir) + status = test.expected_status + signature = test.crash_signature + + if not settings.reduce_tool_crashes and status == fuzz.STATUS_TOOL_CRASH: + return False + if ( + not settings.reduce_crashes + and status == fuzz.STATUS_CRASH + and signature != signature_util.BAD_IMAGE_SIGNATURE + ): + return False + if ( + not settings.reduce_bad_images + and status == fuzz.STATUS_CRASH + and signature_util == signature_util.BAD_IMAGE_SIGNATURE + ): + return False + + return True + + def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: test = test_util.metadata_read(test_dir) @@ -356,7 +380,10 @@ def handle_test( # For each report, run a reduction on the target device with the device-specific crash signature. for test_dir_in_reports in report_paths: - run_reduction_on_report(test_dir_in_reports, reports_dir) + if should_reduce_report(settings, test_dir_in_reports): + run_reduction_on_report(test_dir_in_reports, reports_dir) + else: + log("Skipping reduction due to settings.") # For each report, create a summary and reproduce the bug. for test_dir_in_reports in report_paths: diff --git a/gfauto/gfauto/fuzz_spirv_test.py b/gfauto/gfauto/fuzz_spirv_test.py index 0c4360656..cbc262754 100644 --- a/gfauto/gfauto/fuzz_spirv_test.py +++ b/gfauto/gfauto/fuzz_spirv_test.py @@ -41,6 +41,7 @@ ) from gfauto.device_pb2 import Device from gfauto.fuzz_glsl_test import ReductionFailedError +from gfauto.gflogging import log from gfauto.settings_pb2 import Settings from gfauto.test_pb2 import Test, TestSpirvFuzz from gfauto.util import check @@ -382,7 +383,10 @@ def handle_test( # For each report, run a reduction on the target device with the device-specific crash signature. for test_dir_in_reports in report_paths: - run_reduction_on_report(test_dir_in_reports, reports_dir) + if fuzz_glsl_test.should_reduce_report(settings, test_dir_in_reports): + run_reduction_on_report(test_dir_in_reports, reports_dir) + else: + log("Skipping reduction due to settings.") # For each report, create a summary and reproduce the bug. for test_dir_in_reports in report_paths: @@ -413,12 +417,17 @@ def fuzz_spirv( # TODO: Allow using downloaded spirv-fuzz. try: - spirv_fuzz_util.run_generate_on_shader_job( - util.tool_on_path("spirv-fuzz"), - reference_spirv_shader_job, - template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, - seed=str(random.getrandbits(spirv_fuzz_util.GENERATE_SEED_BITS)), - ) + with util.file_open_text(staging_dir / "log.txt", "w") as log_file: + try: + gflogging.push_stream_for_logging(log_file) + spirv_fuzz_util.run_generate_on_shader_job( + util.tool_on_path("spirv-fuzz"), + reference_spirv_shader_job, + template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, + seed=str(random.getrandbits(spirv_fuzz_util.GENERATE_SEED_BITS)), + ) + finally: + gflogging.pop_stream_for_logging() except subprocess.CalledProcessError: util.mkdirs_p(fuzz_failures_dir) if len(list(fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures: diff --git a/gfauto/gfauto/host_device_util.py b/gfauto/gfauto/host_device_util.py index 9aaef93e0..77740145f 100644 --- a/gfauto/gfauto/host_device_util.py +++ b/gfauto/gfauto/host_device_util.py @@ -69,7 +69,8 @@ def run_amber_helper( # TODO: Use binary paths. - image_file = output_dir / fuzz.IMAGE_FILE_NAME + variant_image_file = output_dir / fuzz.VARIANT_IMAGE_FILE_NAME + reference_image_file = output_dir / fuzz.REFERENCE_IMAGE_FILE_NAME buffer_file = output_dir / fuzz.BUFFER_FILE_NAME cmd = [ @@ -87,10 +88,14 @@ def run_amber_helper( cmd.append("-ps") else: if dump_image: - cmd.append("-i") - cmd.append(str(image_file)) cmd.append("-I") cmd.append("variant_framebuffer") + cmd.append("-i") + cmd.append(str(variant_image_file)) + cmd.append("-I") + cmd.append("reference_framebuffer") + cmd.append("-i") + cmd.append(str(reference_image_file)) if dump_buffer: cmd.append("-b") cmd.append(str(buffer_file)) diff --git a/gfauto/gfauto/settings.proto b/gfauto/gfauto/settings.proto index a95596a53..aef1a16e6 100644 --- a/gfauto/gfauto/settings.proto +++ b/gfauto/gfauto/settings.proto @@ -37,4 +37,14 @@ message Settings { // Store at most this many fuzzing failure files. uint32 maximum_fuzz_failures = 4; + + // Reduce tool crashes. E.g. spirv-opt or glslangValidator crashes. You can disable this if you would rather focus on + // bugs in your actual test device. + bool reduce_tool_crashes = 5; + + // Reduce crashes. E.g. shader compilation errors, shader compilation crashes, runtime crashes. + bool reduce_crashes = 6; + + // Reduce bad images. E.g. your test device renders an incorrect image. + bool reduce_bad_images = 7; } diff --git a/gfauto/gfauto/settings_pb2.py b/gfauto/gfauto/settings_pb2.py index 0e277b787..7c733a3f1 100644 --- a/gfauto/gfauto/settings_pb2.py +++ b/gfauto/gfauto/settings_pb2.py @@ -22,7 +22,7 @@ package='gfauto', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\x15gfauto/settings.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\x9e\x01\n\x08Settings\x12\'\n\x0b\x64\x65vice_list\x18\x01 \x01(\x0b\x32\x12.gfauto.DeviceList\x12\'\n\x0f\x63ustom_binaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binary\x12!\n\x19maximum_duplicate_crashes\x18\x03 \x01(\r\x12\x1d\n\x15maximum_fuzz_failures\x18\x04 \x01(\rb\x06proto3') + serialized_pb=_b('\n\x15gfauto/settings.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\xee\x01\n\x08Settings\x12\'\n\x0b\x64\x65vice_list\x18\x01 \x01(\x0b\x32\x12.gfauto.DeviceList\x12\'\n\x0f\x63ustom_binaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binary\x12!\n\x19maximum_duplicate_crashes\x18\x03 \x01(\r\x12\x1d\n\x15maximum_fuzz_failures\x18\x04 \x01(\r\x12\x1b\n\x13reduce_tool_crashes\x18\x05 \x01(\x08\x12\x16\n\x0ereduce_crashes\x18\x06 \x01(\x08\x12\x19\n\x11reduce_bad_images\x18\x07 \x01(\x08\x62\x06proto3') , dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,gfauto_dot_device__pb2.DESCRIPTOR,]) @@ -64,6 +64,27 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reduce_tool_crashes', full_name='gfauto.Settings.reduce_tool_crashes', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reduce_crashes', full_name='gfauto.Settings.reduce_crashes', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reduce_bad_images', full_name='gfauto.Settings.reduce_bad_images', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -77,7 +98,7 @@ oneofs=[ ], serialized_start=76, - serialized_end=234, + serialized_end=314, ) _SETTINGS.fields_by_name['device_list'].message_type = gfauto_dot_device__pb2._DEVICELIST diff --git a/gfauto/gfauto/settings_pb2.pyi b/gfauto/gfauto/settings_pb2.pyi index 4b92adecb..d59d898b7 100644 --- a/gfauto/gfauto/settings_pb2.pyi +++ b/gfauto/gfauto/settings_pb2.pyi @@ -34,6 +34,9 @@ class Settings(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... maximum_duplicate_crashes = ... # type: int maximum_fuzz_failures = ... # type: int + reduce_tool_crashes = ... # type: bool + reduce_crashes = ... # type: bool + reduce_bad_images = ... # type: bool @property def device_list(self) -> gfauto___device_pb2___DeviceList: ... @@ -47,6 +50,9 @@ class Settings(google___protobuf___message___Message): custom_binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, maximum_duplicate_crashes : typing___Optional[int] = None, maximum_fuzz_failures : typing___Optional[int] = None, + reduce_tool_crashes : typing___Optional[bool] = None, + reduce_crashes : typing___Optional[bool] = None, + reduce_bad_images : typing___Optional[bool] = None, ) -> None: ... @classmethod def FromString(cls, s: bytes) -> Settings: ... @@ -54,7 +60,7 @@ class Settings(google___protobuf___message___Message): def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): def HasField(self, field_name: typing_extensions___Literal[u"device_list"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",u"device_list",u"maximum_duplicate_crashes",u"maximum_fuzz_failures"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",u"device_list",u"maximum_duplicate_crashes",u"maximum_fuzz_failures",u"reduce_bad_images",u"reduce_crashes",u"reduce_tool_crashes"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"device_list",b"device_list"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",b"custom_binaries",u"device_list",b"device_list",u"maximum_duplicate_crashes",b"maximum_duplicate_crashes",u"maximum_fuzz_failures",b"maximum_fuzz_failures"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",b"custom_binaries",u"device_list",b"device_list",u"maximum_duplicate_crashes",b"maximum_duplicate_crashes",u"maximum_fuzz_failures",b"maximum_fuzz_failures",u"reduce_bad_images",b"reduce_bad_images",u"reduce_crashes",b"reduce_crashes",u"reduce_tool_crashes",b"reduce_tool_crashes"]) -> None: ... diff --git a/gfauto/gfauto/settings_util.py b/gfauto/gfauto/settings_util.py index 3589e82f2..f336e4431 100644 --- a/gfauto/gfauto/settings_util.py +++ b/gfauto/gfauto/settings_util.py @@ -27,7 +27,13 @@ DEFAULT_SETTINGS_FILE_PATH = Path("settings.json") -DEFAULT_SETTINGS = Settings(maximum_duplicate_crashes=3, maximum_fuzz_failures=10) +DEFAULT_SETTINGS = Settings( + maximum_duplicate_crashes=3, + maximum_fuzz_failures=10, + reduce_tool_crashes=True, + reduce_crashes=True, + reduce_bad_images=True, +) class NoSettingsFile(Exception): @@ -38,20 +44,18 @@ def read_or_create(settings_path: Path) -> Settings: try: return read(settings_path) except FileNotFoundError as exception: - message = f"Could not find settings file at: {settings_path}" + message = f'gfauto could not find "{settings_path}"' if not DEFAULT_SETTINGS_FILE_PATH.exists(): write_default(DEFAULT_SETTINGS_FILE_PATH) message += ( - f"; a default settings file has been created for you at {str(DEFAULT_SETTINGS_FILE_PATH)}. " - f"Please review it and then try again. " + f'; a default settings file "{str(DEFAULT_SETTINGS_FILE_PATH)}" has been created for you. ' + f"Please review it and then try again. \n" ) raise NoSettingsFile(message) from exception def read(settings_path: Path) -> Settings: result = proto_util.file_to_message(settings_path, Settings()) - if not result.maximum_duplicate_crashes: - result.maximum_duplicate_crashes = DEFAULT_SETTINGS.maximum_duplicate_crashes return result diff --git a/gfauto/gfauto/signature_util.py b/gfauto/gfauto/signature_util.py index 6cfa0e956..814bc06fc 100644 --- a/gfauto/gfauto/signature_util.py +++ b/gfauto/gfauto/signature_util.py @@ -332,12 +332,20 @@ def get_signature_from_catchsegv_frame_address(log_contents: str) -> Optional[st address = match.group(2) function_signature = get_function_signature_from_address(module, address) if not function_signature: - return None + # As a last resort, we can use the module name + offset as the signature. + return get_hex_signature_from_frame(module, address) function_signature = clean_up(function_signature) function_signature = reduce_length(function_signature) return function_signature +def get_hex_signature_from_frame(module: Path, address: str) -> str: + signature = f"{module.name}+{address}" + signature = clean_up(signature) + signature = reduce_length(signature) + return signature + + def get_function_signature_from_address(module: Path, address: str) -> Optional[str]: try: address_tool = util.tool_on_path("addr2line") diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index 4a9179e46..eff39299a 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -394,10 +394,12 @@ def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( binary_list=list(test.device.binaries) + list(test.binaries) ) - spirv_opt_args = list(test.glsl.spirv_opt_args) - spirv_opt_hash = binary_manager.get_binary_by_name( - binaries_util.SPIRV_OPT_NAME - ).version + spirv_opt_args = list(test.glsl.spirv_opt_args) or None + spirv_opt_hash: Optional[str] = None + if spirv_opt_args: + spirv_opt_hash = binary_manager.get_binary_by_name( + binaries_util.SPIRV_OPT_NAME + ).version # Compile all shader jobs shader_job_files = [ diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py index 59deb728c..7a44cd970 100644 --- a/gfauto/gfauto/util.py +++ b/gfauto/gfauto/util.py @@ -286,11 +286,11 @@ def check_field_truthy(field: Any, field_name: str) -> None: def check_file_exists(path: Path) -> None: - check(path.is_file(), FileNotFoundError(f"Could not find file {str(path)}")) + check(path.is_file(), FileNotFoundError(f'Could not find file "{str(path)}"')) def check_dir_exists(path: Path) -> None: - check(path.is_dir(), FileNotFoundError(f"Could not find directory {str(path)}")) + check(path.is_dir(), FileNotFoundError(f'Could not find directory "{str(path)}"')) def get_platform() -> str: diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index cc5c82ab2..de33c07bb 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -193,3 +193,5 @@ makedirs unlink fsync fileno +Debian +prepended From 8e48ba781394daf03b53ec35ed2ff69f863023b0 Mon Sep 17 00:00:00 2001 From: jarisiru <53901722+jarisiru@users.noreply.github.com> Date: Wed, 30 Oct 2019 16:46:05 +0200 Subject: [PATCH 159/180] Add constant folding to array size (#777) This change adds constant folder in array size fields which supports most of the operators as well as constant variables. --- .../common/ast/decl/ArrayInfo.java | 97 +++++++++-- .../common/ast/visitors/AstBuilder.java | 160 +++++++++++++++++- .../common/tool/PrettyPrinterVisitor.java | 24 ++- .../com/graphicsfuzz/common/typing/Typer.java | 8 +- .../ast/expr/ArrayConstructorExprTest.java | 4 +- .../common/tool/PrettyPrinterVisitorTest.java | 11 ++ .../common/util/ParseHelperTest.java | 84 ++++++++- .../common/util/PipelineInfo.java | 4 +- .../common/util/PruneUniforms.java | 4 +- .../graphicsfuzz/generator/fuzzer/Fuzzer.java | 7 +- .../semanticschanging/AddArrayMutation.java | 5 +- .../generator/tool/GenerateShaderFamily.java | 2 +- .../donation/MakeArrayAccessesInBounds.java | 2 +- .../generator/util/FloatLiteralReplacer.java | 2 +- .../MakeArrayAccessesInBoundsTest.java | 112 ++++++++++++ 15 files changed, 479 insertions(+), 47 deletions(-) diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInfo.java b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInfo.java index 2eff72fc8..0bddbf1d4 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInfo.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInfo.java @@ -17,27 +17,102 @@ package com.graphicsfuzz.common.ast.decl; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.visitors.IAstVisitor; +import com.graphicsfuzz.common.ast.visitors.UnsupportedLanguageFeatureException; import java.util.Optional; public class ArrayInfo implements IAstNode { - private Optional size; + /** + * Size, set after folding. + */ + private Optional constantSize; + /** + * Original size expression before folding, for pretty printing. + */ + private Optional sizeExpr; - public ArrayInfo(Integer size) { - this.size = Optional.of(size); + /** + * "originalSize" is the expression in the original shader representing the array size, + * and is empty if no size was given. If "originalSize" is present, constant folding can + * be used to turn the expression into an integer value, which can be stored in + * "constantSize". It is useful to keep "originalSize" around to allow the shader to + * be pretty-printed in its original form. + * @param originalSize Array size expression + */ + public ArrayInfo(Expr originalSize) { + this.constantSize = Optional.empty(); + this.sizeExpr = Optional.of(originalSize); } + /** + * Private constructor for cloning, needed since the constant size expression may have been + * folded by the time the expression is cloned. + * @param constantSize Possible constant-folded size + * @param originalSize Original size, for pretty printing + */ + private ArrayInfo(Optional constantSize, Optional originalSize) { + this.constantSize = constantSize; + this.sizeExpr = originalSize; + } + + /** + * Default constructor, for arrays with no size definition. + */ public ArrayInfo() { - this.size = Optional.empty(); + this.constantSize = Optional.empty(); + this.sizeExpr = Optional.empty(); + } + + /** + * Query for whether this array has a size definition. + * @return true if size definition is available + */ + public boolean hasConstantSize() { + return constantSize.isPresent(); + } + + /** + * Query whether this array has a size expression + * @return Size expression + */ + public boolean hasSizeExpr() { + return sizeExpr.isPresent(); + } + + /** + * Get the constant size. + * If constant folding was not performed yet (for example, the AST is stil being built) or if + * the array has no size declaration, calling this function will throw an exception. + * @return Integer value of the array size + * @throws UnsupportedLanguageFeatureException if folding was not done + */ + public Integer getConstantSize() throws UnsupportedLanguageFeatureException { + if (hasConstantSize()) { + return constantSize.get(); + } + // TODO(https://github.com/google/graphicsfuzz/issues/784) Until array parameter support + // is overhauled there could be array parameters to which constant folding has not been + // applied. + throw new UnsupportedLanguageFeatureException("Not a constant expression"); } - public boolean hasSize() { - return size.isPresent(); + /** + * Set constant expression after folding. + * @param foldedSize Completely folded size + */ + public void setConstantSizeExpr(int foldedSize) { + constantSize = Optional.of(foldedSize); } - public Integer getSize() { - return size.get(); + /** + * Get the original expression for pretty printing. + * @return Original, non-folded size expression + */ + public Expr getSizeExpr() { + return sizeExpr.get(); } @Override @@ -47,17 +122,17 @@ public void accept(IAstVisitor visitor) { @Override public ArrayInfo clone() { - return hasSize() ? new ArrayInfo(getSize()) : new ArrayInfo(); + return new ArrayInfo(constantSize, sizeExpr); } @Override public boolean equals(Object obj) { - return obj instanceof ArrayInfo && size.equals(((ArrayInfo) obj).size); + return obj instanceof ArrayInfo && sizeExpr.equals(((ArrayInfo) obj).sizeExpr); } @Override public int hashCode() { - return size.hashCode(); + return sizeExpr.hashCode(); } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java index 178ac1bdc..f0294b93f 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java @@ -16,6 +16,8 @@ package com.graphicsfuzz.common.ast.visitors; +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.ArrayInfo; import com.graphicsfuzz.common.ast.decl.Declaration; @@ -85,6 +87,7 @@ import com.graphicsfuzz.common.ast.type.UnknownLayoutQualifier; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.parser.GLSLBaseVisitor; import com.graphicsfuzz.parser.GLSLParser; @@ -195,9 +198,158 @@ void addTopLevelDeclaration(Declaration decl) { topLevelDeclarations.add(decl); } + /** + * Performs constant folding for array size fields + * @param tu Parsed syntax tree to scan and modify + * @return Modified syntax tree + */ + private static TranslationUnit fixUpArraySizes(TranslationUnit tu) { + // Use a private class that extends ScopeTrackingVisitor to do the patching up. + new ScopeTrackingVisitor() { + /** + * Attempt to reduce expression, or throw exception if failed. + * @param Expression to fold + * @return Folded expression + */ + public Expr reduce(Expr expr) { + + if (expr instanceof IntConstantExpr) { + return expr; + } + if (expr instanceof VariableIdentifierExpr) { + Expr newexpr = + getCurrentScope() + .lookupScopeEntry(((VariableIdentifierExpr) expr).getName()) + .getVariableDeclInfo() + .getInitializer() + .getExpr(); + return reduce(newexpr); + } + if (expr instanceof ParenExpr) { + return reduce(((ParenExpr)expr).getExpr()); + } + if (expr instanceof BinaryExpr) { + BinaryExpr bexpr = (BinaryExpr) expr; + + Expr lexpr = reduce(bexpr.getLhs()); + Expr rexpr = reduce(bexpr.getRhs()); + + if (!(lexpr instanceof IntConstantExpr && rexpr instanceof IntConstantExpr)) { + throw new RuntimeException("Unable to fold constant (leaf of binary expression did " + + "not fold)" + bexpr.getText()); + } + + int lval = ((IntConstantExpr) lexpr).getNumericValue(); + int rval = ((IntConstantExpr) rexpr).getNumericValue(); + int fval = 0; + switch (bexpr.getOp()) { + case MOD: + fval = lval % rval; + break; + case MUL: + fval = lval * rval; + break; + case DIV: + if (rval == 0) { + throw new RuntimeException("Division by zero while folding constant " + + bexpr.getText()); + } + fval = lval / rval; + break; + case ADD: + fval = lval + rval; + break; + case SUB: + fval = lval - rval; + break; + case BAND: + fval = lval & rval; + break; + case BOR: + fval = lval | rval; + break; + case BXOR: + fval = lval ^ rval; + break; + case LAND: + fval = ((lval != 0) && (rval != 0)) ? 1 : 0; + break; + case LOR: + fval = ((lval != 0) || (rval != 0)) ? 1 : 0; + break; + case LXOR: + fval = ((lval == 0) != (rval == 0)) ? 1 : 0; + break; + case SHL: + fval = lval << rval; + break; + case SHR: + fval = lval >> rval; + break; + case LT: + fval = (lval < rval) ? 1 : 0; + break; + case GT: + fval = (lval > rval) ? 1 : 0; + break; + case LE: + fval = (lval <= rval) ? 1 : 0; + break; + case GE: + fval = (lval >= rval) ? 1 : 0; + break; + case EQ: + fval = (lval == rval) ? 1 : 0; + break; + case NE: + fval = (lval != rval) ? 1 : 0; + break; + default: + throw new RuntimeException("Unable to fold constant (unimplemented binary " + + "expression) " + bexpr.getText()); + } + return new IntConstantExpr(Integer.toString(fval)); + } + throw new RuntimeException("Unable to fold constant (unimplemented expression) " + + expr.getText()); + } + + private void handleArrayInfo(ArrayInfo arrayInfo) { + if (arrayInfo.hasSizeExpr()) { + arrayInfo.setConstantSizeExpr(((IntConstantExpr)reduce(arrayInfo.getSizeExpr())) + .getNumericValue()); + } + } + + @Override + public void visitVariableDeclInfo(VariableDeclInfo variableDeclInfo) { + if (variableDeclInfo.hasArrayInfo()) { + handleArrayInfo(variableDeclInfo.getArrayInfo()); + } + super.visitVariableDeclInfo(variableDeclInfo); + } + + @Override + public void visitStructDefinitionType(StructDefinitionType structDefinitionType) { + super.visitStructDefinitionType(structDefinitionType); + for (Type fieldType : structDefinitionType.getFieldTypes()) { + if (fieldType.getWithoutQualifiers() instanceof ArrayType) { + handleArrayInfo(((ArrayType) fieldType.getWithoutQualifiers()).getArrayInfo()); + } + } + } + + }.visit(tu); + return tu; + } + public static TranslationUnit getTranslationUnit(Translation_unitContext ctx, ShaderKind shaderKind, boolean hasWebGlHint) { - return new AstBuilder(shaderKind, hasWebGlHint).visitTranslation_unit(ctx); + /* Scan parsed AST and perform constant folding for array sizes. + * Doing the constant folding during AST build is not as convenient as + * not all of the required information may be available when needed. + */ + return fixUpArraySizes(new AstBuilder(shaderKind, hasWebGlHint).visitTranslation_unit(ctx)); } @Override @@ -426,11 +578,7 @@ private ArrayInfo getArrayInfo(Array_specifierContext arraySpecifierContext) { return new ArrayInfo(); } final Expr expr = (Expr) visit(arraySpecifierContext.constant_expression()); - if (expr instanceof IntConstantExpr) { - return new ArrayInfo(Integer.parseInt(((IntConstantExpr) expr).getValue())); - } - throw new UnsupportedLanguageFeatureException("Unable to construct array info for array with " - + "size " + expr.getText()); + return new ArrayInfo(expr); } private BuiltinType getBuiltinType(Builtin_type_specifier_nonarrayContext ctx) { diff --git a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java index 2a26dd91e..8be692ac3 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java +++ b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java @@ -175,10 +175,14 @@ public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) first = false; out.append(vdi.getName()); if (vdi.hasArrayInfo()) { - out.append("[" + vdi.getArrayInfo().getSize() + "]"); + out.append("["); + visit(vdi.getArrayInfo().getSizeExpr()); + out.append("]"); assert !(baseType instanceof ArrayType); } else if (baseType instanceof ArrayType) { - out.append("[" + ((ArrayType) baseType).getArrayInfo().getSize() + "]"); + out.append("["); + visit(((ArrayType) baseType).getArrayInfo().getSizeExpr()); + out.append("]"); } if (vdi.hasInitializer()) { out.append(" = "); @@ -222,7 +226,9 @@ public void visitParameterDecl(ParameterDecl parameterDecl) { out.append(" " + parameterDecl.getName()); } if (parameterDecl.getArrayInfo() != null) { - out.append("[" + parameterDecl.getArrayInfo().getSize() + "]"); + out.append("["); + visit(parameterDecl.getArrayInfo().getSizeExpr()); + out.append("]"); } } @@ -515,7 +521,9 @@ public void visitArrayType(ArrayType arrayType) { @Override public void visitArrayConstructorExpr(ArrayConstructorExpr arrayConstructorExpr) { visit(arrayConstructorExpr.getArrayType()); - out.append("[" + arrayConstructorExpr.getArrayType().getArrayInfo().getSize() + "]("); + out.append("["); + visit(arrayConstructorExpr.getArrayType().getArrayInfo().getSizeExpr()); + out.append("]("); boolean first = true; for (Expr e : arrayConstructorExpr.getArgs()) { if (!first) { @@ -553,9 +561,11 @@ private void processArrayInfo(Type type) { } ArrayType arrayType = (ArrayType) type.getWithoutQualifiers(); while (true) { - out.append("[" - + (arrayType.getArrayInfo().hasSize() ? arrayType.getArrayInfo().getSize() : "") - + "]"); + out.append("["); + if (arrayType.getArrayInfo().hasSizeExpr()) { + visit(arrayType.getArrayInfo().getSizeExpr()); + } + out.append("]"); if (!(arrayType.getBaseType().getWithoutQualifiers() instanceof ArrayType)) { break; } diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java index 9d5f322d1..7f276871c 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java @@ -40,6 +40,7 @@ import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.ast.visitors.UnsupportedLanguageFeatureException; import com.graphicsfuzz.common.util.OpenGlConstants; import java.util.Collection; import java.util.Collections; @@ -151,8 +152,11 @@ public boolean prototypeMatches(FunctionPrototype prototype, FunctionCallExpr fu if (argType == null) { return false; } - // Not yet worked out how to deal with array info - assert prototype.getParameters().get(i).getArrayInfo() == null; + // TODO(https://github.com/google/graphicsfuzz/issues/784) Not yet worked out how to deal with + // array info + if (prototype.getParameters().get(i).getArrayInfo() != null) { + throw new UnsupportedLanguageFeatureException("Array parameters are not yet supported."); + } if (!argType.getWithoutQualifiers() .equals(prototype.getParameters().get(i).getType().getWithoutQualifiers())) { return false; diff --git a/ast/src/test/java/com/graphicsfuzz/common/ast/expr/ArrayConstructorExprTest.java b/ast/src/test/java/com/graphicsfuzz/common/ast/expr/ArrayConstructorExprTest.java index c7c312d14..fcc3e7ef6 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/ast/expr/ArrayConstructorExprTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/ast/expr/ArrayConstructorExprTest.java @@ -36,7 +36,7 @@ public void setup() { "vec4", new FloatConstantExpr("0.0")); arrayConstructor = new ArrayConstructorExpr(new ArrayType( BasicType.VEC4, - new ArrayInfo(3)), + new ArrayInfo(new IntConstantExpr("3"))), Arrays.asList(temp.clone(), temp.clone(), temp.clone()) ); } @@ -88,4 +88,4 @@ public void setChildBad() { arrayConstructor.setChild(3, new VariableIdentifierExpr("z")); } -} \ No newline at end of file +} diff --git a/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java b/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java index 7850251a0..1bee1fe00 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java @@ -46,6 +46,17 @@ public class PrettyPrinterVisitorTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testArraySizeExpression() throws Exception { + final String program = "" + + "void main()\n" + + "{\n" + + " int a[3 + 4];\n" + + "}\n"; + assertEquals(program, PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(program + ))); + } + @Test public void testParseAndPrint() throws Exception { final String program = "" diff --git a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java index 2ae7cd2ac..24a00207d 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java @@ -26,8 +26,10 @@ import com.graphicsfuzz.common.ast.expr.FloatConstantExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; import com.graphicsfuzz.common.ast.stmt.ReturnStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; @@ -565,18 +567,82 @@ private String getStringFromInputStream(InputStream strippedIs) throws IOExcepti return writer.toString(); } - @Test - public void testUnsupportedArrayLength() throws Exception { - - // Change this test to check for support if it is eventually introduced. - + /** + * Validate folding in testSupportedArrayLength. We know that the array is the last statement + * in the function, but it may not be the first. + * @param shaderSource Source code for the shader to test + * @param expectedSize Expected size of the array + * @throws Exception Producing the AST may throw exceptions + */ + private void + validateFolding(String shaderSource, int expectedSize) throws Exception + { + final BlockStmt block = ParseHelper.parse(shaderSource) + .getMainFunction().getBody(); + assertTrue(((DeclarationStmt)block.getStmt(block.getNumStmts()-1)).getVariablesDeclaration() + .getDeclInfo(0).getArrayInfo().getConstantSize() == expectedSize); + } + + /** + * Test various forms of statements that may occur inside array size declaration + * @throws Exception Producing the AST may throw exceptions + */ + @Test + public void + testSupportedArrayLength() throws Exception { try { - ParseHelper.parse("void main() {\n" + validateFolding("void main() {\n" + " int A[3 + 4];\n" - + "}\n"); - fail("Exception was expected"); + + "}\n",7); + validateFolding("void main() {\n" + + " int A[4 - 3];\n" + + "}\n",1); + validateFolding("void main() {\n" + + " int A[3 * 4];\n" + + "}\n", 12); + validateFolding("void main() {\n" + + " int A[4 / 2];\n" + + "}\n", 2); + validateFolding("void main() {\n" + + " int A[3 << 2];\n" + + "}\n", 12); + validateFolding("void main() {\n" + + " int A[8 >> 2];\n" + + "}\n", 2); + validateFolding("void main() {\n" + + " int A[(3 + 4)];\n" + + "}\n", 7); + validateFolding("void main() {\n" + + " int A[3 && 4];\n" + + "}\n", 1); + validateFolding("void main() {\n" + + " int A[3 & 5];\n" + + "}\n", 1); + validateFolding("void main() {\n" + + " int A[3 || 4];\n" + + "}\n", 1); + validateFolding("void main() {\n" + + " int A[3 | 4];\n" + + "}\n", 7); + validateFolding("void main() {\n" + + " int A[3 ^^ 4];\n" + + "}\n", 0); + validateFolding("void main() {\n" + + " int A[3 ^ 4];\n" + + "}\n", 7); + validateFolding("void main() {\n" + + " int A[3 + 4 + 5];\n" + + "}\n", 12); + validateFolding("void main() {\n" + + " const int v = 5;\n" + + " int A[3 + v];\n" + + "}\n", 8); + validateFolding("void main() {\n" + + " const int v = 5;\n" + + " int A[(((3 + 4) * v) >> 2) && 7];\n" + + "}\n", 1); } catch (UnsupportedLanguageFeatureException exception) { - assertTrue(exception.getMessage().contains("Unable to construct array info")); + fail(exception.getMessage()); } } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java b/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java index 0e63ae2bc..b334f097a 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java @@ -143,7 +143,7 @@ public void setUniforms(TranslationUnit tu, } int arrayLength; if (vdi.hasArrayInfo()) { - arrayLength = vdi.getArrayInfo().getSize(); + arrayLength = vdi.getArrayInfo().getConstantSize(); } else { arrayLength = 1; } @@ -167,7 +167,7 @@ public void setUniforms(TranslationUnit tu, private Optional maybeGetArrayCount(VariableDeclInfo vdi) { if (vdi.hasArrayInfo()) { - return Optional.of(vdi.getArrayInfo().getSize()); + return Optional.of(vdi.getArrayInfo().getConstantSize()); } return Optional.empty(); } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java b/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java index 49e75cdd5..7e243a63f 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java @@ -123,9 +123,9 @@ private static Initializer makeInitializer(BasicType baseType, ArrayInfo arrayInfo, List args) { if (arrayInfo != null) { - assert arrayInfo.getSize() * baseType.getNumElements() == args.size(); + assert arrayInfo.getConstantSize() * baseType.getNumElements() == args.size(); List argExprs = new ArrayList<>(); - for (int index = 0; index < arrayInfo.getSize(); index++) { + for (int index = 0; index < arrayInfo.getConstantSize(); index++) { argExprs.add(getBasicTypeLiteralExpr(baseType, args.subList(index * baseType.getNumElements(), (index + 1) * baseType.getNumElements()))); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java index 00e26f03f..814529344 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java @@ -27,6 +27,7 @@ import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.BreakStmt; @@ -176,7 +177,7 @@ private Expr makeExpr(Type targetType, boolean isLValue, boolean constContext, f if (!shadingLanguageVersion.restrictedArrayIndexing()) { ArrayType arrayType = (ArrayType) targetType; List args = new ArrayList<>(); - for (int i = 0; i < arrayType.getArrayInfo().getSize(); i++) { + for (int i = 0; i < arrayType.getArrayInfo().getConstantSize(); i++) { args.add(makeExpr(arrayType.getBaseType(), isLValue, constContext, depth + 1)); } return new ArrayConstructorExpr((ArrayType) stripQualifiers(targetType), args); @@ -452,7 +453,9 @@ private DeclarationStmt fuzzDeclarationStmt() { final String name = createName("v"); ArrayInfo arrayInfo = null; if (generator.nextInt(10) < 3) { // TODO Hack for now, needs thought - arrayInfo = new ArrayInfo(generator.nextPositiveInt(MAX_ARRAY_SIZE)); + arrayInfo = + new ArrayInfo(new IntConstantExpr(Integer.toString( + generator.nextPositiveInt(MAX_ARRAY_SIZE)))); } fuzzingContext.addLocal(name, arrayInfo == null ? baseType : getType(baseType, arrayInfo)); decls.add(new VariableDeclInfo(name, arrayInfo, null)); // TODO: no initializer for now diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/AddArrayMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/AddArrayMutation.java index 59097110a..ee052f257 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/AddArrayMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/AddArrayMutation.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.ast.decl.ArrayInfo; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.generator.mutateapi.Mutation; @@ -40,7 +41,9 @@ public AddArrayMutation(TranslationUnit tu, String name, BasicType baseType, int @Override public void apply() { tu.addDeclaration(new VariablesDeclaration( - baseType, new VariableDeclInfo(name, new ArrayInfo(numElements), null))); + baseType, new VariableDeclInfo(name, + new ArrayInfo(new IntConstantExpr(Integer.toString(numElements))), + null))); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java index 386f1f283..5900c31b4 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java @@ -226,7 +226,7 @@ public static void mainHelper(String[] args) throws ArgumentParserException, Generate.generateVariant(fileOps, referenceShaderJob, variantShaderJobFile, generatorArguments, childRandom, writeProbabilities); } catch (Exception exception) { - if (verbose) { + /*if (verbose)*/ { LOGGER.error("Failed generating variant: " + "\nGenerator arguments: " + generatorArguments + "\nReference shader job: " + referenceShaderJob diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java index f8a161ca3..6c8ad07fc 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java @@ -88,7 +88,7 @@ public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { private static Integer getSize(Type type) { assert isArrayVectorOrMatrix(type); if (type instanceof ArrayType) { - return ((ArrayType) type).getArrayInfo().getSize(); + return ((ArrayType) type).getArrayInfo().getConstantSize(); } if (BasicType.allVectorTypes().contains(type)) { return ((BasicType) type).getNumElements(); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/FloatLiteralReplacer.java b/generator/src/main/java/com/graphicsfuzz/generator/util/FloatLiteralReplacer.java index 1a1cfb2b0..aa6d29a8a 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/FloatLiteralReplacer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/FloatLiteralReplacer.java @@ -79,7 +79,7 @@ private FloatLiteralReplacer(TranslationUnit tu) { tu.addDeclaration(new VariablesDeclaration( new QualifiedType(BasicType.FLOAT, Arrays.asList(TypeQualifier.UNIFORM)), new VariableDeclInfo(Constants.FLOAT_CONST, - new ArrayInfo(uniformIndex), null) + new ArrayInfo(new IntConstantExpr(Integer.toString(uniformIndex))), null) )); } } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java index 074cc2e03..150363c02 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java @@ -20,8 +20,10 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.typing.Typer; +import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.util.Constants; +import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -137,4 +139,114 @@ public void testUIntStaticallyInBounds() throws Exception { PrettyPrinterVisitor.prettyPrintAsString(tu)); } + @Test + public void testMakeStructMemberArrayAccessInBounds1() throws Exception { + final String shader = "#version 310 es\n" + + "struct S {\n" + + " int A[3];\n" + + "};\n" + + "void main() {\n" + + " int x;\n" + + " x = 22;\n" + + " S myS;\n" + + " myS.A[x] = 6;\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "struct S {\n" + + " int A[3];\n" + + "};\n" + + "void main() {\n" + + " int x;\n" + + " x = 22;\n" + + " S myS;\n" + + " myS.A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 3)] = 6;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + MakeArrayAccessesInBounds.makeInBounds(tu, new Typer(tu), tu); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testMakeStructMemberArrayAccessInBounds2() throws Exception { + final String shader = "#version 310 es\n" + + "const int N = 3;\n" + + "struct S {\n" + + " int A[N];\n" + + "};\n" + + "void main() {\n" + + " int x;\n" + + " x = 22;\n" + + " S myS;\n" + + " myS.A[x] = 6;\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "const int N = 3;\n" + + "struct S {\n" + + " int A[N];\n" + + "};\n" + + "void main() {\n" + + " int x;\n" + + " x = 22;\n" + + " S myS;\n" + + " myS.A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 3)] = 6;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + MakeArrayAccessesInBounds.makeInBounds(tu, new Typer(tu), tu); + CompareAsts.assertEqualAsts(expected, tu); + } + + // TODO(https://github.com/google/graphicsfuzz/issues/784) Enable once array parameter support is + // overhauled. + @Ignore + @Test + public void testMakeArrayParameterAccessInBounds1() throws Exception { + final String shader = "#version 310 es\n" + + "int foo(int A[3], int x) {\n" + + " return A[x];\n" + + "}\n" + + "void main() {\n" + + " int A[3];\n" + + " foo(A, 7);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "int foo(int A[3], int x) {\n" + + " return A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 3)];\n" + + "}\n" + + "void main() {\n" + + " int A[3];\n" + + " foo(A, 7);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + MakeArrayAccessesInBounds.makeInBounds(tu, new Typer(tu), tu); + CompareAsts.assertEqualAsts(expected, tu); + } + + // TODO(https://github.com/google/graphicsfuzz/issues/784) Enable once array parameter support is + // overhauled. + @Ignore + @Test + public void testMakeArrayParameterAccessInBounds2() throws Exception { + final String shader = "#version 310 es\n" + + "const int N = 3;\n" + + "int foo(int A[N], int x) {\n" + + " return A[x];\n" + + "}\n" + + "void main() {\n" + + " int A[N];\n" + + " foo(A, 7);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "const int N = 3;\n" + + "int foo(int A[N], int x) {\n" + + " return A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 3)];\n" + + "}\n" + + "void main() {\n" + + " int A[N];\n" + + " foo(A, 7);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + MakeArrayAccessesInBounds.makeInBounds(tu, new Typer(tu), tu); + CompareAsts.assertEqualAsts(expected, tu); + } + } From b25cf72aa4fbfd0d500e2a848dc430c1a065a06b Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Wed, 30 Oct 2019 15:12:29 +0000 Subject: [PATCH 160/180] Always make array accesses in bounds during code donation (#779) This change makes all array accesses of a donor shader be clamped to be in-bounds as soon as the shader is parsed. Fixes #778. --- .../DonateCodeTransformation.java | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java index a5f6e24f2..0979e00dc 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java @@ -67,6 +67,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -126,20 +127,19 @@ public DonateCodeTransformation(Function probabilityOfDonation private TranslationUnit prepareTranslationUnit(File donorFile, IRandom generator) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - TranslationUnit tu = ParseHelper.parse(donorFile); + final TranslationUnit tu = ParseHelper.parse(donorFile); + final Typer typer = new Typer(tu); + // To avoid undefined behaviours, make all array access in bounds for every donor. + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); addPrefixes(tu, getDeclaredFunctionNames(tu)); // Add prefixed versions of these builtins, in case they are used. // Use explicit precision qualifier to avoid introducing errors if there are no float precision // qualifiers. - List coordQualifiers = new ArrayList<>(); - coordQualifiers.add(TypeQualifier.MEDIUMP); tu.addDeclaration(new VariablesDeclaration( - new QualifiedType(BasicType.VEC4, coordQualifiers), + new QualifiedType(BasicType.VEC4, Collections.singletonList(TypeQualifier.MEDIUMP)), new VariableDeclInfo(addPrefix(OpenGlConstants.GL_FRAG_COORD), null, null))); - List colorQualifiers = new ArrayList<>(); - colorQualifiers.add(TypeQualifier.MEDIUMP); tu.addDeclaration(new VariablesDeclaration( - new QualifiedType(BasicType.VEC4, colorQualifiers), + new QualifiedType(BasicType.VEC4, Collections.singletonList(TypeQualifier.MEDIUMP)), new VariableDeclInfo(addPrefix(OpenGlConstants.GL_FRAG_COLOR), null, null))); adaptTranslationUnitForSpecificDonation(tu, generator); translationUnitCount++; @@ -269,8 +269,6 @@ public boolean apply(TranslationUnit tu, } donateFunctionsAndGlobals(tu); eliminateUsedDonors(); - makeInjectedArrayAccessesInBounds(tu, injectedStmts); - return !injectionPoints.isEmpty(); } @@ -292,14 +290,6 @@ private void eliminateUsedDonors() { donorsToTranslationUnits = new HashMap<>(); } - private void makeInjectedArrayAccessesInBounds(TranslationUnit tu, - List injectedStmts) { - Typer typer = new Typer(tu); - for (Stmt stmt : injectedStmts) { - MakeArrayAccessesInBounds.makeInBounds(stmt, typer, tu); - } - } - private boolean incompatible(IInjectionPoint injectionPoint, DonationContext donationContext, ShadingLanguageVersion shadingLanguageVersion) { // It is a problem if the injection point has an available variable that has the same name From f098d1db3740edc0c65a4991a84cab56cbb48b19 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 31 Oct 2019 10:44:59 +0000 Subject: [PATCH 161/180] Add braces when preparing reference (#787) Fixes #782. --- .../com/graphicsfuzz/generator/tool/PrepareReference.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java index edf877f9b..8def96ba2 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java @@ -18,6 +18,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.AddBraces; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.PipelineInfo; @@ -150,6 +151,11 @@ private static void prepareReferenceShader(TranslationUnit tu, if (replaceFloatLiterals) { FloatLiteralReplacer.replace(tu, pipelineInfo); } + + // Ensure that all if-then-else statements have braces. This makes the reference easier to + // compare with a reduced variant. + AddBraces.transform(tu); + } } From e2ea5f9a53e2401073d208b38ea8840930008196 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Thu, 31 Oct 2019 10:45:35 +0000 Subject: [PATCH 162/180] gfauto: do not generate Amber text for uniforms if there are none (#786) If a shader has no uniforms, this change avoids generating "uniforms for ..." followed by no uniform information. This was motivated by such redundant text in Amber files being flagged up in Vulkan CTS reviews. --- gfauto/gfauto/amber_converter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py index d8241db3e..83635287a 100644 --- a/gfauto/gfauto/amber_converter.py +++ b/gfauto/gfauto/amber_converter.py @@ -267,11 +267,16 @@ def amberscript_uniform_buffer_def(uniform_json_contents: str, prefix: str) -> s "glUniformMatrix4fv": "mat4x4", } + uniforms = json.loads(uniform_json_contents) + + # If there are no uniforms, do not generate anything. + if not uniforms: + return "" + result = f"# uniforms for {prefix}\n" result += "\n" - uniforms = json.loads(uniform_json_contents) for name, entry in uniforms.items(): if name == "$compute": From 42ca80396adfd92cd4f50db0a5372985669c0c3d Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 1 Nov 2019 11:56:34 +0000 Subject: [PATCH 163/180] Restrict initializer inlining when preserving semantics (#788) The reducer tries to be clever and do some initializer inlining when preserving semantics. But this can be problematic for a couple of reasons: - It can make references and variants hard to compare if initializer inlining has been applied to one and not the other - There are subtle cases where initializer inlining does not preserve semantics; for example: highp int x = 16777216; findLSB(x); might not be equivalen to: findLSB(16777216); since findLSB can operate at low precision when passed a literal argument. This change disables initializer inlining when semantics are being preserved, unless the initializer occurs in dead code, or is the initializer for a live-injected variable. The change contributes some clean-up related to how dead code and live-injected variables are identified in general. Partially fixes #639. Partially fixes #783. --- ...ttenControlFlowReductionOpportunities.java | 4 +- ...lineInitializerReductionOpportunities.java | 31 +------- .../ReductionOpportunitiesBase.java | 27 +++++-- .../SimplifyExprReductionOpportunities.java | 10 +-- .../StmtReductionOpportunities.java | 18 +---- .../VariableDeclReductionOpportunities.java | 9 +-- ...InitializerReductionOpportunitiesTest.java | 76 ++++++++++++++++--- 7 files changed, 99 insertions(+), 76 deletions(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java index 27e67d9de..fd0c6de55 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java @@ -120,11 +120,9 @@ private boolean allowedToReduce(Stmt compoundStmt) { } return context.reduceEverywhere() - || injectionTracker.enclosedByDeadCodeInjection() - || injectionTracker.underUnreachableSwitchCase() + || currentProgramPointIsDeadCode() || (StmtReductionOpportunities.isLiveCodeInjection(compoundStmt) && !isLoopLimiterCheck(compoundStmt)) - || enclosingFunctionIsDead() || SideEffectChecker.isSideEffectFree(compoundStmt, context.getShadingLanguageVersion(), shaderKind); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java index 23b093175..6240237e3 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java @@ -112,7 +112,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie } if (inLValueContext()) { - if (!context.reduceEverywhere() && !currentProgramPointHasNoEffect()) { + if (!context.reduceEverywhere() && !currentProgramPointIsDeadCode()) { // The declaration is used as an l-value. To preserve semantics we cannot inline its // initializer. For example, in: // int x = 2; @@ -137,39 +137,14 @@ private boolean allowedToReduce(VariableDeclInfo variableDeclInfo) { if (context.reduceEverywhere()) { return true; } - if (currentProgramPointHasNoEffect()) { + if (currentProgramPointIsDeadCode()) { return true; } if (StmtReductionOpportunities.isLooplimiter(variableDeclInfo.getName())) { // Do not mess with loop limiters. return false; } - if (initializerIsScalarAndSideEffectFree(variableDeclInfo) - && !referencesVariableIdentifier(variableDeclInfo.getInitializer())) { - // We need to be careful about inlining e.g.: "int x = y;", because if y is then modified, - // inlined uses of x would get the new value of y. - return true; - } - return false; - } - - private boolean referencesVariableIdentifier(Initializer initializer) { - return new CheckPredicateVisitor() { - @Override - public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { - predicateHolds(); - } - }.test(initializer); - } - - private boolean currentProgramPointHasNoEffect() { - if (injectionTracker.enclosedByDeadCodeInjection()) { - return true; - } - if (injectionTracker.underUnreachableSwitchCase()) { - return true; - } - if (enclosingFunctionIsDead()) { + if (isLiveInjectedVariableName(variableDeclInfo.getName())) { return true; } return false; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java index aeb4c722d..2a2f49561 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java @@ -98,7 +98,6 @@ public void visitExprCaseLabel(ExprCaseLabel exprCaseLabel) { // to simplify the literal expression that is used as a case label. Literals are simple enough // already, and attempting to change the value of a case label literal runs the risk of // introducing duplicate case labels. - injectionTracker.notifySwitchCase(exprCaseLabel); } protected void visitChildOfBlock(BlockStmt block, int childIndex) { @@ -111,25 +110,33 @@ public void visitBlockStmt(BlockStmt block) { for (int i = 0; i < block.getNumStmts(); i++) { - visitChildOfBlock(block, i); - final Stmt child = block.getStmt(i); + if (child instanceof ExprCaseLabel) { + // It is important to notify the injection tracker of a switch case *before* we visit the + // switch case so that we can identify when we have entered the reachable part of an + // injected switch statement. + injectionTracker.notifySwitchCase((ExprCaseLabel) child); + } + + visitChildOfBlock(block, i); + if (child instanceof BreakStmt && parentMap.getParent(block) instanceof SwitchStmt) { injectionTracker.notifySwitchBreak(); } visit(child); + } leaveBlockStmt(block); } - public static boolean isDeadCodeInjection(Stmt stmt) { + static boolean isDeadCodeInjection(Stmt stmt) { return stmt instanceof IfStmt && MacroNames .isDeadByConstruction(((IfStmt) stmt).getCondition()); } - protected boolean enclosingFunctionIsDead() { + private boolean enclosingFunctionIsDead() { if (enclosingFunctionName == null) { // We are in global scope return false; @@ -138,6 +145,12 @@ protected boolean enclosingFunctionIsDead() { || notReferencedFromLiveContext.neverCalledFromLiveContext(enclosingFunctionName); } + boolean currentProgramPointIsDeadCode() { + return injectionTracker.enclosedByDeadCodeInjection() + || injectionTracker.underUnreachableSwitchCase() + || enclosingFunctionIsDead(); + } + @Override public void visitBinaryExpr(BinaryExpr binaryExpr) { boolean isSideEffecting = binaryExpr.getOp().isSideEffecting(); @@ -265,4 +278,8 @@ final void addOpportunity( } } + static boolean isLiveInjectedVariableName(String name) { + return name.startsWith(Constants.LIVE_PREFIX); + } + } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java index 63bb31cbb..014f4260a 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java @@ -74,8 +74,8 @@ public void visitDeclarationStmt(DeclarationStmt declarationStmt) { boolean allowedToReduceExpr(IAstNode parent, Expr child) { if (child instanceof VariableIdentifierExpr) { - String name = ((VariableIdentifierExpr) child).getName(); - if (name.startsWith(Constants.LIVE_PREFIX) + final String name = ((VariableIdentifierExpr) child).getName(); + if (isLiveInjectedVariableName(name) && !StmtReductionOpportunities.isLooplimiter(name)) { return true; } @@ -102,7 +102,7 @@ boolean allowedToReduceExpr(IAstNode parent, Expr child) { return true; } - if (enclosingFunctionIsDead()) { + if (currentProgramPointIsDeadCode()) { return true; } @@ -110,10 +110,6 @@ boolean allowedToReduceExpr(IAstNode parent, Expr child) { return true; } - if (injectionTracker.enclosedByDeadCodeInjection()) { - return true; - } - if (inLiveInjectedStmtOrDeclaration && !referencesLoopLimiter(child)) { return true; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java index a0fcd284d..493127e12 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java @@ -115,11 +115,7 @@ private boolean allowedToReduceStmt(BlockStmt block, int childIndex) { return true; } - if (injectionTracker.enclosedByDeadCodeInjection()) { - return true; - } - - if (injectionTracker.underUnreachableSwitchCase() && !isZeroSwitchCase(stmt)) { + if (currentProgramPointIsDeadCode()) { return true; } @@ -135,8 +131,7 @@ private boolean allowedToReduceStmt(BlockStmt block, int childIndex) { } return context.reduceEverywhere() - || (isLiveCodeInjection(stmt) && !referencesLoopLimiter(stmt)) - || enclosingFunctionIsDead(); + || (isLiveCodeInjection(stmt) && !referencesLoopLimiter(stmt)); } @@ -175,11 +170,6 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie }.test(stmt); } - private boolean isZeroSwitchCase(Stmt stmt) { - return stmt instanceof ExprCaseLabel - && ((IntConstantExpr) ((ExprCaseLabel) stmt).getExpr()).getValue().equals("0"); - } - /** * Determines whether the given statement came from a live code injection. This is the case if * one of the following holds: @@ -259,10 +249,6 @@ private static boolean isLiveCodeVariableDeclaration(VariableDeclInfo vdi) { return isLiveInjectedVariableName(name); } - private static boolean isLiveInjectedVariableName(String name) { - return name.startsWith(Constants.LIVE_PREFIX); - } - /** *

* Determines whether the given statement came from a live code injection. diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java index c3c69dee8..dcd5f4187 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java @@ -128,17 +128,12 @@ private boolean allowedToReduceLocalDecl(VariableDeclInfo variableDeclInfo) { } // Fine to remove if in a dead context, a live context, if no initializer, or if // initializer does not have side effects. - return context.reduceEverywhere() || enclosingFunctionIsDead() - || injectionTracker.enclosedByDeadCodeInjection() - || isLiveInjection(variableDeclInfo) + return context.reduceEverywhere() || currentProgramPointIsDeadCode() + || isLiveInjectedVariableName(variableDeclInfo.getName()) || !variableDeclInfo.hasInitializer() || initializerIsScalarAndSideEffectFree(variableDeclInfo); } - private boolean isLiveInjection(VariableDeclInfo variableDeclInfo) { - return variableDeclInfo.getName().startsWith(Constants.LIVE_PREFIX); - } - /** * Find all unused declaration opportunities for the given translation unit. * diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java index 3b0e08bd3..745ce05a2 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java @@ -17,6 +17,7 @@ package com.graphicsfuzz.reducer.reductionopportunities; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; @@ -24,6 +25,7 @@ import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.util.Constants; import java.util.List; import org.junit.Test; @@ -40,6 +42,10 @@ public void testGlobalScope() throws Exception { assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); + + // If we are preserving semantics, we do not want to apply this transformation. + assertTrue(InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())).isEmpty()); } @Test @@ -86,23 +92,20 @@ private String make1Plus1Plus1Expr(int n) { } @Test - public void testCanInlineAllExceptLValueIfReducingEverywhere() throws Exception { + public void testDoNotInlineWhenPreservingSemantics() throws Exception { final String program = "void main() { int i = 4 + 2; i += i + i + i; i -= i * i; }"; - final String expected = "void main() { int i = 4 + 2; i += (4 + 2) + (4 + 2) + (4 + 2); i -= (4 + 2) * (4 + 2); }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); - assertEquals(5, ops.size()); - ops.forEach(SimplifyExprReductionOpportunity::applyReduction); - CompareAsts.assertEqualAsts(expected, tu); + assertTrue(ops.isEmpty()); } @Test - public void testNoInliningIfLValueUsageExistsWhenPreservingSemantics() throws Exception { + public void testDoNotInlineWhenPreservingSemantics2() throws Exception { final String program = "void main() { int i = 4 + 2; i += i + i + i; i -= i * i; }"; @@ -111,18 +114,18 @@ public void testNoInliningIfLValueUsageExistsWhenPreservingSemantics() throws Ex List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); - assertEquals(0, ops.size()); + assertTrue(ops.isEmpty()); } @Test - public void testNoInliningIfPartsOfInitializerAreModifiedWhenPreservingSemantics() throws Exception { + public void testDoNotInlineWhenPreservingSemantics3() throws Exception { final String program = "void main() { int x = 2; int y = x; x = 3; y; }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); - assertEquals(0, ops.size()); + assertTrue(ops.isEmpty()); } @Test @@ -148,4 +151,57 @@ public void testNoInliningWithNameShadowing2() throws Exception { assertEquals(0, ops.size()); } + @Test + public void testInliningInDeadCodeWithPreserveSemantics() throws Exception { + // We do want to be able to inline an initializer if we are in dead code, when preserving + // semantics + final String program = "void main() {\n" + + " if(" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false, false))) {\n" + + " int do_inline_me = 5 + 2;\n" + + " do_inline_me * do_inline_me;\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " if(" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false, false))) {\n" + + " int do_inline_me = 5 + 2;\n" + + " (5 + 2) * (5 + 2);\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertEquals(2, ops.size()); + ops.get(0).applyReduction(); + ops.get(1).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testInliningInLiveCodeWithPreserveSemantics() throws Exception { + // We do want to be able to inline an initializer if we are in live code, when preserving + // semantics + final String variableName = Constants.LIVE_PREFIX + "do_inline_me"; + final String program = "void main() {\n" + + " {\n" + + " int " + variableName + " = 5 + 2;\n" + + " " + variableName + " * " + variableName + ";\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " {\n" + + " int " + variableName + " = 5 + 2;\n" + + " (5 + 2) * (5 + 2);\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertEquals(2, ops.size()); + ops.get(0).applyReduction(); + ops.get(1).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + } From a27c03d9801b798c31c93e2203a384c981258167 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 1 Nov 2019 11:56:55 +0000 Subject: [PATCH 164/180] Add an 'AMBER_COMMAND_EXPECT_BLACK' constant to gfauto. (#789) --- gfauto/gfauto/tool.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index eff39299a..f2b5d062d 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -46,6 +46,10 @@ "EXPECT variant_framebuffer IDX 0 0 SIZE 256 256 EQ_RGBA 255 0 0 255\n" ) +AMBER_COMMAND_EXPECT_BLACK = ( + "EXPECT variant_framebuffer IDX 0 0 SIZE 256 256 EQ_RGBA 0 0 0 255\n" +) + @dataclass class NameAndShaderJob: From 0a563bd67a3e96e0fe69b42357b9138ebd0b3249 Mon Sep 17 00:00:00 2001 From: Ilkka Saarelainen Date: Mon, 4 Nov 2019 15:28:35 +0200 Subject: [PATCH 165/180] Fix cts test list generator ordering logic (#790) add_amber_test_to_cts.py used amber file names (including .amber suffix) to order the lines in the cpp file and test names to order the lines in the must pass lists. This may lead to inconsistent order in cpp and must pass lists in some rare cases. For example the following test would be ordered in the wrong way if the amber file names are used: unreachable-discard-statement-in-if.amber unreachable-discard-statement.amber This commit changes the cpp file adder code to order the lines by test name. --- gfauto/gfauto/add_amber_tests_to_cts.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/gfauto/gfauto/add_amber_tests_to_cts.py b/gfauto/gfauto/add_amber_tests_to_cts.py index 970c52ee3..370e45336 100644 --- a/gfauto/gfauto/add_amber_tests_to_cts.py +++ b/gfauto/gfauto/add_amber_tests_to_cts.py @@ -23,6 +23,7 @@ import argparse import os +import re import shutil import sys from typing import TextIO, cast @@ -214,12 +215,20 @@ def add_amber_test_to_cpp( break cpp_out.write(cpp_in.readline()) + # Compile regex for finding test name from a cpp file line. + search_pattern = re.compile(r"(?:\")(.*)(?:\.amber\")") + # Get to the point where we should insert our line. line = "" for line in cpp_in: if not line.startswith(" {"): break - elif line >= line_to_write: + + # Search the test name from the line. + result = search_pattern.search(line) + # Group 1 is the test name. + test_name_in_line = result.group(1) if result else None + if test_name_in_line and test_name_in_line >= amber_test_name: break else: cpp_out.write(line) From 16b0c293d4c21a0c589b41ceb55dbc948a379530 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 5 Nov 2019 09:28:06 +0000 Subject: [PATCH 166/180] gfauto: switch to new download-on-demand binaries (#791) --- gfauto/README.md | 4 +- gfauto/gfauto/binaries_util.py | 189 +++++++++++++++++++++++++++++---- gfauto/gfauto/tool.py | 19 ---- gfauto/whitelist.dic | 2 + 4 files changed, 174 insertions(+), 40 deletions(-) diff --git a/gfauto/README.md b/gfauto/README.md index 18f926047..df3599eb8 100644 --- a/gfauto/README.md +++ b/gfauto/README.md @@ -133,7 +133,7 @@ To start fuzzing, create and change to a directory outside the `gfauto/` directo You can get some samples from the GraphicsFuzz project. ```sh -mkdir references/ donors/ +mkdir references/ donors/ cp /data/graphicsfuzz_zip/samples/310es/* references/ cp /data/graphicsfuzz_zip/samples/310es/* donors/ ``` @@ -195,7 +195,7 @@ export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH # In a new terminal: -eval "$(pyenv init -) +eval "$(pyenv init -)" pyenv install 3.6.9 pyenv global 3.6.9 diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py index 3e515d946..0b17b3821 100644 --- a/gfauto/gfauto/binaries_util.py +++ b/gfauto/gfauto/binaries_util.py @@ -49,6 +49,14 @@ CUSTOM_BINARY_RECIPES_PATH_PREFIX = f"{BINARY_RECIPES_PREFIX}/custom" +PLATFORMS = ["Linux", "Mac", "Windows"] + +PLATFORMS_SET = set(PLATFORMS) + +CONFIGS = ["Release", "Debug"] + +CONFIGS_SET = set(CONFIGS) + PLATFORM_SUFFIXES_DEBUG = ["Linux_x64_Debug", "Windows_x64_Debug", "Mac_x64_Debug"] PLATFORM_SUFFIXES_RELEASE = [ "Linux_x64_Release", @@ -61,13 +69,13 @@ "Mac_x64_RelWithDebInfo", ] -DEFAULT_SPIRV_TOOLS_VERSION = "6b072126595dd8c2448eb1fda616251c5e6d7079" +DEFAULT_SPIRV_TOOLS_VERSION = "f1e5cd73f658abcc23ee96d78f2dc27c4b7028c1" DEFAULT_BINARIES = [ Binary( name="glslangValidator", tags=["Debug"], - version="fe0b2bd694bb07004a2db859c5714c321c26b751", + version="18d6b6b63e9adc2aa2cce1ce85d1c348f9475118", ), Binary(name="spirv-opt", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), Binary(name="spirv-dis", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), @@ -77,7 +85,7 @@ Binary( name="swift_shader_icd", tags=["Debug"], - version="b6fa949c45397bd1fbfda769a104b9e8884f343e", + version="aaa64b76c0b40c2958a18cfdc623157c8c6e1b7d", ), ] @@ -450,6 +458,122 @@ def get_graphics_fuzz_121() -> List[recipe_wrap.RecipeWrap]: } +def get_platform_from_binary(binary: Binary) -> str: + tags = list(binary.tags) + platforms = [p for p in tags if p in PLATFORMS_SET] + if platforms: + check( + len(platforms) == 1, AssertionError(f"More than one platform in: {binary}") + ) + platform = platforms[0] + else: + platform = util.get_platform() + return platform + + +def get_config_from_binary(binary: Binary) -> str: + tags = list(binary.tags) + configs = [c for c in tags if c in CONFIGS_SET] + if not configs: + raise AssertionError(f"Could not find a config in tags: {tags}") + check(len(configs) == 1, AssertionError(f"More than one config in: {binary}")) + config = configs[0] + return config + + +def get_github_release_recipe(binary: Binary) -> recipe_wrap.RecipeWrap: + + if binary.name == "glslangValidator": + project_name = "glslang" + elif binary.name in ("spirv-opt", "spirv-as", "spirv-dis", "spirv-val"): + project_name = "SPIRV-Tools" + elif binary.name == "swift_shader_icd": + project_name = "swiftshader" + else: + raise AssertionError(f'Could not map "{binary.name}" to a project.') + + platform = get_platform_from_binary(binary) + config = get_config_from_binary(binary) + arch = "x64" + + tags = [platform, config, arch] + + repo_name = f"gfbuild-{project_name}" + version = binary.version + artifact_name = f"gfbuild-{project_name}-{version}-{platform}_{arch}_{config}" + + recipe = recipe_wrap.RecipeWrap( + path=f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/{artifact_name}", + recipe=Recipe( + download_and_extract_archive_set=RecipeDownloadAndExtractArchiveSet( + archive_set=ArchiveSet( + archives=[ + Archive( + url=f"https://github.com/google/{repo_name}/releases/download/github/google/{repo_name}/{version}/{artifact_name}.zip", + output_file=f"{project_name}.zip", + output_directory=f"{project_name}", + ) + ], + binaries=[], + ) + ) + ), + ) + + executable_suffix = ".exe" if platform == "Windows" else "" + + if project_name == "glslang": + binaries = [ + Binary( + name="glslangValidator", + tags=tags, + path=f"{project_name}/bin/glslangValidator" + executable_suffix, + version=version, + ) + ] + elif project_name == "SPIRV-Tools": + binaries = [ + Binary( + name="spirv-opt", + tags=tags, + path=f"{project_name}/bin/spirv-opt" + executable_suffix, + version=version, + ), + Binary( + name="spirv-as", + tags=tags, + path=f"{project_name}/bin/spirv-as" + executable_suffix, + version=version, + ), + Binary( + name="spirv-dis", + tags=tags, + path=f"{project_name}/bin/spirv-dis" + executable_suffix, + version=version, + ), + Binary( + name="spirv-val", + tags=tags, + path=f"{project_name}/bin/spirv-val" + executable_suffix, + version=version, + ), + ] + elif project_name == "swiftshader": + binaries = [ + Binary( + name="swift_shader_icd", + tags=tags, + path=f"{project_name}/lib/vk_swiftshader_icd.json", + version=version, + ) + ] + else: + raise AssertionError(f"Unknown project name: {project_name}") + + recipe.recipe.download_and_extract_archive_set.archive_set.binaries.extend(binaries) + return recipe + + class BinaryManager(BinaryGetter): """ Implements BinaryGetter. @@ -477,7 +601,6 @@ def __init__( binary_list: Optional[List[Binary]] = None, platform: Optional[str] = None, built_in_binary_recipes: Optional[Dict[str, Recipe]] = None, - custom_binary_artifacts_prefix: Optional[str] = None, ): self._binary_list = binary_list or DEFAULT_BINARIES self._resolved_paths = {} @@ -485,6 +608,10 @@ def __init__( self._binary_artifacts = [] self._built_in_binary_recipes = {} + self._binary_artifacts.extend( + artifact_util.binary_artifacts_find(BINARY_RECIPES_PREFIX) + ) + # When changing this constructor, check self.get_child_binary_manager(). if built_in_binary_recipes: @@ -498,11 +625,6 @@ def __init__( archive_set: RecipeDownloadAndExtractArchiveSet = recipe.download_and_extract_archive_set self._binary_artifacts.append((archive_set.archive_set, artifact_path)) - if custom_binary_artifacts_prefix: - self._binary_artifacts.extend( - artifact_util.binary_artifacts_find(custom_binary_artifacts_prefix) - ) - @staticmethod def get_binary_list_from_test_metadata(test_json_path: Path) -> List[Binary]: test_metadata = test_util.metadata_read_from_path(test_json_path) @@ -512,11 +634,7 @@ def get_binary_list_from_test_metadata(test_json_path: Path) -> List[Binary]: result.extend(test_metadata.binaries) return result - def get_binary_path(self, binary: Binary) -> Path: - result = self._resolved_paths.get(binary.SerializePartialToString()) - if result: - return result - log(f"Finding path of binary:\n{binary}") + def _get_binary_path_from_binary_artifacts(self, binary: Binary) -> Optional[Path]: binary_tags = set(binary.tags) binary_tags.add(self._platform) for (archive_set, artifact_path) in self._binary_artifacts: @@ -536,7 +654,43 @@ def get_binary_path(self, binary: Binary) -> Path: ) self._resolved_paths[binary.SerializePartialToString()] = result return result - raise BinaryPathNotFound(binary) + return None + + def get_binary_path(self, binary: Binary) -> Path: + # Try resolved cache first. + result = self._resolved_paths.get(binary.SerializePartialToString()) + if result: + return result + log(f"Finding path of binary:\n{binary}") + + # Try list (cache) of binary artifacts on disk. + result = self._get_binary_path_from_binary_artifacts(binary) + if result: + return result + + # Try online. + wrapped_recipe = get_github_release_recipe(binary) + # Execute the recipe to download the binaries. + artifact_util.artifact_execute_recipe_if_needed( + wrapped_recipe.path, {wrapped_recipe.path: wrapped_recipe.recipe} + ) + # Add to binary artifacts list (cache). + self._binary_artifacts.append( + ( + wrapped_recipe.recipe.download_and_extract_archive_set.archive_set, + wrapped_recipe.path, + ) + ) + # Now we should be able to find it in the binary artifacts list. + result = self._get_binary_path_from_binary_artifacts(binary) + check( + bool(result), + AssertionError( + f"Could not find:\n{binary} even though we just added it:\n{wrapped_recipe}" + ), + ) + assert result # noqa + return result @staticmethod def get_binary_by_name_from_list(name: str, binary_list: List[Binary]) -> Binary: @@ -571,7 +725,4 @@ def get_child_binary_manager( def get_default_binary_manager() -> BinaryManager: - return BinaryManager( - built_in_binary_recipes=BUILT_IN_BINARY_RECIPES_MAP, - custom_binary_artifacts_prefix=CUSTOM_BINARY_RECIPES_PATH_PREFIX, - ) + return BinaryManager(built_in_binary_recipes=BUILT_IN_BINARY_RECIPES_MAP) diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index f2b5d062d..ed3710920 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -26,7 +26,6 @@ from gfauto import ( amber_converter, - artifact_util, binaries_util, glslang_validator_util, shader_job_util, @@ -86,24 +85,6 @@ def get_copyright_header_google(year: str) -> str: """ -def get_binary_paths_using_artifact_system( - artifact_path: str -) -> binaries_util.BinaryManager: - - # Deprecated. - - artifact_util.artifact_execute_recipe_if_needed( - artifact_path, binaries_util.BUILT_IN_BINARY_RECIPES_MAP - ) - artifact_metadata = artifact_util.artifact_read_metadata(artifact_path) - - return binaries_util.BinaryManager( - list(artifact_metadata.data.extracted_archive_set.archive_set.binaries), - built_in_binary_recipes=binaries_util.BUILT_IN_BINARY_RECIPES_MAP, - custom_binary_artifacts_prefix=artifact_path, - ) - - def amberfy( input_json: Path, output_amber: Path, diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index de33c07bb..21db37883 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -195,3 +195,5 @@ fsync fileno Debian prepended +config +configs From b3d4225c9bf55ff7f2f053d3c703905227ccce7e Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Fri, 8 Nov 2019 07:06:32 +0000 Subject: [PATCH 167/180] gfauto: run_cts_gf_tests: support android and host (#796) Also, fix bug where SwiftShader results were incorrect. --- gfauto/gfauto/run_cts_gf_tests.py | 38 +++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/gfauto/gfauto/run_cts_gf_tests.py b/gfauto/gfauto/run_cts_gf_tests.py index 6c4a98ba3..abcfa952d 100644 --- a/gfauto/gfauto/run_cts_gf_tests.py +++ b/gfauto/gfauto/run_cts_gf_tests.py @@ -23,10 +23,12 @@ from typing import Optional from gfauto import ( + android_device, binaries_util, devices_util, fuzz, host_device_util, + result_util, settings_util, shader_compiler_util, spirv_opt_util, @@ -106,40 +108,68 @@ def write_newline() -> None: # Enumerate tests and devices, writing the results. for test in sorted(tests_dir.glob("*.amber")): - write_entry(test.name) + test_name = util.remove_end(test.name, ".amber") + write_entry(test_name) spirv_shaders = sorted( tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv") ) for device in active_devices: + test_run_dir = work_dir / f"{test_name}_{device.name}" + util.mkdirs_p(test_run_dir) try: + # Confusingly, some functions below will raise on an error; others will write e.g. CRASH to the + # STATUS file in the output directory. In the latter case, we update |status|. We check |status| at + # the end of this if-else chain and raise fake exceptions if appropriate. + status = fuzz.STATUS_SUCCESS + if device.HasField("preprocess"): # This just means spirv-op for now. assert spirv_opt_path # noqa for spirv_shader in spirv_shaders: spirv_opt_util.run_spirv_opt_on_spirv_shader( - spirv_shader, work_dir, ["-O"], spirv_opt_path + spirv_shader, test_run_dir, ["-O"], spirv_opt_path ) elif device.HasField("shader_compiler"): for spirv_shader in spirv_shaders: shader_compiler_util.run_shader( shader_compiler_device=device.shader_compiler, shader_path=spirv_shader, - output_dir=work_dir, + output_dir=test_run_dir, timeout=DEFAULT_TIMEOUT, ) elif device.HasField("swift_shader"): assert swift_shader_path # noqa host_device_util.run_amber( test, - work_dir, + test_run_dir, dump_image=False, dump_buffer=False, icd=swift_shader_path, ) + status = result_util.get_status(test_run_dir) + elif device.HasField("host"): + host_device_util.run_amber( + test, test_run_dir, dump_image=False, dump_buffer=False + ) + status = result_util.get_status(test_run_dir) + elif device.HasField("android"): + android_device.run_amber_on_device( + test, + test_run_dir, + dump_image=False, + dump_buffer=False, + serial=device.android.serial, + ) + status = result_util.get_status(test_run_dir) else: raise AssertionError(f"Unsupported device {device.name}") + if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): + raise CalledProcessError(1, "??") + if status != fuzz.STATUS_SUCCESS: + raise TimeoutExpired("??", fuzz.AMBER_RUN_TIME_LIMIT) + write_entry("P") except CalledProcessError: write_entry("F") From 9d22f575b2e1d4b392e059075c2b717601452ec9 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 8 Nov 2019 10:48:13 +0000 Subject: [PATCH 168/180] Reducer: inline uniforms less when preserving semantics (#794) In order to make reference and variant shaders easier to compare, this stops the reducer from inlining uniforms when preserving semantics, unless the uniforms are live-injected, or the use being inlined is in dead code. Partly fixes #783. Fixes #639. --- .../InlineUniformReductionOpportunities.java | 8 + ...lineUniformReductionOpportunitiesTest.java | 166 +++++++++++++++++- 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java index 9cf4a4559..0d71fd4d6 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java @@ -59,6 +59,14 @@ private InlineUniformReductionOpportunities(TranslationUnit tu, @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); + + // We only inline uniforms if we are not preserving semantics, if the current program point is + // is dead code, or if the uniform is a live-injected variable. + if (!(context.reduceEverywhere() || currentProgramPointIsDeadCode() + || isLiveInjectedVariableName(variableIdentifierExpr.getName()))) { + return; + } + final String name = variableIdentifierExpr.getName(); final ScopeEntry se = getCurrentScope().lookupScopeEntry(name); if (se == null) { diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java index da4945e08..9f41b156e 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java @@ -23,12 +23,15 @@ import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.util.Constants; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import org.junit.Test; @@ -197,12 +200,11 @@ public void inlineUniforms() throws Exception { private ShaderJob checkCanReduceToTarget(ShaderJob shaderJob, int expectedSize, String target) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - boolean found = false; for (int i = 0; i < expectedSize; i++) { final ShaderJob temp = shaderJob.clone(); List ops = InlineUniformReductionOpportunities.findOpportunities(temp, - new ReducerContext(false, + new ReducerContext(true, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null)); assertEquals(expectedSize, ops.size()); ops.get(i).applyReduction(); @@ -214,4 +216,164 @@ private ShaderJob checkCanReduceToTarget(ShaderJob shaderJob, int expectedSize, return null; } + @Test + public void testDoNotInlineWhenPreservingSemantics() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " u;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertTrue(opportunities.isEmpty()); + } + + @Test + public void testDoInlineWhenPreservingSemanticsDeadCode() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " u;\n" + + " if(" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false, false))) {\n" + + " u;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " u;\n" + + " if(" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false, false))) {\n" + + " 2;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoInlineWhenPreservingSemanticsDeadFunction() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "void " + Constants.GLF_DEAD + "_foo() {\n" + + " u;\n" + + "}\n" + + "void main() {\n" + + " u;\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "uniform int u;\n" + + "void " + Constants.GLF_DEAD + "_foo() {\n" + + " 2;\n" + + "}\n" + + "void main() {\n" + + " u;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoInlineWhenPreservingSemanticsUnreachableSwitchCase() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " switch(" + Constants.GLF_SWITCH + "(0)) {\n" + + " case 2:\n" + + " u;\n" + + " case 3:\n" + + " u;\n" + + " case 0:\n" + + " u;\n" + + " case 5:\n" + + " u;\n" + + " break;\n" + + " case 6:\n" + + " u;\n" + + " default:\n" + + " u;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " switch(" + Constants.GLF_SWITCH + "(0)) {\n" + + " case 2:\n" + + " 2;\n" + + " case 3:\n" + + " 2;\n" + + " case 0:\n" + + " u;\n" + + " case 5:\n" + + " u;\n" + + " break;\n" + + " case 6:\n" + + " 2;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertEquals(4, opportunities.size()); + for (SimplifyExprReductionOpportunity op : opportunities) { + op.applyReduction(); + } + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoInlineWhenPreservingSemanticsLiveCode() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "uniform int " + Constants.LIVE_PREFIX + "v;\n" + + "void main() {\n" + + " u;\n" + + " " + Constants.LIVE_PREFIX + "v;\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "uniform int u;\n" + + "uniform int " + Constants.LIVE_PREFIX + "v;\n" + + "void main() {\n" + + " u;\n" + + " 3;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + pipelineInfo.addUniform(Constants.LIVE_PREFIX + "v", BasicType.INT, Optional.empty(), Collections.singletonList(3)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + } From 2584a469d121aa0b1304115a8640cc4e7aedfcf4 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 8 Nov 2019 10:51:16 +0000 Subject: [PATCH 169/180] Reducer: limit constant folding when preserving semantics (#798) To make references and reduced variants easier to compare, this change supresses constant folding in non-injected code when semantics are being preserved. --- .../FoldConstantReductionOpportunities.java | 4 ++++ .../FoldConstantReductionOpportunitiesTest.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java index a7a062236..167fe02e5 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java @@ -52,6 +52,10 @@ private FoldConstantReductionOpportunities(TranslationUnit tu, @Override void identifyReductionOpportunitiesForChild(IAstNode parent, Expr child) { + if (!allowedToReduceExpr(parent, child)) { + return; + } + Optional maybeFce = asFunctionCallExpr(child); if (maybeFce.isPresent()) { switch (maybeFce.get().getCallee()) { diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java index 7334ff9b1..f1deaa8da 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java @@ -628,7 +628,7 @@ private void check(String before, int numOps, String after) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { final TranslationUnit tu = ParseHelper.parse(before); final List ops = FoldConstantReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); ops.forEach(item -> item.applyReduction()); CompareAsts.assertEqualAsts(after, tu); From 8b1443cbdc63e9dee0cd38fedf16b8a28c8f5002 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Mon, 18 Nov 2019 07:40:30 -0800 Subject: [PATCH 170/180] gfauto: latest downloaded binaries and other fixes (#799) * gfauto: fix typo that prevented skipping reductions * gfauto: small fix to README * Add more downloaded binaries, use downloaded amber * Use downloaded graphicsfuzz and other fixes * Use downloaded llpc * Support downloading latest binary versions * Continue if downloading versions fails * Use downloaded spirv-fuzz/reduce * run_cts_...: Output shader tests for llpc --- gfauto/README.md | 2 +- gfauto/gfauto/amber_converter.py | 20 ++ gfauto/gfauto/android_device.py | 2 +- gfauto/gfauto/binaries_util.py | 207 +++++++++++++++++-- gfauto/gfauto/devices_util.py | 2 +- gfauto/gfauto/download_cts_gf_tests.py | 12 +- gfauto/gfauto/fuzz.py | 36 ++-- gfauto/gfauto/fuzz_glsl_test.py | 44 +++- gfauto/gfauto/fuzz_spirv_test.py | 18 +- gfauto/gfauto/gfauto_interestingness_test.py | 25 ++- gfauto/gfauto/host_device_util.py | 7 +- gfauto/gfauto/run_cts_gf_tests.py | 36 +++- gfauto/gfauto/settings.proto | 9 +- gfauto/gfauto/settings_pb2.py | 12 +- gfauto/gfauto/settings_pb2.pyi | 8 +- gfauto/gfauto/settings_util.py | 39 +++- gfauto/gfauto/shader_compiler_util.py | 25 +-- gfauto/gfauto/test_create_readme.py | 5 +- gfauto/gfauto/tool.py | 5 +- gfauto/gfauto/util.py | 4 +- gfauto/whitelist.dic | 5 + 21 files changed, 415 insertions(+), 108 deletions(-) diff --git a/gfauto/README.md b/gfauto/README.md index df3599eb8..b54810c22 100644 --- a/gfauto/README.md +++ b/gfauto/README.md @@ -192,7 +192,7 @@ git clone https://github.com/pyenv/pyenv.git ~/.pyenv # Add the following two lines to your ~/.bashrc file. export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$PATH +export PATH="$PYENV_ROOT/bin:$PATH" # In a new terminal: eval "$(pyenv init -)" diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py index 83635287a..ac6fefbf1 100644 --- a/gfauto/gfauto/amber_converter.py +++ b/gfauto/gfauto/amber_converter.py @@ -712,6 +712,12 @@ def write_shader( f"{amber_file.stem}.{shader_name}{shader_type_suffix}{shader_job_util.SUFFIX_SPIRV}" ) + # E.g. dEQP-VK.graphicsfuzz.ifs-and-whiles.variant_fragment_shader.spvas + # These files can be added to the llpc repo as a shader test. + shader_llpc_asm_test_file_path = output_dir / ( + f"dEQP-VK.graphicsfuzz.{amber_file.stem}.{shader_name}.spvas" + ) + util.file_write_text(shader_asm_file_path, shader_asm) files_written.append(shader_asm_file_path) @@ -731,6 +737,20 @@ def write_shader( files_written.append(shader_spirv_file_path) + util.file_write_text( + shader_llpc_asm_test_file_path, + """; BEGIN_SHADERTEST +; RUN: amdllpc -verify-ir -spvgen-dir=%spvgendir% -v %gfxip %s | FileCheck -check-prefix=SHADERTEST %s +; SHADERTEST-LABEL: {{^// LLPC.*}} SPIRV-to-LLVM translation results +; SHADERTEST: AMDLLPC SUCCESS +; END_SHADERTEST +; +""" + + f"; Based on dEQP-VK.graphicsfuzz.{amber_file.stem}\n\n" + + shader_asm, + ) + files_written.append(shader_llpc_asm_test_file_path) + return files_written diff --git a/gfauto/gfauto/android_device.py b/gfauto/gfauto/android_device.py index ec351c6b8..da04fda4e 100644 --- a/gfauto/gfauto/android_device.py +++ b/gfauto/gfauto/android_device.py @@ -137,7 +137,7 @@ def is_screen_off_or_locked(serial: Optional[str] = None) -> bool: def get_all_android_devices() -> List[Device]: result: List[Device] = [] - log("Getting the list of connected Android devices via adb") + log("Getting the list of connected Android devices via adb\n") adb_devices = adb_check(None, ["devices", "-l"], verbose=True) stdout: str = adb_devices.stdout diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py index 0b17b3821..195ef9723 100644 --- a/gfauto/gfauto/binaries_util.py +++ b/gfauto/gfauto/binaries_util.py @@ -26,11 +26,13 @@ from typing import Dict, List, Optional, Tuple import attr +import requests from gfauto import artifact_util, recipe_wrap, test_util, util from gfauto.common_pb2 import Archive, ArchiveSet, Binary from gfauto.gflogging import log from gfauto.recipe_pb2 import Recipe, RecipeDownloadAndExtractArchiveSet +from gfauto.settings_pb2 import Settings from gfauto.util import check BINARY_RECIPES_PREFIX = "//binaries" @@ -42,6 +44,7 @@ SPIRV_VAL_NAME = "spirv-val" SPIRV_DIS_NAME = "spirv-dis" SWIFT_SHADER_NAME = "swift_shader_icd" +AMBER_NAME = "amber" SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG = "no-validate-after-all" @@ -82,11 +85,25 @@ Binary(name="spirv-as", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), Binary(name="spirv-val", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), Binary(name="spirv-fuzz", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-reduce", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), Binary( name="swift_shader_icd", tags=["Debug"], version="aaa64b76c0b40c2958a18cfdc623157c8c6e1b7d", ), + Binary( + name="amber", tags=["Debug"], version="2bade8f0a3608872962c0e9e451ccdd63c3332f9" + ), + Binary( + name="graphicsfuzz-tool", + tags=[], + version="2584a469d121aa0b1304115a8640cc4e7aedfcf4", + ), + Binary( + name="amdllpc", + tags=["Debug"], + version="06e4d24336a16ed10d305931804d75a4104dce35", + ), ] @@ -481,26 +498,58 @@ def get_config_from_binary(binary: Binary) -> str: return config -def get_github_release_recipe(binary: Binary) -> recipe_wrap.RecipeWrap: - - if binary.name == "glslangValidator": +def binary_name_to_project_name(binary_name: str) -> str: + if binary_name == "glslangValidator": project_name = "glslang" - elif binary.name in ("spirv-opt", "spirv-as", "spirv-dis", "spirv-val"): + elif binary_name in ( + "spirv-opt", + "spirv-as", + "spirv-dis", + "spirv-val", + "spirv-fuzz", + "spirv-reduce", + ): project_name = "SPIRV-Tools" - elif binary.name == "swift_shader_icd": + elif binary_name == "swift_shader_icd": project_name = "swiftshader" + elif binary_name == "amber": + project_name = "amber" + elif binary_name == "graphicsfuzz-tool": + project_name = "graphicsfuzz" + elif binary_name == "amdllpc": + project_name = "llpc" else: - raise AssertionError(f'Could not map "{binary.name}" to a project.') + raise AssertionError( + f"Could not find {binary_name}. Could not map {binary_name} to a gfbuild- repo." + ) + + return project_name + + +def get_github_release_recipe( # pylint: disable=too-many-branches; + binary: Binary +) -> recipe_wrap.RecipeWrap: + + project_name = binary_name_to_project_name(binary.name) - platform = get_platform_from_binary(binary) - config = get_config_from_binary(binary) - arch = "x64" + if project_name == "graphicsfuzz": + # Special case: + platform = util.get_platform() + tags = PLATFORMS[:] + repo_name = f"gfbuild-{project_name}" + version = binary.version + artifact_name = f"gfbuild-{project_name}-{version}" + else: + # Normal case: + platform = get_platform_from_binary(binary) + config = get_config_from_binary(binary) + arch = "x64" - tags = [platform, config, arch] + tags = [platform, config, arch] - repo_name = f"gfbuild-{project_name}" - version = binary.version - artifact_name = f"gfbuild-{project_name}-{version}-{platform}_{arch}_{config}" + repo_name = f"gfbuild-{project_name}" + version = binary.version + artifact_name = f"gfbuild-{project_name}-{version}-{platform}_{arch}_{config}" recipe = recipe_wrap.RecipeWrap( path=f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/{artifact_name}", @@ -527,7 +576,7 @@ def get_github_release_recipe(binary: Binary) -> recipe_wrap.RecipeWrap: Binary( name="glslangValidator", tags=tags, - path=f"{project_name}/bin/glslangValidator" + executable_suffix, + path=f"{project_name}/bin/glslangValidator{executable_suffix}", version=version, ) ] @@ -536,25 +585,37 @@ def get_github_release_recipe(binary: Binary) -> recipe_wrap.RecipeWrap: Binary( name="spirv-opt", tags=tags, - path=f"{project_name}/bin/spirv-opt" + executable_suffix, + path=f"{project_name}/bin/spirv-opt{executable_suffix}", version=version, ), Binary( name="spirv-as", tags=tags, - path=f"{project_name}/bin/spirv-as" + executable_suffix, + path=f"{project_name}/bin/spirv-as{executable_suffix}", version=version, ), Binary( name="spirv-dis", tags=tags, - path=f"{project_name}/bin/spirv-dis" + executable_suffix, + path=f"{project_name}/bin/spirv-dis{executable_suffix}", version=version, ), Binary( name="spirv-val", tags=tags, - path=f"{project_name}/bin/spirv-val" + executable_suffix, + path=f"{project_name}/bin/spirv-val{executable_suffix}", + version=version, + ), + Binary( + name="spirv-fuzz", + tags=tags, + path=f"{project_name}/bin/spirv-fuzz{executable_suffix}", + version=version, + ), + Binary( + name="spirv-reduce", + tags=tags, + path=f"{project_name}/bin/spirv-reduce{executable_suffix}", version=version, ), ] @@ -567,6 +628,35 @@ def get_github_release_recipe(binary: Binary) -> recipe_wrap.RecipeWrap: version=version, ) ] + elif project_name == "amber": + binaries = [ + Binary( + name="amber", + tags=tags, + path=f"{project_name}/bin/amber{executable_suffix}", + version=version, + ) + ] + elif project_name == "graphicsfuzz": + binaries = [ + Binary( + name="graphicsfuzz-tool", + tags=tags, + path=f"{project_name}/python/drivers/graphicsfuzz-tool", + version=version, + ) + ] + elif project_name == "llpc": + if platform != "Linux": + raise AssertionError("amdllpc is only available on Linux") + binaries = [ + Binary( + name="amdllpc", + tags=tags, + path=f"{project_name}/bin/amdllpc{executable_suffix}", + version=version, + ) + ] else: raise AssertionError(f"Unknown project name: {project_name}") @@ -724,5 +814,82 @@ def get_child_binary_manager( return result -def get_default_binary_manager() -> BinaryManager: - return BinaryManager(built_in_binary_recipes=BUILT_IN_BINARY_RECIPES_MAP) +def get_default_binary_manager(settings: Settings) -> BinaryManager: + """ + Gets the default binary manager. + + :param settings: Passing just "Settings()" will use the hardcoded (slightly out-of-date) default binary_list, which + may be fine, especially if you plan to use specific versions anyway by immediately overriding the binary_list using + get_child_binary_manager(). + :return: + """ + return BinaryManager( + binary_list=list(settings.latest_binary_versions) or DEFAULT_BINARIES, + built_in_binary_recipes=BUILT_IN_BINARY_RECIPES_MAP, + ) + + +class DownloadVersionError(Exception): + pass + + +def _download_latest_version_number(project_name: str) -> str: + + url = f"https://api.github.com/repos/google/gfbuild-{project_name}/releases" + log(f"Checking: {url}") + response = requests.get(url) + if not response: + raise DownloadVersionError(f"Failed to find version of {project_name}") + + result = response.json() + + expected_num_assets_map = { + "amber": 17, + "glslang": 15, + "SPIRV-Tools": 15, + "swiftshader": 15, + "graphicsfuzz": 5, + "llpc": 7, + } + + expected_num_assets = expected_num_assets_map[project_name] + + for release in result: + assets = release["assets"] + if len(assets) != expected_num_assets: + log( + f"SKIPPING a release of {project_name} with {len(assets)} assets (expected {expected_num_assets})" + ) + continue + + tag_name: str = release["tag_name"] + last_slash = tag_name.rfind("/") + if last_slash == -1: + raise DownloadVersionError( + f"Failed to find version of {project_name}; tag name: {tag_name}" + ) + version = tag_name[last_slash + 1 :] + log(f"Found {project_name} version {version}") + return version + + raise DownloadVersionError( + f"Failed to find version of {project_name} with {expected_num_assets} assets" + ) + + +def download_latest_binary_version_numbers() -> List[Binary]: + log("Downloading the latest binary version numbers...") + + # Deep copy of DEFAULT_BINARIES. + binaries: List[Binary] = [] + for binary in DEFAULT_BINARIES: + new_binary = Binary() + new_binary.CopyFrom(binary) + binaries.append(new_binary) + + # Update version numbers. + for binary in binaries: + project_name = binary_name_to_project_name(binary.name) + binary.version = _download_latest_version_number(project_name) + + return binaries diff --git a/gfauto/gfauto/devices_util.py b/gfauto/gfauto/devices_util.py index 202962e32..331ff96c0 100644 --- a/gfauto/gfauto/devices_util.py +++ b/gfauto/gfauto/devices_util.py @@ -88,7 +88,7 @@ def get_device_list(device_list: Optional[DeviceList] = None) -> DeviceList: # Offline compiler. device = Device( - name="offline_compiler_1", + name="amdllpc", shader_compiler=DeviceShaderCompiler( binary="amdllpc", args=["-gfxip=9.0.0", "-verify-ir", "-auto-layout-desc"] ), diff --git a/gfauto/gfauto/download_cts_gf_tests.py b/gfauto/gfauto/download_cts_gf_tests.py index e5265e6e2..0c712ebe8 100644 --- a/gfauto/gfauto/download_cts_gf_tests.py +++ b/gfauto/gfauto/download_cts_gf_tests.py @@ -28,6 +28,7 @@ binaries_util, fuzz, gerrit_util, + settings_util, subprocess_util, util, ) @@ -137,14 +138,23 @@ def main() -> None: parser.add_argument("gerrit_cookie", help=GERRIT_COOKIE_ARGUMENT_DESCRIPTION) + parser.add_argument( + "--settings", + help="Path to the settings JSON file for this instance.", + default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), + ) + parsed_args = parser.parse_args(sys.argv[1:]) cookie: str = parsed_args.gerrit_cookie + settings_path: Path = Path(parsed_args.settings) # Need git. git_tool = util.tool_on_path("git") - binaries = binaries_util.get_default_binary_manager() + settings = settings_util.read_or_create(settings_path) + + binaries = binaries_util.get_default_binary_manager(settings=settings) download_cts_graphicsfuzz_tests(git_tool, cookie, binaries) diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index 067aa7838..b4d975b11 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -220,26 +220,28 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many ref for ref in references if shader_job_util.get_related_files(ref) ] - binary_manager = binaries_util.get_default_binary_manager().get_child_binary_manager( - list(settings.custom_binaries), prepend=True - ) + binary_manager = binaries_util.get_default_binary_manager( + settings=settings + ).get_child_binary_manager(list(settings.custom_binaries), prepend=True) - # For convenience, we add the default (i.e. newest) SwiftShader ICD (binary) to any swift_shader devices - # so that we don't need to specify it and update it in the device list (on disk). - # Thus, when we save the test, the device will contain the version of SwiftShader we used. + # For convenience, for some virtual devices, we mutate the device proto (in memory) to add the newest relevant + # Binary protos (e.g. SwiftShader binary info). + # When we save a report to disk, the test's device proto will contain the binaries we used. for device in active_devices: - # noinspection PyTypeChecker - if device.HasField("swift_shader"): - swift_binaries = [ - binary - for binary in device.binaries - if "swift" not in binary.name.lower() - ] - if not swift_binaries: - device.binaries.extend( - [binary_manager.get_binary_by_name("swift_shader_icd")] - ) + if device.HasField("swift_shader") and not [ + binary for binary in device.binaries if binary.name == "swift_shader_icd" + ]: + device.binaries.extend( + [binary_manager.get_binary_by_name("swift_shader_icd")] + ) + + if ( + device.HasField("shader_compiler") + and device.shader_compiler.binary == "amdllpc" + and not [binary for binary in device.binaries if binary.name == "amdllpc"] + ): + device.binaries.extend([binary_manager.get_binary_by_name("amdllpc")]) while True: diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py index 5520c2e7f..eeeb185fe 100644 --- a/gfauto/gfauto/fuzz_glsl_test.py +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -46,7 +46,7 @@ from gfauto.gflogging import log from gfauto.settings_pb2 import Settings from gfauto.test_pb2 import Test, TestGlsl -from gfauto.util import check, tool_on_path +from gfauto.util import check class ReductionFailedError(Exception): @@ -72,7 +72,12 @@ def fuzz_glsl( # Pick a randomly chosen reference. unprepared_reference_shader_job = random.choice(references) - # TODO: Allow GraphicsFuzz to be downloaded. + # The "graphicsfuzz-tool" tool is designed to be on your PATH so that e.g. ".bat" will be appended on Windows. + # So we use tool_on_path with a custom PATH to get the actual file we want to execute. + graphicsfuzz_tool_path = util.tool_on_path( + "graphicsfuzz-tool", + str(binary_manager.get_binary_path_by_name("graphicsfuzz-tool").path.parent), + ) try: with util.file_open_text(staging_dir / "log.txt", "w") as log_file: @@ -81,7 +86,7 @@ def fuzz_glsl( # Create the prepared (for Vulkan GLSL) reference. glsl_generate_util.run_prepare_reference( - util.tool_on_path("graphicsfuzz-tool"), + graphicsfuzz_tool_path, unprepared_reference_shader_job, template_source_dir / test_util.REFERENCE_DIR @@ -90,7 +95,7 @@ def fuzz_glsl( # Generate the variant (GraphicsFuzz requires the unprepared reference as input). glsl_generate_util.run_generate( - util.tool_on_path("graphicsfuzz-tool"), + graphicsfuzz_tool_path, unprepared_reference_shader_job, donors_dir, template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, @@ -300,14 +305,16 @@ def should_reduce_report(settings: Settings, test_dir: Path) -> bool: if ( not settings.reduce_bad_images and status == fuzz.STATUS_CRASH - and signature_util == signature_util.BAD_IMAGE_SIGNATURE + and signature == signature_util.BAD_IMAGE_SIGNATURE ): return False return True -def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: +def run_reduction_on_report( + test_dir: Path, reports_dir: Path, binary_manager: binaries_util.BinaryManager +) -> None: test = test_util.metadata_read(test_dir) try: @@ -316,6 +323,7 @@ def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: test_dir_reduction_output=test_dir, test_dir_to_reduce=reduced_test, preserve_semantics=True, + binary_manager=binary_manager, reduction_name="1", ) @@ -324,6 +332,7 @@ def run_reduction_on_report(test_dir: Path, reports_dir: Path) -> None: test_dir_reduction_output=test_dir, test_dir_to_reduce=reduced_test, preserve_semantics=False, + binary_manager=binary_manager, reduction_name="2", ) @@ -381,7 +390,7 @@ def handle_test( # For each report, run a reduction on the target device with the device-specific crash signature. for test_dir_in_reports in report_paths: if should_reduce_report(settings, test_dir_in_reports): - run_reduction_on_report(test_dir_in_reports, reports_dir) + run_reduction_on_report(test_dir_in_reports, reports_dir, binary_manager) else: log("Skipping reduction due to settings.") @@ -427,6 +436,13 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc log(f"Running test on device:\n{device.name}") + # We will create a binary_manager child with a restricted set of binaries so that we only use the binaries + # specified in the test and by the device; if some required binaries are not specified by the test nor the + # device, there will be an error instead of falling back to our default binaries. But we keep a reference to + # the parent so we can still access certain "test-independent" binaries like Amber. + + binary_manager_parent = binary_manager + if not ignore_test_and_device_binaries: binary_manager = binary_manager.get_child_binary_manager( list(device.binaries) + list(test.binaries) @@ -492,6 +508,7 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc device.shader_compiler, combined_spirv_shader_job.spirv_shader_job, output_dir, + binary_manager=binary_manager, ) except subprocess.CalledProcessError: result_util.write_status( @@ -571,6 +588,9 @@ def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branc host_device_util.run_amber( amber_script_file, output_dir, + amber_path=binary_manager_parent.get_binary_path_by_name( + binaries_util.AMBER_NAME + ).path, dump_image=(not is_compute), dump_buffer=is_compute, icd=icd, @@ -606,6 +626,7 @@ def run_reduction( test_dir_reduction_output: Path, test_dir_to_reduce: Path, preserve_semantics: bool, + binary_manager: binaries_util.BinaryManager, reduction_name: str = "reduction1", ) -> Path: test = test_util.metadata_read(test_dir_to_reduce) @@ -649,6 +670,7 @@ def run_reduction( output_dir=test_util.get_reduction_work_directory( reduced_test_dir, name_of_shader_to_reduce ), + binary_manager=binary_manager, preserve_semantics=preserve_semantics, ) @@ -679,13 +701,19 @@ def run_glsl_reduce( source_dir: Path, name_of_shader_to_reduce: str, output_dir: Path, + binary_manager: binaries_util.BinaryManager, preserve_semantics: bool = False, ) -> Path: input_shader_job = source_dir / name_of_shader_to_reduce / test_util.SHADER_JOB + glsl_reduce_path = util.tool_on_path( + "glsl-reduce", + str(binary_manager.get_binary_path_by_name("graphicsfuzz-tool").path.parent), + ) + cmd = [ - str(tool_on_path("glsl-reduce")), + str(glsl_reduce_path), str(input_shader_job), "--output", str(output_dir), diff --git a/gfauto/gfauto/fuzz_spirv_test.py b/gfauto/gfauto/fuzz_spirv_test.py index cbc262754..8d2379efa 100644 --- a/gfauto/gfauto/fuzz_spirv_test.py +++ b/gfauto/gfauto/fuzz_spirv_test.py @@ -95,6 +95,7 @@ def run_spirv_reduce_or_shrink( extension_to_reduce: str, output_dir: Path, preserve_semantics: bool, + binary_manager: binaries_util.BinaryManager, ) -> Path: input_shader_job = source_dir / name_of_shader_job_to_reduce / test_util.SHADER_JOB @@ -120,7 +121,7 @@ def run_spirv_reduce_or_shrink( if preserve_semantics: cmd = [ - str(util.tool_on_path("spirv-fuzz")), + str(binary_manager.get_binary_path_by_name("spirv-fuzz").path), str(original_spirv_file), "-o", str(final_shader), @@ -137,7 +138,7 @@ def run_spirv_reduce_or_shrink( ] else: cmd = [ - str(util.tool_on_path("spirv-reduce")), + str(binary_manager.get_binary_path_by_name("spirv-reduce").path), str(transformed_spirv_file), "-o", str(final_shader), @@ -170,6 +171,7 @@ def run_reduction( shader_job_name_to_reduce: str, extension_to_reduce: str, preserve_semantics: bool, + binary_manager: binaries_util.BinaryManager, reduction_name: str = "reduction1", ) -> Path: test = test_util.metadata_read(test_dir_to_reduce) @@ -206,6 +208,7 @@ def run_reduction( extension_to_reduce=extension_to_reduce, output_dir=output_dir, preserve_semantics=preserve_semantics, + binary_manager=binary_manager, ) else: final_shader_path = run_spirv_reduce_or_shrink( @@ -214,6 +217,7 @@ def run_reduction( extension_to_reduce=extension_to_reduce, output_dir=output_dir, preserve_semantics=preserve_semantics, + binary_manager=binary_manager, ) check( @@ -257,7 +261,7 @@ def run_reduction( def run_reduction_on_report( # pylint: disable=too-many-locals; - test_dir: Path, reports_dir: Path + test_dir: Path, reports_dir: Path, binary_manager: binaries_util.BinaryManager ) -> None: test = test_util.metadata_read(test_dir) @@ -313,6 +317,7 @@ def run_reduction_on_report( # pylint: disable=too-many-locals; shader_job_name_to_reduce=shader_job_to_reduce.name, extension_to_reduce=extension_to_reduce, preserve_semantics=True, + binary_manager=binary_manager, reduction_name=f"0_{index}_{suffix.split('.')[1]}", ) @@ -326,6 +331,7 @@ def run_reduction_on_report( # pylint: disable=too-many-locals; shader_job_name_to_reduce=shader_job_to_reduce.name, extension_to_reduce=extension_to_reduce, preserve_semantics=False, + binary_manager=binary_manager, reduction_name=f"1_{index}_{suffix.split('.')[1]}", ) @@ -384,7 +390,9 @@ def handle_test( # For each report, run a reduction on the target device with the device-specific crash signature. for test_dir_in_reports in report_paths: if fuzz_glsl_test.should_reduce_report(settings, test_dir_in_reports): - run_reduction_on_report(test_dir_in_reports, reports_dir) + run_reduction_on_report( + test_dir_in_reports, reports_dir, binary_manager=binary_manager + ) else: log("Skipping reduction due to settings.") @@ -421,7 +429,7 @@ def fuzz_spirv( try: gflogging.push_stream_for_logging(log_file) spirv_fuzz_util.run_generate_on_shader_job( - util.tool_on_path("spirv-fuzz"), + binary_manager.get_binary_path_by_name("spirv-fuzz").path, reference_spirv_shader_job, template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, seed=str(random.getrandbits(spirv_fuzz_util.GENERATE_SEED_BITS)), diff --git a/gfauto/gfauto/gfauto_interestingness_test.py b/gfauto/gfauto/gfauto_interestingness_test.py index 8af729862..ba48f1f6e 100644 --- a/gfauto/gfauto/gfauto_interestingness_test.py +++ b/gfauto/gfauto/gfauto_interestingness_test.py @@ -33,14 +33,14 @@ fuzz, fuzz_glsl_test, result_util, + settings_util, signature_util, test_util, tool, util, ) from gfauto.gflogging import log - -# TODO: Maybe add helper method and throw exceptions instead of calling sys.exit. +from gfauto.settings_pb2 import Settings # TODO: Could we make the interestingness test the only way of running a shader job (or indeed, any test)? # We would want to pass the output directory (default is one will be created, as is currently the case), the test_json, @@ -48,6 +48,8 @@ # A device (or test?) could then even specify a custom interestingness command, although the default one would probably # be the same for all devices and it would look at the device info in the test_json? +# TODO: Maybe add helper method and throw exceptions instead of calling sys.exit. + def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too-many-branches; parser = argparse.ArgumentParser( @@ -79,7 +81,7 @@ def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too parser.add_argument( "--use_default_binaries", help="Use the latest binaries, ignoring those defined in the test.json. " - "Implies --fallback_binaries.", + "Implies --fallback_binaries. Passing --settings is recommended to ensure the latest binaries are used.", action="store_true", ) @@ -95,11 +97,26 @@ def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too default=None, ) + parser.add_argument( + "--settings", + help="Path to a settings JSON file for this instance. " + "Unlike with gfauto_fuzz, the default value is an empty string, which is ignored. " + "You only need to use a settings file if you pass --use_default_binaries and you want to use the latest binary versions. " + 'In this case, use e.g. "--settings settings.json" so that a default settings file is generated with the latest binary version numbers ' + "and then run gfauto_interestingness_test again to use those latest binaries.", + default="", + ) + parsed_args = parser.parse_args(sys.argv[1:]) source_dir: Path = Path(parsed_args.source_dir) override_shader_job: Optional[Tuple[str, str]] = parsed_args.override_shader_job override_shader: Optional[Tuple[str, str, str]] = parsed_args.override_shader + settings_str: str = parsed_args.settings + + settings = Settings() + if settings_str: + settings = settings_util.read_or_create(Path(settings_str)) use_default_binaries: bool = parsed_args.use_default_binaries fallback_binaries: bool = parsed_args.fallback_binaries or use_default_binaries @@ -113,7 +130,7 @@ def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too else: raise AssertionError("Need --output or --override_shader[_job] parameter.") - binary_manager = binaries_util.get_default_binary_manager() + binary_manager = binaries_util.get_default_binary_manager(settings=settings) if not fallback_binaries: binary_manager = binary_manager.get_child_binary_manager(binary_list=[]) diff --git a/gfauto/gfauto/host_device_util.py b/gfauto/gfauto/host_device_util.py index 77740145f..fd38c1620 100644 --- a/gfauto/gfauto/host_device_util.py +++ b/gfauto/gfauto/host_device_util.py @@ -32,6 +32,7 @@ def run_amber( output_dir: Path, dump_image: bool, dump_buffer: bool, + amber_path: Path, skip_render: bool = False, debug_layers: bool = False, icd: Optional[Path] = None, @@ -47,6 +48,7 @@ def run_amber( output_dir, dump_image, dump_buffer, + amber_path, skip_render, debug_layers, icd, @@ -62,19 +64,18 @@ def run_amber_helper( output_dir: Path, dump_image: bool, dump_buffer: bool, + amber_path: Path, skip_render: bool = False, debug_layers: bool = False, icd: Optional[Path] = None, ) -> Path: - # TODO: Use binary paths. - variant_image_file = output_dir / fuzz.VARIANT_IMAGE_FILE_NAME reference_image_file = output_dir / fuzz.REFERENCE_IMAGE_FILE_NAME buffer_file = output_dir / fuzz.BUFFER_FILE_NAME cmd = [ - str(util.tool_on_path("amber")), + str(amber_path), str(amber_script_file), "--log-graphics-calls-time", "--disable-spirv-val", diff --git a/gfauto/gfauto/run_cts_gf_tests.py b/gfauto/gfauto/run_cts_gf_tests.py index abcfa952d..164d07e46 100644 --- a/gfauto/gfauto/run_cts_gf_tests.py +++ b/gfauto/gfauto/run_cts_gf_tests.py @@ -46,7 +46,7 @@ def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-man parser.add_argument( "--settings", - help="Path to the settings JSON file for this fuzzing instance.", + help="Path to the settings JSON file for this instance.", default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), ) @@ -67,7 +67,7 @@ def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-man active_devices = devices_util.get_active_devices(settings.device_list) # Binaries. - binaries = binaries_util.get_default_binary_manager() + binaries = binaries_util.get_default_binary_manager(settings=settings) work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" @@ -86,22 +86,32 @@ def write_newline() -> None: spirv_opt_path: Optional[Path] = None swift_shader_path: Optional[Path] = None + amber_path: Optional[Path] = None # Enumerate active devices, writing their name and storing binary paths if needed. write_entry("test") for device in active_devices: - if device.HasField("preprocess"): + + if device.name == "host_preprocessor": + # We are actually just running spirv-opt on the SPIR-V shaders. write_entry("spirv-opt") + else: + write_entry(device.name) + + if device.HasField("preprocess"): spirv_opt_path = binaries.get_binary_path_by_name( binaries_util.SPIRV_OPT_NAME ).path - elif device.HasField("swift_shader"): - write_entry("SwiftShader") + + if device.HasField("swift_shader"): swift_shader_path = binaries.get_binary_path_by_name( binaries_util.SWIFT_SHADER_NAME ).path - else: - write_entry(device.name) + + if device.HasField("swift_shader") or device.HasField("host"): + amber_path = binaries.get_binary_path_by_name( + binaries_util.AMBER_NAME + ).path write_newline() @@ -136,21 +146,31 @@ def write_newline() -> None: shader_compiler_device=device.shader_compiler, shader_path=spirv_shader, output_dir=test_run_dir, + compiler_path=binaries.get_binary_path_by_name( + device.shader_compiler.binary + ).path, timeout=DEFAULT_TIMEOUT, ) elif device.HasField("swift_shader"): assert swift_shader_path # noqa + assert amber_path # noqa host_device_util.run_amber( test, test_run_dir, + amber_path=amber_path, dump_image=False, dump_buffer=False, icd=swift_shader_path, ) status = result_util.get_status(test_run_dir) elif device.HasField("host"): + assert amber_path # noqa host_device_util.run_amber( - test, test_run_dir, dump_image=False, dump_buffer=False + test, + test_run_dir, + amber_path=amber_path, + dump_image=False, + dump_buffer=False, ) status = result_util.get_status(test_run_dir) elif device.HasField("android"): diff --git a/gfauto/gfauto/settings.proto b/gfauto/gfauto/settings.proto index aef1a16e6..e2c8f6416 100644 --- a/gfauto/gfauto/settings.proto +++ b/gfauto/gfauto/settings.proto @@ -26,9 +26,9 @@ message Settings { // A list of devices, including the list of active devices names that will actually be used in this session. DeviceList device_list = 1; - // A list of custom binaries; this should rarely be set, but can be used, for example, if you want to override the - // default versions of some binaries for all devices. It is probably more common to override binaries for certain - // devices, or to just update the hardcoded set of default binaries (in binaries_util.py) to their newer versions. + // A list of custom binary versions; this should rarely be set, but can be used, for example, if you want to override + // the default versions of some binaries for all devices. It is probably more common to specify binaries for certain + // devices. repeated Binary custom_binaries = 2; // Allow at most this many duplicate crashes per device. @@ -47,4 +47,7 @@ message Settings { // Reduce bad images. E.g. your test device renders an incorrect image. bool reduce_bad_images = 7; + + // A list of built-in binary versions that will be generated automatically. + repeated Binary latest_binary_versions = 8; } diff --git a/gfauto/gfauto/settings_pb2.py b/gfauto/gfauto/settings_pb2.py index 7c733a3f1..9697d091f 100644 --- a/gfauto/gfauto/settings_pb2.py +++ b/gfauto/gfauto/settings_pb2.py @@ -22,7 +22,7 @@ package='gfauto', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\x15gfauto/settings.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\xee\x01\n\x08Settings\x12\'\n\x0b\x64\x65vice_list\x18\x01 \x01(\x0b\x32\x12.gfauto.DeviceList\x12\'\n\x0f\x63ustom_binaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binary\x12!\n\x19maximum_duplicate_crashes\x18\x03 \x01(\r\x12\x1d\n\x15maximum_fuzz_failures\x18\x04 \x01(\r\x12\x1b\n\x13reduce_tool_crashes\x18\x05 \x01(\x08\x12\x16\n\x0ereduce_crashes\x18\x06 \x01(\x08\x12\x19\n\x11reduce_bad_images\x18\x07 \x01(\x08\x62\x06proto3') + serialized_pb=_b('\n\x15gfauto/settings.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\x9e\x02\n\x08Settings\x12\'\n\x0b\x64\x65vice_list\x18\x01 \x01(\x0b\x32\x12.gfauto.DeviceList\x12\'\n\x0f\x63ustom_binaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binary\x12!\n\x19maximum_duplicate_crashes\x18\x03 \x01(\r\x12\x1d\n\x15maximum_fuzz_failures\x18\x04 \x01(\r\x12\x1b\n\x13reduce_tool_crashes\x18\x05 \x01(\x08\x12\x16\n\x0ereduce_crashes\x18\x06 \x01(\x08\x12\x19\n\x11reduce_bad_images\x18\x07 \x01(\x08\x12.\n\x16latest_binary_versions\x18\x08 \x03(\x0b\x32\x0e.gfauto.Binaryb\x06proto3') , dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,gfauto_dot_device__pb2.DESCRIPTOR,]) @@ -85,6 +85,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='latest_binary_versions', full_name='gfauto.Settings.latest_binary_versions', index=7, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -98,11 +105,12 @@ oneofs=[ ], serialized_start=76, - serialized_end=314, + serialized_end=362, ) _SETTINGS.fields_by_name['device_list'].message_type = gfauto_dot_device__pb2._DEVICELIST _SETTINGS.fields_by_name['custom_binaries'].message_type = gfauto_dot_common__pb2._BINARY +_SETTINGS.fields_by_name['latest_binary_versions'].message_type = gfauto_dot_common__pb2._BINARY DESCRIPTOR.message_types_by_name['Settings'] = _SETTINGS _sym_db.RegisterFileDescriptor(DESCRIPTOR) diff --git a/gfauto/gfauto/settings_pb2.pyi b/gfauto/gfauto/settings_pb2.pyi index d59d898b7..099633771 100644 --- a/gfauto/gfauto/settings_pb2.pyi +++ b/gfauto/gfauto/settings_pb2.pyi @@ -44,6 +44,9 @@ class Settings(google___protobuf___message___Message): @property def custom_binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... + @property + def latest_binary_versions(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... + def __init__(self, *, device_list : typing___Optional[gfauto___device_pb2___DeviceList] = None, @@ -53,6 +56,7 @@ class Settings(google___protobuf___message___Message): reduce_tool_crashes : typing___Optional[bool] = None, reduce_crashes : typing___Optional[bool] = None, reduce_bad_images : typing___Optional[bool] = None, + latest_binary_versions : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, ) -> None: ... @classmethod def FromString(cls, s: bytes) -> Settings: ... @@ -60,7 +64,7 @@ class Settings(google___protobuf___message___Message): def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): def HasField(self, field_name: typing_extensions___Literal[u"device_list"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",u"device_list",u"maximum_duplicate_crashes",u"maximum_fuzz_failures",u"reduce_bad_images",u"reduce_crashes",u"reduce_tool_crashes"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",u"device_list",u"latest_binary_versions",u"maximum_duplicate_crashes",u"maximum_fuzz_failures",u"reduce_bad_images",u"reduce_crashes",u"reduce_tool_crashes"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"device_list",b"device_list"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",b"custom_binaries",u"device_list",b"device_list",u"maximum_duplicate_crashes",b"maximum_duplicate_crashes",u"maximum_fuzz_failures",b"maximum_fuzz_failures",u"reduce_bad_images",b"reduce_bad_images",u"reduce_crashes",b"reduce_crashes",u"reduce_tool_crashes",b"reduce_tool_crashes"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",b"custom_binaries",u"device_list",b"device_list",u"latest_binary_versions",b"latest_binary_versions",u"maximum_duplicate_crashes",b"maximum_duplicate_crashes",u"maximum_fuzz_failures",b"maximum_fuzz_failures",u"reduce_bad_images",b"reduce_bad_images",u"reduce_crashes",b"reduce_crashes",u"reduce_tool_crashes",b"reduce_tool_crashes"]) -> None: ... diff --git a/gfauto/gfauto/settings_util.py b/gfauto/gfauto/settings_util.py index f336e4431..3f3824813 100644 --- a/gfauto/gfauto/settings_util.py +++ b/gfauto/gfauto/settings_util.py @@ -19,10 +19,11 @@ Used to read and write the Settings proto. See settings.proto. """ - +import traceback from pathlib import Path -from gfauto import devices_util, proto_util +from gfauto import binaries_util, devices_util, proto_util +from gfauto.gflogging import log from gfauto.settings_pb2 import Settings DEFAULT_SETTINGS_FILE_PATH = Path("settings.json") @@ -44,14 +45,15 @@ def read_or_create(settings_path: Path) -> Settings: try: return read(settings_path) except FileNotFoundError as exception: - message = f'gfauto could not find "{settings_path}"' - if not DEFAULT_SETTINGS_FILE_PATH.exists(): - write_default(DEFAULT_SETTINGS_FILE_PATH) - message += ( - f'; a default settings file "{str(DEFAULT_SETTINGS_FILE_PATH)}" has been created for you. ' - f"Please review it and then try again. \n" - ) - raise NoSettingsFile(message) from exception + if settings_path.exists(): + raise + log( + f'\ngfauto could not find "{settings_path}" so one will be created for you\n' + ) + write_default(settings_path) + raise NoSettingsFile( + f'\ngfauto could not find "{settings_path}" so one was created for you. Please review "{settings_path}" and try again.\n' + ) from exception def read(settings_path: Path) -> Settings: @@ -67,4 +69,21 @@ def write_default(settings_path: Path) -> Path: settings = Settings() settings.CopyFrom(DEFAULT_SETTINGS) devices_util.get_device_list(settings.device_list) + + # noinspection PyBroadException + try: + + settings.latest_binary_versions.extend( + binaries_util.download_latest_binary_version_numbers() + ) + except Exception: # pylint: disable=broad-except; + message = "WARNING: Could not download the latest binary version numbers. We will just use the (older) hardcoded versions." + details = traceback.format_exc() # noqa: SC100, SC200 (spelling of exc) + + log(message) + log(f"\nDetails:\n{details}\n\n") + log(message) + + settings.latest_binary_versions.extend(binaries_util.DEFAULT_BINARIES) + return write(settings, settings_path) diff --git a/gfauto/gfauto/shader_compiler_util.py b/gfauto/gfauto/shader_compiler_util.py index 18684959a..0ff2df2fa 100644 --- a/gfauto/gfauto/shader_compiler_util.py +++ b/gfauto/gfauto/shader_compiler_util.py @@ -20,38 +20,24 @@ """ from pathlib import Path -from typing import List, Optional +from typing import List -from gfauto import shader_job_util, subprocess_util, util +from gfauto import binaries_util, shader_job_util, subprocess_util, util from gfauto.device_pb2 import DeviceShaderCompiler from gfauto.gflogging import log -from gfauto.util import check_file_exists DEFAULT_TIMEOUT = 600 -def resolve_compiler_path(shader_compiler_device: DeviceShaderCompiler) -> Path: - try: - compiler_path: Path = util.tool_on_path(shader_compiler_device.binary) - except util.ToolNotOnPathError: - # If not found on PATH, then assume it is an absolute path. - compiler_path = Path(shader_compiler_device.binary) - check_file_exists(compiler_path) - return compiler_path - - def run_shader( shader_compiler_device: DeviceShaderCompiler, shader_path: Path, output_dir: Path, - compiler_path: Optional[Path] = None, + compiler_path: Path, timeout: int = DEFAULT_TIMEOUT, ) -> Path: output_file_path = output_dir / (shader_path.name + ".out") - if not compiler_path: - compiler_path = resolve_compiler_path(shader_compiler_device) - cmd = [str(compiler_path)] cmd += list(shader_compiler_device.args) cmd += [str(shader_path), "-o", str(output_file_path)] @@ -64,8 +50,11 @@ def run_shader_job( shader_compiler_device: DeviceShaderCompiler, spirv_shader_job_path: Path, output_dir: Path, + binary_manager: binaries_util.BinaryManager, ) -> List[Path]: - compiler_path = resolve_compiler_path(shader_compiler_device) + compiler_path = binary_manager.get_binary_path_by_name( + shader_compiler_device.binary + ).path log(f"Running {str(compiler_path)} on shader job {str(spirv_shader_job_path)}") diff --git a/gfauto/gfauto/test_create_readme.py b/gfauto/gfauto/test_create_readme.py index 860ee55e3..24d7fb1a4 100644 --- a/gfauto/gfauto/test_create_readme.py +++ b/gfauto/gfauto/test_create_readme.py @@ -24,6 +24,7 @@ from pathlib import Path from gfauto import binaries_util, fuzz_glsl_test, test_util +from gfauto.settings_pb2 import Settings from gfauto.util import check, check_dir_exists @@ -58,7 +59,9 @@ def main() -> None: test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) - binary_manager = binaries_util.get_default_binary_manager().get_child_binary_manager( + binary_manager = binaries_util.get_default_binary_manager( + settings=Settings() + ).get_child_binary_manager( binary_list=list(test.binaries) + list(test.device.binaries) ) diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py index ed3710920..74d44b497 100644 --- a/gfauto/gfauto/tool.py +++ b/gfauto/gfauto/tool.py @@ -35,6 +35,7 @@ test_util, util, ) +from gfauto.settings_pb2 import Settings from gfauto.util import check AMBER_COMMAND_PROBE_TOP_LEFT_RED = "probe rgba (0, 0) (1, 0, 0, 1)\n" @@ -375,7 +376,9 @@ def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( shader_jobs = get_shader_jobs(source_dir) test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) - binary_manager = binaries_util.get_default_binary_manager().get_child_binary_manager( + binary_manager = binaries_util.get_default_binary_manager( + settings=Settings() + ).get_child_binary_manager( binary_list=list(test.device.binaries) + list(test.binaries) ) diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py index 7a44cd970..f5d6e8dc4 100644 --- a/gfauto/gfauto/util.py +++ b/gfauto/gfauto/util.py @@ -129,8 +129,8 @@ class ToolNotOnPathError(Exception): pass -def tool_on_path(tool: str) -> pathlib.Path: # noqa VNE002 - result = shutil.which(tool) +def tool_on_path(tool: str, path: Optional[str] = None) -> pathlib.Path: # noqa VNE002 + result = shutil.which(tool, path=path) if result is None: raise ToolNotOnPathError( "Could not find {} on PATH. Please add to PATH.".format(tool) diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index 21db37883..a520b5fe1 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -197,3 +197,8 @@ Debian prepended config configs +amdllpc +protos +llpc +rfind +spvas From 726e202ed835ebe6783efaf2718bfc00d0d1b84c Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 21 Nov 2019 16:43:49 +0000 Subject: [PATCH 171/180] gfauto: get device details and other changes (#800) * gfauto: device fingerprint and driver details * gfauto: fix for driver details on host * gfauto: add tool to run arbitrary binary * gfauto: fix add_amber_tests_to_cts.py for new format --- gfauto/gfauto/add_amber_tests_to_cts.py | 92 +++++++++++-------------- gfauto/gfauto/android_device.py | 50 +++++++++++++- gfauto/gfauto/device.proto | 5 ++ gfauto/gfauto/device_pb2.py | 38 ++++++---- gfauto/gfauto/device_pb2.pyi | 12 ++-- gfauto/gfauto/devices_util.py | 66 ++++++++++++++---- gfauto/gfauto/fuzz.py | 37 +++------- gfauto/gfauto/host_device_util.py | 29 +++++++- gfauto/gfauto/run_bin.py | 69 +++++++++++++++++++ gfauto/gfauto/settings_util.py | 5 +- gfauto/setup.py | 1 + 11 files changed, 291 insertions(+), 113 deletions(-) create mode 100644 gfauto/gfauto/run_bin.py diff --git a/gfauto/gfauto/add_amber_tests_to_cts.py b/gfauto/gfauto/add_amber_tests_to_cts.py index 370e45336..c25d2331d 100644 --- a/gfauto/gfauto/add_amber_tests_to_cts.py +++ b/gfauto/gfauto/add_amber_tests_to_cts.py @@ -26,7 +26,7 @@ import re import shutil import sys -from typing import TextIO, cast +from typing import Pattern, TextIO, cast SHORT_DESCRIPTION_LINE_PREFIX = "# Short description: " @@ -38,6 +38,8 @@ os.path.join("external", "vulkancts", "mustpass", "master", "vk-default.txt"), ] +TEST_NAME_IN_LINE_PATTERN: Pattern[str] = re.compile(r"\"(.*?)\.amber\"") + def check(condition: bool, exception: Exception) -> None: if not condition: @@ -86,15 +88,16 @@ def get_amber_test_file_path(vk_gl_cts: str, amber_test_name: str) -> str: ) -def get_graphics_fuzz_tests_cpp_file_path(vk_gl_cts: str) -> str: +def get_graphics_fuzz_tests_index_file_path(vk_gl_cts: str) -> str: return os.path.join( vk_gl_cts, "external", "vulkancts", - "modules", + "data", "vulkan", "amber", - "vktAmberGraphicsFuzzTests.cpp", + "graphicsfuzz", + "index.txt", ) @@ -136,7 +139,7 @@ def check_and_add_tabs( return line -def get_cpp_line_to_write(amber_test_name: str, short_description: str) -> str: +def get_index_line_to_write(amber_test_name: str, short_description: str) -> str: # A test line has the following form, except with tabs aligning each field. # { "name.amber", "name", "description" }, @@ -144,17 +147,17 @@ def get_cpp_line_to_write(amber_test_name: str, short_description: str) -> str: # 1 2 3 4 # 1 - test_file_name_start_index = 13 + test_file_name_start_index = 5 # 2 - test_name_start_index = 61 + test_name_start_index = 53 # 3 - test_description_start_index = 101 + test_description_start_index = 93 # 4 - test_close_bracket_index = 189 + test_close_bracket_index = 181 tab_size = 4 - line = "\t\t{" + line = "{" line = check_and_add_tabs( line, "internal", "internal", test_file_name_start_index, tab_size @@ -183,71 +186,54 @@ def get_cpp_line_to_write(amber_test_name: str, short_description: str) -> str: return line -def add_amber_test_to_cpp( +def add_amber_test_to_index_file( vk_gl_cts: str, amber_test_name: str, input_amber_test_file_path: str ) -> None: - log("Adding Amber test to the C++.") + log("Adding Amber test to the index file.") short_description = get_amber_test_short_description(input_amber_test_file_path) - cpp_file = get_graphics_fuzz_tests_cpp_file_path(vk_gl_cts) - cpp_file_bak = cpp_file + ".bak" + index_file = get_graphics_fuzz_tests_index_file_path(vk_gl_cts) + index_file_bak = index_file + ".bak" - copyfile(cpp_file, cpp_file_bak) + copyfile(index_file, index_file_bak) - line_to_write = get_cpp_line_to_write(amber_test_name, short_description) + line_to_write = get_index_line_to_write(amber_test_name, short_description) - log("Writing from {} to {}.".format(cpp_file_bak, cpp_file)) + log("Writing from {} to {}.".format(index_file_bak, index_file)) - with open_helper(cpp_file_bak, "r") as cpp_in: - with open_helper(cpp_file, "w") as cpp_out: + with open_helper(index_file_bak, "r") as index_file_in: + with open_helper(index_file, "w") as index_file_out: - # The start of the tests look like this (except with tabs!): - # tests[] = - # { - # { "continue-and-merge.amber",... - # { "control-flow-switch.amber",... - - # Get to just before the first test line, writing lines as we go. - for line in cpp_in: - cpp_out.write(line) - if line.startswith("\ttests[] ="): - break - cpp_out.write(cpp_in.readline()) - - # Compile regex for finding test name from a cpp file line. - search_pattern = re.compile(r"(?:\")(.*)(?:\.amber\")") + # { "continue-and-merge.amber",... # Get to the point where we should insert our line. line = "" - for line in cpp_in: - if not line.startswith(" {"): - break - - # Search the test name from the line. - result = search_pattern.search(line) - # Group 1 is the test name. - test_name_in_line = result.group(1) if result else None - if test_name_in_line and test_name_in_line >= amber_test_name: - break - else: - cpp_out.write(line) + for line in index_file_in: + if line.startswith("{"): + result = TEST_NAME_IN_LINE_PATTERN.search(line) + if result: + # Group 1 is the test name. + test_name_in_line = result.group(1) + if test_name_in_line >= amber_test_name: + break + index_file_out.write(line) # Write our line and then the previously read line. # Don't write the line if it already exists; idempotent. if line != line_to_write: log("Writing line: {}".format(line_to_write[:-1])) - cpp_out.write(line_to_write) + index_file_out.write(line_to_write) else: log("Line already exists.") - cpp_out.write(line) + index_file_out.write(line) # Write the remaining lines. - for line in cpp_in: - cpp_out.write(line) + for line in index_file_in: + index_file_out.write(line) - remove(cpp_file_bak) + remove(index_file_bak) def add_amber_test_to_must_pass(amber_test_name: str, must_pass_file_path: str) -> None: @@ -322,7 +308,7 @@ def add_amber_test(input_amber_test_file_path: str, vk_gl_cts: str) -> None: log('Using test name "{}"'.format(amber_test_name)) - add_amber_test_to_cpp(vk_gl_cts, amber_test_name, input_amber_test_file_path) + add_amber_test_to_index_file(vk_gl_cts, amber_test_name, input_amber_test_file_path) copy_amber_test_file(vk_gl_cts, amber_test_name, input_amber_test_file_path) @@ -351,7 +337,7 @@ def main() -> None: amber_files = parsed_args.amber_files check_dir_exists(vk_gl_cts) - check_file_exists(get_graphics_fuzz_tests_cpp_file_path(vk_gl_cts)) + check_file_exists(get_graphics_fuzz_tests_index_file_path(vk_gl_cts)) for amber_file in amber_files: add_amber_test(amber_file, vk_gl_cts) diff --git a/gfauto/gfauto/android_device.py b/gfauto/gfauto/android_device.py index da04fda4e..2beb126b3 100644 --- a/gfauto/gfauto/android_device.py +++ b/gfauto/gfauto/android_device.py @@ -27,7 +27,7 @@ from subprocess import CompletedProcess from typing import List, Optional -from gfauto import fuzz, gflogging, result_util, subprocess_util, util +from gfauto import devices_util, fuzz, gflogging, result_util, subprocess_util, util from gfauto.device_pb2 import Device, DeviceAndroid from gfauto.gflogging import log from gfauto.util import check, file_open_text, file_write_text @@ -134,6 +134,31 @@ def is_screen_off_or_locked(serial: Optional[str] = None) -> bool: return False +def get_device_driver_details(serial: str) -> str: + prepare_device(wait_for_screen=True, serial=serial) + cmd = [ + "shell", + # The following is a single string: + f"cd {ANDROID_DEVICE_RESULT_DIR} && " + # -d disables Vulkan validation layers. + f"{ANDROID_DEVICE_AMBER} -d -V", + ] + try: + result = adb_check(serial, cmd, verbose=True) + + except subprocess.SubprocessError as ex: + raise devices_util.GetDeviceDetailsError() from ex + + match = devices_util.AMBER_DEVICE_DETAILS_PATTERN.search(result.stdout) + + if not match: + raise devices_util.GetDeviceDetailsError( + "Could not find device details in stdout: " + result.stdout + ) + + return match.group(1) + + def get_all_android_devices() -> List[Device]: result: List[Device] = [] @@ -163,9 +188,30 @@ def get_all_android_devices() -> List[Device]: device_model = util.remove_start(fields[field_index], "model:") break + build_fingerprint: str = "" + try: + adb_fingerprint_result = adb_check( + device_serial, ["adb", "shell", "getprop ro.build.fingerprint"] + ) + build_fingerprint = adb_fingerprint_result.stdout + build_fingerprint = build_fingerprint.strip() + except subprocess.CalledProcessError: + log("Failed to get device fingerprint") + + driver_details = "" + try: + driver_details = get_device_driver_details(device_serial) + except devices_util.GetDeviceDetailsError as ex: + log(f"WARNING: Failed to get device driver details: {ex}") + device = Device( name=f"{device_model}_{device_serial}", - android=DeviceAndroid(serial=device_serial, model=device_model), + android=DeviceAndroid( + serial=device_serial, + model=device_model, + build_fingerprint=build_fingerprint, + ), + device_properties=driver_details, ) log(f"Found Android device:\n{str(device)}") result.append(device) diff --git a/gfauto/gfauto/device.proto b/gfauto/gfauto/device.proto index a45055619..e75e8df71 100644 --- a/gfauto/gfauto/device.proto +++ b/gfauto/gfauto/device.proto @@ -39,6 +39,9 @@ message Device { // A device can specify/override some binaries. E.g. you may want several SwiftShader "devices" with different // versions or configurations of SwiftShader. repeated Binary binaries = 6; + + // The Vulkan device properties, most likely obtained by running "amber -d -V". + string device_properties = 9; } message DeviceSwiftShader { @@ -58,6 +61,8 @@ message DeviceHost { message DeviceAndroid { string serial = 1; string model = 2; + string build_fingerprint = 3; + } // A virtual device that just executes an offline shader compiler. diff --git a/gfauto/gfauto/device_pb2.py b/gfauto/gfauto/device_pb2.py index f93fab568..b9db15aca 100644 --- a/gfauto/gfauto/device_pb2.py +++ b/gfauto/gfauto/device_pb2.py @@ -21,7 +21,7 @@ package='gfauto', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\x13gfauto/device.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"J\n\nDeviceList\x12\x1b\n\x13\x61\x63tive_device_names\x18\x01 \x03(\t\x12\x1f\n\x07\x64\x65vices\x18\x02 \x03(\x0b\x32\x0e.gfauto.Device\"\xac\x02\n\x06\x44\x65vice\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\npreprocess\x18\x02 \x01(\x0b\x32\x18.gfauto.DevicePreprocessH\x00\x12\x31\n\x0cswift_shader\x18\x03 \x01(\x0b\x32\x19.gfauto.DeviceSwiftShaderH\x00\x12\"\n\x04host\x18\x04 \x01(\x0b\x32\x12.gfauto.DeviceHostH\x00\x12(\n\x07\x61ndroid\x18\x05 \x01(\x0b\x32\x15.gfauto.DeviceAndroidH\x00\x12\x37\n\x0fshader_compiler\x18\x07 \x01(\x0b\x32\x1c.gfauto.DeviceShaderCompilerH\x00\x12 \n\x08\x62inaries\x18\x06 \x03(\x0b\x32\x0e.gfauto.BinaryB\x08\n\x06\x64\x65vice\"\x13\n\x11\x44\x65viceSwiftShader\"\x12\n\x10\x44\x65vicePreprocess\"\x0c\n\nDeviceHost\".\n\rDeviceAndroid\x12\x0e\n\x06serial\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\t\"4\n\x14\x44\x65viceShaderCompiler\x12\x0e\n\x06\x62inary\x18\x01 \x01(\t\x12\x0c\n\x04\x61rgs\x18\x02 \x03(\tb\x06proto3') + serialized_pb=_b('\n\x13gfauto/device.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"J\n\nDeviceList\x12\x1b\n\x13\x61\x63tive_device_names\x18\x01 \x03(\t\x12\x1f\n\x07\x64\x65vices\x18\x02 \x03(\x0b\x32\x0e.gfauto.Device\"\xc7\x02\n\x06\x44\x65vice\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\npreprocess\x18\x02 \x01(\x0b\x32\x18.gfauto.DevicePreprocessH\x00\x12\x31\n\x0cswift_shader\x18\x03 \x01(\x0b\x32\x19.gfauto.DeviceSwiftShaderH\x00\x12\"\n\x04host\x18\x04 \x01(\x0b\x32\x12.gfauto.DeviceHostH\x00\x12(\n\x07\x61ndroid\x18\x05 \x01(\x0b\x32\x15.gfauto.DeviceAndroidH\x00\x12\x37\n\x0fshader_compiler\x18\x07 \x01(\x0b\x32\x1c.gfauto.DeviceShaderCompilerH\x00\x12 \n\x08\x62inaries\x18\x06 \x03(\x0b\x32\x0e.gfauto.Binary\x12\x19\n\x11\x64\x65vice_properties\x18\t \x01(\tB\x08\n\x06\x64\x65vice\"\x13\n\x11\x44\x65viceSwiftShader\"\x12\n\x10\x44\x65vicePreprocess\"\x0c\n\nDeviceHost\"I\n\rDeviceAndroid\x12\x0e\n\x06serial\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\t\x12\x19\n\x11\x62uild_fingerprint\x18\x03 \x01(\t\"4\n\x14\x44\x65viceShaderCompiler\x12\x0e\n\x06\x62inary\x18\x01 \x01(\t\x12\x0c\n\x04\x61rgs\x18\x02 \x03(\tb\x06proto3') , dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,]) @@ -122,6 +122,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='device_properties', full_name='gfauto.Device.device_properties', index=7, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -138,7 +145,7 @@ index=0, containing_type=None, fields=[]), ], serialized_start=129, - serialized_end=429, + serialized_end=456, ) @@ -161,8 +168,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=431, - serialized_end=450, + serialized_start=458, + serialized_end=477, ) @@ -185,8 +192,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=452, - serialized_end=470, + serialized_start=479, + serialized_end=497, ) @@ -209,8 +216,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=472, - serialized_end=484, + serialized_start=499, + serialized_end=511, ) @@ -235,6 +242,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='build_fingerprint', full_name='gfauto.DeviceAndroid.build_fingerprint', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -247,8 +261,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=486, - serialized_end=532, + serialized_start=513, + serialized_end=586, ) @@ -285,8 +299,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=534, - serialized_end=586, + serialized_start=588, + serialized_end=640, ) _DEVICELIST.fields_by_name['devices'].message_type = _DEVICE diff --git a/gfauto/gfauto/device_pb2.pyi b/gfauto/gfauto/device_pb2.pyi index 3cb8345e9..dedd5c86b 100644 --- a/gfauto/gfauto/device_pb2.pyi +++ b/gfauto/gfauto/device_pb2.pyi @@ -52,6 +52,7 @@ class DeviceList(google___protobuf___message___Message): class Device(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... name = ... # type: typing___Text + device_properties = ... # type: typing___Text @property def preprocess(self) -> DevicePreprocess: ... @@ -80,6 +81,7 @@ class Device(google___protobuf___message___Message): android : typing___Optional[DeviceAndroid] = None, shader_compiler : typing___Optional[DeviceShaderCompiler] = None, binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, + device_properties : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod def FromString(cls, s: bytes) -> Device: ... @@ -87,10 +89,10 @@ class Device(google___protobuf___message___Message): def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): def HasField(self, field_name: typing_extensions___Literal[u"android",u"device",u"host",u"preprocess",u"shader_compiler",u"swift_shader"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"android",u"binaries",u"device",u"host",u"name",u"preprocess",u"shader_compiler",u"swift_shader"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"android",u"binaries",u"device",u"device_properties",u"host",u"name",u"preprocess",u"shader_compiler",u"swift_shader"]) -> None: ... else: def HasField(self, field_name: typing_extensions___Literal[u"android",b"android",u"device",b"device",u"host",b"host",u"preprocess",b"preprocess",u"shader_compiler",b"shader_compiler",u"swift_shader",b"swift_shader"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"android",b"android",u"binaries",b"binaries",u"device",b"device",u"host",b"host",u"name",b"name",u"preprocess",b"preprocess",u"shader_compiler",b"shader_compiler",u"swift_shader",b"swift_shader"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"android",b"android",u"binaries",b"binaries",u"device",b"device",u"device_properties",b"device_properties",u"host",b"host",u"name",b"name",u"preprocess",b"preprocess",u"shader_compiler",b"shader_compiler",u"swift_shader",b"swift_shader"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"device",b"device"]) -> typing_extensions___Literal["preprocess","swift_shader","host","android","shader_compiler"]: ... class DeviceSwiftShader(google___protobuf___message___Message): @@ -127,20 +129,22 @@ class DeviceAndroid(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... serial = ... # type: typing___Text model = ... # type: typing___Text + build_fingerprint = ... # type: typing___Text def __init__(self, *, serial : typing___Optional[typing___Text] = None, model : typing___Optional[typing___Text] = None, + build_fingerprint : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod def FromString(cls, s: bytes) -> DeviceAndroid: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"model",u"serial"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"build_fingerprint",u"model",u"serial"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"model",b"model",u"serial",b"serial"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"build_fingerprint",b"build_fingerprint",u"model",b"model",u"serial",b"serial"]) -> None: ... class DeviceShaderCompiler(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... diff --git a/gfauto/gfauto/devices_util.py b/gfauto/gfauto/devices_util.py index 331ff96c0..9a5b33b6b 100644 --- a/gfauto/gfauto/devices_util.py +++ b/gfauto/gfauto/devices_util.py @@ -18,10 +18,10 @@ Used to enumerate the available devices and work with device lists. """ +import re +from typing import Dict, List, Optional, Pattern -from typing import Dict, List, Optional - -from gfauto import android_device +from gfauto import android_device, binaries_util, host_device_util from gfauto.device_pb2 import ( Device, DeviceHost, @@ -33,10 +33,41 @@ from gfauto.gflogging import log from gfauto.util import ToolNotOnPathError +# [\s\S] matches anything, including newlines. +# *? is non-greedy. + +AMBER_DEVICE_DETAILS_PATTERN: Pattern[str] = re.compile( + r"Physical device properties:\n([\s\S]*?)End of physical device properties." +) + + +class GetDeviceDetailsError(Exception): + pass + + +def swift_shader_device(binary_manager: binaries_util.BinaryManager) -> Device: + + amber_path = binary_manager.get_binary_path_by_name(binaries_util.AMBER_NAME).path + swift_shader_binary_and_path = binary_manager.get_binary_path_by_name( + binaries_util.SWIFT_SHADER_NAME + ) + + driver_details = "" + try: + driver_details = host_device_util.get_driver_details( + amber_path, swift_shader_binary_and_path.path + ) + except GetDeviceDetailsError as ex: + log(f"WARNING: Failed to get device driver details: {ex}") -def swift_shader_device() -> Device: - # TODO: version hash, maybe a better name. - return Device(name="swift_shader", swift_shader=DeviceSwiftShader()) + device = Device( + name="swift_shader", + swift_shader=DeviceSwiftShader(), + binaries=[swift_shader_binary_and_path.binary], + device_properties=driver_details, + ) + + return device def device_preprocessor() -> Device: @@ -47,12 +78,22 @@ def device_preprocessor() -> Device: return Device(name="host_preprocessor", preprocess=DevicePreprocess()) -def device_host() -> Device: - # TODO: add details to host (and indeed, to all devices). - return Device(name="host", host=DeviceHost()) +def device_host(binary_manager: binaries_util.BinaryManager) -> Device: + amber_path = binary_manager.get_binary_path_by_name(binaries_util.AMBER_NAME).path + + driver_details = "" + try: + driver_details = host_device_util.get_driver_details(amber_path) + except GetDeviceDetailsError as ex: + log(f"WARNING: Failed to get device driver details: {ex}") + + return Device(name="host", host=DeviceHost(), device_properties=driver_details) -def get_device_list(device_list: Optional[DeviceList] = None) -> DeviceList: +def get_device_list( + binary_manager: binaries_util.BinaryManager, + device_list: Optional[DeviceList] = None, +) -> DeviceList: if not device_list: device_list = DeviceList() @@ -66,12 +107,12 @@ def get_device_list(device_list: Optional[DeviceList] = None) -> DeviceList: device_list.active_device_names.append(device.name) # SwiftShader. - device = swift_shader_device() + device = swift_shader_device(binary_manager) device_list.devices.extend([device]) device_list.active_device_names.append(device.name) # Host device. - device = device_host() + device = device_host(binary_manager) device_list.devices.extend([device]) device_list.active_device_names.append(device.name) @@ -92,6 +133,7 @@ def get_device_list(device_list: Optional[DeviceList] = None) -> DeviceList: shader_compiler=DeviceShaderCompiler( binary="amdllpc", args=["-gfxip=9.0.0", "-verify-ir", "-auto-layout-desc"] ), + binaries=[binary_manager.get_binary_path_by_name("amdllpc").binary], ) device_list.devices.extend([device]) # Don't add to active devices, since this is mostly just an example. diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index b4d975b11..ac3560f1d 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -179,6 +179,15 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many force_no_stack_traces: bool, ) -> None: + try: + artifact_util.artifact_path_get_root() + except FileNotFoundError: + log( + "Could not find ROOT file (in the current directory or above) to mark where binaries should be stored. " + "Creating a ROOT file in the current directory." + ) + util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") + settings = settings_util.read_or_create(settings_path) active_devices = devices_util.get_active_devices(settings.device_list) @@ -190,15 +199,6 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many donors_dir = Path() / "donors" spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders" - try: - artifact_util.artifact_path_get_root() - except FileNotFoundError: - log( - "Could not find ROOT file (in the current directory or above) to mark where binaries should be stored. " - "Creating a ROOT file in the current directory." - ) - util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") - # Log a warning if there is no tool on the PATH for printing stack traces. prepended = util.prepend_catchsegv_if_available([], log_warning=True) if not force_no_stack_traces and not prepended: @@ -224,25 +224,6 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many settings=settings ).get_child_binary_manager(list(settings.custom_binaries), prepend=True) - # For convenience, for some virtual devices, we mutate the device proto (in memory) to add the newest relevant - # Binary protos (e.g. SwiftShader binary info). - # When we save a report to disk, the test's device proto will contain the binaries we used. - for device in active_devices: - - if device.HasField("swift_shader") and not [ - binary for binary in device.binaries if binary.name == "swift_shader_icd" - ]: - device.binaries.extend( - [binary_manager.get_binary_by_name("swift_shader_icd")] - ) - - if ( - device.HasField("shader_compiler") - and device.shader_compiler.binary == "amdllpc" - and not [binary for binary in device.binaries if binary.name == "amdllpc"] - ): - device.binaries.extend([binary_manager.get_binary_by_name("amdllpc")]) - while True: # We have to use "is not None" because the seed could be 0. diff --git a/gfauto/gfauto/host_device_util.py b/gfauto/gfauto/host_device_util.py index fd38c1620..41d39e483 100644 --- a/gfauto/gfauto/host_device_util.py +++ b/gfauto/gfauto/host_device_util.py @@ -23,10 +23,37 @@ from pathlib import Path from typing import Dict, Optional -from gfauto import fuzz, gflogging, result_util, subprocess_util, util +from gfauto import devices_util, fuzz, gflogging, result_util, subprocess_util, util from gfauto.gflogging import log +def get_driver_details(amber_path: Path, icd: Optional[Path] = None) -> str: + + env: Optional[Dict[str, str]] = None + + if icd: + env = {"VK_ICD_FILENAMES": str(icd)} + + try: + result = subprocess_util.run( + [str(amber_path), "-d", "-V"], + timeout=fuzz.AMBER_RUN_TIME_LIMIT, + verbose=True, + env=env, + ) + except subprocess.SubprocessError as ex: + raise devices_util.GetDeviceDetailsError() from ex + + match = devices_util.AMBER_DEVICE_DETAILS_PATTERN.search(result.stdout) + + if not match: + raise devices_util.GetDeviceDetailsError( + "Could not find device details in stdout: " + result.stdout + ) + + return match.group(1) + + def run_amber( amber_script_file: Path, output_dir: Path, diff --git a/gfauto/gfauto/run_bin.py b/gfauto/gfauto/run_bin.py new file mode 100644 index 000000000..bdd4781fb --- /dev/null +++ b/gfauto/gfauto/run_bin.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runs a binary from the given binary name and settings file.""" + +import argparse +import subprocess +import sys +from pathlib import Path +from typing import List + +from gfauto import binaries_util, settings_util + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Runs a binary given the binary name and settings.json file." + ) + + parser.add_argument( + "--settings", + help="Path to the settings JSON file for this instance.", + default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), + ) + + parser.add_argument( + "binary_name", + help="The name of the binary to run. E.g. spirv-opt, glslangValidator", + type=str, + ) + + parser.add_argument( + "arguments", + metavar="arguments", + type=str, + nargs="*", + help="The arguments to pass to the binary", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + # Args. + settings_path: Path = Path(parsed_args.settings) + binary_name: str = parsed_args.binary_name + arguments: List[str] = parsed_args.arguments + + settings = settings_util.read_or_create(settings_path) + binary_manager = binaries_util.get_default_binary_manager(settings=settings) + + cmd = [str(binary_manager.get_binary_path_by_name(binary_name).path)] + cmd.extend(arguments) + return subprocess.run(cmd).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/gfauto/gfauto/settings_util.py b/gfauto/gfauto/settings_util.py index 3f3824813..1aa862655 100644 --- a/gfauto/gfauto/settings_util.py +++ b/gfauto/gfauto/settings_util.py @@ -68,7 +68,6 @@ def write(settings: Settings, settings_path: Path) -> Path: def write_default(settings_path: Path) -> Path: settings = Settings() settings.CopyFrom(DEFAULT_SETTINGS) - devices_util.get_device_list(settings.device_list) # noinspection PyBroadException try: @@ -86,4 +85,8 @@ def write_default(settings_path: Path) -> Path: settings.latest_binary_versions.extend(binaries_util.DEFAULT_BINARIES) + binary_manager = binaries_util.get_default_binary_manager(settings=settings) + + devices_util.get_device_list(binary_manager, settings.device_list) + return write(settings, settings_path) diff --git a/gfauto/setup.py b/gfauto/setup.py index eb922d95a..cd1140a18 100644 --- a/gfauto/setup.py +++ b/gfauto/setup.py @@ -52,5 +52,6 @@ "gfauto_test_create_readme = gfauto.test_create_readme:main", "gfauto_download_cts_gf_tests = gfauto.download_cts_gf_tests:main", "gfauto_run_cts_gf_tests = gfauto.run_cts_gf_tests:main", + "gfauto_run_bin = gfauto.run_bin:main", ]}, ) From 718066e803fff061a17ee5efb8055d372dde36c5 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Thu, 5 Dec 2019 16:47:58 +0000 Subject: [PATCH 172/180] gfauto: add code coverage tools (#804) Added tools for processing code coverage. The main tools are: * gfauto_cov_from_gcov: processes .gcno and .gcda files using gcov to collect line coverage into a .cov file. * gfauto_cov_new: compares two .cov files to output the newly covered lines into a .cov file. * gfauto_cov_to_source: given a .cov file, output annotated source files with the coverage information. Added support in gfauto_fuzz for collecting SwiftShader code coverage in parallel using the environment variable `GCOV_PREFIX=/output/path/PROC_ID`. Also added `docs/coverage.md` as a guide on how to collect SwiftShader coverage from gfauto and dEQP (using the Regres dEQP runner). --- gfauto/.flake8 | 15 + gfauto/Pipfile | 5 +- gfauto/Pipfile.lock | 615 ++++++++++-------- gfauto/check_all.sh | 2 +- gfauto/docs/coverage.md | 104 +++ gfauto/gfauto/add_amber_tests_to_cts.py | 2 +- gfauto/gfauto/android_device.py | 19 +- gfauto/gfauto/binaries_util.py | 2 +- gfauto/gfauto/cov_from_gcov.py | 126 ++++ gfauto/gfauto/cov_merge.py | 69 ++ gfauto/gfauto/cov_new.py | 70 ++ gfauto/gfauto/cov_to_source.py | 78 +++ gfauto/gfauto/cov_util.py | 299 +++++++++ gfauto/gfauto/fuzz.py | 2 + gfauto/gfauto/gflogging.py | 4 +- gfauto/gfauto/host_device_util.py | 12 +- ...recipe_download_and_extract_archive_set.py | 4 +- gfauto/gfauto/run_bin.py | 2 +- gfauto/gfauto/signature_util.py | 4 +- gfauto/gfauto/subprocess_util.py | 13 +- gfauto/gfauto/types.py | 30 + gfauto/gfauto/util.py | 17 +- gfauto/pylintrc | 19 +- gfauto/setup.py | 4 + gfauto/whitelist.dic | 25 + 25 files changed, 1229 insertions(+), 313 deletions(-) create mode 100644 gfauto/docs/coverage.md create mode 100644 gfauto/gfauto/cov_from_gcov.py create mode 100644 gfauto/gfauto/cov_merge.py create mode 100644 gfauto/gfauto/cov_new.py create mode 100644 gfauto/gfauto/cov_to_source.py create mode 100644 gfauto/gfauto/cov_util.py create mode 100644 gfauto/gfauto/types.py diff --git a/gfauto/.flake8 b/gfauto/.flake8 index 6cde794b4..a081794ee 100644 --- a/gfauto/.flake8 +++ b/gfauto/.flake8 @@ -57,10 +57,25 @@ ignore = P102 P103 + # Consider possible security implications associated with pickle module + S403 + # Pickle and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue. + S301 + +per-file-ignores = + # Ignore spelling errors in signatures. + gfauto/signature_util.py:SC100,SC200 + + # Ignore use of print in coverage code. + gfauto/cov_*.py:T001 # For flake8-quotes: inline-quotes = " +# Use the formatter provided by flake8_formatter_abspath. +# This will output the full path of files with warnings so they will be clickable from PyCharm's terminal. +format = abspath + # For flake8-spellcheck. Default is whitelist.txt but using whitelist.dic means # we can add the file to PyCharm/IntelliJ. whitelist=whitelist.dic diff --git a/gfauto/Pipfile b/gfauto/Pipfile index 233bfabcb..085a37f0e 100644 --- a/gfauto/Pipfile +++ b/gfauto/Pipfile @@ -32,7 +32,7 @@ jedi = "*" # Code formatter. Explicit version given because it is a pre-release and # otherwise we get errors. -black = "==19.3b0" +black = "==19.10b0" # Plugin to protoc that outputs .pyi files (type information). mypy-protobuf = "*" @@ -51,6 +51,7 @@ pylint = "*" # Flake8 linter. flake8 = "*" +flake8_formatter_abspath = "*" # cohesion = "*" # A tool for measuring Python class cohesion # flake8-alfred = "*" # Can be used to blacklist symbols. # flake8-copyright = "*" # Not maintained. @@ -69,8 +70,6 @@ flake8-comprehensions = "*" flake8-commas = "*" flake8-debugger = "*" flake8-docstrings = "*" -# TODO: Remove once bug in flake8-docstrings is fixed: https://gitlab.com/pycqa/flake8-docstrings/issues/36 -pydocstyle = "==3.0.0" flake8-isort = "*" flake8-logging-format = "*" flake8-mock = "*" diff --git a/gfauto/Pipfile.lock b/gfauto/Pipfile.lock index df11525aa..9eb587289 100644 --- a/gfauto/Pipfile.lock +++ b/gfauto/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8427f9b747e8467d42d63c04936cbc6235900a98a6d97686ac8456cf4a3084b6" + "sha256": "dfa4f6c61ae08c5c770b0a1eb05ecf76bb31bb1d4013710c2206d26b54717c56" }, "pipfile-spec": 6, "requires": {}, @@ -16,10 +16,10 @@ "default": { "certifi": { "hashes": [ - "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", - "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.6.16" + "version": "==2019.11.28" }, "chardet": { "hashes": [ @@ -41,31 +41,31 @@ }, "protobuf": { "hashes": [ - "sha256:00a1b0b352dc7c809749526d1688a64b62ea400c5b05416f93cfb1b11a036295", - "sha256:01acbca2d2c8c3f7f235f1842440adbe01bbc379fa1cbdd80753801432b3fae9", - "sha256:0a795bca65987b62d6b8a2d934aa317fd1a4d06a6dd4df36312f5b0ade44a8d9", - "sha256:0ec035114213b6d6e7713987a759d762dd94e9f82284515b3b7331f34bfaec7f", - "sha256:31b18e1434b4907cb0113e7a372cd4d92c047ce7ba0fa7ea66a404d6388ed2c1", - "sha256:32a3abf79b0bef073c70656e86d5bd68a28a1fbb138429912c4fc07b9d426b07", - "sha256:55f85b7808766e5e3f526818f5e2aeb5ba2edcc45bcccede46a3ccc19b569cb0", - "sha256:64ab9bc971989cbdd648c102a96253fdf0202b0c38f15bd34759a8707bdd5f64", - "sha256:64cf847e843a465b6c1ba90fb6c7f7844d54dbe9eb731e86a60981d03f5b2e6e", - "sha256:917c8662b585470e8fd42f052661fc66d59fccaae450a60044307dcbf82a3335", - "sha256:afed9003d7f2be2c3df20f64220c30faec441073731511728a2cb4cab4cd46a6", - "sha256:bf8e05d638b585d1752c5a84247134a0350d3a8b73d3632489a014a9f6f1e758", - "sha256:d831b047bd69becaf64019a47179eb22118a50dd008340655266a906c69c6417", - "sha256:de2760583ed28749ff885789c1cbc6c9c06d6de92fc825740ab99deb2f25ea4d", - "sha256:eabc4cf1bc19689af8022ba52fd668564a8d96e0d08f3b4732d26a64255216a4", - "sha256:fcff6086c86fb1628d94ea455c7b9de898afc50378042927a59df8065a79a549" - ], - "version": "==3.9.1" + "sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd", + "sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed", + "sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057", + "sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce", + "sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03", + "sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46", + "sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33", + "sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c", + "sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9", + "sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef", + "sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b", + "sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d", + "sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8", + "sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6", + "sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941", + "sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13" + ], + "version": "==3.11.1" }, "python-dateutil": { "hashes": [ - "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", - "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "version": "==2.8.0" + "version": "==2.8.1" }, "requests": { "hashes": [ @@ -76,17 +76,17 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "urllib3": { "hashes": [ - "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", - "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" ], - "version": "==1.25.3" + "version": "==1.25.7" } }, "develop": { @@ -99,24 +99,17 @@ }, "astroid": { "hashes": [ - "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", - "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" + "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", + "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" ], - "version": "==2.2.5" - }, - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" + "version": "==2.3.3" }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.1.0" + "version": "==19.3.0" }, "backcall": { "hashes": [ @@ -134,11 +127,11 @@ }, "black": { "hashes": [ - "sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf", - "sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c" + "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", + "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" ], "index": "pypi", - "version": "==19.3b0" + "version": "==19.10b0" }, "click": { "hashes": [ @@ -149,10 +142,10 @@ }, "decorator": { "hashes": [ - "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", - "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" + "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", + "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d" ], - "version": "==4.4.0" + "version": "==4.4.1" }, "entrypoints": { "hashes": [ @@ -163,18 +156,18 @@ }, "flake8": { "hashes": [ - "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", - "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" + "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", + "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" ], "index": "pypi", - "version": "==3.7.8" + "version": "==3.7.9" }, "flake8-bandit": { "hashes": [ - "sha256:52a67f453ba765398098b39a2273e138983b3499465ef39a12e8ee544c4598fc" + "sha256:687fc8da2e4a239b206af2e54a90093572a60d0954f3054e23690739b0b0de3b" ], "index": "pypi", - "version": "==2.1.1" + "version": "==2.1.2" }, "flake8-black": { "hashes": [ @@ -233,26 +226,34 @@ }, "flake8-comprehensions": { "hashes": [ - "sha256:7b174ded3d7e73edf587e942458b6c1a7c3456d512d9c435deae367236b9562c", - "sha256:e36fc12bd3833e0b34fe0639b7a817d32c86238987f532078c57eafdc7a8a219" + "sha256:6de428c3ac67d3614d527456469c8a3d6638960e9ad7e1222358526a2507400a", + "sha256:e3a8350a35d7bc71f8a1f64cf3c633a418a26b0edace2219d26ea4dd78ac21f3" ], "index": "pypi", - "version": "==2.2.0" + "version": "==3.1.4" }, "flake8-debugger": { "hashes": [ - "sha256:be4fb88de3ee8f6dd5053a2d347e2c0a2b54bab6733a2280bb20ebd3c4ca1d97" + "sha256:712d7c1ff69ddf3f0130e94cc88c2519e720760bce45e8c330bfdcb61ab4090d" ], "index": "pypi", - "version": "==3.1.0" + "version": "==3.2.1" }, "flake8-docstrings": { "hashes": [ - "sha256:1666dd069c9c457ee57e80af3c1a6b37b00cc1801c6fde88e455131bb2e186cd", - "sha256:9c0db5a79a1affd70fdf53b8765c8a26bf968e59e0252d7f2fc546b41c0cda06" + "sha256:3d5a31c7ec6b7367ea6506a87ec293b94a0a46c0bce2bb4975b7f1d09b6f3717", + "sha256:a256ba91bc52307bef1de59e2a009c3cf61c3d0952dbe035d6ff7208940c2edc" ], "index": "pypi", - "version": "==1.4.0" + "version": "==1.5.0" + }, + "flake8-formatter-abspath": { + "hashes": [ + "sha256:694d0874d5d047ed57c82a10213f75604475e4525ee8bbaad53417a7d6f8442c", + "sha256:7ff3d9186eb61b4c78a118acf70cd6ef26fb5e323d9a927b47f062b5e8bf31ed" + ], + "index": "pypi", + "version": "==1.0.1" }, "flake8-isort": { "hashes": [ @@ -308,24 +309,25 @@ }, "flake8-print": { "hashes": [ - "sha256:5010e6c138b63b62400da4b06afa33becc5e08bd1fcce9af3752445cf3342f54" + "sha256:324f9e59a522518daa2461bacd7f82da3c34eb26a4314c2a54bd493f8b394a68" ], "index": "pypi", - "version": "==3.1.0" + "version": "==3.1.4" }, "flake8-quotes": { "hashes": [ - "sha256:5dbaf668887873f28346fb87943d6da2e4b9f77ce9f2169cff21764a0a4934ed" + "sha256:11a15d30c92ca5f04c2791bd7019cf62b6f9d3053eb050d02a135557eb118bfc" ], "index": "pypi", - "version": "==2.1.0" + "version": "==2.1.1" }, "flake8-spellcheck": { "hashes": [ - "sha256:0805f1d7ac6ae8d6ab84ef68244042cd6adbdee0b7a3dc653a39f3e9780d6a07" + "sha256:a82429d1e479f4efbd20870be1963ed0d9b389ddc3b087ff2261eb137ab8988a", + "sha256:f92e6da46fd30d805c28e22f705b5da2a719b9dca8a9abd669bdcd9cc6158e09" ], "index": "pypi", - "version": "==0.9.0" + "version": "==0.9.1" }, "flake8-string-format": { "hashes": [ @@ -345,115 +347,147 @@ }, "flake8-variables-names": { "hashes": [ - "sha256:f9ee2edf0892a73fff33ef3eda37a7182cb91cd4c3a19a592ad432b68b261927" + "sha256:d109f5a8fe8c20d64e165287330f1b0160b442d7f96e1527124ba1b63c438347" ], "index": "pypi", - "version": "==0.0.2" + "version": "==0.0.3" }, "gitdb2": { "hashes": [ - "sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2", - "sha256:e3a0141c5f2a3f635c7209d56c496ebe1ad35da82fe4d3ec4aaa36278d70648a" + "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350", + "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b" ], - "version": "==2.0.5" + "version": "==2.0.6" }, "gitpython": { "hashes": [ - "sha256:947cc75913e7b6da108458136607e2ee0e40c20be1e12d4284e7c6c12956c276", - "sha256:d2f4945f8260f6981d724f5957bc076398ada55cb5d25aaee10108bcdc894100" + "sha256:9c2398ffc3dcb3c40b27324b316f08a4f93ad646d5a6328cafbb871aa79f5e42", + "sha256:c155c6a2653593ccb300462f6ef533583a913e17857cfef8fc617c246b6dc245" ], - "version": "==3.0.2" + "version": "==3.0.5" }, "grpcio": { "hashes": [ - "sha256:1303578092f1f6e4bfbc354c04ac422856c393723d3ffa032fff0f7cb5cfd693", - "sha256:229c6b313cd82bec8f979b059d87f03cc1a48939b543fe170b5a9c5cf6a6bc69", - "sha256:3cd3d99a8b5568d0d186f9520c16121a0f2a4bcad8e2b9884b76fb88a85a7774", - "sha256:41cfb222db358227521f9638a6fbc397f310042a4db5539a19dea01547c621cd", - "sha256:43330501660f636fd6547d1e196e395cd1e2c2ae57d62219d6184a668ffebda0", - "sha256:45d7a2bd8b4f25a013296683f4140d636cdbb507d94a382ea5029a21e76b1648", - "sha256:47dc935658a13b25108823dabd010194ddea9610357c5c1ef1ad7b3f5157ebee", - "sha256:480aa7e2b56238badce0b9413a96d5b4c90c3bfbd79eba5a0501e92328d9669e", - "sha256:4a0934c8b0f97e1d8c18e76c45afc0d02d33ab03125258179f2ac6c7a13f3626", - "sha256:5624dab19e950f99e560400c59d87b685809e4cfcb2c724103f1ab14c06071f7", - "sha256:60515b1405bb3dadc55e6ca99429072dad3e736afcf5048db5452df5572231ff", - "sha256:610f97ebae742a57d336a69b09a9c7d7de1f62aa54aaa8adc635b38f55ba4382", - "sha256:64ea189b2b0859d1f7b411a09185028744d494ef09029630200cc892e366f169", - "sha256:686090c6c1e09e4f49585b8508d0a31d58bc3895e4049ea55b197d1381e9f70f", - "sha256:7745c365195bb0605e3d47b480a2a4d1baa8a41a5fd0a20de5fa48900e2c886a", - "sha256:79491e0d2b77a1c438116bf9e5f9e2e04e78b78524615e2ce453eff62db59a09", - "sha256:825177dd4c601c487836b7d6b4ba268db59787157911c623ba59a7c03c8d3adc", - "sha256:8a060e1f72fb94eee8a035ed29f1201ce903ad14cbe27bda56b4a22a8abda045", - "sha256:90168cc6353e2766e47b650c963f21cfff294654b10b3a14c67e26a4e3683634", - "sha256:94b7742734bceeff6d8db5edb31ac844cb68fc7f13617eca859ff1b78bb20ba1", - "sha256:962aebf2dd01bbb2cdb64580e61760f1afc470781f9ecd5fe8f3d8dcd8cf4556", - "sha256:9c8d9eacdce840b72eee7924c752c31b675f8aec74790e08cff184a4ea8aa9c1", - "sha256:af5b929debc336f6bab9b0da6915f9ee5e41444012aed6a79a3c7e80d7662fdf", - "sha256:b9cdb87fc77e9a3eabdc42a512368538d648fa0760ad30cf97788076985c790a", - "sha256:c5e6380b90b389454669dc67d0a39fb4dc166416e01308fcddd694236b8329ef", - "sha256:d60c90fe2bfbee735397bf75a2f2c4e70c5deab51cd40c6e4fa98fae018c8db6", - "sha256:d8582c8b1b1063249da1588854251d8a91df1e210a328aeb0ece39da2b2b763b", - "sha256:ddbf86ba3aa0ad8fed2867910d2913ee237d55920b55f1d619049b3399f04efc", - "sha256:e46bc0664c5c8a0545857aa7a096289f8db148e7f9cca2d0b760113e8994bddc", - "sha256:f6437f70ec7fed0ca3a0eef1146591bb754b418bb6c6b21db74f0333d624e135", - "sha256:f71693c3396530c6b00773b029ea85e59272557e9bd6077195a6593e4229892a", - "sha256:f79f7455f8fbd43e8e9d61914ecf7f48ba1c8e271801996fef8d6a8f3cc9f39f" - ], - "version": "==1.23.0" + "sha256:0419ae5a45f49c7c40d9ae77ae4de9442431b7822851dfbbe56ee0eacb5e5654", + "sha256:1e8631eeee0fb0b4230aeb135e4890035f6ef9159c2a3555fa184468e325691a", + "sha256:24db2fa5438f3815a4edb7a189035051760ca6aa2b0b70a6a948b28bfc63c76b", + "sha256:2adb1cdb7d33e91069517b41249622710a94a1faece1fed31cd36904e4201cde", + "sha256:2cd51f35692b551aeb1fdeb7a256c7c558f6d78fcddff00640942d42f7aeba5f", + "sha256:3247834d24964589f8c2b121b40cd61319b3c2e8d744a6a82008643ef8a378b1", + "sha256:3433cb848b4209717722b62392e575a77a52a34d67c6730138102abc0a441685", + "sha256:39671b7ff77a962bd745746d9d2292c8ed227c5748f16598d16d8631d17dd7e5", + "sha256:40a0b8b2e6f6dd630f8b267eede2f40a848963d0f3c40b1b1f453a4a870f679e", + "sha256:40f9a74c7aa210b3e76eb1c9d56aa8d08722b73426a77626967019df9bbac287", + "sha256:423f76aa504c84cb94594fb88b8a24027c887f1c488cf58f2173f22f4fbd046c", + "sha256:43bd04cec72281a96eb361e1b0232f0f542b46da50bcfe72ef7e5a1b41d00cb3", + "sha256:43e38762635c09e24885d15e3a8e374b72d105d4178ee2cc9491855a8da9c380", + "sha256:4413b11c2385180d7de03add6c8845dd66692b148d36e27ec8c9ef537b2553a1", + "sha256:4450352a87094fd58daf468b04c65a9fa19ad11a0ac8ac7b7ff17d46f873cbc1", + "sha256:49ffda04a6e44de028b3b786278ac9a70043e7905c3eea29eed88b6524d53a29", + "sha256:4a38c4dde4c9120deef43aaabaa44f19186c98659ce554c29788c4071ab2f0a4", + "sha256:50b1febdfd21e2144b56a9aa226829e93a79c354ef22a4e5b013d9965e1ec0ed", + "sha256:559b1a3a8be7395ded2943ea6c2135d096f8cc7039d6d12127110b6496f251fe", + "sha256:5de86c182667ec68cf84019aa0d8ceccf01d352cdca19bf9e373725204bdbf50", + "sha256:5fc069bb481fe3fad0ba24d3baaf69e22dfa6cc1b63290e6dfeaf4ac1e996fb7", + "sha256:6a19d654da49516296515d6f65de4bbcbd734bc57913b21a610cfc45e6df3ff1", + "sha256:7535b3e52f498270e7877dde1c8944d6b7720e93e2e66b89c82a11447b5818f5", + "sha256:7c4e495bcabc308198b8962e60ca12f53b27eb8f03a21ac1d2d711d6dd9ecfca", + "sha256:8a8fc4a0220367cb8370cedac02272d574079ccc32bffbb34d53aaf9e38b5060", + "sha256:8b008515e067232838daca020d1af628bf6520c8cc338bf383284efe6d8bd083", + "sha256:8d1684258e1385e459418f3429e107eec5fb3d75e1f5a8c52e5946b3f329d6ea", + "sha256:8eb5d54b87fb561dc2e00a5c5226c33ffe8dbc13f2e4033a412bafb7b37b194d", + "sha256:94cdef0c61bd014bb7af495e21a1c3a369dd0399c3cd1965b1502043f5c88d94", + "sha256:9d9f3be69c7a5e84c3549a8c4403fa9ac7672da456863d21e390b2bbf45ccad1", + "sha256:9fb6fb5975a448169756da2d124a1beb38c0924ff6c0306d883b6848a9980f38", + "sha256:a5eaae8700b87144d7dfb475aa4675e500ff707292caba3deff41609ddc5b845", + "sha256:aaeac2d552772b76d24eaff67a5d2325bc5205c74c0d4f9fbe71685d4a971db2", + "sha256:bb611e447559b3b5665e12a7da5160c0de6876097f62bf1d23ba66911564868e", + "sha256:bc0d41f4eb07da8b8d3ea85e50b62f6491ab313834db86ae2345be07536a4e5a", + "sha256:bf51051c129b847d1bb63a9b0826346b5f52fb821b15fe5e0d5ef86f268510f5", + "sha256:c948c034d8997526011960db54f512756fb0b4be1b81140a15b4ef094c6594a4", + "sha256:d435a01334157c3b126b4ee5141401d44bdc8440993b18b05e2f267a6647f92d", + "sha256:d46c1f95672b73288e08cdca181e14e84c6229b5879561b7b8cfd48374e09287", + "sha256:d5d58309b42064228b16b0311ff715d6c6e20230e81b35e8d0c8cfa1bbdecad8", + "sha256:dc6e2e91365a1dd6314d615d80291159c7981928b88a4c65654e3fefac83a836", + "sha256:e0dfb5f7a39029a6cbec23affa923b22a2c02207960fd66f109e01d6f632c1eb", + "sha256:eb4bf58d381b1373bd21d50837a53953d625d1693f1b58fed12743c75d3dd321", + "sha256:ebb211a85248dbc396b29320273c1ffde484b898852432613e8df0164c091006", + "sha256:ec759ece4786ae993a5b7dc3b3dead6e9375d89a6c65dfd6860076d2eb2abe7b", + "sha256:f55108397a8fa164268238c3e69cc134e945d1f693572a2f05a028b8d0d2b837", + "sha256:f6c706866d424ff285b85a02de7bbe5ed0ace227766b2c42cbe12f3d9ea5a8aa", + "sha256:f8370ad332b36fbad117440faf0dd4b910e80b9c49db5648afd337abdde9a1b6" + ], + "version": "==1.25.0" }, "grpcio-tools": { "hashes": [ - "sha256:056f2a274edda4315e825ac2e3a9536f5415b43aa51669196860c8de6e76d847", - "sha256:0c953251585fdcd422072e4b7f4243fce215f22e21db94ec83c5970e41db6e18", - "sha256:142a73f5769f37bf2e4a8e4a77ef60f7af5f55635f60428322b49c87bd8f9cc0", - "sha256:1b333e2a068d8ef89a01eb23a098d2a789659c3178de79da9bd3d0ffb944cc6d", - "sha256:2124f19cc51d63405a0204ae38ef355732ab0a235975ab41ff6f6f9701905432", - "sha256:24c3a04adfb6c6f1bc4a2f8498d7661ca296ae352b498e538832c22ddde7bf81", - "sha256:3a2054e9640cbdd0ce8a345afb86be52875c5a8f9f5973a5c64791a8002da2dd", - "sha256:3fd15a09eecef83440ac849dcda2ff522f8ee1603ebfcdbb0e9b320ef2012e41", - "sha256:457e7a7dfa0b6bb608a766edba6f20c9d626a790df802016b930ad242fec4470", - "sha256:49ad5661d54ff0d164e4b441ee5e05191187d497380afa16d36d72eb8ef048de", - "sha256:561078e425d21a6720c3c3828385d949e24c0765e2852a46ecc3ad3fca2706e5", - "sha256:5a4f65ab06b32dc34112ed114dee3b698c8463670474334ece5b0b531073804c", - "sha256:8883e0e34676356d219a4cd37d239c3ead655cc550836236b52068e091416fff", - "sha256:8d2b45b1faf81098780e07d6a1c207b328b07e913160b8baa7e2e8d89723e06c", - "sha256:b0ebddb6ecc4c161369f93bb3a74c6120a498d3ddc759b64679709a885dd6d4f", - "sha256:b786ba4842c50de865dd3885b5570690a743e84a327b7213dd440eb0e6b996f8", - "sha256:be8efa010f5a80f1862ead80c3b19b5eb97dc954a0f59a1e2487078576105e03", - "sha256:c29106eaff0e2e708a9a89628dc0134ef145d0d3631f0ef421c09f380c30e354", - "sha256:c3c71236a056ec961b2b8b3b7c0b3b5a826283bc69c4a1c6415d23b70fea8243", - "sha256:cbc35031ec2b29af36947d085a7fbbcd8b79b84d563adf6156103d82565f78db", - "sha256:d47307c22744918e803c1eec7263a14f36aaf34fe496bff9ccbcae67c02b40ae", - "sha256:db088c98e563c1bb070da5984c8df08b45b61e4d9c6d2a8a1ffeed2af89fd1f3", - "sha256:df4dd1cb670062abdacc1fbce41cae4e08a4a212d28dd94fdbbf90615d027f73", - "sha256:e3adcf1499ca08d1e036ff44aedf55ed78653d946f4c4426b6e72ab757cc4dec", - "sha256:e3b3e32e0cda4dc382ec5bed8599dab644e4b3fc66a9ab54eb58248e207880b9", - "sha256:ed524195b35304c670755efa1eca579e5c290a66981f97004a5b2c0d12d6897d", - "sha256:edb42432790b1f8ec9f08faf9326d7e5dfe6e1d8c8fe4db39abc0a49c1c76537", - "sha256:eff1f995e5aa4cc941b6bbc45b5b57842f8f62bbe1a99427281c2c70cf42312c", - "sha256:f2fcdc2669662d77b400f80e20315a3661466e3cb3df1730f8083f9e49465cbc", - "sha256:f52ec9926daf48f41389d39d01570967b99c7dbc12bffc134cc3a3c5b5540ba2", - "sha256:fd007d67fdfbd2a13bf8a8c8ced8353b42a92ca72dbee54e951d8ddbc6ca12bc", - "sha256:ff9045e928dbb7943ea8559bfabebee95a43a830e00bf52c16202d2d805780fb" - ], - "index": "pypi", - "version": "==1.23.0" + "sha256:007c075eb9611379fa8f520a1865b9afd850469495b0e4a46e1349b2dc1744ce", + "sha256:02ae9708bdd3f329b1abe1ee16b1d768b2dd7a036a8a57e342d08ee8ca054cec", + "sha256:2f10226bfea4f947de355008b14fb4711c85fc1121570833a96f0e2cd8de580f", + "sha256:314354c7321c84a6e176a99afe1945c933b8a38b4f837255c8decfef8d07f24e", + "sha256:406b530c283a2bb804a10ee97928290b0b60788cd114ddfce0faa681cccfe4b8", + "sha256:49e7682e505e6a1d35459dae1d8a616a08d5cfa6f05de00235aff2e15786af14", + "sha256:4a5c2b38078fc4b949e4e70f7e25cb80443d1ee9a648ce4223aa3c040a0d3b9b", + "sha256:4b40291d67a1fecb5170ed9ec32016e2ae07908a8fa143d2d37311b2bcbeb2c5", + "sha256:4b72b04cba6ecd1940d6eda07886f80fe71fb2e669f1095ebab58b1eb17a53fa", + "sha256:4cc95d5fddebb9348fafcc4c0147745882794ded7cfd5282b2aa158596c77a8a", + "sha256:4ce0261bd4426482a96467ed9ad8411417a6932c331a5bb35aa1907f618f34f6", + "sha256:5226371a2b569c62be0d0590ccff7bbb9566762f243933efbd4b695f9f108cd5", + "sha256:52aab4cbab10683f8830420c0b55ccdc6344702b4a0940913d71fe928dd731c9", + "sha256:532a19419535a92a1b621222f70d6da7624151fe69afa4a1063be56e7a2b884a", + "sha256:5a8d44add097e0a3a7c27e66a8ed0aa2fd561cda77381e818cf7862d4ad0f629", + "sha256:64f6027887e32a938f00b2344c337c6d4f7c4cf157ec2e84b1dd6b6fddad8e50", + "sha256:651b0441e8d8f302b44fb50397fe73dcd5e61b790533438e690055abdef3b234", + "sha256:67d12ec4548dd2b1f15c9e3a953c8f48d8c3441c2d8bd143fc3af95a1c041c2b", + "sha256:6c029341132a0e64cbd2dba1dda9a125e06a798b9ec864569afdecce626dd5d5", + "sha256:6e64214709f37b347875ac83cfed4e9cfd287f255dab2836521f591620412c40", + "sha256:6f70fc9a82a0145296358720cf24f83a657a745e8b51ec9564f4c9e678c5b872", + "sha256:6fb4739eb5eef051945b16b3c434d08653ea05f0313cf88495ced5d9db641745", + "sha256:79b5b1c172dafb0e76aa95bf572d4c7afc0bf97a1669b2228a0bc151071c4666", + "sha256:7d02755480cec3c0222f35397e810bfaf4cf9f2bf2e626f7f6efc1d40fffb7fa", + "sha256:818f2b8168760cf16e66fe85894a37afcff5378a64939549663a371216618498", + "sha256:834564c2fba02c31179af081bd80aada8dfdcca52c80e241353f6063b6154bd2", + "sha256:8b17347a90a14386641ffe57743bbb01a16a7149c95905364d3c8091ad377bd8", + "sha256:902e13dbaca9733e4668928967b301526197ecffacb8c7a0acc0c7045de8836f", + "sha256:988014c714ca654b3b7ca9f4dabfe487b00e023bfdd9eaf1bb0fed82bf8c4255", + "sha256:9a83d39e198cbed5d093f43790b92945ab74140357ec00e53ae13b421489ffb7", + "sha256:ac7649cff7354d2f04ebe2872f786a1d07547deded61f3d39036ebb569de91bc", + "sha256:b013d93bc6dc5c7bf3642bf30e673daee46f9a4984fbd9588a9cda1071278414", + "sha256:b02701d40f1ccf16bc8c46f56bdbf89e03110bd8fd570c854e72299ce2920c35", + "sha256:b0ef0da2eec959def8ba508b2a763c492f1fb989446a422d1456ac17dc1b19f4", + "sha256:bb8264ccf8ff904a1a396dc757ac1560b24f270b90e7dabb0ae3f637cb351bb3", + "sha256:bbfb58f5c0aa27b599141bb5eacaf8116b55ad89bc5a2c3afd5e965d840ad341", + "sha256:c1a482fdd8952a7f0098f78161a4deef8a500e54babef302548cd9f1e326d42c", + "sha256:c40efc662fa037898488e31756242af68a8ab5729f939bc8c9ba259bc32e7d6a", + "sha256:c5ad07adae3fe62761bc662c554c2734203f0f700616fc58138b852a7ef5e40e", + "sha256:c765512cb5cb4afaf652837b8cc69229dee14c8e92f15a6ea0f4dfd646902dd2", + "sha256:c871f5a89012ae44d9233305d74dfdd2059a78f0cb0303d38a4b6a562c6f9ba7", + "sha256:cc950fb17c1172d0c0129e8c6e787206e7ef8c24a8e39005f8cc297e9faa4f9a", + "sha256:d3619b43009a5c82cb7ef11847518236140d7ffdcc6600e1a151b8b49350693a", + "sha256:dc17a8a8b39cb37380d927d4669882af4ccc7d3ee298a15a3004f4b18ecd2ac3", + "sha256:eab3684ce9dec3a934a36ba79e8435210d07c50906425ab157eeb4b14503a925", + "sha256:f258b32dffd27ef1eb5f5f01ebb115dfad07677b0510b41f786c511a62ded033", + "sha256:f550c94728b67a7eeddc35b03c99552f2d7aac09c52935ad4b0552d0843fd03c", + "sha256:f7fc690a517c8f3765796ed005bb3273895a985a8593977291bad24568e018e3" + ], + "index": "pypi", + "version": "==1.25.0" }, "importlib-metadata": { "hashes": [ - "sha256:0c505102757e7fa28b9f0958d8bc81301159dea16e2649858c92edc158b78a83", - "sha256:9a9f75ce32e78170905888acbf2376a81d3f21ecb3bb4867050413411d3ca7a9" + "sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21", + "sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742" ], "markers": "python_version < '3.8'", - "version": "==0.21" + "version": "==1.1.0" }, "ipython": { "hashes": [ - "sha256:c4ab005921641e40a68e405e286e7a1fcc464497e14d81b6914b4fd95e5dee9b", - "sha256:dd76831f065f17bddd7eaa5c781f5ea32de5ef217592cf019e34043b56895aa1" + "sha256:c66c7e27239855828a764b1e8fc72c24a6f4498a2637572094a78c5551fb9d51", + "sha256:f186b01b36609e0c5d0de27c7ef8e80c990c70478f8c880863004b3489a9030e" ], "index": "pypi", - "version": "==7.8.0" + "version": "==7.10.1" }, "ipython-genutils": { "hashes": [ @@ -479,26 +513,29 @@ }, "lazy-object-proxy": { "hashes": [ - "sha256:02b260c8deb80db09325b99edf62ae344ce9bc64d68b7a634410b8e9a568edbf", - "sha256:18f9c401083a4ba6e162355873f906315332ea7035803d0fd8166051e3d402e3", - "sha256:1f2c6209a8917c525c1e2b55a716135ca4658a3042b5122d4e3413a4030c26ce", - "sha256:2f06d97f0ca0f414f6b707c974aaf8829c2292c1c497642f63824119d770226f", - "sha256:616c94f8176808f4018b39f9638080ed86f96b55370b5a9463b2ee5c926f6c5f", - "sha256:63b91e30ef47ef68a30f0c3c278fbfe9822319c15f34b7538a829515b84ca2a0", - "sha256:77b454f03860b844f758c5d5c6e5f18d27de899a3db367f4af06bec2e6013a8e", - "sha256:83fe27ba321e4cfac466178606147d3c0aa18e8087507caec78ed5a966a64905", - "sha256:84742532d39f72df959d237912344d8a1764c2d03fe58beba96a87bfa11a76d8", - "sha256:874ebf3caaf55a020aeb08acead813baf5a305927a71ce88c9377970fe7ad3c2", - "sha256:9f5caf2c7436d44f3cec97c2fa7791f8a675170badbfa86e1992ca1b84c37009", - "sha256:a0c8758d01fcdfe7ae8e4b4017b13552efa7f1197dd7358dc9da0576f9d0328a", - "sha256:a4def978d9d28cda2d960c279318d46b327632686d82b4917516c36d4c274512", - "sha256:ad4f4be843dace866af5fc142509e9b9817ca0c59342fdb176ab6ad552c927f5", - "sha256:ae33dd198f772f714420c5ab698ff05ff900150486c648d29951e9c70694338e", - "sha256:b4a2b782b8a8c5522ad35c93e04d60e2ba7f7dcb9271ec8e8c3e08239be6c7b4", - "sha256:c462eb33f6abca3b34cdedbe84d761f31a60b814e173b98ede3c81bb48967c4f", - "sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1" - ], - "version": "==1.4.2" + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" }, "mccabe": { "hashes": [ @@ -510,49 +547,52 @@ }, "more-itertools": { "hashes": [ - "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", - "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + "sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", + "sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" ], - "version": "==7.2.0" + "version": "==8.0.0" }, "mypy": { "hashes": [ - "sha256:0107bff4f46a289f0e4081d59b77cef1c48ea43da5a0dbf0005d54748b26df2a", - "sha256:07957f5471b3bb768c61f08690c96d8a09be0912185a27a68700f3ede99184e4", - "sha256:10af62f87b6921eac50271e667cc234162a194e742d8e02fc4ddc121e129a5b0", - "sha256:11fd60d2f69f0cefbe53ce551acf5b1cec1a89e7ce2d47b4e95a84eefb2899ae", - "sha256:15e43d3b1546813669bd1a6ec7e6a11d2888db938e0607f7b5eef6b976671339", - "sha256:352c24ba054a89bb9a35dd064ee95ab9b12903b56c72a8d3863d882e2632dc76", - "sha256:437020a39417e85e22ea8edcb709612903a9924209e10b3ec6d8c9f05b79f498", - "sha256:49925f9da7cee47eebf3420d7c0e00ec662ec6abb2780eb0a16260a7ba25f9c4", - "sha256:6724fcd5777aa6cebfa7e644c526888c9d639bd22edd26b2a8038c674a7c34bd", - "sha256:7a17613f7ea374ab64f39f03257f22b5755335b73251d0d253687a69029701ba", - "sha256:cdc1151ced496ca1496272da7fc356580e95f2682be1d32377c22ddebdf73c91" + "sha256:02d9bdd3398b636723ecb6c5cfe9773025a9ab7f34612c1cde5c7f2292e2d768", + "sha256:088f758a50af31cf8b42688118077292370c90c89232c783ba7979f39ea16646", + "sha256:28e9fbc96d13397a7ddb7fad7b14f373f91b5cff538e0772e77c270468df083c", + "sha256:30e123b24931f02c5d99307406658ac8f9cd6746f0d45a3dcac2fe5fbdd60939", + "sha256:3294821b5840d51a3cd7a2bb63b40fc3f901f6a3cfb3c6046570749c4c7ef279", + "sha256:41696a7d912ce16fdc7c141d87e8db5144d4be664a0c699a2b417d393994b0c2", + "sha256:4f42675fa278f3913340bb8c3371d191319704437758d7c4a8440346c293ecb2", + "sha256:54d205ccce6ed930a8a2ccf48404896d456e8b87812e491cb907a355b1a9c640", + "sha256:6992133c95a2847d309b4b0c899d7054adc60481df6f6b52bb7dee3d5fd157f7", + "sha256:6ecbd0e8e371333027abca0922b0c2c632a5b4739a0c61ffbd0733391e39144c", + "sha256:83fa87f556e60782c0fc3df1b37b7b4a840314ba1ac27f3e1a1e10cb37c89c17", + "sha256:c87ac7233c629f305602f563db07f5221950fe34fe30af072ac838fa85395f78", + "sha256:de9ec8dba773b78c49e7bec9a35c9b6fc5235682ad1fc2105752ae7c22f4b931", + "sha256:f385a0accf353ca1bca4bbf473b9d83ed18d923fdb809d3a70a385da23e25b6a" ], "index": "pypi", - "version": "==0.720" + "version": "==0.750" }, "mypy-extensions": { "hashes": [ - "sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812", - "sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e" + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" ], - "version": "==0.4.1" + "version": "==0.4.3" }, "mypy-protobuf": { "hashes": [ - "sha256:69f37e166ace422ef8c8266ff73be5384968152e1d7769f3d866d0e923c354ba", - "sha256:a10bb9f24cc2c1f5077a5e64fc5e22b7f7e230a1a399a34cebc57c80a8a532e6" + "sha256:4c18a300054abdd6d635b02059d36bc165a166dab2bde4b899d76e1118f63d9a", + "sha256:72ab724299aebd930b88476f6545587bff5bf480697c016097bd188841a56276" ], "index": "pypi", - "version": "==1.15" + "version": "==1.16" }, "packaging": { "hashes": [ - "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", - "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" ], - "version": "==19.1" + "version": "==19.2" }, "parso": { "hashes": [ @@ -561,20 +601,26 @@ ], "version": "==0.5.1" }, + "pathspec": { + "hashes": [ + "sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c" + ], + "version": "==0.6.0" + }, "pbr": { "hashes": [ - "sha256:2c8e420cd4ed4cec4e7999ee47409e876af575d4c35a45840d59e8b5f3155ab8", - "sha256:b32c8ccaac7b1a20c0ce00ce317642e6cf231cf038f9875e0280e28af5bf7ac9" + "sha256:139d2625547dbfa5fb0b81daebb39601c478c21956dc57e2e07b74450a8c506b", + "sha256:61aa52a0f18b71c5cc58232d2cf8f8d09cd67fcad60b742a60124cb8d6951488" ], - "version": "==5.4.3" + "version": "==5.4.4" }, "pep8-naming": { "hashes": [ - "sha256:01cb1dab2f3ce9045133d08449f1b6b93531dceacb9ef04f67087c11c723cea9", - "sha256:0ec891e59eea766efd3059c3d81f1da304d858220678bdc351aab73c533f2fbb" + "sha256:45f330db8fcfb0fba57458c77385e288e7a3be1d01e8ea4268263ef677ceea5f", + "sha256:a33d38177056321a167decd6ba70b890856ba5025f0a8eca6a3eda607da93caf" ], "index": "pypi", - "version": "==0.8.2" + "version": "==0.9.1" }, "pexpect": { "hashes": [ @@ -593,39 +639,38 @@ }, "pluggy": { "hashes": [ - "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", - "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], - "version": "==0.12.0" + "version": "==0.13.1" }, "prompt-toolkit": { "hashes": [ - "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", - "sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1", - "sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55" + "sha256:0278d2f51b5ceba6ea8da39f76d15684e84c996b325475f6e5720edc584326a7", + "sha256:63daee79aa8366c8f1c637f1a4876b890da5fc92a19ebd2f7080ebacb901e990" ], - "version": "==2.0.9" + "version": "==3.0.2" }, "protobuf": { "hashes": [ - "sha256:00a1b0b352dc7c809749526d1688a64b62ea400c5b05416f93cfb1b11a036295", - "sha256:01acbca2d2c8c3f7f235f1842440adbe01bbc379fa1cbdd80753801432b3fae9", - "sha256:0a795bca65987b62d6b8a2d934aa317fd1a4d06a6dd4df36312f5b0ade44a8d9", - "sha256:0ec035114213b6d6e7713987a759d762dd94e9f82284515b3b7331f34bfaec7f", - "sha256:31b18e1434b4907cb0113e7a372cd4d92c047ce7ba0fa7ea66a404d6388ed2c1", - "sha256:32a3abf79b0bef073c70656e86d5bd68a28a1fbb138429912c4fc07b9d426b07", - "sha256:55f85b7808766e5e3f526818f5e2aeb5ba2edcc45bcccede46a3ccc19b569cb0", - "sha256:64ab9bc971989cbdd648c102a96253fdf0202b0c38f15bd34759a8707bdd5f64", - "sha256:64cf847e843a465b6c1ba90fb6c7f7844d54dbe9eb731e86a60981d03f5b2e6e", - "sha256:917c8662b585470e8fd42f052661fc66d59fccaae450a60044307dcbf82a3335", - "sha256:afed9003d7f2be2c3df20f64220c30faec441073731511728a2cb4cab4cd46a6", - "sha256:bf8e05d638b585d1752c5a84247134a0350d3a8b73d3632489a014a9f6f1e758", - "sha256:d831b047bd69becaf64019a47179eb22118a50dd008340655266a906c69c6417", - "sha256:de2760583ed28749ff885789c1cbc6c9c06d6de92fc825740ab99deb2f25ea4d", - "sha256:eabc4cf1bc19689af8022ba52fd668564a8d96e0d08f3b4732d26a64255216a4", - "sha256:fcff6086c86fb1628d94ea455c7b9de898afc50378042927a59df8065a79a549" - ], - "version": "==3.9.1" + "sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd", + "sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed", + "sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057", + "sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce", + "sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03", + "sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46", + "sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33", + "sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c", + "sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9", + "sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef", + "sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b", + "sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d", + "sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8", + "sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6", + "sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941", + "sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13" + ], + "version": "==3.11.1" }, "ptyprocess": { "hashes": [ @@ -650,12 +695,10 @@ }, "pydocstyle": { "hashes": [ - "sha256:2258f9b0df68b97bf3a6c29003edc5238ff8879f1efb6f1999988d934e432bd8", - "sha256:5741c85e408f9e0ddf873611085e819b809fca90b619f5fd7f34bd4959da3dd4", - "sha256:ed79d4ec5e92655eccc21eb0c6cf512e69512b4a97d215ace46d17e4990f2039" + "sha256:04c84e034ebb56eb6396c820442b8c4499ac5eb94a3bda88951ac3dc519b6058", + "sha256:66aff87ffe34b1e49bff2dd03a88ce6843be2f3346b0c9814410d34987fbab59" ], - "index": "pypi", - "version": "==3.0.0" + "version": "==4.0.1" }, "pyflakes": { "hashes": [ @@ -666,58 +709,74 @@ }, "pygments": { "hashes": [ - "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", - "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", + "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" ], - "version": "==2.4.2" + "version": "==2.5.2" }, "pylint": { "hashes": [ - "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", - "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" + "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", + "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" ], "index": "pypi", - "version": "==2.3.1" + "version": "==2.4.4" }, "pyparsing": { "hashes": [ - "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", - "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" + "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", + "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" ], - "version": "==2.4.2" + "version": "==2.4.5" }, "pytest": { "hashes": [ - "sha256:95d13143cc14174ca1a01ec68e84d76ba5d9d493ac02716fd9706c949a505210", - "sha256:b78fe2881323bd44fd9bd76e5317173d4316577e7b1cddebae9136a4495ec865" + "sha256:63344a2e3bce2e4d522fd62b4fdebb647c019f1f9e4ca075debbd13219db4418", + "sha256:f67403f33b2b1d25a6756184077394167fe5e2f9d8bdaab30707d19ccec35427" ], "index": "pypi", - "version": "==5.1.2" + "version": "==5.3.1" }, "pyyaml": { "hashes": [ - "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", - "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", - "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", - "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", - "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", - "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", - "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", - "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", - "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", - "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", - "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", - "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", - "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" - ], - "version": "==5.1.2" + "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", + "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", + "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", + "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", + "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", + "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", + "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", + "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", + "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", + "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", + "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" + ], + "version": "==5.2" + }, + "regex": { + "hashes": [ + "sha256:15454b37c5a278f46f7aa2d9339bda450c300617ca2fca6558d05d870245edc7", + "sha256:1ad40708c255943a227e778b022c6497c129ad614bb7a2a2f916e12e8a359ee7", + "sha256:5e00f65cc507d13ab4dfa92c1232d004fa202c1d43a32a13940ab8a5afe2fb96", + "sha256:604dc563a02a74d70ae1f55208ddc9bfb6d9f470f6d1a5054c4bd5ae58744ab1", + "sha256:720e34a539a76a1fedcebe4397290604cc2bdf6f81eca44adb9fb2ea071c0c69", + "sha256:7caf47e4a9ac6ef08cabd3442cc4ca3386db141fb3c8b2a7e202d0470028e910", + "sha256:7faf534c1841c09d8fefa60ccde7b9903c9b528853ecf41628689793290ca143", + "sha256:b4e0406d822aa4993ac45072a584d57aa4931cf8288b5455bbf30c1d59dbad59", + "sha256:c31eaf28c6fe75ea329add0022efeed249e37861c19681960f99bbc7db981fb2", + "sha256:c7393597191fc2043c744db021643549061e12abe0b3ff5c429d806de7b93b66", + "sha256:d2b302f8cdd82c8f48e9de749d1d17f85ce9a0f082880b9a4859f66b07037dc6", + "sha256:e3d8dd0ec0ea280cf89026b0898971f5750a7bd92cb62c51af5a52abd020054a", + "sha256:ec032cbfed59bd5a4b8eab943c310acfaaa81394e14f44454ad5c9eba4f24a74" + ], + "version": "==2019.11.1" }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "smmap2": { "hashes": [ @@ -728,9 +787,10 @@ }, "snowballstemmer": { "hashes": [ - "sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e" + "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0", + "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52" ], - "version": "==1.9.1" + "version": "==2.0.0" }, "stevedore": { "hashes": [ @@ -741,10 +801,10 @@ }, "testfixtures": { "hashes": [ - "sha256:665a298976c8d77f311b65c46f16b7cda7229a47dff5ad7c822e5b3371a439e2", - "sha256:9d230c5c80746f9f86a16a1f751a5cf5d8e317d4cc48243a19fb180d22303bce" + "sha256:8f22100d4fb841b958f64e71c8820a32dc46f57d4d7e077777b932acd87b7327", + "sha256:9334f64d4210b734d04abff516d6ddaab7328306a0c4c1268ce4624df51c4f6d" ], - "version": "==6.10.0" + "version": "==6.10.3" }, "toml": { "hashes": [ @@ -755,39 +815,44 @@ }, "traitlets": { "hashes": [ - "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835", - "sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9" + "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44", + "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7" ], - "version": "==4.3.2" + "version": "==4.3.3" }, "typed-ast": { "hashes": [ + "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161", "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47", "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2", + "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e", "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" ], - "markers": "implementation_name == 'cpython'", + "markers": "implementation_name == 'cpython' and python_version < '3.8'", "version": "==1.4.0" }, "typing-extensions": { "hashes": [ - "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", - "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", - "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" ], - "version": "==3.7.4" + "version": "==3.7.4.1" }, "wcwidth": { "hashes": [ diff --git a/gfauto/check_all.sh b/gfauto/check_all.sh index 5133e139f..0255d73a1 100755 --- a/gfauto/check_all.sh +++ b/gfauto/check_all.sh @@ -22,7 +22,7 @@ if [ -z ${VIRTUAL_ENV+x} ]; then source .venv/bin/activate fi -mypy --strict gfauto gfautotests +mypy --strict --show-absolute-path gfauto gfautotests pylint gfauto gfautotests # Flake checks formatting via black. flake8 . diff --git a/gfauto/docs/coverage.md b/gfauto/docs/coverage.md new file mode 100644 index 000000000..fe3c8a872 --- /dev/null +++ b/gfauto/docs/coverage.md @@ -0,0 +1,104 @@ +# Coverage + +```sh +# Make sure gfauto_* is available. +# E.g. +# source /path/to/gfauto/.venv/activate +# gfauto_cov_from_gcov -h + +COV_ROOT=$(pwd) + + +### Build SwiftShader ### + +git clone https://swiftshader.googlesource.com/SwiftShader +cd SwiftShader + +mkdir -p out/build +cd out/build + +BUILD_DIR=$(pwd) + +export CFLAGS=--coverage +export CXXFLAGS=--coverage +export LDFLAGS=--coverage + +cmake -G Ninja ../.. -DCMAKE_BUILD_TYPE=Debug +cmake --build . --config Debug + +unset CFLAGS +unset CXXFLAGS +unset LDFLAGS + +cd $COV_ROOT + +export VK_ICD_FILENAMES=$BUILD_DIR/Linux/vk_swiftshader_icd.json + + +### Build dEQP ### +DEQP_USERNAME=paulthomson +git clone ssh://$DEQP_USERNAME@gerrit.khronos.org:29418/vk-gl-cts && scp -p -P 29418 $DEQP_USERNAME@gerrit.khronos.org:hooks/commit-msg vk-gl-cts/.git/hooks/ + +cd vk-gl-cts +python external/fetch_sources.py +cd .. + +mkdir vk-gl-cts-build +cd vk-gl-cts-build +cmake -G Ninja ../vk-gl-cts -DCMAKE_BUILD_TYPE=Release DCMAKE_C_FLAGS=-m64 -DCMAKE_CXX_FLAGS=-m64 +cmake --build . --config Release --target deqp-vk + +cd $COV_ROOT + + +### Run gfauto ### + +export GCOV_PREFIX=$COV_ROOT/prefix_gfauto/PROC_ID + +mkdir gfauto_run +cd gfauto_run + +# Run gfauto. See ../README.md. +# In settings.json, set `active_devices` to `host`. +# Of course, `host` will actually be your built version of SwiftShader +# because of VK_ICD_FILENAMES. +# In settings.json, set `reduce_*` to false. E.g. "reduce_bad_images": false. +# Once `gfauto_fuzz` seems to work, you can try running e.g. 32 instances in parallel: + +parallel -j 32 -i gfauto_fuzz -- $(seq 100) +# Output coverage files are in: $COV_ROOT/prefix_gfauto/*/ + +cd $COV_ROOT + + +### Run dEQP ### + +export GCOV_PREFIX=$COV_ROOT/prefix_deqp/PROC_ID + +# TODO: where do we find rundeqp.go? + +go run rundeqpvk.go -deqp-vk $COV_ROOT/vk-gl-cts-build/external/vulkancts/modules/vulkan/deqp-vk -num-threads 32 + +cd $COV_ROOT + + +### Process coverage info ### + +gfauto_cov_from_gcov -h +# Check help text. +# You may need to add --gcov_uses_json depending on your version of GCC. +# This flag has not been tested yet. + +gfauto_cov_from_gcov --out deqp.cov $BUILD_DIR $COV_ROOT/prefix_deqp/PROC_ID --num_threads 32 + +gfauto_cov_from_gcov --out gfauto.cov $BUILD_DIR $COV_ROOT/prefix_gfauto/PROC_ID --num_threads 32 + +gfauto_cov_new deqp.cov gfauto.cov new.cov + +gfauto_cov_to_source --coverage_out new_cov --zero_coverage_out new_cov_zero --cov new.cov $BUILD_DIR + +# Note: Install meld if needed. + +meld new_cov_zero/ new_cov/ + +``` diff --git a/gfauto/gfauto/add_amber_tests_to_cts.py b/gfauto/gfauto/add_amber_tests_to_cts.py index c25d2331d..83f845ecb 100644 --- a/gfauto/gfauto/add_amber_tests_to_cts.py +++ b/gfauto/gfauto/add_amber_tests_to_cts.py @@ -47,7 +47,7 @@ def check(condition: bool, exception: Exception) -> None: def log(message: str = "") -> None: - print(message, flush=True) # noqa T001 + print(message, flush=True) # noqa: T001 def remove_start(string: str, start: str) -> str: diff --git a/gfauto/gfauto/android_device.py b/gfauto/gfauto/android_device.py index 2beb126b3..db45a5767 100644 --- a/gfauto/gfauto/android_device.py +++ b/gfauto/gfauto/android_device.py @@ -24,10 +24,17 @@ import subprocess import time from pathlib import Path -from subprocess import CompletedProcess from typing import List, Optional -from gfauto import devices_util, fuzz, gflogging, result_util, subprocess_util, util +from gfauto import ( + devices_util, + fuzz, + gflogging, + result_util, + subprocess_util, + types, + util, +) from gfauto.device_pb2 import Device, DeviceAndroid from gfauto.gflogging import log from gfauto.util import check, file_open_text, file_write_text @@ -66,7 +73,7 @@ def adb_helper( check_exit_code: bool, verbose: bool = False, timeout: Optional[int] = ADB_DEFAULT_TIME_LIMIT, -) -> subprocess.CompletedProcess: +) -> types.CompletedProcess: adb_cmd = [str(adb_path())] if serial: @@ -85,7 +92,7 @@ def adb_check( adb_args: List[str], verbose: bool = False, timeout: Optional[int] = ADB_DEFAULT_TIME_LIMIT, -) -> subprocess.CompletedProcess: +) -> types.CompletedProcess: return adb_helper( serial, adb_args, check_exit_code=True, verbose=verbose, timeout=timeout @@ -97,7 +104,7 @@ def adb_can_fail( adb_args: List[str], verbose: bool = False, timeout: Optional[int] = ADB_DEFAULT_TIME_LIMIT, -) -> subprocess.CompletedProcess: +) -> types.CompletedProcess: return adb_helper( serial, adb_args, check_exit_code=False, verbose=verbose, timeout=timeout @@ -366,7 +373,7 @@ def run_amber_on_device_helper( status = "UNEXPECTED_ERROR" - result: Optional[CompletedProcess] = None + result: Optional[types.CompletedProcess] = None try: result = adb_can_fail( diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py index 195ef9723..34632b249 100644 --- a/gfauto/gfauto/binaries_util.py +++ b/gfauto/gfauto/binaries_util.py @@ -527,7 +527,7 @@ def binary_name_to_project_name(binary_name: str) -> str: def get_github_release_recipe( # pylint: disable=too-many-branches; - binary: Binary + binary: Binary, ) -> recipe_wrap.RecipeWrap: project_name = binary_name_to_project_name(binary.name) diff --git a/gfauto/gfauto/cov_from_gcov.py b/gfauto/gfauto/cov_from_gcov.py new file mode 100644 index 000000000..29184553b --- /dev/null +++ b/gfauto/gfauto/cov_from_gcov.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Processes .gcda and .gcno files to get a .cov file. + +Unlike most code in gfauto, we use str instead of pathlib.Path because the increased speed is probably worthwhile. +""" + +import argparse +import os +import pickle +import shutil +import sys + +from gfauto import cov_util + + +def main() -> None: + + parser = argparse.ArgumentParser( + description="Processes .gcda and .gcno files to get a .cov file that contains the line coverage data.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--gcov_path", type=str, help="Path to gcov", default=shutil.which("gcov") + ) + + parser.add_argument( + "--gcov_uses_json", + help="Pass to indicate that your gcov version is 9+ and so uses the newer JSON intermediate format. " + "This is faster, but you must have gcc 9+.", + action="store_true", + ) + + parser.add_argument( + "--num_threads", + type=int, + help="Thead pool size for running gcov in parallel.", + default=8, + ) + + parser.add_argument( + "--out", type=str, help="The output .cov file", default="output.cov" + ) + + parser.add_argument( + "build_dir", + type=str, + help="The build directory where the compiler was invoked.", + ) + + parser.add_argument( + "gcov_prefix_dir", + type=str, + help="The GCOV_PREFIX directory that was used when running the target application. " + 'If the directory ends with "PROC_ID" then "PROC_ID" will be replaced with each directory that exists ' + 'and the coverage results will be merged. E.g. Given "--gcov_prefix_dir /cov/PROC_ID", the results from ' + "/cov/001 /cov/002 /cov/blah etc. will be computed and the results will be merged.", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + if not parsed_args.gcov_path: + parser.error("Please provide gcov_path") + + gcov_path: str = parsed_args.gcov_path + + build_dir = parsed_args.build_dir + build_dir = os.path.abspath(build_dir) + build_dir = os.path.normpath(build_dir) + + gcov_prefix_dir = parsed_args.gcov_prefix_dir + gcov_prefix_dir = os.path.abspath(gcov_prefix_dir) + gcov_prefix_dir = os.path.normpath(gcov_prefix_dir) + + data = cov_util.GetLineCountsData( + gcov_path=gcov_path, + gcov_uses_json_output=parsed_args.gcov_uses_json, + build_dir=build_dir, + gcov_prefix_dir=gcov_prefix_dir, + num_threads=parsed_args.num_threads, + ) + + output_coverage_path: str = parsed_args.out + + # Special case for "PROC_ID". + if "PROC_ID" in gcov_prefix_dir: + print("Detected PROC_ID in gcov_prefix_dir") + if os.path.exists(gcov_prefix_dir): + raise AssertionError(f"Unexpected file/directory: {gcov_prefix_dir}.") + gcov_prefix_prefix = os.path.dirname(gcov_prefix_dir) + if "PROC_ID" in gcov_prefix_prefix: + raise AssertionError( + f"Can only handle PROC_ID as the last component of the path: {gcov_prefix_dir}" + ) + for proc_dir in os.listdir(gcov_prefix_prefix): + proc_dir = os.path.join(gcov_prefix_prefix, proc_dir) + if not os.path.isdir(proc_dir): + continue + data.gcov_prefix_dir = proc_dir + print(f"Consuming {data.gcov_prefix_dir}") + cov_util.get_line_counts(data) + else: + print(f"Consuming {data.gcov_prefix_dir}") + cov_util.get_line_counts(data) + + with open(output_coverage_path, mode="wb") as f: + pickle.dump(data.line_counts, f, protocol=pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/cov_merge.py b/gfauto/gfauto/cov_merge.py new file mode 100644 index 000000000..6cc4fe4c1 --- /dev/null +++ b/gfauto/gfauto/cov_merge.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Outputs merged coverage data (merged.cov) given a list of .cov files. + +Unlike most code in gfauto, we use str instead of pathlib.Path because the increased speed is probably worthwhile. +""" + +import argparse +import pickle +import sys +from typing import List + +from gfauto import cov_util + + +def main() -> None: + + parser = argparse.ArgumentParser( + description="Outputs merged coverage data given a list of .cov files.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--out", type=str, default="merged.cov", help="The output merged coverage file." + ) + parser.add_argument( + "coverage_files", + metavar="coverage_files", + type=str, + nargs="*", + help="The .cov files to merge.", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + output_file: str = parsed_args.out + input_files: List[str] = parsed_args.coverage_files + + output_line_counts: cov_util.LineCounts = {} + + for input_file in input_files: + with open(input_file, mode="rb") as f: + input_line_counts: cov_util.LineCounts = pickle.load(f) + for file_path, line_counts in input_line_counts.items(): + if file_path not in line_counts: + output_line_counts[file_path] = line_counts + else: + output_line_counts[file_path].update(line_counts) + + with open(output_file, mode="wb") as f: + pickle.dump(output_line_counts, f, protocol=pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/cov_new.py b/gfauto/gfauto/cov_new.py new file mode 100644 index 000000000..75d1ed614 --- /dev/null +++ b/gfauto/gfauto/cov_new.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Outputs the "new coverage" given A.cov (baseline) and B.cov. + +Unlike most code in gfauto, we use str instead of pathlib.Path because the increased speed is probably worthwhile. +""" + +import argparse +import pickle +import sys + +from gfauto import cov_util + + +def main() -> None: + + parser = argparse.ArgumentParser( + description='Outputs the "new coverage" from A.cov (baseline) to B.cov. ' + "The output coverage file will only include the counts from B.cov for newly-covered lines; " + "all other lines will have a count of zero.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument("a_coverage", type=str, help="The baseline A.cov file.") + parser.add_argument("b_coverage", type=str, help="The B.cov file.") + parser.add_argument("output_coverage", type=str, help="The output .cov file.") + + parsed_args = parser.parse_args(sys.argv[1:]) + + a_coverage: str = parsed_args.a_coverage + b_coverage: str = parsed_args.b_coverage + output_coverage: str = parsed_args.output_coverage + + with open(a_coverage, mode="rb") as f: + a_line_counts: cov_util.LineCounts = pickle.load(f) + + with open(b_coverage, mode="rb") as f: + b_line_counts: cov_util.LineCounts = pickle.load(f) + + # We modify b_line_counts so that lines already covered by A are set to 0. + # Note that line counts appear to be able to overflow, so we use "!= 0" instead of "> 0". + for source_file_path, b_counts in b_line_counts.items(): + if source_file_path in a_line_counts: + a_counts = a_line_counts[source_file_path] + for line_number, b_count in b_counts.items(): + if b_count != 0: + # Defaults to 0 if not present. + if a_counts[line_number] != 0: + b_counts[line_number] = 0 + + with open(output_coverage, mode="wb") as f: + pickle.dump(b_line_counts, f, protocol=pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/cov_to_source.py b/gfauto/gfauto/cov_to_source.py new file mode 100644 index 000000000..579ace126 --- /dev/null +++ b/gfauto/gfauto/cov_to_source.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Takes a .cov file and outputs annotated source files. + +Unlike most code in gfauto, we use str instead of pathlib.Path because the increased speed is probably worthwhile. +""" + +import argparse +import pickle +import sys + +from gfauto import cov_util + + +def main() -> None: + + parser = argparse.ArgumentParser( + description="Takes a .cov file and outputs annotated source files.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--coverage_out", + type=str, + help="The output directory for source files annotated with line coverage.", + default="", + ) + + parser.add_argument( + "--zero_coverage_out", + type=str, + help="The output directory for source files annotated with line coverage, assuming zero coverage.", + default="", + ) + + parser.add_argument("--cov", type=str, help="The .cov file.", default="output.cov") + + parser.add_argument( + "build_dir", + type=str, + help="The build directory where the compiler was invoked.", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + coverage_out: str = parsed_args.coverage_out + zero_coverage_out: str = parsed_args.zero_coverage_out + coverage_file: str = parsed_args.cov + build_dir: str = parsed_args.build_dir + + with open(coverage_file, mode="rb") as f: + line_counts: cov_util.LineCounts = pickle.load(f) + + if coverage_out: + cov_util.output_source_files(build_dir, coverage_out, line_counts) + + if zero_coverage_out: + cov_util.output_source_files( + build_dir, zero_coverage_out, line_counts, force_zero_coverage=True + ) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/cov_util.py b/gfauto/gfauto/cov_util.py new file mode 100644 index 000000000..472603d51 --- /dev/null +++ b/gfauto/gfauto/cov_util.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Processes coverage files.""" + +import io +import os +import subprocess +import threading +import typing +from collections import Counter +from queue import Queue +from typing import Dict, List, Tuple + +from attr import dataclass + +# Type: +from gfauto import util + +LineCounts = Dict[str, typing.Counter[int]] + +DirAndItsFiles = Tuple[str, List[str]] + +DirAndItsOutput = Tuple[str, str] + +IGNORED_MISSING_FILES = ["CMakeCXXCompilerId.cpp", "CMakeCCompilerId.c"] + + +@dataclass # pylint: disable=too-many-instance-attributes; +class GetLineCountsData: + gcov_path: str + gcov_uses_json_output: bool + build_dir: str + gcov_prefix_dir: str + num_threads: int + gcda_files_queue: "Queue[DirAndItsFiles]" = Queue() + stdout_queue: "Queue[DirAndItsOutput]" = Queue() + line_counts: LineCounts = {} + + +def _thread_gcov(data: GetLineCountsData) -> None: + # Keep getting files and processing until the special "done" ("", []) message. + + while True: + root, files = data.gcda_files_queue.get() + if not root: + # This is the special "done" message. + break + cmd = [data.gcov_path, "-i"] + if data.gcov_uses_json_output: + cmd.append("-t") + cmd.extend(files) + # I.e.: cd $root && gcov -i -t file_1.gcda file_2.gcda ... + result = subprocess.run( + cmd, + encoding="utf-8", + errors="ignore", + check=True, + cwd=root, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if data.gcov_uses_json_output: + data.stdout_queue.put((root, result.stdout)) + else: + gcov_files = [file + ".gcov" for file in files] + gcov_contents = [] + for gcov_file in gcov_files: + with open( + os.path.join(root, gcov_file), + "r", + encoding="utf-8", + errors="ignore", + ) as f: + gcov_contents.append(f.read()) + gcov_contents_combined = "\n".join(gcov_contents) + data.stdout_queue.put((root, gcov_contents_combined)) + + +def _thread_gcovs(data: GetLineCountsData) -> None: + threads = [ + threading.Thread(target=_thread_gcov, args=(data,)) + for _ in range(data.num_threads) + ] + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + # send "done" message to adder thread. + data.stdout_queue.put(("", "")) + + +def _process_text_lines(data: GetLineCountsData, lines: typing.TextIO) -> None: + current_file = "" + current_file_line_counts: typing.Counter[int] = Counter() + for line in lines: + line = line.strip() + # file:file_path + if line.startswith("file:"): + current_file = line[5:] + current_file_line_counts = data.line_counts.setdefault( + current_file, Counter() + ) + continue + # lcount:line number,execution_count,has_unexecuted_block + if line.startswith("lcount:"): + assert current_file + line = line[7:] + parts = line.split(sep=",") + line_number = int(parts[0]) + count = int(parts[1]) + # Confusingly, |update| adds counts. + current_file_line_counts.update({line_number: count}) + continue + + +def _process_json_lines(data: GetLineCountsData, lines: typing.TextIO) -> None: + current_file = "" + current_file_line_counts: typing.Counter[int] = Counter() + # The count value given by the previous line, which may or may not be used depending on the next line. + current_count = -1 + for line in lines: + line = line.strip() + # "file": "file_name", + if line.startswith('"file": "'): + assert current_count < 0 + current_file = line[9:-2] + current_file_line_counts = data.line_counts.setdefault( + current_file, Counter() + ) + continue + # "count": count, + if line.startswith('"count": '): + assert current_file + current_count = int(line[9:-1]) + continue + # "line_number": line_number, + if line.startswith('"line_number": '): + assert current_file + assert current_count >= 0 + line_number = int(line[15:-1]) + # Confusingly, |update| adds counts. + current_file_line_counts.update({line_number: current_count}) + # Fallthrough. + + # Reset current_count; the "count" field can occur in a few places, but we only want to use the count that + # is immediately followed by "line_number". + current_count = -1 + + +def _thread_adder(data: GetLineCountsData) -> None: + # Keep processing stdout entries until we get the special "done" message. + + while True: + root, stdout = data.stdout_queue.get() + if not root: + # This is the special "done" message. + break + lines = io.StringIO(stdout) + + if data.gcov_uses_json_output: + _process_json_lines(data, lines) + else: + _process_text_lines(data, lines) + + +def get_line_counts(data: GetLineCountsData) -> None: + + root: str + files: List[str] + + # In gcov_prefix_dir, add symlinks to build_dir .gcno files. + print("Adding symlinks.") + + # If the symlinks already exist, we will get an error. + # os.symlink does not provide an option to overwrite. + # We instead create the symlink with a randomized name and then rename it using os.replace, which atomically + # overwrites any existing file. + random_text = util.get_random_name()[:10] + + for root, _, files in os.walk(data.build_dir): + gcno_files = [f for f in files if f.endswith(".gcno")] + if gcno_files: + root_rel = strip_root(root) + os.makedirs(os.path.join(data.gcov_prefix_dir, root_rel), exist_ok=True) + for file_name in gcno_files: + source = os.path.join(root, file_name) + dest = os.path.join(data.gcov_prefix_dir, root_rel, file_name) + temp = dest + random_text + os.symlink(source, temp) + os.replace(temp, dest) + + print("Done.") + + print("Processing .gcno files.") + + gcovs_thread = threading.Thread(target=_thread_gcovs, args=(data,)) + adder_thread = threading.Thread(target=_thread_adder, args=(data,)) + + gcovs_thread.start() + adder_thread.start() + + for root, _, files in os.walk( + os.path.join(data.gcov_prefix_dir, strip_root(data.build_dir)) + ): + gcno_files = [f for f in files if f.endswith(".gcno")] + # TODO: Could split further if necessary. + if gcno_files: + data.gcda_files_queue.put((os.path.join(root), gcno_files)) + + # Send a "done" message for each thread. + for _ in range(data.num_threads): + data.gcda_files_queue.put(("", [])) + + # wait for threads. + gcovs_thread.join() + adder_thread.join() + + print("Done.") + + +INDENT = 8 + + +def strip_root(path: str) -> str: + path_stripped = path + if os.path.isabs(path_stripped): + # Most of this coverage code only works on Linux, so we assume Linux here. + # If we had Windows paths that could be on different drives etc., we would need to be more careful. + util.check( + path_stripped.startswith("/"), + AssertionError(f"Non-posix absolute file path? {path}"), + ) + path_stripped = path_stripped[1:] + util.check( + not path_stripped.startswith("/"), + AssertionError( + f"Internal error trying to make a relative path: {path_stripped}" + ), + ) + return path_stripped + + +def output_source_files( + build_dir: str, + output_dir: str, + line_counts: LineCounts, + force_zero_coverage: bool = False, +) -> None: + build_dir = os.path.abspath(build_dir) + + for source_path, counts in line_counts.items(): + source_path = os.path.join(build_dir, source_path) + source_path = os.path.normpath(source_path) + + if not os.path.isfile(source_path): + if not os.path.basename(source_path) in IGNORED_MISSING_FILES: + print(f"WARNING: Could not find source file: {source_path}") + continue + + dest_path = os.path.join(output_dir, strip_root(source_path)) + + with open(source_path, "r", encoding="utf-8", errors="ignore") as source_file: + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + with open(dest_path, "w", encoding="utf-8", errors="ignore") as dest_file: + line_number = 1 + while True: + line = source_file.readline() + if not line: + break + # .get() returns None if the line is not executable + line_count = counts.get(line_number) + if line_count: + if force_zero_coverage: + line_count = 0 + line_count_str = str(line_count) + " " + else: + line_count_str = " " + line_count_str = ( + " " * (INDENT - len(line_count_str)) + line_count_str + ) + line = line_count_str + line + dest_file.write(line) + line_number += 1 diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py index ac3560f1d..24edb6476 100644 --- a/gfauto/gfauto/fuzz.py +++ b/gfauto/gfauto/fuzz.py @@ -179,6 +179,8 @@ def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many force_no_stack_traces: bool, ) -> None: + util.update_gcov_environment_variable_if_needed() + try: artifact_util.artifact_path_get_root() except FileNotFoundError: diff --git a/gfauto/gfauto/gflogging.py b/gfauto/gfauto/gflogging.py index 0bc5949d5..e181d10d5 100644 --- a/gfauto/gfauto/gflogging.py +++ b/gfauto/gfauto/gflogging.py @@ -39,9 +39,9 @@ def pop_stream_for_logging() -> None: def log(message: str, skip_newline: bool = False) -> None: if _LOG_TO_STDOUT: if not skip_newline: - print(message, flush=True) # noqa T001 + print(message, flush=True) # noqa: T001 else: - print(message, end="", flush=True) # noqa T001 + print(message, end="", flush=True) # noqa: T001 for stream in _LOG_TO_STREAM: stream.write(message) if not skip_newline: diff --git a/gfauto/gfauto/host_device_util.py b/gfauto/gfauto/host_device_util.py index 41d39e483..ab6c2f8c9 100644 --- a/gfauto/gfauto/host_device_util.py +++ b/gfauto/gfauto/host_device_util.py @@ -23,7 +23,15 @@ from pathlib import Path from typing import Dict, Optional -from gfauto import devices_util, fuzz, gflogging, result_util, subprocess_util, util +from gfauto import ( + devices_util, + fuzz, + gflogging, + result_util, + subprocess_util, + types, + util, +) from gfauto.gflogging import log @@ -134,7 +142,7 @@ def run_amber_helper( status = "UNEXPECTED_ERROR" - result: Optional[subprocess.CompletedProcess] = None + result: Optional[types.CompletedProcess] = None env: Optional[Dict[str, str]] = None if icd: diff --git a/gfauto/gfauto/recipe_download_and_extract_archive_set.py b/gfauto/gfauto/recipe_download_and_extract_archive_set.py index c25b9547c..fc50b8f0a 100644 --- a/gfauto/gfauto/recipe_download_and_extract_archive_set.py +++ b/gfauto/gfauto/recipe_download_and_extract_archive_set.py @@ -32,7 +32,9 @@ from gfauto.gflogging import log from gfauto.recipe_pb2 import RecipeDownloadAndExtractArchiveSet -ALL_EXECUTABLE_PERMISSION_BITS = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH +ALL_EXECUTABLE_PERMISSION_BITS = ( + stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH # noqa: SC200 +) def recipe_download_and_extract_archive_set( diff --git a/gfauto/gfauto/run_bin.py b/gfauto/gfauto/run_bin.py index bdd4781fb..cea48e916 100644 --- a/gfauto/gfauto/run_bin.py +++ b/gfauto/gfauto/run_bin.py @@ -62,7 +62,7 @@ def main() -> int: cmd = [str(binary_manager.get_binary_path_by_name(binary_name).path)] cmd.extend(arguments) - return subprocess.run(cmd).returncode + return subprocess.run(cmd, check=False).returncode if __name__ == "__main__": diff --git a/gfauto/gfauto/signature_util.py b/gfauto/gfauto/signature_util.py index 814bc06fc..b27b3995c 100644 --- a/gfauto/gfauto/signature_util.py +++ b/gfauto/gfauto/signature_util.py @@ -20,8 +20,6 @@ stack trace. """ -# Disable spell-checking for this file. -# flake8: noqa: SC100 import re from pathlib import Path @@ -163,7 +161,7 @@ def basic_match(pattern: Pattern[str], log_contents: str) -> Optional[str]: def get_signature_from_log_contents( # pylint: disable=too-many-return-statements, too-many-branches, too-many-statements; - log_contents: str + log_contents: str, ) -> str: # noinspection PyUnusedLocal diff --git a/gfauto/gfauto/subprocess_util.py b/gfauto/gfauto/subprocess_util.py index 28d8b6eda..a4bace436 100644 --- a/gfauto/gfauto/subprocess_util.py +++ b/gfauto/gfauto/subprocess_util.py @@ -28,6 +28,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union +from gfauto import types from gfauto.gflogging import log from gfauto.util import check @@ -49,7 +50,7 @@ def log_stdout_stderr_helper(stdout: str, stderr: str) -> None: def log_stdout_stderr( result: Union[ subprocess.CalledProcessError, - subprocess.CompletedProcess, + types.CompletedProcess, subprocess.TimeoutExpired, ], ) -> None: @@ -61,14 +62,12 @@ def log_returncode_helper(returncode: int) -> None: def log_returncode( - result: Union[ - subprocess.CalledProcessError, subprocess.CompletedProcess, subprocess.Popen - ], + result: Union[subprocess.CalledProcessError, types.CompletedProcess, types.Popen], ) -> None: log_returncode_helper(result.returncode) -def posix_kill_group(process: subprocess.Popen) -> None: +def posix_kill_group(process: types.Popen) -> None: # Work around type warnings that will only show up on Windows: os_alias: Any = os os_alias.killpg(process.pid, signal.SIGTERM) @@ -82,7 +81,7 @@ def run_helper( timeout: Optional[float] = None, env: Optional[Dict[str, str]] = None, working_dir: Optional[Path] = None, -) -> subprocess.CompletedProcess: +) -> types.CompletedProcess: check( bool(cmd) and cmd[0] is not None and isinstance(cmd[0], str), AssertionError("run takes a list of str, not a str"), @@ -134,7 +133,7 @@ def run( verbose: bool = False, env: Optional[Dict[str, str]] = None, working_dir: Optional[Path] = None, -) -> subprocess.CompletedProcess: +) -> types.CompletedProcess: log("Exec" + (" (verbose):" if verbose else ":") + str(cmd)) try: result = run_helper(cmd, check_exit_code, timeout, env, working_dir) diff --git a/gfauto/gfauto/types.py b/gfauto/gfauto/types.py new file mode 100644 index 000000000..ccdb6280a --- /dev/null +++ b/gfauto/gfauto/types.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Types module. + +Defines types that cause issues with the type checker and/or other error checking. +""" + +import subprocess +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + CompletedProcess = subprocess.CompletedProcess[Any] # pylint: disable=invalid-name; + Popen = subprocess.Popen[Any] # pylint: disable=invalid-name; +else: + CompletedProcess = subprocess.CompletedProcess # pylint: disable=invalid-name; + Popen = subprocess.Popen # pylint: disable=invalid-name; diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py index f5d6e8dc4..64a89c6cf 100644 --- a/gfauto/gfauto/util.py +++ b/gfauto/gfauto/util.py @@ -233,7 +233,7 @@ def make_directory_symlink(new_symlink_file_path: Path, existing_dir: Path) -> P # Retry using junctions under Windows. try: # noinspection PyUnresolvedReferences - import _winapi # pylint: disable=import-error; + import _winapi # pylint: disable=import-error,import-outside-toplevel; # Unlike symlink_to, CreateJunction takes a path relative to the current directory. _winapi.CreateJunction(str(existing_dir), str(new_symlink_file_path)) @@ -333,3 +333,18 @@ def create_zip(output_file_path: Path, entries: List[ZipEntry]) -> Path: def get_random_name() -> str: return uuid.uuid4().hex + + +def update_gcov_environment_variable_if_needed() -> None: + if "GCOV_PREFIX" in os.environ.keys(): + gcov_prefix: str = os.environ["GCOV_PREFIX"] + if "PROC_ID" in gcov_prefix: + pid = str(os.getpid()) + check( + bool(pid), + AssertionError( + "Failed to get process ID to replace PROC_ID in GCOV_PREFIX environment variable." + ), + ) + gcov_prefix = gcov_prefix.replace("PROC_ID", pid) + os.environ["GCOV_PREFIX"] = gcov_prefix diff --git a/gfauto/pylintrc b/gfauto/pylintrc index 7f81be881..153828859 100644 --- a/gfauto/pylintrc +++ b/gfauto/pylintrc @@ -77,14 +77,15 @@ confidence= # --disable=W". disable=C0330, C0111, - W0511, # Allow to-dos. - E1101, # Missing member; mypy (type checking) should cover this. - R0913, # Too many arguments; could re-enable this. - C0301, # Line too long; we use an auto formatter. - R0903, # too-few-public-methods; good, but I want to store data in classes in case I later add methods. - ungrouped-imports, # We use an imports formatter. - cyclic-import, # We allow cyclic imports, but we should import modules, not functions and variables. - duplicate-code, # Unfortunately, disabling this on a case-by-case basis is broken: https://github.com/PyCQA/pylint/issues/214 + W0511, # Allow to-dos. + E1101, # Missing member; mypy (type checking) should cover this. + R0913, # Too many arguments; could re-enable this. + C0301, # Line too long; we use an auto formatter. + R0903, # too-few-public-methods; good, but I want to store data in classes in case I later add methods. + ungrouped-imports, # We use an imports formatter. + cyclic-import, # We allow cyclic imports, but we should import modules, not functions and variables. + duplicate-code, # Unfortunately, disabling this on a case-by-case basis is broken: https://github.com/PyCQA/pylint/issues/214 + unsubscriptable-object, # False-positives for type hints. E.g. CompletedProcess[Any]. Mypy should cover this. @@ -106,7 +107,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details. -#msg-template= +msg-template='{abspath}:{line:d}:{column}: {obj}: [{msg_id} {symbol}] {msg}' # Set the output format. Available formats are text, parseable, colorized, json # and msvs (visual studio). You can also give a reporter class, e.g. diff --git a/gfauto/setup.py b/gfauto/setup.py index cd1140a18..1199bc9d0 100644 --- a/gfauto/setup.py +++ b/gfauto/setup.py @@ -53,5 +53,9 @@ "gfauto_download_cts_gf_tests = gfauto.download_cts_gf_tests:main", "gfauto_run_cts_gf_tests = gfauto.run_cts_gf_tests:main", "gfauto_run_bin = gfauto.run_bin:main", + "gfauto_cov_merge = gfauto.cov_merge:main", + "gfauto_cov_new = gfauto.cov_new:main", + "gfauto_cov_to_source = gfauto.cov_to_source:main", + "gfauto_cov_from_gcov = gfauto.cov_from_gcov:main", ]}, ) diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic index a520b5fe1..49df03a0b 100644 --- a/gfauto/whitelist.dic +++ b/gfauto/whitelist.dic @@ -202,3 +202,28 @@ protos llpc rfind spvas +gcov +gcovs +gcda +gcno +mScreenState +SIGTERM +SIGKILL +cov +formatter +Formatter +TODO +setdefault +lcount +unexecuted +Fallthrough +symlinks +T001 +isabs +SC200 +SC100 +PROC +listdir +RNG +toplevel +getpid From ccdbf72d781f820fb50d2d5abe0205626ba2619e Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 6 Dec 2019 11:59:20 +0000 Subject: [PATCH 173/180] Stable version of colorgrid modulo. (#806) --- .../310es/stable_colorgrid_modulo.frag | 61 +++++++++++++++++++ .../310es/stable_colorgrid_modulo.json | 9 +++ 2 files changed, 70 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.frag create mode 100644 shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.json diff --git a/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.frag b/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.frag new file mode 100644 index 000000000..86d654bd6 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.frag @@ -0,0 +1,61 @@ +#version 310 es + +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +float compute_value(float limit, float thirty_two) { + float result = -0.5; + + // The loop will not really execute 800 times, as 'limit' is bounded by gl_FragCoord, and there is an early exit based on 'limit'. + for (int i = 1; i < 800; i++) { + if ((i % 32) == 0) { + // Avoid computing e.g. mod(float(32), round(32.0)), which could be sensitive to round-off if mutated. + result += 0.4; + } else if (mod(float(i), round(thirty_two)) <= 0.01) { + // This should never get executed, because the previous if condition would get triggered. + result += 100.0; + } + if (float(i) >= limit) { + return result; + } + } + return result; +} + +void main() +{ + vec3 c = vec3(7.0, 8.0, 9.0); + + // This is guaranteed to be 32.0, as resolution.x is 256.0. + float thirty_two = round(resolution.x / 8.0); + + c.x = compute_value(gl_FragCoord.x, thirty_two); + c.y = compute_value(gl_FragCoord.y, thirty_two); + c.z = c.x + c.y; + + for (int i = 0; i < 3; i++) { + if (c[i] >= 1.0) { + c[i] = c[i] * c[i]; + } + } + _GLF_color = vec4(normalize(abs(c)), 1.0); +} diff --git a/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.json b/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.json new file mode 100644 index 000000000..269b946d9 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From 6aedac73c1636b66c14c27774ffb2bc40b2718ba Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 6 Dec 2019 11:59:43 +0000 Subject: [PATCH 174/180] Stable version of mergesort shader. (#803) --- .../glsl/samples/310es/mergesort_mosaic.frag | 28 ++-- .../glsl/samples/310es/stable_mergesort.frag | 140 ++++++++++++++++++ .../glsl/samples/310es/stable_mergesort.json | 16 ++ 3 files changed, 170 insertions(+), 14 deletions(-) create mode 100644 shaders/src/main/glsl/samples/310es/stable_mergesort.frag create mode 100644 shaders/src/main/glsl/samples/310es/stable_mergesort.json diff --git a/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag index ba5da010a..2ce17a2ee 100644 --- a/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag +++ b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag @@ -77,34 +77,34 @@ void main() { switch(i) { case 0: data[i] = 4; - break; + break; case 1: - data[i] = 3 ; - break; + data[i] = 3; + break; case 2: - data[i] = 2 ; - break; + data[i] = 2; + break; case 3: - data[i] = 1 ; - break; + data[i] = 1; + break; case 4: - data[i] = 0 ; - break; + data[i] = 0; + break; case 5: data[i] = -1; - break; + break; case 6: data[i] = -2; - break; + break; case 7: data[i] = -3; - break; + break; case 8: data[i] = -4; - break; + break; case 9: data[i] = -5; - break; + break; } i++; } while (i < 10); diff --git a/shaders/src/main/glsl/samples/310es/stable_mergesort.frag b/shaders/src/main/glsl/samples/310es/stable_mergesort.frag new file mode 100644 index 000000000..54e8ffe82 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_mergesort.frag @@ -0,0 +1,140 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +// Size of an array. +const int N = 10; +// An array and its temperary array whose elements will be sorted. +int data[10], temp[10]; + +// Merge two sorted subarrays data[from ... mid] and data[mid+1 ... to]. +void merge(int from, int mid, int to) { + int k = from, i = from, j = mid +1; + + while (i <= mid && j <= to) { + if (data[i] < data[j]) { + temp[k++] = data[i++]; + } else { + temp[k++] = data[j++]; + } + } + // Copy remaining elements. + while (i < N && i <= mid) { + temp[k++] = data[i++]; + } + // Copy back to the original array. + for (int i = from; i <= to; i++) { + data[i] = temp[i]; + } +} + +// Sort array using the iterative approach. +void mergeSort() { + int low = 0; + int high = N - 1; + + // Devide the array into blocks of size m. + // m = [1, 2, 4 ,8, 16...]. + for (int m = 1; m <= high; m = 2 * m) { + + // For m = 1, i = [0, 2, 4, 6, 8]. + // For m = 2, i = [0, 4, 8]. + // For m = 4, i = [0, 8]. + for (int i = low; i< high; i += 2 * m) { + int from = i; + int mid = i + m - 1; + int to = min (i + 2 * m - 1, high); + merge(from, mid, to); + } + } +} + +void main() { + int i = int(injectionSwitch.x); + do { + switch(i) { + case 0: + data[i] = 4; + break; + case 1: + data[i] = 3; + break; + case 2: + data[i] = 2; + break; + case 3: + data[i] = 1; + break; + case 4: + data[i] = 0; + break; + case 5: + data[i] = -1; + break; + case 6: + data[i] = -2; + break; + case 7: + data[i] = -3; + break; + case 8: + data[i] = -4; + break; + case 9: + data[i] = -5; + break; + } + i++; + } while (i < 10); + + for (int j = 0; j < 10; j++) { + temp[j] = data[j]; + } + mergeSort(); + + float grey; + if(int(gl_FragCoord[1]) < 30) { + grey = 0.5 + float(data[0]) / 10.0; // data[0] should be -5, this should end up being 0.0 + } else if(int(gl_FragCoord[1]) < 60) { + grey = 0.5 + float(data[1]) / 10.0; // data[1] should be -4, this should end up being 0.1 + } else if(int(gl_FragCoord[1]) < 90) { + grey = 0.5 + float(data[2]) / 10.0; // similar + } else if(int(gl_FragCoord[1]) < 120) { + grey = 0.5 + float(data[3]) / 10.0; // similar + } else if(int(gl_FragCoord[1]) < 150) { + discard; // Deliberately discard this one + } else if(int(gl_FragCoord[1]) < 180) { + grey = 0.5 + float(data[5]) / 10.0; + } else if(int(gl_FragCoord[1]) < 210) { + grey = 0.5 + float(data[6]) / 10.0; + } else if(int(gl_FragCoord[1]) < 240) { + grey = 0.5 + float(data[7]) / 10.0; + } else if(int(gl_FragCoord[1]) < 270) { + grey = 0.5 + float(data[8]) / 10.0; + } else { + discard; + } + _GLF_color = vec4(vec3(grey), 1.0); +} diff --git a/shaders/src/main/glsl/samples/310es/stable_mergesort.json b/shaders/src/main/glsl/samples/310es/stable_mergesort.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_mergesort.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From edd1b8db8bd67ad0e6ede4485090640e44d7e859 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 6 Dec 2019 16:23:07 +0000 Subject: [PATCH 175/180] Removed redundant assembly-public module. (#809) --- assembly-public/pom.xml | 196 --------------------- assembly-public/src/main/resources/HASH | 1 - assembly-public/src/main/scripts/README.md | 0 parent-all/pom.xml | 5 - pom.xml | 1 - 5 files changed, 203 deletions(-) delete mode 100644 assembly-public/pom.xml delete mode 100644 assembly-public/src/main/resources/HASH delete mode 100644 assembly-public/src/main/scripts/README.md diff --git a/assembly-public/pom.xml b/assembly-public/pom.xml deleted file mode 100644 index 329a7ac4b..000000000 --- a/assembly-public/pom.xml +++ /dev/null @@ -1,196 +0,0 @@ - - - - 4.0.0 - assembly-public - assembly-public - pom - - - com.graphicsfuzz - parent-all - 1.0 - ../parent-all/pom.xml - - - - - - - com.graphicsfuzz - server-public - - - - com.graphicsfuzz - assembly-binaries - zip - provided - - - - com.graphicsfuzz - thrift - python - zip - provided - - - - com.graphicsfuzz - server-static-public - zip - provided - - - - com.graphicsfuzz - thrift - js - zip - provided - - - - com.graphicsfuzz.thirdparty - jquery-js - zip - provided - - - - ant-contrib - ant-contrib - provided - - - - - - - - - pl.project13.maven - git-commit-id-plugin - - - get-the-git-infos - - revision - - initialize - - - - false - - 40 - - - - - - maven-dependency-plugin - - - dep-1 - - copy-dependencies - - generate-sources - - runtime - true - - - - - - - maven-antrun-plugin - - - run-ant - compile - - run - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/assembly-public/src/main/resources/HASH b/assembly-public/src/main/resources/HASH deleted file mode 100644 index f53be555b..000000000 --- a/assembly-public/src/main/resources/HASH +++ /dev/null @@ -1 +0,0 @@ -${git.commit.id.describe} \ No newline at end of file diff --git a/assembly-public/src/main/scripts/README.md b/assembly-public/src/main/scripts/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/parent-all/pom.xml b/parent-all/pom.xml index 853b0261a..c9f156544 100644 --- a/parent-all/pom.xml +++ b/parent-all/pom.xml @@ -266,11 +266,6 @@ limitations under the License. assembly-desktop-client 1.0 - - com.graphicsfuzz - assembly-public - 1.0 - com.graphicsfuzz assembly-webglfuzz diff --git a/pom.xml b/pom.xml index 84707bc4a..9b1e2ed68 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,6 @@ limitations under the License. assembly-binaries - assembly-public ast checkstyle-config client-tests From 2c5ff33ba67cd47ad9901b78af852d3a96bfe98f Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 6 Dec 2019 16:23:26 +0000 Subject: [PATCH 176/180] Apply rounding to length of normalized vector (#807) We compute 1.0 as the length of a normalized vector. Round-off might mean that we get a result slightly different from 1.0. This change introduces a call to 'round', so we compute 1.0 as 'round(length(normalize(non_zero_vector)', to guard against this. The use of 'round' means that the transformation is only applicable for GLSL versions that support 'round'. --- .../fuzzer/OpaqueExpressionGenerator.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index 614f45a20..586f2f867 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -136,7 +136,9 @@ List waysToMakeOne() { opaqueOneFactories.addAll(waysToMakeZeroOrOne()); opaqueOneFactories.add(this::opaqueOneExponential); opaqueOneFactories.add(this::opaqueOneCosine); - opaqueOneFactories.add(this::opaqueOneNormalizedVectorLength); + if (shadingLanguageVersion.supportedRound()) { + opaqueOneFactories.add(this::opaqueOneRoundedNormalizedVectorLength); + } return opaqueOneFactories; } @@ -646,10 +648,12 @@ private Optional opaqueOneCosine(BasicType type, boolean constContext, fin fuzzer))); } - private Optional opaqueOneNormalizedVectorLength(BasicType type, boolean constContext, - final int depth, - Fuzzer fuzzer, boolean isZero) { - // represent 1 as the length of a normalized vector + private Optional opaqueOneRoundedNormalizedVectorLength(BasicType type, + boolean constContext, + final int depth, + Fuzzer fuzzer, + boolean isZero) { + // represent 1 as the rounded length of a normalized vector assert !isZero; if (type != BasicType.FLOAT) { // 'length' has return type 'float', so we can only create a scalar floating-point zero. @@ -660,11 +664,13 @@ private Optional opaqueOneNormalizedVectorLength(BasicType type, boolean c final BasicType vectorType = BasicType.allGenTypes().get(generator.nextInt(BasicType.allGenTypes().size())); - // We create a vector of ones and normalize it. Note that we could be more general and create - // any non-zero vector and normalize it. - Expr normalizedExpr = new FunctionCallExpr("normalize", makeOpaqueZeroOrOne(false, - vectorType, constContext, depth, fuzzer)); - return Optional.of(new FunctionCallExpr("length", normalizedExpr)); + // We create a vector of ones and normalize it, rounding the result to guard against the case + // where round-off leads to a result that is not quite one. Note that we could be more general + // here and normalize any non-zero vector. + return Optional.of(new FunctionCallExpr("round", + new FunctionCallExpr("length", + new FunctionCallExpr("normalize", + makeOpaqueZeroOrOne(false, vectorType, constContext, depth, fuzzer))))); } private List numericTypesOrGenTypesIfEssl100() { From 59c73bf0593856cb96d212e5296a8f73688d38a7 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 6 Dec 2019 16:24:11 +0000 Subject: [PATCH 177/180] Stable bubblesort (#808) Change name of bubblesort_flag shader to indicate that it is stable. Partly fixes #802. --- build/travis/cmd-tests.sh | 8 ++++---- .../generator/tool/GenerateShaderFamilyTest.java | 4 ++-- .../com/graphicsfuzz/generator/tool/GlslGenerateTest.java | 2 +- .../{bubblesort_flag.frag => stable_bubblesort_flag.frag} | 0 .../{bubblesort_flag.json => stable_bubblesort_flag.json} | 0 .../{bubblesort_flag.frag => stable_bubblesort_flag.frag} | 0 .../{bubblesort_flag.json => stable_bubblesort_flag.json} | 0 .../{bubblesort_flag.frag => stable_bubblesort_flag.frag} | 0 .../{bubblesort_flag.json => stable_bubblesort_flag.json} | 0 .../{bubblesort_flag.frag => stable_bubblesort_flag.frag} | 0 .../{bubblesort_flag.json => stable_bubblesort_flag.json} | 0 .../{bubblesort_flag.frag => stable_bubblesort_flag.frag} | 0 .../{bubblesort_flag.json => stable_bubblesort_flag.json} | 0 .../{bubblesort_flag.frag => stable_bubblesort_flag.frag} | 0 .../{bubblesort_flag.json => stable_bubblesort_flag.json} | 0 15 files changed, 7 insertions(+), 7 deletions(-) rename shaders/src/main/glsl/samples/100/{bubblesort_flag.frag => stable_bubblesort_flag.frag} (100%) rename shaders/src/main/glsl/samples/100/{bubblesort_flag.json => stable_bubblesort_flag.json} (100%) rename shaders/src/main/glsl/samples/300es/{bubblesort_flag.frag => stable_bubblesort_flag.frag} (100%) rename shaders/src/main/glsl/samples/300es/{bubblesort_flag.json => stable_bubblesort_flag.json} (100%) rename shaders/src/main/glsl/samples/310es/{bubblesort_flag.frag => stable_bubblesort_flag.frag} (100%) rename shaders/src/main/glsl/samples/310es/{bubblesort_flag.json => stable_bubblesort_flag.json} (100%) rename shaders/src/main/glsl/samples/donors/{bubblesort_flag.frag => stable_bubblesort_flag.frag} (100%) rename shaders/src/main/glsl/samples/donors/{bubblesort_flag.json => stable_bubblesort_flag.json} (100%) rename shaders/src/main/glsl/samples/webgl1/{bubblesort_flag.frag => stable_bubblesort_flag.frag} (100%) rename shaders/src/main/glsl/samples/webgl1/{bubblesort_flag.json => stable_bubblesort_flag.json} (100%) rename shaders/src/main/glsl/samples/webgl2/{bubblesort_flag.frag => stable_bubblesort_flag.frag} (100%) rename shaders/src/main/glsl/samples/webgl2/{bubblesort_flag.json => stable_bubblesort_flag.json} (100%) diff --git a/build/travis/cmd-tests.sh b/build/travis/cmd-tests.sh index c0345ff5d..1bf943750 100755 --- a/build/travis/cmd-tests.sh +++ b/build/travis/cmd-tests.sh @@ -59,9 +59,9 @@ glsl-generate --seed 0 samples/300es samples/donors 10 family_300es work/shaderf glsl-generate --seed 0 samples/100 samples/donors 10 family_100 work/shaderfamilies >/dev/null glsl-generate --seed 0 --generate-uniform-bindings --max-uniforms 10 samples/310es samples/donors 10 family_vulkan work/shaderfamilies >/dev/null -test -d "work/shaderfamilies/family_100_bubblesort_flag" -test -d "work/shaderfamilies/family_300es_bubblesort_flag" -test -d "work/shaderfamilies/family_vulkan_bubblesort_flag" +test -d "work/shaderfamilies/family_100_stable_bubblesort_flag" +test -d "work/shaderfamilies/family_300es_stable_bubblesort_flag" +test -d "work/shaderfamilies/family_vulkan_stable_bubblesort_flag" ### Reduce examples. @@ -102,7 +102,7 @@ test "${EXIT_CODE}" -eq 143 ### Check some binaries. -SHADER=work/shaderfamilies/family_100_bubblesort_flag/reference.frag +SHADER=work/shaderfamilies/family_100_stable_bubblesort_flag/reference.frag glslangValidator "${SHADER}" shader_translator "${SHADER}" >/dev/null diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java index 32929461c..6a2fa282c 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java @@ -42,7 +42,7 @@ public class GenerateShaderFamilyTest { @Test public void testGenerateSmall100ShaderFamily() throws Exception { final String samplesSubdir = "100"; - final String referenceShaderName = "bubblesort_flag"; + final String referenceShaderName = "stable_bubblesort_flag"; final int numVariants = 3; int seed = 0; checkShaderFamilyGeneration(samplesSubdir, referenceShaderName, numVariants, @@ -136,7 +136,7 @@ public void testGenerateSmallWebGL2ShaderFamilyMultiPass() throws Exception { @Test public void testGenerateSmallVulkanShaderFamilyMultiPass() throws Exception { final String samplesSubdir = "310es"; - final String referenceShaderName = "bubblesort_flag"; + final String referenceShaderName = "stable_bubblesort_flag"; final int numVariants = 3; int seed = 9; checkShaderFamilyGeneration(samplesSubdir, referenceShaderName, numVariants, diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java index 3986984fa..679cbf4ea 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java @@ -150,7 +150,7 @@ private void checkFragmentShaderFamilyGeneration(String references, generateShaderFamily(references, donors, numVariants, prefix, outputDir, seed, extraArgs, true); - for (String reference : Arrays.asList("bubblesort_flag", "colorgrid_modulo", + for (String reference : Arrays.asList("stable_bubblesort_flag", "colorgrid_modulo", "mandelbrot_zoom", "prefix_sum", "squares")) { final File expectedOutputDirectory = new File(temporaryFolder.getRoot(), prefix + "_" + reference); diff --git a/shaders/src/main/glsl/samples/100/bubblesort_flag.frag b/shaders/src/main/glsl/samples/100/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/100/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/100/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/100/bubblesort_flag.json b/shaders/src/main/glsl/samples/100/stable_bubblesort_flag.json similarity index 100% rename from shaders/src/main/glsl/samples/100/bubblesort_flag.json rename to shaders/src/main/glsl/samples/100/stable_bubblesort_flag.json diff --git a/shaders/src/main/glsl/samples/300es/bubblesort_flag.frag b/shaders/src/main/glsl/samples/300es/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/300es/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/300es/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/300es/bubblesort_flag.json b/shaders/src/main/glsl/samples/300es/stable_bubblesort_flag.json similarity index 100% rename from shaders/src/main/glsl/samples/300es/bubblesort_flag.json rename to shaders/src/main/glsl/samples/300es/stable_bubblesort_flag.json diff --git a/shaders/src/main/glsl/samples/310es/bubblesort_flag.frag b/shaders/src/main/glsl/samples/310es/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/310es/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/310es/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/310es/bubblesort_flag.json b/shaders/src/main/glsl/samples/310es/stable_bubblesort_flag.json similarity index 100% rename from shaders/src/main/glsl/samples/310es/bubblesort_flag.json rename to shaders/src/main/glsl/samples/310es/stable_bubblesort_flag.json diff --git a/shaders/src/main/glsl/samples/donors/bubblesort_flag.frag b/shaders/src/main/glsl/samples/donors/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/donors/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/donors/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/donors/bubblesort_flag.json b/shaders/src/main/glsl/samples/donors/stable_bubblesort_flag.json similarity index 100% rename from shaders/src/main/glsl/samples/donors/bubblesort_flag.json rename to shaders/src/main/glsl/samples/donors/stable_bubblesort_flag.json diff --git a/shaders/src/main/glsl/samples/webgl1/bubblesort_flag.frag b/shaders/src/main/glsl/samples/webgl1/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/webgl1/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/webgl1/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/webgl1/bubblesort_flag.json b/shaders/src/main/glsl/samples/webgl1/stable_bubblesort_flag.json similarity index 100% rename from shaders/src/main/glsl/samples/webgl1/bubblesort_flag.json rename to shaders/src/main/glsl/samples/webgl1/stable_bubblesort_flag.json diff --git a/shaders/src/main/glsl/samples/webgl2/bubblesort_flag.frag b/shaders/src/main/glsl/samples/webgl2/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/webgl2/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/webgl2/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/webgl2/bubblesort_flag.json b/shaders/src/main/glsl/samples/webgl2/stable_bubblesort_flag.json similarity index 100% rename from shaders/src/main/glsl/samples/webgl2/bubblesort_flag.json rename to shaders/src/main/glsl/samples/webgl2/stable_bubblesort_flag.json From 7b143bcb3ad38b64ddc17d132886636b229b6684 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 6 Dec 2019 16:24:38 +0000 Subject: [PATCH 178/180] Stable version of quicksort shader. (#805) Partly fixes #802. --- .../glsl/samples/310es/stable_quicksort.frag | 112 ++++++++++++++++++ .../glsl/samples/310es/stable_quicksort.json | 9 ++ 2 files changed, 121 insertions(+) create mode 100644 shaders/src/main/glsl/samples/310es/stable_quicksort.frag create mode 100644 shaders/src/main/glsl/samples/310es/stable_quicksort.json diff --git a/shaders/src/main/glsl/samples/310es/stable_quicksort.frag b/shaders/src/main/glsl/samples/310es/stable_quicksort.frag new file mode 100644 index 000000000..0078f1984 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_quicksort.frag @@ -0,0 +1,112 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +struct QuicksortObject{ + int numbers[10]; +}; + +QuicksortObject obj; + + void swap(int i, int j) { + int temp = obj.numbers[i]; + obj.numbers[i] = obj.numbers[j]; + obj.numbers[j] = temp; + } + + // Since "partition" is the preserved word, we add prefix to this function name to prevent an error. + int performPartition(int l, int h) { + // The rightmost element is chosen as a pivot. + int pivot = obj.numbers[h]; + int i = (l - 1); + + for (int j = l; j <= h - 1; j++) { + if (obj.numbers[j] <= pivot) { + i++; + swap(i, j); + } + } + swap(i + 1, h); + return (i + 1); + } + + void quicksort() { + int l = 0, h = 9; + int stack[10]; + int top = -1; + + stack[++top] = l; + stack[++top] = h; + + while (top >= 0) { + h = stack[top--]; + l = stack[top--]; + + int p = performPartition(l, h); + if (p - 1 > l) { + stack[++top] = l; + stack[++top] = p - 1; + } + if (p + 1 < h) { + stack[++top] = p + 1; + stack[++top] = h; + } + } + } + +void main() { + // Initialize decreasing values to an array starting from 10. + for (int i = 0; i < 10; i ++) { + obj.numbers[i] = (10 - i); + obj.numbers[i] = obj.numbers[i] * obj.numbers[i]; + } + quicksort(); + vec2 uv = gl_FragCoord.xy / resolution; + vec3 color = vec3(1.0, 2.0, 3.0); + color.x += float(obj.numbers[0]); + if (uv.x > (1.0 / 4.0)) { + color.x += float(obj.numbers[1]); + } + if (uv.x > (2.0 / 4.0)) { + color.y += float(obj.numbers[2]); + } + if(uv.x > (3.0 / 4.0)) { + color.z += float(obj.numbers[3]); + } + color.y += float(obj.numbers[4]); + if (uv.y > (1.0 / 4.0)) { + color.x += float(obj.numbers[5]); + } + if (uv.y > (2.0 / 4.0)) { + color.y += float(obj.numbers[6]); + } + if(uv.y > (3.0 / 4.0)) { + color.z += float(obj.numbers[7]); + } + color.z += float(obj.numbers[8]); + if (abs(uv.x - uv.y) < 0.25) { + color.x += float(obj.numbers[9]); + } + _GLF_color = vec4(normalize(color), 1.0); + +} diff --git a/shaders/src/main/glsl/samples/310es/stable_quicksort.json b/shaders/src/main/glsl/samples/310es/stable_quicksort.json new file mode 100644 index 000000000..28c3c3e61 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_quicksort.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} From fc0c75414064aec6100f5ac6fb1d9da5325e6315 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Tue, 10 Dec 2019 14:50:58 +0000 Subject: [PATCH 179/180] Stable version of binary search tree shader (#810) Adds an obviously numerically stable version of the binary search tree shader. Eliminates the "simple" version of the shader, which was very similar to another version of the shader and not obviously numerically stable. Avoids using "injectionSwitch" in the stable and original versions of the binary search tree shader. Partly fixes #802. --- .../300es/binarysearch_tree_simple.frag | 173 ------------------ .../300es/binarysearch_tree_simple.json | 16 -- .../glsl/samples/310es/binarysearch_tree.frag | 12 +- .../glsl/samples/310es/binarysearch_tree.json | 7 - .../310es/binarysearch_tree_simple.json | 16 -- ...ple.frag => stable_binarysearch_tree.frag} | 74 +++----- .../310es/stable_binarysearch_tree.json | 1 + 7 files changed, 36 insertions(+), 263 deletions(-) delete mode 100644 shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.frag delete mode 100644 shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.json delete mode 100644 shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.json rename shaders/src/main/glsl/samples/310es/{binarysearch_tree_simple.frag => stable_binarysearch_tree.frag} (76%) create mode 100644 shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.json diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.frag b/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.frag deleted file mode 100644 index 25e67e16f..000000000 --- a/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.frag +++ /dev/null @@ -1,173 +0,0 @@ -#version 300 es - -/* - * Copyright 2019 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -precision highp float; - -layout(location = 0) out vec4 _GLF_color; - -uniform vec2 injectionSwitch; - -uniform vec2 resolution; - -struct BST{ - int data; - int leftIndex; - int rightIndex; -}; - -BST tree[10]; - -void makeTreeNode(inout BST tree, int data) -{ - tree.data = data; - tree.leftIndex = -1; - tree.rightIndex = -1; -} - -void insert(int treeIndex, int data) -{ - int baseIndex = 0; - while (baseIndex <= treeIndex) { - // If new value is smaller thatn the current node, we known that we will have - // add this element in the left side. - if (data <= tree[baseIndex].data) { - // If a left subtree of the current node is empty, the new node is added as - // a left subtree of the current node. - if (tree[baseIndex].leftIndex == -1) { - tree[baseIndex].leftIndex = treeIndex; - makeTreeNode(tree[treeIndex], data); - return; - } else { - baseIndex = tree[baseIndex].leftIndex; - continue; - } - } else { - // If a right subtree of the current node is empty, the new node is added as - // a right subtree of the current node. - if (tree[baseIndex].rightIndex == -1) { - tree[baseIndex].rightIndex = treeIndex; - makeTreeNode(tree[treeIndex], data); - return; - } else { - baseIndex = tree[baseIndex].rightIndex; - continue; - } - } - } -} - -// Return element data if the given target exists in a tree. Otherwise, we simply return -1. -int search(int target){ - BST currentNode; - int index = 0; - while (index != -1) { - currentNode = tree[index]; - if (currentNode.data == target) { - return target; - } - index = target > currentNode.data ? currentNode.rightIndex : currentNode.leftIndex; - } - return -1; -} - -vec3 hueColor(float angle) { - float nodeData = float(search(15)); - return (30.0 + angle * vec3(1.0, 5.0, nodeData)) / 50.0; -} - -float makeFrame(float v) { - v *= 6.5; - if (v < 1.5) { - return float(search(100)); - } - if (v < 4.0) { - return injectionSwitch.x; - } - if (v < float(search(6))) { - return 1.0; - } - return 10.0 + float(search(30)); -} - -/* -* This shader implements binary search tree using an array data structure. The elements of -* tree are kept in the array that contains a list of BST object holding indices of left and -* right subtree in the array. -* -* - Tree representation of the number used in this shader: -* 9 -* / \ -* 5 12 -* / \ \ -* 2 7 15 -* / \ / \ -* 6 8 13 17 -* -* - Array representation: -* [9, 5, 12, 15, 7, 8, 2, 6, 17, 13] -* -*/ - -void main() { - int treeIndex = int(injectionSwitch.x); - // Initialize root node. - makeTreeNode(tree[int(injectionSwitch.x)], 9); - // Each time we insert a new node into the tree, we increment one. - treeIndex++; - - insert(treeIndex, 5); - treeIndex++; - insert(treeIndex, 12); - treeIndex++; - insert(treeIndex, 15); - treeIndex++; - insert(treeIndex, 7); - treeIndex++; - insert(treeIndex, 8); - treeIndex++; - insert(treeIndex, 2); - treeIndex++; - insert(treeIndex, 6); - treeIndex++; - insert(treeIndex, 17); - treeIndex++; - insert(treeIndex, 13); - - vec2 z = (gl_FragCoord.yx / resolution); - float x = makeFrame(z.x); - float y = makeFrame(z.y); - - int sum = -100; - for (int target = 0; target < 20; target ++) { - int result = search(target); - if (result > 0) { - sum += result; - } else { - switch (result) { - case -1: - sum += int(injectionSwitch.y); - break; - case 0: - return; - } - } - } - float a = x + y * float(sum); - _GLF_color = vec4(hueColor(a), 1.); - -} diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.json b/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.json deleted file mode 100644 index d7b58d068..000000000 --- a/shaders/src/main/glsl/samples/300es/binarysearch_tree_simple.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "injectionSwitch": { - "func": "glUniform2f", - "args": [ - 0.0, - 1.0 - ] - }, - "resolution": { - "func": "glUniform2f", - "args": [ - 256.0, - 256.0 - ] - } -} diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag b/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag index 3810b48b6..47a58490c 100644 --- a/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag +++ b/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag @@ -20,8 +20,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; - uniform vec2 resolution; struct BST{ @@ -89,7 +87,7 @@ vec3 hueColor(float angle) { float nodeData = float(search(15)); vec3 color; color = clamp(fract(angle * vec3(1.0, 5.0, nodeData)), 0.0, 1.0); - color.x *= cosh(isnan(float(search(30))) ? injectionSwitch.x : injectionSwitch.y); + color.x *= cosh(isnan(float(search(30))) ? 0.0 : 1.0); return color; } @@ -99,7 +97,7 @@ float makeFrame(float v) { return float(search(100)); } if (v < 4.0) { - return injectionSwitch.x; + return 0.0; } if (v < float(search(6))) { return 1.0; @@ -127,9 +125,9 @@ float makeFrame(float v) { */ void main() { - int treeIndex = int(injectionSwitch.x); + int treeIndex = 0; // Initialize root node. - makeTreeNode(tree[int(injectionSwitch.x)], 9); + makeTreeNode(tree[0], 9); // Each time we insert a new node into the tree, we increment one. treeIndex++; @@ -163,7 +161,7 @@ void main() { } else { switch (result) { case -1: - sum += int(injectionSwitch.y); + sum += 1; break; case 0: return; diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree.json b/shaders/src/main/glsl/samples/310es/binarysearch_tree.json index d7b58d068..269b946d9 100644 --- a/shaders/src/main/glsl/samples/310es/binarysearch_tree.json +++ b/shaders/src/main/glsl/samples/310es/binarysearch_tree.json @@ -1,11 +1,4 @@ { - "injectionSwitch": { - "func": "glUniform2f", - "args": [ - 0.0, - 1.0 - ] - }, "resolution": { "func": "glUniform2f", "args": [ diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.json b/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.json deleted file mode 100644 index d7b58d068..000000000 --- a/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "injectionSwitch": { - "func": "glUniform2f", - "args": [ - 0.0, - 1.0 - ] - }, - "resolution": { - "func": "glUniform2f", - "args": [ - 256.0, - 256.0 - ] - } -} diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.frag b/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.frag similarity index 76% rename from shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.frag rename to shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.frag index 0df0eee8b..5829c1b26 100644 --- a/shaders/src/main/glsl/samples/310es/binarysearch_tree_simple.frag +++ b/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.frag @@ -20,10 +20,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; - -uniform vec2 resolution; - struct BST{ int data; int leftIndex; @@ -85,25 +81,6 @@ int search(int target){ return -1; } -vec3 hueColor(float angle) { - float nodeData = float(search(15)); - return (30.0 + angle * vec3(1.0, 5.0, nodeData)) / 50.0; -} - -float makeFrame(float v) { - v *= 6.5; - if (v < 1.5) { - return float(search(100)); - } - if (v < 4.0) { - return injectionSwitch.x; - } - if (v < float(search(6))) { - return 1.0; - } - return 10.0 + float(search(30)); -} - /* * This shader implements binary search tree using an array data structure. The elements of * tree are kept in the array that contains a list of BST object holding indices of left and @@ -124,9 +101,9 @@ float makeFrame(float v) { */ void main() { - int treeIndex = int(injectionSwitch.x); + int treeIndex = 0; // Initialize root node. - makeTreeNode(tree[int(injectionSwitch.x)], 9); + makeTreeNode(tree[0], 9); // Each time we insert a new node into the tree, we increment one. treeIndex++; @@ -148,26 +125,35 @@ void main() { treeIndex++; insert(treeIndex, 13); - vec2 z = (gl_FragCoord.yx / resolution); - float x = makeFrame(z.x); - float y = makeFrame(z.y); - - int sum = -100; - for (int target = 0; target < 20; target ++) { - int result = search(target); - if (result > 0) { - sum += result; - } else { - switch (result) { - case -1: - sum += int(injectionSwitch.y); + int count = 0; + for (int i = 0; i < 20; i++) { + int result = search(i); + switch (i) { + case 9: + case 5: + case 12: + case 15: + case 7: + case 8: + case 2: + case 6: + case 17: + case 13: + if (result == i) { + count++; + } + break; + default: + if (result == -1) { + count++; + } break; - case 0: - return; - } } } - float a = x + y * float(sum); - _GLF_color = vec4(hueColor(a), 1.); - + if (count == 20) { + _GLF_color = vec4(1.0, 0.0, 0.0, 1.0); + } else { + _GLF_color = vec4(0.0, 0.0, 1.0, 1.0); + } } + diff --git a/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.json b/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.json @@ -0,0 +1 @@ +{} From 61bdedcb6566bfdbcf8bc0c759f7e21d997dd9e6 Mon Sep 17 00:00:00 2001 From: Paul Thomson Date: Tue, 10 Dec 2019 14:52:36 +0000 Subject: [PATCH 180/180] Fix .bat launchers (#811) Fixes #801 --- build/travis/python-launch.bat | 18 ++++++++++------- .../glsl-reduce-walkthrough/fake_compiler.bat | 18 ++++++++++------- .../interestingness_test.bat | 18 ++++++++++------- .../weak_interestingness_test.bat | 18 ++++++++++------- .../main/python/drivers/backtrace-summary.bat | 18 ++++++++++------- python/src/main/python/drivers/gapidfuzz.bat | 18 ++++++++++------- .../src/main/python/drivers/glsl-generate.bat | 18 ++++++++++------- .../src/main/python/drivers/glsl-reduce.bat | 18 ++++++++++------- .../src/main/python/drivers/glsl-server.bat | 18 ++++++++++------- .../python/drivers/glsl-to-spv-worker.bat | 18 ++++++++++------- .../drivers/graphicsfuzz-piglit-converter.bat | 20 +++++++++++-------- .../main/python/drivers/graphicsfuzz-tool.bat | 18 ++++++++++------- .../drivers/inspect-compute-results.bat | 18 ++++++++++------- .../drivers/knownvalue-shader-generator.bat | 18 ++++++++++------- .../src/main/python/drivers/piglit-worker.bat | 18 ++++++++++------- .../report-compute-shader-family-results.bat | 18 ++++++++++------- .../main/python/drivers/run-shader-family.bat | 18 ++++++++++------- python/src/main/python/drivers/runspv.bat | 18 ++++++++++------- .../compute-interesting-diff.bat | 18 ++++++++++------- .../compute-interesting-same.bat | 18 ++++++++++------- 20 files changed, 221 insertions(+), 141 deletions(-) diff --git a/build/travis/python-launch.bat b/build/travis/python-launch.bat index 2d02e64a6..eb8d5908e 100644 --- a/build/travis/python-launch.bat +++ b/build/travis/python-launch.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 %* ) ELSE ( - python %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 %* + ) ELSE ( + python %* + ) ) ) diff --git a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/fake_compiler.bat b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/fake_compiler.bat index cafd86cce..8d598fb54 100644 --- a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/fake_compiler.bat +++ b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/fake_compiler.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/interestingness_test.bat b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/interestingness_test.bat index cafd86cce..8d598fb54 100644 --- a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/interestingness_test.bat +++ b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/interestingness_test.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/weak_interestingness_test.bat b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/weak_interestingness_test.bat index cafd86cce..8d598fb54 100644 --- a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/weak_interestingness_test.bat +++ b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/weak_interestingness_test.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/backtrace-summary.bat b/python/src/main/python/drivers/backtrace-summary.bat index aded83e44..4860a636c 100644 --- a/python/src/main/python/drivers/backtrace-summary.bat +++ b/python/src/main/python/drivers/backtrace-summary.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/gapidfuzz.bat b/python/src/main/python/drivers/gapidfuzz.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/gapidfuzz.bat +++ b/python/src/main/python/drivers/gapidfuzz.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/glsl-generate.bat b/python/src/main/python/drivers/glsl-generate.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/glsl-generate.bat +++ b/python/src/main/python/drivers/glsl-generate.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/glsl-reduce.bat b/python/src/main/python/drivers/glsl-reduce.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/glsl-reduce.bat +++ b/python/src/main/python/drivers/glsl-reduce.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/glsl-server.bat b/python/src/main/python/drivers/glsl-server.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/glsl-server.bat +++ b/python/src/main/python/drivers/glsl-server.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/glsl-to-spv-worker.bat b/python/src/main/python/drivers/glsl-to-spv-worker.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/glsl-to-spv-worker.bat +++ b/python/src/main/python/drivers/glsl-to-spv-worker.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat index 619bd62d5..4860a636c 100644 --- a/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat +++ b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) -) \ No newline at end of file +) diff --git a/python/src/main/python/drivers/graphicsfuzz-tool.bat b/python/src/main/python/drivers/graphicsfuzz-tool.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/graphicsfuzz-tool.bat +++ b/python/src/main/python/drivers/graphicsfuzz-tool.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/inspect-compute-results.bat b/python/src/main/python/drivers/inspect-compute-results.bat index aded83e44..4860a636c 100644 --- a/python/src/main/python/drivers/inspect-compute-results.bat +++ b/python/src/main/python/drivers/inspect-compute-results.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/knownvalue-shader-generator.bat b/python/src/main/python/drivers/knownvalue-shader-generator.bat index aded83e44..4860a636c 100644 --- a/python/src/main/python/drivers/knownvalue-shader-generator.bat +++ b/python/src/main/python/drivers/knownvalue-shader-generator.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/piglit-worker.bat b/python/src/main/python/drivers/piglit-worker.bat index aded83e44..4860a636c 100644 --- a/python/src/main/python/drivers/piglit-worker.bat +++ b/python/src/main/python/drivers/piglit-worker.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/report-compute-shader-family-results.bat b/python/src/main/python/drivers/report-compute-shader-family-results.bat index aded83e44..4860a636c 100644 --- a/python/src/main/python/drivers/report-compute-shader-family-results.bat +++ b/python/src/main/python/drivers/report-compute-shader-family-results.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/run-shader-family.bat b/python/src/main/python/drivers/run-shader-family.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/run-shader-family.bat +++ b/python/src/main/python/drivers/run-shader-family.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/runspv.bat b/python/src/main/python/drivers/runspv.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/runspv.bat +++ b/python/src/main/python/drivers/runspv.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/interestingness/compute-interesting-diff.bat b/python/src/main/python/interestingness/compute-interesting-diff.bat index aded83e44..4860a636c 100755 --- a/python/src/main/python/interestingness/compute-interesting-diff.bat +++ b/python/src/main/python/interestingness/compute-interesting-diff.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/interestingness/compute-interesting-same.bat b/python/src/main/python/interestingness/compute-interesting-same.bat index aded83e44..4860a636c 100755 --- a/python/src/main/python/interestingness/compute-interesting-same.bat +++ b/python/src/main/python/interestingness/compute-interesting-same.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) )

Worker", "", - f.getName().replace(".frag", ""), "
", worker, "", - ""); + "'>"); + if (shaderFamily.isCompute) { + htmlAppendLn("COMPUTE"); + + } else { + htmlAppendLn(""); + } + htmlAppendLn(""); } else { htmlAppendLn("", "No result yet
Variant is identical to reference
Variant is significantly different from reference
Variant is similar but not identical to reference
Rendering the variant led to an error
Variant leads to non-deterministic rendering
The image comparison metrics used to compare variant and reference ", "disagree on whether they are different or not
", - "", + "", "