From 6c2ba4006b7e4d33b8d69fca1b4e7b4f039b1d47 Mon Sep 17 00:00:00 2001 From: Chris Bieneman Date: Thu, 18 Jul 2024 16:46:46 -0500 Subject: [PATCH] [Metal] Add experimental Metal support This adds a new `-metal` flag to DXC which can be used to generate Metal's IR directly from DXC after compilation. There are some limitations in this flag which are worth noting: 1) It does not support library shaders (yet) 2) It does not support disassembly (yet) 3) It is _wildly_ under tested because wtihout (2) we can't do anything to really verify correct output (yay?) --- README.md | 10 +++ cmake/config-ix.cmake | 9 ++ cmake/modules/FindMetalIRConverter.cmake | 16 ++++ include/dxc/Support/HLSLOptions.h | 2 + include/dxc/Support/HLSLOptions.td | 3 + lib/DxcSupport/HLSLOptions.cpp | 17 ++++ tools/clang/test/DXC/metal.test | 5 ++ tools/clang/test/DXC/no_metal.test | 4 + .../clang/test/DXC/no_metal_disassembly.test | 4 + tools/clang/test/lit.cfg | 3 + tools/clang/test/lit.site.cfg.in | 1 + tools/clang/tools/dxcompiler/CMakeLists.txt | 8 ++ .../clang/tools/dxcompiler/dxcompilerobj.cpp | 89 ++++++++++++++++++- 13 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 cmake/modules/FindMetalIRConverter.cmake create mode 100644 tools/clang/test/DXC/metal.test create mode 100644 tools/clang/test/DXC/no_metal.test create mode 100644 tools/clang/test/DXC/no_metal_disassembly.test diff --git a/README.md b/README.md index b9e556bae7..72a34fcb94 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,16 @@ Development kits containing only the dxc.exe driver app, the dxcompiler.dll, and As an example of community contribution, this project can also target the [SPIR-V](https://www.khronos.org/registry/spir-v/) intermediate representation. Please see the [doc](docs/SPIR-V.rst) for how HLSL features are mapped to SPIR-V, and the [wiki](https://github.com/microsoft/DirectXShaderCompiler/wiki/SPIR%E2%80%90V-CodeGen) page for how to build, use, and contribute to the SPIR-V CodeGen. +### Metal CodeGen + +When built from source DXC can utilize the [Metal Shader +Converter](https://developer.apple.com/metal/shader-converter/) if it is +available during build and configuration time. This allows DXC to generate Metal +shader libraries directly using the `-metal` flag. + +Note: DXC cannot currently disassemble Metal shaders so the `-Fc` flag cannot be +used in conjunction with the `-Fo` flag. + ## Building Sources See the full documentation for [Building and testing DXC](docs/BuildingAndTestingDXC.rst) for detailed instructions. diff --git a/cmake/config-ix.cmake b/cmake/config-ix.cmake index 01b30568a9..4541d08162 100644 --- a/cmake/config-ix.cmake +++ b/cmake/config-ix.cmake @@ -568,3 +568,12 @@ else() endif() string(REPLACE " " ";" LLVM_BINDINGS_LIST "${LLVM_BINDINGS}") + +# HLSL Change Begin - Metal IR Converter +find_package(MetalIRConverter) +if (METAL_IRCONVERTER_FOUND) + set(ENABLE_METAL_CODEGEN On) + message(STATUS "Enabling Metal Support") + add_definitions(-DENABLE_METAL_CODEGEN) +endif() +# HLSL Change End - Metal IR Converter diff --git a/cmake/modules/FindMetalIRConverter.cmake b/cmake/modules/FindMetalIRConverter.cmake new file mode 100644 index 0000000000..fc7df1d6cc --- /dev/null +++ b/cmake/modules/FindMetalIRConverter.cmake @@ -0,0 +1,16 @@ +find_path(METAL_IRCONVERTER_INCLUDE_DIR metal_irconverter.h + HINTS /usr/local/include/metal_irconverter + DOC "Path to metal IR converter headers" + ) + +find_library(METAL_IRCONVERTER_LIB NAMES metalirconverter + PATH_SUFFIXES lib + ) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(METAL_IRCONVERTER + REQUIRED_VARS METAL_IRCONVERTER_LIB METAL_IRCONVERTER_INCLUDE_DIR) + +message(STATUS "Metal IR Converter Include Dir: ${METAL_IRCONVERTER_INCLUDE_DIR}") +message(STATUS "Metal IR Converter Library: ${METAL_IRCONVERTER_LIB}") +mark_as_advanced(METAL_IRCONVERTER_LIB METAL_IRCONVERTER_INCLUDE_DIR) diff --git a/include/dxc/Support/HLSLOptions.h b/include/dxc/Support/HLSLOptions.h index 887591ae82..56e95a1659 100644 --- a/include/dxc/Support/HLSLOptions.h +++ b/include/dxc/Support/HLSLOptions.h @@ -274,6 +274,8 @@ class DxcOpts { SpirvOptions; // All SPIR-V CodeGen-related options #endif // SPIRV Change Ends + + bool GenMetal = false; // OPT_metal }; /// Use this class to capture, convert and handle the lifetime for the diff --git a/include/dxc/Support/HLSLOptions.td b/include/dxc/Support/HLSLOptions.td index cd4eb830d0..ac49e9b294 100644 --- a/include/dxc/Support/HLSLOptions.td +++ b/include/dxc/Support/HLSLOptions.td @@ -346,6 +346,9 @@ def disable_exception_handling : Flag<["-", "/"], "disable-exception-handling">, def skip_serialization : Flag<["-", "/"], "skip-serialization">, Group, Flags<[CoreOption, HelpHidden]>, HelpText<"Return a module interface instead of serialized output">; +def metal : Flag<["-"], "metal">, Group, Flags<[CoreOption, DriverOption]>, + HelpText<"Generate Metal code">; + // SPIRV Change Starts def spirv : Flag<["-"], "spirv">, Group, Flags<[CoreOption, DriverOption]>, HelpText<"Generate SPIR-V code">; diff --git a/lib/DxcSupport/HLSLOptions.cpp b/lib/DxcSupport/HLSLOptions.cpp index 94103077f2..396e13bcec 100644 --- a/lib/DxcSupport/HLSLOptions.cpp +++ b/lib/DxcSupport/HLSLOptions.cpp @@ -1044,6 +1044,8 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude, addDiagnosticArgs(Args, OPT_W_Group, OPT_W_value_Group, opts.Warnings); + opts.GenMetal = Args.hasFlag(OPT_metal, OPT_INVALID, false); + // SPIRV Change Starts #ifdef ENABLE_SPIRV_CODEGEN opts.GenSPIRV = Args.hasFlag(OPT_spirv, OPT_INVALID, false); @@ -1253,6 +1255,21 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude, #endif // ENABLE_SPIRV_CODEGEN // SPIRV Change Ends + #ifndef ENABLE_METAL_CODEGEN + if (opts.GenMetal) { + errors << "Metal CodeGen not available. " + "Please rebuild with Metal IR Converter installed."; + return 1; + } + #endif + + if (opts.GenMetal) { + if (!opts.OutputObject.empty() && opts.AssemblyCode.empty()) { + errors << "Disassembly of Metal IR not supported (yet)."; + return 1; + } + } + // Validation for DebugInfo here because spirv uses same DebugInfo opt, // and legacy wrappers will add EmbedDebug in this case, leading to this // failing if placed before spirv path sets DebugInfo to true. diff --git a/tools/clang/test/DXC/metal.test b/tools/clang/test/DXC/metal.test new file mode 100644 index 0000000000..446477137e --- /dev/null +++ b/tools/clang/test/DXC/metal.test @@ -0,0 +1,5 @@ +// REQUIRES: metal + +// RUN: %dxc /T ps_6_0 %S/Inputs/smoke.hlsl -metal | FileCheck %s + +// CHECK: define void @main() diff --git a/tools/clang/test/DXC/no_metal.test b/tools/clang/test/DXC/no_metal.test new file mode 100644 index 0000000000..37af16cad5 --- /dev/null +++ b/tools/clang/test/DXC/no_metal.test @@ -0,0 +1,4 @@ +// UNSUPPORTED: metal + +// RUN:not %dxc %S/Inputs/smoke.hlsl /T ps_6_0 -metal 2>&1 | FileCheck %s +// CHECK:Metal CodeGen not available diff --git a/tools/clang/test/DXC/no_metal_disassembly.test b/tools/clang/test/DXC/no_metal_disassembly.test new file mode 100644 index 0000000000..7f24e8630d --- /dev/null +++ b/tools/clang/test/DXC/no_metal_disassembly.test @@ -0,0 +1,4 @@ +// REQUIRES: metal + +// RUN:not %dxc %S/Inputs/smoke.hlsl /T ps_6_0 -metal -Fo Tmp.metal -Fc Tmp.air 2>&1 | FileCheck %s +// CHECK: Disassembly of Metal IR not supported (yet). diff --git a/tools/clang/test/lit.cfg b/tools/clang/test/lit.cfg index e42297d2c9..5a93ad9552 100644 --- a/tools/clang/test/lit.cfg +++ b/tools/clang/test/lit.cfg @@ -503,6 +503,9 @@ if config.enable_backtrace == "1": if config.spirv: config.available_features.add("spirv") +if config.metal: + config.available_features.add("metal") + # Check supported dxil version def get_dxil_version(): result = subprocess.run([lit.util.which('dxc', llvm_tools_dir), "--version"], stdout=subprocess.PIPE) diff --git a/tools/clang/test/lit.site.cfg.in b/tools/clang/test/lit.site.cfg.in index 6a6fe2bfb0..28decbaa7e 100644 --- a/tools/clang/test/lit.site.cfg.in +++ b/tools/clang/test/lit.site.cfg.in @@ -21,6 +21,7 @@ config.enable_shared = @ENABLE_SHARED@ config.enable_backtrace = "@ENABLE_BACKTRACES@" config.host_arch = "@HOST_ARCH@" config.spirv = "@ENABLE_SPIRV_CODEGEN@" =="ON" +config.metal = "@ENABLE_METAL_CODEGEN@".upper() == "ON" # Support substitution of the tools and libs dirs with user parameters. This is # used when we can't determine the tool dir at configuration time. diff --git a/tools/clang/tools/dxcompiler/CMakeLists.txt b/tools/clang/tools/dxcompiler/CMakeLists.txt index d7aec3276e..0ee920615a 100644 --- a/tools/clang/tools/dxcompiler/CMakeLists.txt +++ b/tools/clang/tools/dxcompiler/CMakeLists.txt @@ -135,6 +135,14 @@ target_link_libraries(dxcompiler PRIVATE ${LIBRARIES}) if (ENABLE_SPIRV_CODEGEN) target_link_libraries(dxcompiler PRIVATE clangSPIRV) endif (ENABLE_SPIRV_CODEGEN) +if (ENABLE_METAL_CODEGEN) + target_link_libraries(dxcompiler PRIVATE ${METAL_IRCONVERTER_LIB}) + target_include_directories(dxcompiler PRIVATE ${METAL_IRCONVERTER_INCLUDE_DIR}) + + get_filename_component(METAL_IRCONVERTER_LIB_DIR ${METAL_IRCONVERTER_LIB} DIRECTORY CACHE) + set_property(TARGET dxcompiler APPEND_STRING + PROPERTY LINK_FLAGS " -Wl,-rpath,${METAL_IRCONVERTER_LIB_DIR}") +endif (ENABLE_METAL_CODEGEN) include_directories(AFTER ${LLVM_INCLUDE_DIR}/dxc/Tracing ${DIASDK_INCLUDE_DIRS} ${HLSL_VERSION_LOCATION}) set_target_properties(dxcompiler diff --git a/tools/clang/tools/dxcompiler/dxcompilerobj.cpp b/tools/clang/tools/dxcompiler/dxcompilerobj.cpp index 0ee0d00765..a909e335bd 100644 --- a/tools/clang/tools/dxcompiler/dxcompilerobj.cpp +++ b/tools/clang/tools/dxcompiler/dxcompilerobj.cpp @@ -70,6 +70,10 @@ #include "clang/Basic/Version.h" #endif // SUPPORT_QUERY_GIT_COMMIT_INFO +#ifdef ENABLE_METAL_CODEGEN +#include "metal_irconverter.h" +#endif + #define CP_UTF16 1200 using namespace llvm; @@ -828,6 +832,10 @@ class DxcCompiler : public IDxcCompiler3, } compiler.getLangOpts().IsHLSLLibrary = opts.IsLibraryProfile(); + if (compiler.getLangOpts().IsHLSLLibrary && opts.GenMetal) + return ErrorWithString("Shader libraries unsupported in Metal (yet)", + riid, ppResult); + // Clear entry function if library target if (compiler.getLangOpts().IsHLSLLibrary) compiler.getLangOpts().HLSLEntryFunction = @@ -1117,7 +1125,86 @@ class DxcCompiler : public IDxcCompiler3, &pHashBlob)); IFT(pResult->SetOutputObject(DXC_OUT_SHADER_HASH, pHashBlob)); } // SUCCEEDED(valHR) - } // compileOK && !opts.CodeGenHighLevel +#ifdef ENABLE_METAL_CODEGEN + // This is a bit hacky because we don't currently have a good way to + // disassemble AIR. + if (opts.GenMetal && produceFullContainer && + !opts.OutputObject.empty()) { + IRCompiler *MetalCompiler = IRCompilerCreate(); + IRCompilerSetEntryPointName( + MetalCompiler, + compiler.getCodeGenOpts().HLSLEntryFunction.c_str()); + + IRObject *DXILObj = IRObjectCreateFromDXIL( + static_cast(pOutputBlob->GetBufferPointer()), + pOutputBlob->GetBufferSize(), IRBytecodeOwnershipNone); + + // Compile DXIL to Metal IR: + IRError *Error = nullptr; + IRObject *AIR = IRCompilerAllocCompileAndLink(MetalCompiler, NULL, + DXILObj, &Error); + + if (!AIR) { + IRObjectDestroy(DXILObj); + IRCompilerDestroy(MetalCompiler); + IRErrorDestroy(Error); + return ErrorWithString( + "Error occurred in Metal Shader Conversion", riid, ppResult); + } + + IRMetalLibBinary *MetalLib = IRMetalLibBinaryCreate(); + IRShaderStage Stage = IRShaderStageInvalid; + const ShaderModel *SM = hlsl::ShaderModel::GetByName( + compiler.getLangOpts().HLSLProfile); + switch (SM->GetKind()) { + case DXIL::ShaderKind::Vertex: + Stage = IRShaderStageVertex; + break; + case DXIL::ShaderKind::Pixel: + Stage = IRShaderStageFragment; + break; + case DXIL::ShaderKind::Hull: + Stage = IRShaderStageHull; + break; + case DXIL::ShaderKind::Domain: + Stage = IRShaderStageDomain; + break; + case DXIL::ShaderKind::Mesh: + Stage = IRShaderStageMesh; + break; + case DXIL::ShaderKind::Amplification: + Stage = IRShaderStageAmplification; + break; + case DXIL::ShaderKind::Geometry: + Stage = IRShaderStageGeometry; + break; + case DXIL::ShaderKind::Compute: + Stage = IRShaderStageCompute; + break; + } + assert(Stage != IRShaderStageInvalid && + "Library targets not supported for Metal (yet)."); + IRObjectGetMetalLibBinary(AIR, Stage, MetalLib); + size_t MetalLibSize = IRMetalLibGetBytecodeSize(MetalLib); + uint8_t *MetalLibBytes = new uint8_t[MetalLibSize]; + IRMetalLibGetBytecode(MetalLib, MetalLibBytes); + + // Store the metallib to custom format or disk, or use to create a + // MTLLibrary. + + CComPtr MetalBlob; + IFT(hlsl::DxcCreateBlobOnHeapCopy( + MetalLibBytes, (uint32_t)MetalLibSize, &MetalBlob)); + std::swap(pOutputBlob, MetalBlob); + + delete[] MetalLibBytes; + IRMetalLibBinaryDestroy(MetalLib); + IRObjectDestroy(DXILObj); + IRObjectDestroy(AIR); + IRCompilerDestroy(MetalCompiler); + } +#endif + } // compileOK && !opts.CodeGenHighLevel } std::string remarks;