From b44b69a9674299e750f20afecdfc40159aff9735 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 31 Jul 2019 16:28:13 +0200 Subject: [PATCH 1/9] [GR-17422] Add dependencies for GraalVM components (cherry picked from commit 8825207e3979780508b9661410728f4d55018b0f) --- mx.fastr/mx_fastr_dists.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mx.fastr/mx_fastr_dists.py b/mx.fastr/mx_fastr_dists.py index 66e477728d..66130d4fc4 100644 --- a/mx.fastr/mx_fastr_dists.py +++ b/mx.fastr/mx_fastr_dists.py @@ -292,6 +292,7 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi short_name='R', license_files=['LICENSE_FASTR'], third_party_license_files=['3rd_party_licenses_fastr.txt'], + dependencies=['Truffle', 'Sulong', 'LLVM.org toolchain'], truffle_jars=['fastr:FASTR'], support_distributions=['fastr:FASTR_GRAALVM_SUPPORT'], provided_executables=[ From 6f9e628da7182e57fef6daa25f7c10b294c1e927 Mon Sep 17 00:00:00 2001 From: stepan Date: Thu, 31 Oct 2019 19:32:21 +0100 Subject: [PATCH 2/9] Revert "[GR-18562] Use DirectCallNode to allow inlining of promises in InlineCacheNode." This reverts commit ab972162cb33733e56b74e75551bb43052fb14f3, reversing changes made to 5a8d0cbfb4fe8a539ea12cd7c988abb124e61758. (cherry picked from commit 46112400a6d9917312fa78c9a90d4ee783a7c2e2) --- .../truffle/r/nodes/InlineCacheNode.java | 101 +++++++++--------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/InlineCacheNode.java b/com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/InlineCacheNode.java index 13c38b5449..7f0c65d28a 100644 --- a/com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/InlineCacheNode.java +++ b/com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/InlineCacheNode.java @@ -22,20 +22,17 @@ */ package com.oracle.truffle.r.nodes; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; -import com.oracle.truffle.api.Truffle; -import com.oracle.truffle.api.TruffleLanguage; -import com.oracle.truffle.api.TruffleRuntime; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.ReportPolymorphism; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.Frame; import com.oracle.truffle.api.frame.MaterializedFrame; import com.oracle.truffle.api.frame.VirtualFrame; -import com.oracle.truffle.api.nodes.DirectCallNode; -import com.oracle.truffle.api.nodes.RootNode; -import com.oracle.truffle.api.profiles.ValueProfile; -import com.oracle.truffle.r.runtime.context.RContext; +import com.oracle.truffle.api.nodes.Node; +import com.oracle.truffle.api.profiles.ConditionProfile; import com.oracle.truffle.r.runtime.data.Closure; import com.oracle.truffle.r.runtime.nodes.RBaseNode; import com.oracle.truffle.r.runtime.nodes.RNode; @@ -55,61 +52,65 @@ protected InlineCacheNode(int maxPicDepth) { this.maxPicDepth = maxPicDepth; } - /** - * Creates an inline cache that will execute promises closures by using a polymorphic inline - * cache (PIC) and falling back to - * {@link InlineCacheNode#evalPromise(MaterializedFrame, Closure)}. - * - * @param maxPicDepth maximum number of entries in the PIC - */ - public static InlineCacheNode create(int maxPicDepth) { - return InlineCacheNodeGen.create(maxPicDepth); - } + @SuppressWarnings("unused") + @Specialization(limit = "maxPicDepth", guards = "value == cachedValue") + protected Object doCached(Frame frame, Object value, + @Cached("value") Object cachedValue, + @Cached("createBinaryProfile()") ConditionProfile isVirtualFrameProfile, + @Cached("cache(cachedValue)") RNode reified, + @Cached("new(reified)") NodeInsertedClosure nodeInsertedClosure) { + VirtualFrame vf; + if (isVirtualFrameProfile.profile(frame instanceof VirtualFrame)) { + vf = (VirtualFrame) frame; + } else { + vf = frame.materialize(); + } - // Cached case: create a root node and direct call node - // This should allow Truffle PE to inline the promise code if it deems it is beneficial + // Use a closure to notify the root node that a new node has just been inserted. The + // closure is necessary to do the notification just once. + nodeInsertedClosure.notifyNodeInserted(); - protected DirectCallNode createInlinableCall(Closure value) { - InlineCacheRootNode rootNode = new InlineCacheRootNode(RContext.getInstance().getLanguage(), RASTUtils.cloneNode((RNode) value.getExpr())); - TruffleRuntime runtime = Truffle.getRuntime(); - runtime.createCallTarget(rootNode); - return runtime.createDirectCallNode(rootNode.getCallTarget()); + return reified.visibleExecute(vf); } - private static final class InlineCacheRootNode extends RootNode { - @Child private RNode node; - - protected InlineCacheRootNode(TruffleLanguage language, RNode node) { - super(language); - this.node = node; - } - - @Override - public Object execute(VirtualFrame frame) { - VirtualFrame execFrame = (VirtualFrame) frame.getArguments()[0]; - return node.visibleExecute(execFrame); - } + @Specialization(replaces = "doCached") + protected Object doGeneric(Frame frame, Object value) { + return evalPromise(frame.materialize(), (Closure) value); } - @Specialization(limit = "maxPicDepth", guards = "value == cachedValue") - protected Object doCached(Frame frame, @SuppressWarnings("unused") Closure value, - @SuppressWarnings("unused") @Cached("value") Closure cachedValue, - @Cached("createClassProfile()") ValueProfile frameClassProfile, - @Cached("createInlinableCall(value)") DirectCallNode callNode) { - return callNode.call(frameClassProfile.profile(frame).materialize()); + protected RNode cache(Object value) { + return RASTUtils.cloneNode((RNode) ((Closure) value).getExpr()); } - // Generic case: execute call target cached in the Closure - // We do not go though call node, so not Truffle inlining can take place, - // but nothing is inserted into this AST, so it doesn't grow without limits - - @Specialization(replaces = "doCached") - protected Object doGeneric(Frame frame, Closure value) { - return evalPromise(frame.materialize(), value); + /** + * Creates an inline cache that will execute promises closures by using a PIC and falling back + * to {@link InlineCacheNode#evalPromise(MaterializedFrame, Closure)}. + * + * @param maxPicDepth maximum number of entries in the polymorphic inline cache + */ + public static InlineCacheNode create(int maxPicDepth) { + return InlineCacheNodeGen.create(maxPicDepth); } @TruffleBoundary protected static Object evalPromise(MaterializedFrame frame, Closure closure) { return closure.eval(frame); } + + protected class NodeInsertedClosure { + private final Node n; + @CompilationFinal boolean notified; + + public NodeInsertedClosure(Node n) { + this.n = n; + } + + void notifyNodeInserted() { + if (!notified) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + InlineCacheNode.this.notifyInserted(n); + notified = true; + } + } + } } From 193f906ddc724a7791c138401d9887f46888312e Mon Sep 17 00:00:00 2001 From: Zbynek Slajchrt Date: Tue, 29 Oct 2019 13:37:50 +0100 Subject: [PATCH 3/9] otool pattern fixed in configure_fastr (cherry picked from commit c2c1b5ef5dd2352d42d5443e96c5e18fd307e10d) --- com.oracle.truffle.r.native/run/configure_fastr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.oracle.truffle.r.native/run/configure_fastr b/com.oracle.truffle.r.native/run/configure_fastr index ea745b36be..f6d9a19624 100755 --- a/com.oracle.truffle.r.native/run/configure_fastr +++ b/com.oracle.truffle.r.native/run/configure_fastr @@ -144,7 +144,7 @@ done # don't look at symlinks if [ ! -L "${TARGET_LIB}" ] ; then # grep for the current path of this gfortran library in this library's linking info - CURRENT_LIB_NAME=`otool -L ${TARGET_LIB} | grep -o "\t.*${GFORTRAN_LIB_BASE}"` + CURRENT_LIB_NAME=`otool -L ${TARGET_LIB} | grep -o "\t.*${GFORTRAN_LIB_BASE}[^ ]*"` if [ ! -z "$CURRENT_LIB_NAME" ] ; then if [ "$CURRENT_LIB_NAME" != "$LIB_LOCATION" ] ; then # change to the new location From 32ea48d7153ce83124f744de2bd0df77c10c7ba2 Mon Sep 17 00:00:00 2001 From: stepan Date: Wed, 30 Oct 2019 19:27:11 +0100 Subject: [PATCH 4/9] configure_fastr show error when --enable-native-compilers-toolchain passed but --configure-etc was not passed (cherry picked from commit 6a0f8857ebc9c14360c6881440dc7e21f2278239) --- com.oracle.truffle.r.native/run/configure_fastr | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/com.oracle.truffle.r.native/run/configure_fastr b/com.oracle.truffle.r.native/run/configure_fastr index f6d9a19624..f119c8bde9 100755 --- a/com.oracle.truffle.r.native/run/configure_fastr +++ b/com.oracle.truffle.r.native/run/configure_fastr @@ -76,6 +76,7 @@ function printHelp { GCC_LIB_PATH="" CONFIGURE_ARGS="" CONFIGURE_ETC=0 +CONFIGURE_NATIVE_TOOLCHAIN=0 while [[ $# -gt 0 ]]; do case $1 in --help) @@ -88,13 +89,22 @@ while [[ $# -gt 0 ]]; do ;; --configure-etc) CONFIGURE_ETC=1 - ;; + ;; + --enable-native-compilers-toolchain) + CONFIGURE_NATIVE_TOOLCHAIN=1 + CONFIGURE_ARGS="$CONFIGURE_ARGS $1" + ;; *) CONFIGURE_ARGS="$CONFIGURE_ARGS $1" ;; esac shift -done +done + +if [[ $CONFIGURE_NATIVE_TOOLCHAIN = 1 ]] && [[ $CONFIGURE_ETC = 0 ]]; then + echo "Error: --enable-native-compilers-toolchain does not have any effect when not run with --configure-etc." + exit 100 +fi ( if [[ "$OSTYPE" == "linux-gnu" ]]; then From 2aa2fdaa47276002af460e454af4cfd23434722d Mon Sep 17 00:00:00 2001 From: stepan Date: Wed, 30 Oct 2019 19:28:04 +0100 Subject: [PATCH 5/9] configure_fastr: overhaul of the GCC runtime libs check on Linux (cherry picked from commit fe65ff22d44f6e8e6a6b639d09ffc3b65ff94eb7) --- .../run/configure_fastr | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/com.oracle.truffle.r.native/run/configure_fastr b/com.oracle.truffle.r.native/run/configure_fastr index f119c8bde9..7ca7635c8f 100755 --- a/com.oracle.truffle.r.native/run/configure_fastr +++ b/com.oracle.truffle.r.native/run/configure_fastr @@ -108,16 +108,45 @@ fi ( if [[ "$OSTYPE" == "linux-gnu" ]]; then - echo "int main(){}" | gcc -x c -Wl,--no-as-needed -lgfortran -lgcc_s -lquadmath -o /dev/null - > /dev/null 2>/dev/null - res=$? + res=0 + for lib in libgfortran.so.3 libm.so.6 libquadmath.so.0 libgcc_s.so.1; do + if ! ldconfig -p | grep --quiet $lib; then + echo "Error: could not find gfortran3 runtime library: $lib" + echo "Please install the gfortran3 runtime libraries:" + echo " On Debian based systems: apt-get install libgfortran3" + echo " On Oracle Linux 7: yum install libgfortran" + echo " On Oracle Linux 8: yum install compat-libgfortran-48" + res=1 + break + fi + done + if ! ldconfig -p | grep --quiet "libgomp.so.1"; then + echo "Error: could not find OpenMP runtime library: libgomp.so.1" + echo "Please install the OpenMP runtime library runtime libraries:" + echo " On Debian based systems: apt-get install libgomp1" + echo " On Oracle Linux 7 and 8: yum install libgomp" + echo " Note: Oracle Linux 8 should contain libgomp by default" + res=1 + fi + if which gfortran > /dev/null 2> /dev/null; then + echo "Note: if you intend to install R packages with Fortran 90 and newer code, you need to install gfortran compiler." + echo "Fortran 77 or any Fortran code that is compilable by the 'f2c' tool does not require gfortran compiler." + echo "You can install the gfortran compiler with:" + echo " On Debian based systems: apt-get install build-essential" + echo " On Oracle Linux 7: yum groupinstall 'Development Tools'" + echo " On Oracle Linux 8: yum groupinstall 'Development Tools'" + if [[ $res != 0 ]]; then + echo "Caution: you still need to install gfortran3 runtime libraries on all systems where this version is not the default (Ubuntu 18 or higher, Oracle Linux 8 or higher)." + fi + fi if [[ $res != 0 ]]; then - echo "Cannot build a simple C program linking with libgfortran, libgcc_s, and libquadmath." - echo "Please install GCC and gfortran using:" - echo " On Debian based systems: sudo apt-get install build-essential gfortran" - echo " On CentOS/RHEL/Oracle Linux: sudo yum groupinstall \"Developent Tools\"; yum install gcc-gfortran" - echo "" - printHelp + echo "The basic configuration of FastR failed." + echo "To learn more:" + echo " run this script with '--help'" + echo " visit https://www.graalvm.org/docs/reference-manual/languages/r" exit 1 + else + echo "The basic configuration of FastR was successfull." fi elif [[ "$OSTYPE" == "darwin"* ]]; then cd -P "$( dirname "$source" )/../lib" From 6c15747b2e9c1788e8815f10b906657521a11b91 Mon Sep 17 00:00:00 2001 From: stepan Date: Thu, 31 Oct 2019 19:37:39 +0100 Subject: [PATCH 6/9] configure_fastr: small improvement in the suggested packages to install on Linux (cherry picked from commit b4907a261b4dfeec44382939eb1435735261fc49) --- com.oracle.truffle.r.native/run/configure_fastr | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/com.oracle.truffle.r.native/run/configure_fastr b/com.oracle.truffle.r.native/run/configure_fastr index 7ca7635c8f..610f2fa896 100755 --- a/com.oracle.truffle.r.native/run/configure_fastr +++ b/com.oracle.truffle.r.native/run/configure_fastr @@ -129,12 +129,10 @@ fi res=1 fi if which gfortran > /dev/null 2> /dev/null; then - echo "Note: if you intend to install R packages with Fortran 90 and newer code, you need to install gfortran compiler." - echo "Fortran 77 or any Fortran code that is compilable by the 'f2c' tool does not require gfortran compiler." - echo "You can install the gfortran compiler with:" - echo " On Debian based systems: apt-get install build-essential" - echo " On Oracle Linux 7: yum groupinstall 'Development Tools'" - echo " On Oracle Linux 8: yum groupinstall 'Development Tools'" + echo "Note: if you intend to install R packages you may need additional packages." + echo "The following packages should cover the most commonly used R packages:" + echo " On Debian based systems: apt-get install build-essential gfortran libxml2 libc++-dev" + echo " On Oracle Linux 7 and 8: yum groupinstall 'Development Tools'" if [[ $res != 0 ]]; then echo "Caution: you still need to install gfortran3 runtime libraries on all systems where this version is not the default (Ubuntu 18 or higher, Oracle Linux 8 or higher)." fi From cd672e49116d6cd464e69e19601a92bd312ea010 Mon Sep 17 00:00:00 2001 From: stepan Date: Thu, 31 Oct 2019 19:47:10 +0100 Subject: [PATCH 7/9] Fix linker issues when running C++ built by the LLVM toolchain in native mode (cherry picked from commit 95fa3698777c569c1ad74a7df8390b313ea1f667) --- com.oracle.truffle.r.native/run/Makefile | 6 +++++- .../r/nodes/builtin/fastr/FastRSetToolchain.java | 12 +++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/com.oracle.truffle.r.native/run/Makefile b/com.oracle.truffle.r.native/run/Makefile index 89cc029f25..4e4f67a29e 100644 --- a/com.oracle.truffle.r.native/run/Makefile +++ b/com.oracle.truffle.r.native/run/Makefile @@ -89,7 +89,11 @@ $(FASTR_BIN_DIR)/R: Makefile R.sh Rscript.sh Rscript_exec.sh Rclasspath.sh cp $(GNUR_HOME_BINARY)/etc/Makeconf Makeconf.etc cp $(GNUR_HOME_BINARY)/etc/javaconf $(FASTR_ETC_DIR)/javaconf cp $(GNUR_HOME_BINARY)/etc/repositories $(FASTR_ETC_DIR)/repositories - cp $(GNUR_HOME_BINARY)/etc/ldpaths $(FASTR_ETC_DIR)/ldpaths + # ldpaths: create two versions: ldpaths.llvm (used also as default ldpaths) and ldpaths.native + # the llvm version adds the LLVM lib folder to the LD_LIBRARY_PATH + sed 's!export LD_LIBRARY_PATH!LD_LIBRARY_PATH="$$\{R_HOME\}/../llvm/native/lib/:$$\{LD_LIBRARY_PATH\}"; export LD_LIBRARY_PATH!' $(GNUR_HOME_BINARY)/etc/ldpaths > $(FASTR_ETC_DIR)/ldpaths + cp $(FASTR_ETC_DIR)/ldpaths $(FASTR_ETC_DIR)/ldpaths.llvm + cp $(GNUR_HOME_BINARY)/etc/ldpaths $(FASTR_ETC_DIR)/ldpaths.native # the ed script adds -DFASTR to compiler options and removes JAVA related variables ed Makeconf.etc < edMakeconf.etc # Backup the current state of Makeconf.etc to allow users to switch back to the native toolchain diff --git a/com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/fastr/FastRSetToolchain.java b/com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/fastr/FastRSetToolchain.java index 63b841aa1f..dddb8edad4 100644 --- a/com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/fastr/FastRSetToolchain.java +++ b/com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/fastr/FastRSetToolchain.java @@ -52,19 +52,25 @@ public abstract class FastRSetToolchain extends RBuiltinNode.Arg1 { @Specialization protected RNull setToolchain(String name) { String srcConf; + String srcLdpaths; if ("native".equals(name)) { srcConf = "Makeconf.native"; + srcLdpaths = "ldpaths.native"; } else if ("llvm".equals(name)) { srcConf = "Makeconf.llvm"; + srcLdpaths = "ldpaths.llvm"; } else { throw error(RError.Message.GENERIC, "Only 'native' or 'llvm' argument values accepted"); } TruffleFile rHome = REnvVars.getRHomeTruffleFile(RContext.getInstance().getEnv()); TruffleFile etc = rHome.resolve("etc"); - TruffleFile src = etc.resolve(srcConf); - TruffleFile dst = etc.resolve("Makeconf"); + TruffleFile srcConfFile = etc.resolve(srcConf); + TruffleFile dstConfFile = etc.resolve("Makeconf"); + TruffleFile srcLdpathsFile = etc.resolve(srcLdpaths); + TruffleFile dstLdpathsFile = etc.resolve("ldpaths"); try { - src.copy(dst, StandardCopyOption.REPLACE_EXISTING); + srcConfFile.copy(dstConfFile, StandardCopyOption.REPLACE_EXISTING); + srcLdpathsFile.copy(dstLdpathsFile, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { throw new RInternalError(String.format("Copying %s over etc/Makeconf failed", srcConf), e); } From 10a54d91b4555444494853dd0af437dcad3a3093 Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Thu, 31 Oct 2019 16:10:42 +0100 Subject: [PATCH 8/9] upgrade jdk8 (cherry picked from commit d5ecfb88da038529016d2eecde53185529e3f701) --- ci_common/common.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_common/common.hocon b/ci_common/common.hocon index c2178d93b7..dfd38bfe1b 100644 --- a/ci_common/common.hocon +++ b/ci_common/common.hocon @@ -1,7 +1,7 @@ # java 7 is needed by Truffle (for now) java7 : {name : oraclejdk, version : "7", platformspecific: true} # java 8 must be a jvmci enabled variant -java8 : {name : oraclejdk, version : "8u221-jvmci-19.3-b02", platformspecific: true} +java8 : {name : oraclejdk, version : "8u231-jvmci-19.3-b04", platformspecific: true} java11 : {name : labsjdk, version : "ce-11.0.5+9-jvmci-19.3-b03", platformspecific: true} java8Downloads : { From 4ee3d2132b71051aebb0a99fe7b634b3aa7fb312 Mon Sep 17 00:00:00 2001 From: stepan Date: Thu, 31 Oct 2019 20:57:12 +0100 Subject: [PATCH 9/9] [GR-19326] Fix potential NPE in REnvVars (cherry picked from commit 9c3ebe65bc75402d935bda0f1638df58ca72bdd9) --- .../truffle/r/nodes/access/vector/CachedExtractVectorNode.java | 2 ++ .../src/com/oracle/truffle/r/runtime/REnvVars.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/access/vector/CachedExtractVectorNode.java b/com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/access/vector/CachedExtractVectorNode.java index aafff7fd88..4680c58038 100644 --- a/com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/access/vector/CachedExtractVectorNode.java +++ b/com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/access/vector/CachedExtractVectorNode.java @@ -308,6 +308,8 @@ private void applyDimensions(RAbstractContainer originalTarget, RAbstractVector } newDimNames[dimIndex] = result; if (newDimNamesNames != null) { + // (newDimNamesNames == null) <=> (originalDimNamesNames == null) + assert originalDimNamesNames != null; newDimNamesNames[dimIndex] = originalDimNamesNames.getDataAt(i); } } diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/REnvVars.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/REnvVars.java index ecbb9da7a7..c869acc970 100644 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/REnvVars.java +++ b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/REnvVars.java @@ -126,7 +126,7 @@ public RContext.ContextState initialize(RContext context) { } } TruffleFile userFile = userFilePath != null ? FileSystemUtils.getSafeTruffleFile(env, userFilePath) : null; - if (userFile.exists()) { + if (userFile != null && userFile.exists()) { safeReadEnvironFile(userFile); } }