[GR-19325] [GR-19296] [GR-19297] [GR-17422] [GR-19326] Backport confi…

…guration related fixes to 19.3.0.

PullRequest: fastr/2209

ci_common/common.hocon

@@ -1,7 +1,7 @@
 # java 7 is needed by Truffle (for now)
 java7 : {name : oraclejdk, version : "7",    platformspecific: true}
 # java 8 must be a jvmci enabled variant
-java8 : {name : oraclejdk, version : "8u221-jvmci-19.3-b02", platformspecific: true}
+java8 : {name : oraclejdk, version : "8u231-jvmci-19.3-b04", platformspecific: true}
 java11 : {name : labsjdk, version : "ce-11.0.5+9-jvmci-19.3-b03", platformspecific: true}
 
 java8Downloads : {

com.oracle.truffle.r.native/run/Makefile

@@ -89,7 +89,11 @@ $(FASTR_BIN_DIR)/R: Makefile R.sh Rscript.sh Rscript_exec.sh Rclasspath.sh
 	cp $(GNUR_HOME_BINARY)/etc/Makeconf Makeconf.etc
 	cp $(GNUR_HOME_BINARY)/etc/javaconf $(FASTR_ETC_DIR)/javaconf
 	cp $(GNUR_HOME_BINARY)/etc/repositories $(FASTR_ETC_DIR)/repositories
-	cp $(GNUR_HOME_BINARY)/etc/ldpaths $(FASTR_ETC_DIR)/ldpaths
+	# ldpaths: create two versions: ldpaths.llvm (used also as default ldpaths) and ldpaths.native
+	# the llvm version adds the LLVM lib folder to the LD_LIBRARY_PATH
+	sed 's!export LD_LIBRARY_PATH!LD_LIBRARY_PATH="$$\{R_HOME\}/../llvm/native/lib/:$$\{LD_LIBRARY_PATH\}"; export LD_LIBRARY_PATH!' $(GNUR_HOME_BINARY)/etc/ldpaths > $(FASTR_ETC_DIR)/ldpaths
+	cp $(FASTR_ETC_DIR)/ldpaths $(FASTR_ETC_DIR)/ldpaths.llvm
+	cp $(GNUR_HOME_BINARY)/etc/ldpaths $(FASTR_ETC_DIR)/ldpaths.native
 	# the ed script adds -DFASTR to compiler options and removes JAVA related variables
 	ed Makeconf.etc < edMakeconf.etc
 	# Backup the current state of Makeconf.etc to allow users to switch back to the native toolchain

com.oracle.truffle.r.native/run/configure_fastr

@@ -76,6 +76,7 @@ function printHelp {
 GCC_LIB_PATH=""
 CONFIGURE_ARGS=""
 CONFIGURE_ETC=0
+CONFIGURE_NATIVE_TOOLCHAIN=0
 while [[ $# -gt 0 ]]; do
   case $1 in
     --help)
@@ -88,26 +89,62 @@ while [[ $# -gt 0 ]]; do
     ;;
     --configure-etc)
       CONFIGURE_ETC=1
-    ;;  
+    ;;
+    --enable-native-compilers-toolchain)
+      CONFIGURE_NATIVE_TOOLCHAIN=1
+      CONFIGURE_ARGS="$CONFIGURE_ARGS $1"
+    ;;
     *)
       CONFIGURE_ARGS="$CONFIGURE_ARGS $1"
     ;;
   esac
   shift
-done  
+done
+
+if [[ $CONFIGURE_NATIVE_TOOLCHAIN = 1 ]] && [[ $CONFIGURE_ETC = 0 ]]; then
+  echo "Error: --enable-native-compilers-toolchain does not have any effect when not run with --configure-etc."
+  exit 100
+fi
 
 (
   if [[ "$OSTYPE" == "linux-gnu" ]]; then
-    echo "int main(){}" | gcc -x c -Wl,--no-as-needed -lgfortran -lgcc_s -lquadmath -o /dev/null - > /dev/null 2>/dev/null
-    res=$?
+    res=0
+    for lib in libgfortran.so.3 libm.so.6 libquadmath.so.0 libgcc_s.so.1; do
+      if ! ldconfig -p | grep --quiet $lib; then
+        echo "Error: could not find gfortran3 runtime library: $lib"
+        echo "Please install the gfortran3 runtime libraries:"
+        echo "    On Debian based systems: apt-get install libgfortran3"
+        echo "    On Oracle Linux 7: yum install libgfortran"
+        echo "    On Oracle Linux 8: yum install compat-libgfortran-48"
+        res=1
+        break
+      fi
+    done
+    if ! ldconfig -p | grep --quiet "libgomp.so.1"; then
+      echo "Error: could not find OpenMP runtime library: libgomp.so.1"
+      echo "Please install the OpenMP runtime library runtime libraries:"
+      echo "    On Debian based systems: apt-get install libgomp1"
+      echo "    On Oracle Linux 7 and 8: yum install libgomp"
+      echo "    Note: Oracle Linux 8 should contain libgomp by default"
+      res=1
+    fi
+    if which gfortran > /dev/null 2> /dev/null; then
+      echo "Note: if you intend to install R packages you may need additional packages."
+      echo "The following packages should cover the most commonly used R packages:"
+      echo "    On Debian based systems: apt-get install build-essential gfortran libxml2 libc++-dev"
+      echo "    On Oracle Linux 7 and 8: yum groupinstall 'Development Tools'"
+      if [[ $res != 0 ]]; then
+        echo "Caution: you still need to install gfortran3 runtime libraries on all systems where this version is not the default (Ubuntu 18 or higher, Oracle Linux 8 or higher)."
+      fi
+    fi
     if [[ $res != 0 ]]; then
-      echo "Cannot build a simple C program linking with libgfortran, libgcc_s, and libquadmath."
-      echo "Please install GCC and gfortran using:"
-      echo "    On Debian based systems: sudo apt-get install build-essential gfortran"
-      echo "    On CentOS/RHEL/Oracle Linux: sudo yum groupinstall \"Developent Tools\"; yum install gcc-gfortran"
-	  echo ""	
-      printHelp
+      echo "The basic configuration of FastR failed."
+      echo "To learn more:"
+      echo "  run this script with '--help'"
+      echo "  visit https://www.graalvm.org/docs/reference-manual/languages/r"
       exit 1
+    else
+      echo "The basic configuration of FastR was successfull."
     fi
   elif [[ "$OSTYPE" == "darwin"* ]]; then
     cd -P "$( dirname "$source" )/../lib"
@@ -144,7 +181,7 @@ done
 			# don't look at symlinks
 			if [ ! -L "${TARGET_LIB}" ] ; then
 			  # grep for the current path of this gfortran library in this library's linking info
-			  CURRENT_LIB_NAME=`otool -L ${TARGET_LIB} | grep -o "\t.*${GFORTRAN_LIB_BASE}"`
+			  CURRENT_LIB_NAME=`otool -L ${TARGET_LIB} | grep -o "\t.*${GFORTRAN_LIB_BASE}[^ ]*"`
 			  if [ ! -z "$CURRENT_LIB_NAME" ] ; then
 				if [ "$CURRENT_LIB_NAME" != "$LIB_LOCATION" ] ; then
 				  # change to the new location

com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/fastr/FastRSetToolchain.java

@@ -52,19 +52,25 @@ public abstract class FastRSetToolchain extends RBuiltinNode.Arg1 {
     @Specialization
     protected RNull setToolchain(String name) {
         String srcConf;
+        String srcLdpaths;
         if ("native".equals(name)) {
             srcConf = "Makeconf.native";
+            srcLdpaths = "ldpaths.native";
         } else if ("llvm".equals(name)) {
             srcConf = "Makeconf.llvm";
+            srcLdpaths = "ldpaths.llvm";
         } else {
             throw error(RError.Message.GENERIC, "Only 'native' or 'llvm' argument values accepted");
         }
         TruffleFile rHome = REnvVars.getRHomeTruffleFile(RContext.getInstance().getEnv());
         TruffleFile etc = rHome.resolve("etc");
-        TruffleFile src = etc.resolve(srcConf);
-        TruffleFile dst = etc.resolve("Makeconf");
+        TruffleFile srcConfFile = etc.resolve(srcConf);
+        TruffleFile dstConfFile = etc.resolve("Makeconf");
+        TruffleFile srcLdpathsFile = etc.resolve(srcLdpaths);
+        TruffleFile dstLdpathsFile = etc.resolve("ldpaths");
         try {
-            src.copy(dst, StandardCopyOption.REPLACE_EXISTING);
+            srcConfFile.copy(dstConfFile, StandardCopyOption.REPLACE_EXISTING);
+            srcLdpathsFile.copy(dstLdpathsFile, StandardCopyOption.REPLACE_EXISTING);
         } catch (IOException e) {
             throw new RInternalError(String.format("Copying %s over etc/Makeconf failed", srcConf), e);
         }

com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/InlineCacheNode.java

@@ -22,20 +22,17 @@
  */
 package com.oracle.truffle.r.nodes;
 
+import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
-import com.oracle.truffle.api.Truffle;
-import com.oracle.truffle.api.TruffleLanguage;
-import com.oracle.truffle.api.TruffleRuntime;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.ReportPolymorphism;
 import com.oracle.truffle.api.dsl.Specialization;
 import com.oracle.truffle.api.frame.Frame;
 import com.oracle.truffle.api.frame.MaterializedFrame;
 import com.oracle.truffle.api.frame.VirtualFrame;
-import com.oracle.truffle.api.nodes.DirectCallNode;
-import com.oracle.truffle.api.nodes.RootNode;
-import com.oracle.truffle.api.profiles.ValueProfile;
-import com.oracle.truffle.r.runtime.context.RContext;
+import com.oracle.truffle.api.nodes.Node;
+import com.oracle.truffle.api.profiles.ConditionProfile;
 import com.oracle.truffle.r.runtime.data.Closure;
 import com.oracle.truffle.r.runtime.nodes.RBaseNode;
 import com.oracle.truffle.r.runtime.nodes.RNode;
@@ -55,61 +52,65 @@ protected InlineCacheNode(int maxPicDepth) {
         this.maxPicDepth = maxPicDepth;
     }
 
-    /**
-     * Creates an inline cache that will execute promises closures by using a polymorphic inline
-     * cache (PIC) and falling back to
-     * {@link InlineCacheNode#evalPromise(MaterializedFrame, Closure)}.
-     *
-     * @param maxPicDepth maximum number of entries in the PIC
-     */
-    public static InlineCacheNode create(int maxPicDepth) {
-        return InlineCacheNodeGen.create(maxPicDepth);
-    }
+    @SuppressWarnings("unused")
+    @Specialization(limit = "maxPicDepth", guards = "value == cachedValue")
+    protected Object doCached(Frame frame, Object value,
+                    @Cached("value") Object cachedValue,
+                    @Cached("createBinaryProfile()") ConditionProfile isVirtualFrameProfile,
+                    @Cached("cache(cachedValue)") RNode reified,
+                    @Cached("new(reified)") NodeInsertedClosure nodeInsertedClosure) {
+        VirtualFrame vf;
+        if (isVirtualFrameProfile.profile(frame instanceof VirtualFrame)) {
+            vf = (VirtualFrame) frame;
+        } else {
+            vf = frame.materialize();
+        }
 
-    // Cached case: create a root node and direct call node
-    // This should allow Truffle PE to inline the promise code if it deems it is beneficial
+        // Use a closure to notify the root node that a new node has just been inserted. The
+        // closure is necessary to do the notification just once.
+        nodeInsertedClosure.notifyNodeInserted();
 
-    protected DirectCallNode createInlinableCall(Closure value) {
-        InlineCacheRootNode rootNode = new InlineCacheRootNode(RContext.getInstance().getLanguage(), RASTUtils.cloneNode((RNode) value.getExpr()));
-        TruffleRuntime runtime = Truffle.getRuntime();
-        runtime.createCallTarget(rootNode);
-        return runtime.createDirectCallNode(rootNode.getCallTarget());
+        return reified.visibleExecute(vf);
     }
 
-    private static final class InlineCacheRootNode extends RootNode {
-        @Child private RNode node;
-
-        protected InlineCacheRootNode(TruffleLanguage<?> language, RNode node) {
-            super(language);
-            this.node = node;
-        }
-
-        @Override
-        public Object execute(VirtualFrame frame) {
-            VirtualFrame execFrame = (VirtualFrame) frame.getArguments()[0];
-            return node.visibleExecute(execFrame);
-        }
+    @Specialization(replaces = "doCached")
+    protected Object doGeneric(Frame frame, Object value) {
+        return evalPromise(frame.materialize(), (Closure) value);
     }
 
-    @Specialization(limit = "maxPicDepth", guards = "value == cachedValue")
-    protected Object doCached(Frame frame, @SuppressWarnings("unused") Closure value,
-                    @SuppressWarnings("unused") @Cached("value") Closure cachedValue,
-                    @Cached("createClassProfile()") ValueProfile frameClassProfile,
-                    @Cached("createInlinableCall(value)") DirectCallNode callNode) {
-        return callNode.call(frameClassProfile.profile(frame).materialize());
+    protected RNode cache(Object value) {
+        return RASTUtils.cloneNode((RNode) ((Closure) value).getExpr());
     }
 
-    // Generic case: execute call target cached in the Closure
-    // We do not go though call node, so not Truffle inlining can take place,
-    // but nothing is inserted into this AST, so it doesn't grow without limits
-
-    @Specialization(replaces = "doCached")
-    protected Object doGeneric(Frame frame, Closure value) {
-        return evalPromise(frame.materialize(), value);
+    /**
+     * Creates an inline cache that will execute promises closures by using a PIC and falling back
+     * to {@link InlineCacheNode#evalPromise(MaterializedFrame, Closure)}.
+     *
+     * @param maxPicDepth maximum number of entries in the polymorphic inline cache
+     */
+    public static InlineCacheNode create(int maxPicDepth) {
+        return InlineCacheNodeGen.create(maxPicDepth);
     }
 
     @TruffleBoundary
     protected static Object evalPromise(MaterializedFrame frame, Closure closure) {
         return closure.eval(frame);
     }
+
+    protected class NodeInsertedClosure {
+        private final Node n;
+        @CompilationFinal boolean notified;
+
+        public NodeInsertedClosure(Node n) {
+            this.n = n;
+        }
+
+        void notifyNodeInserted() {
+            if (!notified) {
+                CompilerDirectives.transferToInterpreterAndInvalidate();
+                InlineCacheNode.this.notifyInserted(n);
+                notified = true;
+            }
+        }
+    }
 }

com.oracle.truffle.r.nodes/src/com/oracle/truffle/r/nodes/access/vector/CachedExtractVectorNode.java

@@ -308,6 +308,8 @@ private void applyDimensions(RAbstractContainer originalTarget, RAbstractVector
                     }
                     newDimNames[dimIndex] = result;
                     if (newDimNamesNames != null) {
+                        // (newDimNamesNames == null) <=> (originalDimNamesNames == null)
+                        assert originalDimNamesNames != null;
                         newDimNamesNames[dimIndex] = originalDimNamesNames.getDataAt(i);
                     }
                 }

com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/REnvVars.java

@@ -126,7 +126,7 @@ public RContext.ContextState initialize(RContext context) {
                 }
             }
             TruffleFile userFile = userFilePath != null ? FileSystemUtils.getSafeTruffleFile(env, userFilePath) : null;
-            if (userFile.exists()) {
+            if (userFile != null && userFile.exists()) {
                 safeReadEnvironFile(userFile);
             }
         }

mx.fastr/mx_fastr_dists.py

@@ -292,6 +292,7 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi
         short_name='R',
         license_files=['LICENSE_FASTR'],
         third_party_license_files=['3rd_party_licenses_fastr.txt'],
+        dependencies=['Truffle', 'Sulong', 'LLVM.org toolchain'],
         truffle_jars=['fastr:FASTR'],
         support_distributions=['fastr:FASTR_GRAALVM_SUPPORT'],
         provided_executables=[