[GR-19137] [GR-17127] [GR-18935] Backport several fixes.

PullRequest: fastr/2199

CHANGELOG.md

@@ -2,16 +2,25 @@
 
 New features:
 
+* new Graal LLVM based back-end for running packages native code.
+  * The default {FASTR_HOME}/etc/Makeconf is configured to use the Graal LLVM toolchain to build the native code of R packages.
+    * The toolchain builds standard native binaries for a given plarform and also embeds the corresponding LLVM bitcode in them.
+    * R builtin `fastr.setToolchain(name)` (`name` can be `llvm` or `native`) sets the compiler toolchain used for package building (modifies etc/Makeconf).
+    * To switch back to the previous toolchain configuration that used `GCC`, execute `fastr.setToolchain("native")`.
+  * Option `--R.BackEnd` specifies the default backend used to execute packages native code.
+    * Different R packages can be run via different backends in one FastR context.
+    * `--R.BackEnd=native`, the default, is JNI based backend that runs directly the actual native code.
+    * `--R.BackEnd=llvm` is the new LLVM backend that loads the LLVM bitcode embedded in package libraries and runs it via Graal LLVM.
+    * `--R.BackEndNative=pkg1,pkg2,...` enumerates packages whose native code will be executed by the native backend.
+    * `--R.BackEndLLVM=pkg1,pkg2,...` enumerates packages whose native code will be executed by the LLVM backend.
+    * Note: All `--R.BackEnd*` options are passed to R subprocesses.
+  * Debugging of packages native code with LLVM backend.
+    * Option `--R.DebugLLVMLibs` activates debugging of native code using the bundled LLVM bitcode.
+    * Builtin `fastr.useDebugMakevars(use)` activates/deactivates a special `etc/Makevars.site` tailored for building the packages native code for debugging.
 * `gc` attempts to invoke Java GC when FastR is run with `--R.EnableExplicitGC=true`
   * this is intended only for testing purposes and it is not recommended to run GC explicitly in FastR.
   * `gc` is not invoking Java GC by default because GNU-R GC and Java GC are fundamentally
     different and this may lead to unintended behavior.
-* Option `--R.BackEnd` specifies an RFFI backend. `--R.BackEnd=native`, the default, activates the JNI based backend, while `--R.BackEnd=llvm` activates the LLVM backend loading the LLVM bitcode bundled with package libraries. Note: All ``--R.BackEnd*` options are passed to R subprocesses.
-* Option `--R.BackEndNative` enumerates packages whose native code will be executed by the native backend (JNI based)
-* Option `--R.BackEndLLVM` enumerates packages whose native code will be executed by the LLVM backend
-* Option `--R.DebugLLVMLibs` activates debugging of native code using the bundled LLVM bitcode
-* Builtin `fastr.useDebugMakevars(use)` activates/deactivates a special `etc/Makevars.site` for debugging native code
-* Builtin `fastr.setToolchain(name)` (`name` can be `llvm` or `native`) sets the compiler toolchain used for package building
 
 Added missing R builtins and C APIs
 
@@ -34,7 +43,6 @@ Bug fixes:
 * failures when working with large vectors (>1GB)
 
 
-
 # 19.2.0
 
 New features:

README.md

@@ -57,7 +57,7 @@ which can be installed with `install.fastr.packages(c("rJava", "data.table"))`.
 Packages that use the [R extensions C API](https://cran.r-project.org/doc/manuals/r-release/R-exts.html) in hot paths,
 especially via `Rcpp`, may exhibit slower performance on FastR due to the high cost of transitions between the native and managed code.
 This will be mitigated by using the Graal LLVM interpreter. The support is available in the development version via
-the `--R.BackEndLLVM` option. Note that most of the times FastR running R code equivalent to given `Rcpp` code is as
+the `--R.BackEnd=llvm` option. Note that most of the times FastR running R code equivalent to given `Rcpp` code is as
 fast as GNU-R and sometimes even faster because the GraalVM dynamic compiler can optimize the program as a whole.
 
 ## Stay connected with the community

com.oracle.truffle.r.ffi.impl/src/com/oracle/truffle/r/ffi/impl/llvm/TruffleLLVM_UserRng.java

@@ -64,7 +64,7 @@ private static final class TruffleLLVMInitNode extends RNGNode implements InitNo
         @Override
         public void execute(VirtualFrame frame, int seed) {
             RFFIContext stateRFFI = RContext.getInstance().getStateRFFI();
-            Object before = stateRFFI.beforeDowncall(frame, RFFIFactory.Type.LLVM);
+            Object before = stateRFFI.beforeDowncall(frame.materialize(), RFFIFactory.Type.LLVM);
             try {
                 userFunctionInterop.execute(userFunctionTarget, seed);
             } catch (InteropException ex) {
@@ -84,7 +84,7 @@ private static final class TruffleLLVM_RandNode extends RNGNode implements RandN
         @Override
         public double execute(VirtualFrame frame) {
             RFFIContext stateRFFI = RContext.getInstance().getStateRFFI();
-            Object before = stateRFFI.beforeDowncall(frame, RFFIFactory.Type.LLVM);
+            Object before = stateRFFI.beforeDowncall(frame.materialize(), RFFIFactory.Type.LLVM);
             try {
                 Object address = userFunctionInterop.execute(userFunctionTarget);
                 return (double) readPointerInterop.execute(readPointerTarget, address);
@@ -105,7 +105,7 @@ private static final class TruffleLLVM_NSeedNode extends RNGNode implements NSee
         @Override
         public int execute(VirtualFrame frame) {
             RFFIContext stateRFFI = RContext.getInstance().getStateRFFI();
-            Object before = stateRFFI.beforeDowncall(frame, RFFIFactory.Type.LLVM);
+            Object before = stateRFFI.beforeDowncall(frame.materialize(), RFFIFactory.Type.LLVM);
             try {
                 Object address = userFunctionInterop.execute(userFunctionTarget);
                 return (int) readPointerInterop.execute(readPointerTarget, address);
@@ -126,7 +126,7 @@ private static final class TruffleLLVM_SeedsNode extends RNGNode implements Seed
         @Override
         public void execute(VirtualFrame frame, int[] n) {
             RFFIContext stateRFFI = RContext.getInstance().getStateRFFI();
-            Object before = stateRFFI.beforeDowncall(frame, RFFIFactory.Type.LLVM);
+            Object before = stateRFFI.beforeDowncall(frame.materialize(), RFFIFactory.Type.LLVM);
             try {
                 Object address = userFunctionInterop.execute(userFunctionTarget);
                 for (int i = 0; i < n.length; i++) {

com.oracle.truffle.r.ffi.impl/src/com/oracle/truffle/r/ffi/impl/mixed/TruffleMixed_Context.java

@@ -22,7 +22,7 @@
  */
 package com.oracle.truffle.r.ffi.impl.mixed;
 
-import com.oracle.truffle.api.frame.VirtualFrame;
+import com.oracle.truffle.api.frame.MaterializedFrame;
 import com.oracle.truffle.api.interop.TruffleObject;
 import com.oracle.truffle.r.ffi.impl.llvm.TruffleLLVM_Context;
 import com.oracle.truffle.r.ffi.impl.llvm.TruffleLLVM_DLL.LLVM_Handle;
@@ -137,7 +137,7 @@ public void afterUpcall(boolean canRunGc, Type rffiType) {
     }
 
     @Override
-    public Object beforeDowncall(VirtualFrame frame, Type rffiType) {
+    public Object beforeDowncall(MaterializedFrame frame, Type rffiType) {
         Type actualRffiType = rffiType == null ? Type.LLVM : rffiType;
         assert rffiType != null;
         switch (rffiType) {

com.oracle.truffle.r.ffi.impl/src/com/oracle/truffle/r/ffi/impl/nfi/TruffleNFI_Context.java

@@ -35,7 +35,7 @@
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
-import com.oracle.truffle.api.frame.VirtualFrame;
+import com.oracle.truffle.api.frame.MaterializedFrame;
 import com.oracle.truffle.api.interop.InteropException;
 import com.oracle.truffle.api.interop.InteropLibrary;
 import com.oracle.truffle.api.interop.TruffleObject;
@@ -399,7 +399,7 @@ public void beforeDispose(RContext context) {
     }
 
     @Override
-    public Object beforeDowncall(VirtualFrame frame, RFFIFactory.Type rffiType) {
+    public Object beforeDowncall(MaterializedFrame frame, RFFIFactory.Type rffiType) {
         Object tokenFromSuper = super.beforeDowncall(frame, RFFIFactory.Type.NFI);
         transientAllocations.push(new ArrayList<>());
         if (hasAccessLock) {

com.oracle.truffle.r.ffi.impl/src/com/oracle/truffle/r/ffi/impl/nfi/TruffleNFI_UserRng.java

@@ -63,7 +63,7 @@ private static final class TruffleNFI_InitNode extends RNGNode implements InitNo
         @Override
         public void execute(VirtualFrame frame, int seed) {
             RFFIContext stateRFFI = RContext.getInstance().getStateRFFI();
-            Object before = stateRFFI.beforeDowncall(frame, RFFIFactory.Type.NFI);
+            Object before = stateRFFI.beforeDowncall(frame.materialize(), RFFIFactory.Type.NFI);
             try {
                 userFunctionInterop.execute(userFunctionTarget, seed);
             } catch (InteropException ex) {
@@ -83,7 +83,7 @@ private static final class TruffleNFI_RandNode extends RNGNode implements RandNo
         @Override
         public double execute(VirtualFrame frame) {
             RFFIContext stateRFFI = RContext.getInstance().getStateRFFI();
-            Object before = stateRFFI.beforeDowncall(frame, RFFIFactory.Type.NFI);
+            Object before = stateRFFI.beforeDowncall(frame.materialize(), RFFIFactory.Type.NFI);
             try {
                 Object address = userFunctionInterop.execute(userFunctionTarget);
                 return (double) readPointerInterop.execute(readPointerTarget, address);
@@ -104,7 +104,7 @@ private static final class TruffleNFI_NSeedNode extends RNGNode implements NSeed
         @Override
         public int execute(VirtualFrame frame) {
             RFFIContext stateRFFI = RContext.getInstance().getStateRFFI();
-            Object before = stateRFFI.beforeDowncall(frame, RFFIFactory.Type.NFI);
+            Object before = stateRFFI.beforeDowncall(frame.materialize(), RFFIFactory.Type.NFI);
             try {
                 Object address = userFunctionInterop.execute(userFunctionTarget);
                 return (int) readPointerInterop.execute(readPointerTarget, address);
@@ -125,7 +125,7 @@ private static final class TruffleNFI_SeedsNode extends RNGNode implements Seeds
         @Override
         public void execute(VirtualFrame frame, int[] n) {
             RFFIContext stateRFFI = RContext.getInstance().getStateRFFI();
-            Object before = stateRFFI.beforeDowncall(frame, RFFIFactory.Type.NFI);
+            Object before = stateRFFI.beforeDowncall(frame.materialize(), RFFIFactory.Type.NFI);
             try {
                 Object address = userFunctionInterop.execute(userFunctionTarget);
                 for (int i = 0; i < n.length; i++) {

com.oracle.truffle.r.ffi.impl/src/com/oracle/truffle/r/ffi/impl/nodes/RForceAndCallNode.java

@@ -180,6 +180,7 @@ private static RArgsValuesAndNames createArgsAndNames(List<Object> argValues, RA
     }
 
     private void flattenFirstArgs(VirtualFrame frame, int n, RArgsValuesAndNames args, PromiseHelperNode promiseHelper) {
+        // In GnuR there appears to be no error checks on n > args.length
         if (args.getLength() < n) {
             CompilerDirectives.transferToInterpreter();
             throw RError.nyi(this, "forceAndCall with insufficient arguments");

com.oracle.truffle.r.native/run/configure_fastr

@@ -102,9 +102,9 @@ done
     res=$?
     if [[ $res != 0 ]]; then
       echo "Cannot build a simple C program linking with libgfortran, libgcc_s, and libquadmath."
-      echo "Please install GCC using:"
-      echo "    On Debian based systems: apt-get install build-essential"
-      echo "    On CentOS/RHEL: yum groupinstall \"Developent Tools\""
+      echo "Please install GCC and gfortran using:"
+      echo "    On Debian based systems: sudo apt-get install build-essential gfortran"
+      echo "    On CentOS/RHEL/Oracle Linux: sudo yum groupinstall \"Developent Tools\"; yum install gcc-gfortran"
 	  echo ""	
       printHelp
       exit 1

com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/base/GrepFunctions.java

@@ -49,8 +49,8 @@
 import com.oracle.truffle.r.nodes.attributes.SetFixedAttributeNode;
 import com.oracle.truffle.r.nodes.builtin.NodeWithArgumentCasts.Casts;
 import com.oracle.truffle.r.nodes.builtin.RBuiltinNode;
-import com.oracle.truffle.r.runtime.RError;
 import com.oracle.truffle.r.runtime.Collections.ArrayListObj;
+import com.oracle.truffle.r.runtime.RError;
 import com.oracle.truffle.r.runtime.RError.Message;
 import com.oracle.truffle.r.runtime.RInternalError;
 import com.oracle.truffle.r.runtime.RRuntime;
@@ -1977,6 +1977,9 @@ private static RIntVector convertFirstMatchedRangeToRIndex(HaystackDescriptor ha
         protected Object grepFixedNoInvert(RRawVector pattern, RRawVector x, int offset, @SuppressWarnings("unused") Object ignoreCase, @SuppressWarnings("unused") boolean fixed, boolean value,
                         boolean all,
                         @SuppressWarnings("unused") boolean invert) {
+            if (x.getLength() == 0) {
+                return RDataFactory.createEmptyIntVector();
+            }
             HaystackDescriptor haystackDescriptor = null;
             if (all) {
                 haystackDescriptor = findAllOccurrences(pattern, x, offset);
@@ -2011,6 +2014,9 @@ protected Object grepFixedNoInvert(RRawVector pattern, RRawVector x, int offset,
         protected Object grepFixedInvert(RRawVector pattern, RRawVector x, int offset, @SuppressWarnings("unused") Object ignoreCase, @SuppressWarnings("unused") boolean fixed,
                         @SuppressWarnings("unused") boolean value, boolean all,
                         @SuppressWarnings("unused") boolean invert) {
+            if (x.getLength() == 0) {
+                return RDataFactory.createEmptyIntVector();
+            }
             HaystackDescriptor haystackDescriptor = null;
             if (all) {
                 haystackDescriptor = findAllOccurrences(pattern, x, offset);
@@ -2029,6 +2035,9 @@ protected Object grepFixedInvert(RRawVector pattern, RRawVector x, int offset, @
         protected Object grepFixedIgnoreInvert(RRawVector pattern, RRawVector x, int offset, @SuppressWarnings("unused") Object ignoreCase, boolean fixed, boolean value, boolean all,
                         boolean invert) {
             warning(Message.ARGUMENT_IGNORED, "invert = TRUE");
+            if (x.getLength() == 0) {
+                return RDataFactory.createEmptyIntVector();
+            }
             return grepFixedNoInvert(pattern, x, offset, ignoreCase, fixed, value, all, invert);
         }
 

com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/base/Rapply.java

@@ -57,6 +57,7 @@
 import com.oracle.truffle.r.runtime.builtins.RBuiltin;
 import com.oracle.truffle.r.runtime.context.RContext;
 import com.oracle.truffle.r.runtime.data.RDataFactory;
+import com.oracle.truffle.r.runtime.data.RExpression;
 import com.oracle.truffle.r.runtime.data.RFunction;
 import com.oracle.truffle.r.runtime.data.RList;
 import com.oracle.truffle.r.runtime.data.model.RAbstractListBaseVector;
@@ -78,22 +79,32 @@ public abstract class Rapply extends RBuiltinNode.Arg5 {
 
     static {
         Casts casts = new Casts(Rapply.class);
-        casts.arg("object").mustBe(RAbstractListVector.class, Message.GENERIC, "'object' must be a list or expression");
+        casts.arg("object").mustBe(RAbstractListBaseVector.class, Message.GENERIC, "'object' must be a list or expression");
         casts.arg("f").mustBe(RFunction.class);
         casts.arg("classes").mapNull(constant("ANY")).mapMissing(constant("ANY")).mustBe(stringValue()).asStringVector().findFirst().mustNotBeNA();
         casts.arg("deflt").allowNull().mapMissing(nullConstant()).mustBe(anyValue());
         casts.arg("how").mapNull(constant("unlist")).mapMissing(constant("unlist")).mustBe(stringValue()).asStringVector().findFirst().mustNotBeNA();
     }
 
     @Specialization(guards = "!isReplace(how)")
-    protected Object rapplyReplace(VirtualFrame frame, RAbstractListVector object, RFunction f, String classes, Object deflt, String how, @Cached("create()") UnaryCopyAttributesNode attri) {
-
+    protected Object rapplyReplace(VirtualFrame frame, RAbstractListVector object, RFunction f, String classes, Object deflt, String how,
+                    @Cached("create()") UnaryCopyAttributesNode attri) {
         return attri.execute(RDataFactory.createList((Object[]) rapply.execute(frame, object, f, classes, deflt, how)), object);
     }
 
     @Specialization(guards = "isReplace(how)")
     protected Object rapply(VirtualFrame frame, RAbstractListVector object, RFunction f, String classes, Object deflt, String how) {
+        return rapply.execute(frame, object, f, classes, deflt, how);
+    }
+
+    @Specialization(guards = "!isReplace(how)")
+    protected Object rapplyReplace(VirtualFrame frame, RExpression object, RFunction f, String classes, Object deflt, String how,
+                    @Cached("create()") UnaryCopyAttributesNode attri) {
+        return attri.execute(RDataFactory.createList((Object[]) rapply.execute(frame, object, f, classes, deflt, how)), object);
+    }
 
+    @Specialization(guards = "isReplace(how)")
+    protected Object rapply(VirtualFrame frame, RExpression object, RFunction f, String classes, Object deflt, String how) {
         return rapply.execute(frame, object, f, classes, deflt, how);
     }
 
@@ -148,7 +159,7 @@ public abstract static class RapplyInternalNode extends RBaseNode implements Int
         protected static final String VECTOR_NAME = "object";
         protected static final String INDEX_NAME = "i";
 
-        public abstract Object execute(VirtualFrame frame, RAbstractListVector object, RFunction f, String classes, Object deflt, String how);
+        public abstract Object execute(VirtualFrame frame, RAbstractListBaseVector object, RFunction f, String classes, Object deflt, String how);
 
         protected static FrameSlot createIndexSlot(Frame frame) {
             return FrameSlotChangeMonitor.findOrAddFrameSlot(frame.getFrameDescriptor(), INDEX_NAME, FrameSlotKind.Int);
@@ -159,7 +170,7 @@ protected static FrameSlot createVectorSlot(Frame frame) {
         }
 
         @Specialization(guards = "isReplace(how)")
-        protected RAbstractListBaseVector cachedLapplyReplace(VirtualFrame frame, RAbstractListVector object, RFunction f, String classes, Object deflt, String how,
+        protected RAbstractListBaseVector cachedRapplyReplace(VirtualFrame frame, RAbstractListBaseVector object, RFunction f, String classes, Object deflt, String how,
                         @Cached("createIndexSlot(frame)") FrameSlot indexSlot,
                         @Cached("createVectorSlot(frame)") FrameSlot vectorSlot,
                         @Cached("create()") RLengthNode lengthNode,
@@ -201,7 +212,7 @@ private RapplyInternalNode getRapply() {
         }
 
         @Specialization(guards = "!isReplace(how)")
-        protected Object[] cachedLapply(VirtualFrame frame, RAbstractListVector object, RFunction f, String classes, Object deflt, String how,
+        protected Object[] cachedRapply(VirtualFrame frame, RAbstractListBaseVector object, RFunction f, String classes, Object deflt, String how,
                         @Cached("createIndexSlot(frame)") FrameSlot indexSlot,
                         @Cached("createVectorSlot(frame)") FrameSlot vectorSlot,
                         @Cached("create()") RLengthNode lengthNode,