Feature/gaugefield unity

include/blas_helper.cuh

@@ -111,9 +111,7 @@ namespace quda
       {}
 
       data_t(const ColorSpinorField &x) :
-        spinor(static_cast<store_t *>(const_cast<ColorSpinorField &>(x).V())),
-        stride(x.VolumeCB()),
-        cb_offset(x.Bytes() / (2 * sizeof(store_t) * N))
+        spinor(x.data<store_t *>()), stride(x.VolumeCB()), cb_offset(x.Bytes() / (2 * sizeof(store_t) * N))
       {}
     };
 
@@ -141,8 +139,8 @@ namespace quda
       {}
 
       data_t(const ColorSpinorField &x) :
-        spinor(static_cast<store_t *>(const_cast<ColorSpinorField &>(x).V())),
-        norm(static_cast<norm_t *>(const_cast<ColorSpinorField &>(x).Norm())),
+        spinor(x.data<store_t *>()),
+        norm(static_cast<norm_t *>(x.Norm())),
         stride(x.VolumeCB()),
         cb_offset(x.Bytes() / (2 * sizeof(store_t) * N)),
         cb_norm_offset(x.Bytes() / (2 * sizeof(norm_t)))

include/blas_quda.h

@@ -23,17 +23,14 @@ namespace quda {
 
     void setParam(int kernel, int prec, int threads, int blocks);
 
-    extern unsigned long long flops;
-    extern unsigned long long bytes;
-
     inline void zero(cvector_ref<ColorSpinorField> &x)
     {
       for (auto i = 0u; i < x.size(); i++) x[i].zero();
     }
 
     inline void copy(ColorSpinorField &dst, const ColorSpinorField &src)
     {
-      if (dst.V() == src.V()) {
+      if (dst.data() == src.data()) {
         // check the fields are equivalent else error
         if (ColorSpinorField::are_compatible(dst, src))
           return;

include/clover_field.h

@@ -178,9 +178,10 @@ namespace quda {
     int nColor = 0;
     int nSpin = 0;
 
-    void *clover = nullptr;
-    void *cloverInv = nullptr;
+    quda_ptr clover = {};
+    quda_ptr cloverInv = {};
 
+    bool inverse = false;
     double diagonal = 0.0;
     array<double, 2> max = {};
 
@@ -213,12 +214,18 @@ namespace quda {
 
   public:
     CloverField(const CloverFieldParam &param);
-    virtual ~CloverField();
 
     static CloverField *Create(const CloverFieldParam &param);
 
-    void* V(bool inverse=false) { return inverse ? cloverInv : clover; }
-    const void* V(bool inverse=false) const { return inverse ? cloverInv : clover; }
+    template <typename T = void *> auto data(bool inverse = false) const
+    {
+      return inverse ? reinterpret_cast<T>(cloverInv.data()) : reinterpret_cast<T>(clover.data());
+    }
+
+    /**
+       @return whether the inverse is explicitly been allocated
+     */
+    bool Inverse() const { return inverse; }
 
     /**
        @return diagonal scaling factor applied to the identity
@@ -406,10 +413,6 @@ namespace quda {
     */
     void copy_from_buffer(void *buffer);
 
-    friend class DiracClover;
-    friend class DiracCloverPC;
-    friend class DiracTwistedClover;
-    friend class DiracTwistedCloverPC;
   };
 
   /**

include/clover_field_order.h

@@ -312,7 +312,7 @@ namespace quda {
       static constexpr int N = nColor * nSpin / 2;
       reconstruct_t<Float, N * N, clover::reconstruct()> recon;
       FloatNAccessor(const CloverField &A, bool inverse = false) :
-        a(static_cast<Float *>(const_cast<void *>(A.V(inverse)))),
+        a(A.Bytes() ? A.data<Float *>(inverse) : nullptr),
         stride(A.VolumeCB()),
         offset_cb(A.Bytes() / (2 * sizeof(Float))),
         compressed_block_size(A.compressed_block_size()),
@@ -403,7 +403,7 @@ namespace quda {
       const int N = nSpin * nColor / 2;
       const complex<Float> zero;
       Accessor(const CloverField &A, bool inverse = false) :
-        a(static_cast<Float *>(const_cast<void *>(A.V(inverse)))),
+        a(A.Bytes() ? A.data<Float *>(inverse) : nullptr),
         offset_cb(A.Bytes() / (2 * sizeof(Float))),
         zero(complex<Float>(0.0, 0.0))
       {
@@ -639,7 +639,7 @@ namespace quda {
           if (clover.max_element(is_inverse) == 0.0 && isFixed<Float>::value)
             errorQuda("%p max_element(%d) appears unset", &clover, is_inverse);
           if (clover.Diagonal() == 0.0 && clover.Reconstruct()) errorQuda("%p diagonal appears unset", &clover);
-          this->clover = clover_ ? clover_ : (Float *)(clover.V(is_inverse));
+          this->clover = clover_ ? clover_ : clover.data<Float *>(is_inverse);
         }
 
         QudaTwistFlavorType TwistFlavor() const { return twist_flavor; }
@@ -844,7 +844,7 @@ namespace quda {
           if (clover.Order() != QUDA_PACKED_CLOVER_ORDER) {
             errorQuda("Invalid clover order %d for this accessor", clover.Order());
           }
-          this->clover = clover_ ? clover_ : (Float *)(clover.V(inverse));
+          this->clover = clover_ ? clover_ : clover.data<Float *>(inverse);
         }
 
         QudaTwistFlavorType TwistFlavor() const { return twist_flavor; }
@@ -892,8 +892,8 @@ namespace quda {
           if (clover.Order() != QUDA_QDPJIT_CLOVER_ORDER) {
             errorQuda("Invalid clover order %d for this accessor", clover.Order());
           }
-          offdiag = clover_ ? ((Float **)clover_)[0] : ((Float **)clover.V(inverse))[0];
-          diag = clover_ ? ((Float **)clover_)[1] : ((Float **)clover.V(inverse))[1];
+          offdiag = clover_ ? ((Float **)clover_)[0] : clover.data<Float **>(inverse)[0];
+          diag = clover_ ? ((Float **)clover_)[1] : clover.data<Float **>(inverse)[1];
         }
 
         QudaTwistFlavorType TwistFlavor() const { return twist_flavor; }
@@ -970,7 +970,7 @@ namespace quda {
           if (clover.Order() != QUDA_BQCD_CLOVER_ORDER) {
             errorQuda("Invalid clover order %d for this accessor", clover.Order());
           }
-          this->clover[0] = clover_ ? clover_ : (Float *)(clover.V(inverse));
+          this->clover[0] = clover_ ? clover_ : clover.data<Float *>(inverse);
           this->clover[1] = (Float *)((char *)this->clover[0] + clover.Bytes() / 2);
         }
 

include/color_spinor_field.h

@@ -121,18 +121,13 @@ namespace quda
     }
   };
 
-  class ColorSpinorParam : public LatticeFieldParam
-  {
-
-  public:
+  struct ColorSpinorParam : public LatticeFieldParam {
     int nColor = 0; // Number of colors of the field
     int nSpin = 0;  // =1 for staggered, =2 for coarse Dslash, =4 for 4d spinor
     int nVec = 1;   // number of packed vectors (for multigrid transfer operator)
 
     QudaTwistFlavorType twistFlavor = QUDA_TWIST_INVALID; // used by twisted mass
-
     QudaSiteOrder siteOrder = QUDA_INVALID_SITE_ORDER; // defined for full fields
-
     QudaFieldOrder fieldOrder = QUDA_INVALID_FIELD_ORDER; // Float, Float2, Float4 etc.
     QudaGammaBasis gammaBasis = QUDA_INVALID_GAMMA_BASIS;
     QudaFieldCreate create = QUDA_INVALID_FIELD_CREATE;
@@ -179,7 +174,6 @@ namespace quda
     ColorSpinorParam() = default;
 
     // used to create cpu params
-
     ColorSpinorParam(void *V, QudaInvertParam &inv_param, const lat_dim_t &X, const bool pc_solution,
                      QudaFieldLocation location = QUDA_CPU_FIELD_LOCATION) :
       LatticeFieldParam(4, X, 0, location, inv_param.cpu_prec),
@@ -188,20 +182,12 @@ namespace quda
              || inv_param.dslash_type == QUDA_LAPLACE_DSLASH) ?
               1 :
               4),
-      nVec(1),
       twistFlavor(inv_param.twist_flavor),
-      siteOrder(QUDA_INVALID_SITE_ORDER),
-      fieldOrder(QUDA_INVALID_FIELD_ORDER),
       gammaBasis(inv_param.gamma_basis),
       create(QUDA_REFERENCE_FIELD_CREATE),
       pc_type(inv_param.dslash_type == QUDA_DOMAIN_WALL_DSLASH ? QUDA_5D_PC : QUDA_4D_PC),
-      v(V),
-      is_composite(false),
-      composite_dim(0),
-      is_component(false),
-      component_id(0)
+      v(V)
     {
-
       if (nDim > QUDA_MAX_DIM) errorQuda("Number of dimensions too great");
       for (int d = 0; d < nDim; d++) x[d] = X[d];
 
@@ -343,8 +329,7 @@ namespace quda
 
     size_t length = 0; // length including pads, but not norm zone
 
-    void *v = nullptr;      // the field elements
-    void *v_h = nullptr;    // the field elements
+    quda_ptr v = {};        // the field elements
     size_t norm_offset = 0; /** offset to the norm (if applicable) */
 
     // multi-GPU parameters
@@ -477,37 +462,19 @@ namespace quda
     /**
        @brief Return pointer to the field allocation
     */
-    void *V()
-    {
-      if (ghost_only) errorQuda("Not defined for ghost-only field");
-      return v;
-    }
-
-    /**
-       @brief Return pointer to the field allocation
-    */
-    const void *V() const
-    {
-      if (ghost_only) errorQuda("Not defined for ghost-only field");
-      return v;
-    }
-
-    /**
-       @brief Return pointer to the norm base pointer in the field allocation
-    */
-    void *Norm()
+    template <typename T = void *> auto data() const
     {
       if (ghost_only) errorQuda("Not defined for ghost-only field");
-      return static_cast<char *>(v) + norm_offset;
+      return reinterpret_cast<T>(v.data());
     }
 
     /**
        @brief Return pointer to the norm base pointer in the field allocation
     */
-    const void *Norm() const
+    void *Norm() const
     {
       if (ghost_only) errorQuda("Not defined for ghost-only field");
-      return static_cast<char *>(v) + norm_offset;
+      return static_cast<char *>(v.data()) + norm_offset;
     }
 
     size_t NormOffset() const { return norm_offset; }
@@ -938,7 +905,7 @@ namespace quda
     static void test_compatible_weak(const ColorSpinorField &a, const ColorSpinorField &b);
 
     friend std::ostream &operator<<(std::ostream &out, const ColorSpinorField &);
-    friend class ColorSpinorParam;
+    friend struct ColorSpinorParam;
   };
 
   /**

include/color_spinor_field_order.h

@@ -877,14 +877,13 @@ namespace quda
       FieldOrderCB(const ColorSpinorField &field, int nFace = 1, void *const v_ = 0, void *const *ghost_ = 0) :
         GhostOrder(field, nFace, ghost_), volumeCB(field.VolumeCB()), accessor(field)
       {
-        v.v = v_ ? static_cast<complex<storeFloat> *>(const_cast<void *>(v_)) :
-                   static_cast<complex<storeFloat> *>(const_cast<void *>(field.V()));
+        v.v = v_ ? static_cast<complex<storeFloat> *>(const_cast<void *>(v_)) : field.data<complex<storeFloat> *>();
         resetScale(field.Scale());
 
         if constexpr (fixed && block_float) {
           if constexpr (nColor == 3 && nSpin == 1 && nVec == 1 && order == 2)
             // special case where the norm is packed into the per site struct
-            v.norm = reinterpret_cast<norm_t *>(const_cast<void *>(field.V()));
+            v.norm = field.data<norm_t *>();
           else
             v.norm = static_cast<norm_t *>(const_cast<void *>(field.Norm()));
           v.norm_offset = field.Bytes() / (2 * sizeof(norm_t));
@@ -1088,7 +1087,7 @@ namespace quda
       size_t bytes;
 
       FloatNOrder(const ColorSpinorField &a, int nFace = 1, Float *buffer = 0, Float **ghost_ = 0) :
-        field(buffer ? buffer : (Float *)a.V()),
+        field(buffer ? buffer : a.data<Float *>()),
         norm(buffer ? reinterpret_cast<norm_type *>(reinterpret_cast<char *>(buffer) + a.NormOffset()) :
                       const_cast<norm_type *>(reinterpret_cast<const norm_type *>(a.Norm()))),
         offset(a.Bytes() / (2 * sizeof(Float) * N)),
@@ -1316,7 +1315,7 @@ namespace quda
       size_t bytes;
 
       FloatNOrder(const ColorSpinorField &a, int nFace = 1, Float *buffer = 0, Float **ghost_ = 0) :
-        field(buffer ? buffer : (Float *)a.V()),
+        field(buffer ? buffer : a.data<Float *>()),
         offset(a.Bytes() / (2 * sizeof(Vector))),
         volumeCB(a.VolumeCB()),
         nParity(a.SiteSubset()),
@@ -1505,7 +1504,7 @@ namespace quda
       int faceVolumeCB[4];
       int nParity;
       SpaceColorSpinorOrder(const ColorSpinorField &a, int nFace = 1, Float *field_ = 0, float * = 0, Float **ghost_ = 0) :
-        field(field_ ? field_ : (Float *)a.V()),
+        field(field_ ? field_ : a.data<Float *>()),
         offset(a.Bytes() / (2 * sizeof(Float))),
         volumeCB(a.VolumeCB()),
         nParity(a.SiteSubset())
@@ -1589,7 +1588,7 @@ namespace quda
       int faceVolumeCB[4];
       int nParity;
       SpaceSpinorColorOrder(const ColorSpinorField &a, int nFace = 1, Float *field_ = 0, float * = 0, Float **ghost_ = 0) :
-        field(field_ ? field_ : (Float *)a.V()),
+        field(field_ ? field_ : a.data<Float *>()),
         offset(a.Bytes() / (2 * sizeof(Float))),
         volumeCB(a.VolumeCB()),
         nParity(a.SiteSubset())
@@ -1668,7 +1667,7 @@ namespace quda
       int exDim[4]; // full field dimensions
       PaddedSpaceSpinorColorOrder(const ColorSpinorField &a, int nFace = 1, Float *field_ = 0, float * = 0,
                                   Float **ghost_ = 0) :
-        field(field_ ? field_ : (Float *)a.V()),
+        field(field_ ? field_ : a.data<Float *>()),
         volumeCB(a.VolumeCB()),
         exVolumeCB(1),
         nParity(a.SiteSubset()),
@@ -1763,7 +1762,7 @@ namespace quda
       int volumeCB;
       int nParity;
       QDPJITDiracOrder(const ColorSpinorField &a, int = 1, Float *field_ = 0, float * = 0) :
-        field(field_ ? field_ : (Float *)a.V()), volumeCB(a.VolumeCB()), nParity(a.SiteSubset())
+        field(field_ ? field_ : a.data<Float *>()), volumeCB(a.VolumeCB()), nParity(a.SiteSubset())
       {
       }