1 #include <petsc/private/vecimpl_kokkos.hpp> 2 #include <petsc/private/dmdaimpl.h> 3 #include <petscdmda_kokkos.hpp> 4 5 /* Use macro instead of inlined function just to avoid annoying warnings like: 'dof' may be used uninitialized in this function [-Wmaybe-uninitialized] */ 6 #define DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof) \ 7 do { \ 8 PetscCall(DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm)); \ 9 PetscCall(DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gxm,&gym,&gzm)); \ 10 PetscCall(DMDAGetInfo(da,&dim,NULL,NULL,NULL,NULL,NULL,NULL,&dof,NULL,NULL,NULL,NULL,NULL)); \ 11 /* Handle case where user passes in global vector as opposed to local */ \ 12 PetscCall(VecGetLocalSize(vec,&N)); \ 13 if (N == xm*ym*zm*dof) { \ 14 gxm = xm; gym = ym; gzm = zm; \ 15 gxs = xs; gys = ys; gzs = zs; \ 16 } else PetscCheck(N == gxm*gym*gzm*dof,PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Vector local size %D is not compatible with DMDA local sizes %D %D",N,xm*ym*zm*dof,gxm*gym*gzm*dof); \ 17 } while (0) 18 19 /* -------------------- 1D ---------------- */ 20 template<class MemorySpace> 21 PetscErrorCode DMDAVecGetKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView1DType<MemorySpace> *ov,PetscBool overwrite) 22 { 23 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 24 PetscScalarKokkosViewType<MemorySpace> kv; 25 26 PetscFunctionBegin; 27 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 28 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 29 PetscValidPointer(ov,3); 30 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 31 PetscCheck(dim == 1,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 1D but DMDA is %dD",(int)dim); 32 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 33 else PetscCall(VecGetKokkosView(vec,&kv)); 34 /* Construct the unmanaged OffsetView with {begin0,begin1,begins2},{end0,end1,end2} */ 35 *ov = PetscScalarKokkosOffsetView1DType<MemorySpace>(kv.data(),{gxs*dof},{(gxs+gxm)*dof}); 36 PetscFunctionReturn(0); 37 } 38 39 template<class MemorySpace> 40 PetscErrorCode DMDAVecRestoreKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView1DType<MemorySpace> *ov,PetscBool overwrite) 41 { 42 PetscScalarKokkosViewType<MemorySpace> kv; 43 44 PetscFunctionBegin; 45 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 46 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 47 PetscValidPointer(ov,3); 48 kv = ov->view(); /* OffsetView to View */ 49 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 50 else PetscCall(VecRestoreKokkosView(vec,&kv)); 51 PetscFunctionReturn(0); 52 } 53 54 template<class MemorySpace> 55 PetscErrorCode DMDAVecGetKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView1DType<MemorySpace> *ov) 56 { 57 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 58 ConstPetscScalarKokkosViewType<MemorySpace> kv; 59 60 PetscFunctionBegin; 61 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 62 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 63 PetscValidPointer(ov,3); 64 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 65 PetscCheck(dim == 1,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 1D but DMDA is %dD",(int)dim); 66 PetscCall(VecGetKokkosView(vec,&kv)); 67 *ov = ConstPetscScalarKokkosOffsetView1DType<MemorySpace>(kv.data(),{gxs*dof},{(gxs+gxm)*dof}); 68 PetscFunctionReturn(0); 69 } 70 71 template<class MemorySpace> 72 PetscErrorCode DMDAVecRestoreKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView1DType<MemorySpace> *ov) 73 { 74 ConstPetscScalarKokkosViewType<MemorySpace> kv; 75 76 PetscFunctionBegin; 77 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 78 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 79 PetscValidPointer(ov,3); 80 kv = ov->view(); 81 PetscCall(VecRestoreKokkosView(vec,&kv)); 82 PetscFunctionReturn(0); 83 } 84 85 /* ============================== 2D ================================= */ 86 template<class MemorySpace> 87 PetscErrorCode DMDAVecGetKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView2DType<MemorySpace> *ov,PetscBool overwrite) 88 { 89 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 90 PetscScalarKokkosViewType<MemorySpace> kv; 91 92 PetscFunctionBegin; 93 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 94 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 95 PetscValidPointer(ov,3); 96 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 97 PetscCheck(dim == 2,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 2D but DMDA is %dD",(int)dim); 98 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 99 else PetscCall(VecGetKokkosView(vec,&kv)); 100 *ov = PetscScalarKokkosOffsetView2DType<MemorySpace>(kv.data(),{gys*dof,gxs*dof},{(gys+gym)*dof,(gxs+gxm)*dof}); 101 PetscFunctionReturn(0); 102 } 103 104 template<class MemorySpace> 105 PetscErrorCode DMDAVecRestoreKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView2DType<MemorySpace> *ov,PetscBool overwrite) 106 { 107 PetscScalarKokkosViewType<MemorySpace> kv; 108 109 PetscFunctionBegin; 110 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 111 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 112 PetscValidPointer(ov,3); 113 // kv = ov->view(); /* 2D OffsetView => 2D View => 1D View. Why does it not work? */ 114 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)); 115 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 116 else PetscCall(VecRestoreKokkosView(vec,&kv)); 117 PetscFunctionReturn(0); 118 } 119 120 template<class MemorySpace> 121 PetscErrorCode DMDAVecGetKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView2DType<MemorySpace> *ov) 122 { 123 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 124 ConstPetscScalarKokkosViewType<MemorySpace> kv; 125 126 PetscFunctionBegin; 127 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 128 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 129 PetscValidPointer(ov,3); 130 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 131 PetscCheck(dim == 2,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 2D but DMDA is %dD",(int)dim); 132 PetscCall(VecGetKokkosView(vec,&kv)); 133 *ov = ConstPetscScalarKokkosOffsetView2DType<MemorySpace>(kv.data(),{gys*dof,gxs*dof},{(gys+gym)*dof,(gxs+gxm)*dof}); 134 PetscFunctionReturn(0); 135 } 136 137 template<class MemorySpace> 138 PetscErrorCode DMDAVecRestoreKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView2DType<MemorySpace> *ov) 139 { 140 ConstPetscScalarKokkosViewType<MemorySpace> kv; 141 142 PetscFunctionBegin; 143 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 144 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 145 PetscValidPointer(ov,3); 146 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)); 147 PetscCall(VecRestoreKokkosView(vec,&kv)); 148 PetscFunctionReturn(0); 149 } 150 151 /* ============================== 3D ================================= */ 152 template<class MemorySpace> 153 PetscErrorCode DMDAVecGetKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView3DType<MemorySpace> *ov,PetscBool overwrite) 154 { 155 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 156 PetscScalarKokkosViewType<MemorySpace> kv; 157 158 PetscFunctionBegin; 159 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 160 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 161 PetscValidPointer(ov,3); 162 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 163 PetscCheck(dim == 3,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 3D but DMDA is %dD",(int)dim); 164 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 165 else PetscCall(VecGetKokkosView(vec,&kv)); 166 *ov = PetscScalarKokkosOffsetView3DType<MemorySpace>(kv.data(),{gzs*dof,gys*dof,gxs*dof},{(gzs+gzm)*dof,(gys+gym)*dof,(gxs+gxm)*dof}); 167 PetscFunctionReturn(0); 168 } 169 170 template<class MemorySpace> 171 PetscErrorCode DMDAVecRestoreKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView3DType<MemorySpace> *ov,PetscBool overwrite) 172 { 173 PetscScalarKokkosViewType<MemorySpace> kv; 174 175 PetscFunctionBegin; 176 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 177 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 178 PetscValidPointer(ov,3); 179 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)); 180 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 181 else PetscCall(VecRestoreKokkosView(vec,&kv)); 182 PetscFunctionReturn(0); 183 } 184 185 template<class MemorySpace> 186 PetscErrorCode DMDAVecGetKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView3DType<MemorySpace> *ov) 187 { 188 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 189 ConstPetscScalarKokkosViewType<MemorySpace> kv; 190 191 PetscFunctionBegin; 192 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 193 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 194 PetscValidPointer(ov,3); 195 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 196 PetscCheck(dim == 3,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 3D but DMDA is %dD",(int)dim); 197 PetscCall(VecGetKokkosView(vec,&kv)); 198 *ov = ConstPetscScalarKokkosOffsetView3DType<MemorySpace>(kv.data(),{gzs*dof,gys*dof,gxs*dof},{(gzs+gzm)*dof,(gys+gym)*dof,(gxs+gxm)*dof}); 199 PetscFunctionReturn(0); 200 } 201 202 template<class MemorySpace> 203 PetscErrorCode DMDAVecRestoreKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView3DType<MemorySpace> *ov) 204 { 205 ConstPetscScalarKokkosViewType<MemorySpace> kv; 206 207 PetscFunctionBegin; 208 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 209 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 210 PetscValidPointer(ov,3); 211 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)); 212 PetscCall(VecRestoreKokkosView(vec,&kv)); 213 PetscFunctionReturn(0); 214 } 215 216 /* Function template explicit instantiation */ 217 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView1D*); 218 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView1D*); 219 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView1D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 220 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView1D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 221 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView1D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 222 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView1D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 223 224 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView2D*); 225 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView2D*); 226 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 227 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 228 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 229 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 230 231 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView3D*); 232 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView3D*); 233 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 234 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 235 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 236 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 237 238 #if !defined(KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_HOST) /* Get host views if the default memory space is not host space */ 239 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView1DHost*); 240 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView1DHost*); 241 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView1DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 242 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView1DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 243 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView1DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 244 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView1DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 245 246 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView2DHost*); 247 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView2DHost*); 248 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 249 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 250 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 251 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 252 253 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView3DHost*); 254 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView3DHost*); 255 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 256 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 257 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 258 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 259 #endif 260 261 /* ============================== 2D including DOF ================================= */ 262 template<class MemorySpace> 263 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView2DType<MemorySpace> *ov,PetscBool overwrite) 264 { 265 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 266 PetscScalarKokkosViewType<MemorySpace> kv; 267 268 PetscFunctionBegin; 269 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 270 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 271 PetscValidPointer(ov,3); 272 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 273 PetscCheck(dim == 1,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 2D but DMDA is %dD",(int)dim); 274 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 275 else PetscCall(VecGetKokkosView(vec,&kv)); 276 *ov = PetscScalarKokkosOffsetView2DType<MemorySpace>(kv.data(),{gxs,0},{gxs+gxm,dof}); 277 PetscFunctionReturn(0); 278 } 279 280 template<class MemorySpace> 281 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView2DType<MemorySpace> *ov,PetscBool overwrite) 282 { 283 PetscScalarKokkosViewType<MemorySpace> kv; 284 285 PetscFunctionBegin; 286 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 287 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 288 PetscValidPointer(ov,3); 289 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)); 290 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 291 else PetscCall(VecRestoreKokkosView(vec,&kv)); 292 PetscFunctionReturn(0); 293 } 294 295 template<class MemorySpace> 296 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView2DType<MemorySpace> *ov) 297 { 298 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 299 ConstPetscScalarKokkosViewType<MemorySpace> kv; 300 301 PetscFunctionBegin; 302 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 303 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 304 PetscValidPointer(ov,3); 305 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 306 PetscCheck(dim == 1,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 2D but DMDA is %dD",(int)dim); 307 PetscCall(VecGetKokkosView(vec,&kv)); 308 *ov = ConstPetscScalarKokkosOffsetView2DType<MemorySpace>(kv.data(),{gxs,0},{gxs+gxm,dof}); 309 PetscFunctionReturn(0); 310 } 311 312 template<class MemorySpace> 313 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView2DType<MemorySpace> *ov) 314 { 315 ConstPetscScalarKokkosViewType<MemorySpace> kv; 316 317 PetscFunctionBegin; 318 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 319 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 320 PetscValidPointer(ov,3); 321 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)); 322 PetscCall(VecRestoreKokkosView(vec,&kv)); 323 PetscFunctionReturn(0); 324 } 325 326 /* ============================== 3D including DOF ================================= */ 327 template<class MemorySpace> 328 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView3DType<MemorySpace> *ov,PetscBool overwrite) 329 { 330 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 331 PetscScalarKokkosViewType<MemorySpace> kv; 332 333 PetscFunctionBegin; 334 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 335 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 336 PetscValidPointer(ov,3); 337 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 338 PetscCheck(dim == 2,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 3D but DMDA is %dD",(int)dim); 339 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 340 else PetscCall(VecGetKokkosView(vec,&kv)); 341 *ov = PetscScalarKokkosOffsetView3DType<MemorySpace>(kv.data(),{gys,gxs,0},{gys+gym,gxs+gxm,dof}); 342 PetscFunctionReturn(0); 343 } 344 345 template<class MemorySpace> 346 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView3DType<MemorySpace> *ov,PetscBool overwrite) 347 { 348 PetscScalarKokkosViewType<MemorySpace> kv; 349 350 PetscFunctionBegin; 351 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 352 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 353 PetscValidPointer(ov,3); 354 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)); 355 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 356 else PetscCall(VecRestoreKokkosView(vec,&kv)); 357 PetscFunctionReturn(0); 358 } 359 360 template<class MemorySpace> 361 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView3DType<MemorySpace> *ov) 362 { 363 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 364 ConstPetscScalarKokkosViewType<MemorySpace> kv; 365 366 PetscFunctionBegin; 367 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 368 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 369 PetscValidPointer(ov,3); 370 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 371 PetscCheck(dim == 2,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 3D but DMDA is %dD",(int)dim); 372 PetscCall(VecGetKokkosView(vec,&kv)); 373 *ov = ConstPetscScalarKokkosOffsetView3DType<MemorySpace>(kv.data(),{gys,gxs,0},{gys+gym,gxs+gxm,dof}); 374 PetscFunctionReturn(0); 375 } 376 377 template<class MemorySpace> 378 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView3DType<MemorySpace> *ov) 379 { 380 ConstPetscScalarKokkosViewType<MemorySpace> kv; 381 382 PetscFunctionBegin; 383 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 384 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 385 PetscValidPointer(ov,3); 386 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)); 387 PetscCall(VecRestoreKokkosView(vec,&kv)); 388 PetscFunctionReturn(0); 389 } 390 391 /* ============================== 4D including DOF ================================= */ 392 template<class MemorySpace> 393 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView4DType<MemorySpace> *ov,PetscBool overwrite) 394 { 395 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 396 PetscScalarKokkosViewType<MemorySpace> kv; 397 398 PetscFunctionBegin; 399 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 400 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 401 PetscValidPointer(ov,3); 402 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 403 PetscCheck(dim == 3,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 4D but DMDA is %dD",(int)dim); 404 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 405 else PetscCall(VecGetKokkosView(vec,&kv)); 406 *ov = PetscScalarKokkosOffsetView4DType<MemorySpace>(kv.data(),{gzs,gys,gxs,0},{gzs+gzm,gys+gym,gxs+gxm,dof}); 407 PetscFunctionReturn(0); 408 } 409 410 template<class MemorySpace> 411 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView4DType<MemorySpace> *ov,PetscBool overwrite) 412 { 413 PetscScalarKokkosViewType<MemorySpace> kv; 414 415 PetscFunctionBegin; 416 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 417 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 418 PetscValidPointer(ov,3); 419 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)*ov->extent(3)); 420 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 421 else PetscCall(VecRestoreKokkosView(vec,&kv)); 422 PetscFunctionReturn(0); 423 } 424 425 template<class MemorySpace> 426 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView4DType<MemorySpace> *ov) 427 { 428 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 429 ConstPetscScalarKokkosViewType<MemorySpace> kv; 430 431 PetscFunctionBegin; 432 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 433 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 434 PetscValidPointer(ov,3); 435 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 436 PetscCheck(dim == 3,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 4D but DMDA is %dD",(int)dim); 437 PetscCall(VecGetKokkosView(vec,&kv)); 438 *ov = ConstPetscScalarKokkosOffsetView4DType<MemorySpace>(kv.data(),{gzs,gys,gxs,0},{gzs+gzm,gys+gym,gxs+gxm,dof}); 439 PetscFunctionReturn(0); 440 } 441 442 template<class MemorySpace> 443 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView4DType<MemorySpace> *ov) 444 { 445 ConstPetscScalarKokkosViewType<MemorySpace> kv; 446 447 PetscFunctionBegin; 448 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 449 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 450 PetscValidPointer(ov,3); 451 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)*ov->extent(3)); 452 PetscCall(VecRestoreKokkosView(vec,&kv)); 453 PetscFunctionReturn(0); 454 } 455 456 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView2D*); 457 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView2D*); 458 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 459 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 460 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 461 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 462 463 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView3D*); 464 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView3D*); 465 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 466 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 467 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 468 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 469 470 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView4D*); 471 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView4D*); 472 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView4D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 473 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView4D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 474 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView4D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 475 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView4D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 476 477 #if !defined(KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_HOST) /* Get host views if the default memory space is not host space */ 478 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView2DHost*); 479 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView2DHost*); 480 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 481 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 482 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 483 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 484 485 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView3DHost*); 486 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView3DHost*); 487 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 488 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 489 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 490 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 491 492 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView4DHost*); 493 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView4DHost*); 494 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView4DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 495 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView4DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 496 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView4DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 497 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView4DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 498 #endif 499