1 #include <petsc/private/vecimpl_kokkos.hpp> 2 #include <petsc/private/dmdaimpl.h> 3 #include <petscdmda_kokkos.hpp> 4 5 /* SUBMANSEC = DMDA */ 6 7 /* Use macro instead of inlined function to avoid annoying warnings like: 'dof' may be used uninitialized in this function [-Wmaybe-uninitialized] */ 8 #define DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof) \ 9 do { \ 10 PetscCall(DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm)); \ 11 PetscCall(DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gxm,&gym,&gzm)); \ 12 PetscCall(DMDAGetInfo(da,&dim,NULL,NULL,NULL,NULL,NULL,NULL,&dof,NULL,NULL,NULL,NULL,NULL)); \ 13 /* Handle case where user passes in global vector as opposed to local */ \ 14 PetscCall(VecGetLocalSize(vec,&N)); \ 15 if (N == xm*ym*zm*dof) { \ 16 gxm = xm; gym = ym; gzm = zm; \ 17 gxs = xs; gys = ys; gzs = zs; \ 18 } else PetscCheck(N == gxm*gym*gzm*dof,PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Vector local size %" PetscInt_FMT " is not compatible with DMDA local sizes %" PetscInt_FMT " %" PetscInt_FMT,N,xm*ym*zm*dof,gxm*gym*gzm*dof); \ 19 } while (0) 20 21 /* -------------------- 1D ---------------- */ 22 template<class MemorySpace> 23 PetscErrorCode DMDAVecGetKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView1DType<MemorySpace> *ov,PetscBool overwrite) 24 { 25 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 26 PetscScalarKokkosViewType<MemorySpace> kv; 27 28 PetscFunctionBegin; 29 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 30 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 31 PetscValidPointer(ov,3); 32 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 33 PetscCheck(dim == 1,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 1D but DMDA is %dD",(int)dim); 34 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 35 else PetscCall(VecGetKokkosView(vec,&kv)); 36 /* Construct the unmanaged OffsetView with {begin0,begin1,begins2},{end0,end1,end2} */ 37 *ov = PetscScalarKokkosOffsetView1DType<MemorySpace>(kv.data(),{gxs*dof},{(gxs+gxm)*dof}); 38 PetscFunctionReturn(0); 39 } 40 41 template<class MemorySpace> 42 PetscErrorCode DMDAVecRestoreKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView1DType<MemorySpace> *ov,PetscBool overwrite) 43 { 44 PetscScalarKokkosViewType<MemorySpace> kv; 45 46 PetscFunctionBegin; 47 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 48 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 49 PetscValidPointer(ov,3); 50 kv = ov->view(); /* OffsetView to View */ 51 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 52 else PetscCall(VecRestoreKokkosView(vec,&kv)); 53 PetscFunctionReturn(0); 54 } 55 56 template<class MemorySpace> 57 PetscErrorCode DMDAVecGetKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView1DType<MemorySpace> *ov) 58 { 59 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 60 ConstPetscScalarKokkosViewType<MemorySpace> kv; 61 62 PetscFunctionBegin; 63 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 64 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 65 PetscValidPointer(ov,3); 66 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 67 PetscCheck(dim == 1,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 1D but DMDA is %dD",(int)dim); 68 PetscCall(VecGetKokkosView(vec,&kv)); 69 *ov = ConstPetscScalarKokkosOffsetView1DType<MemorySpace>(kv.data(),{gxs*dof},{(gxs+gxm)*dof}); 70 PetscFunctionReturn(0); 71 } 72 73 template<class MemorySpace> 74 PetscErrorCode DMDAVecRestoreKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView1DType<MemorySpace> *ov) 75 { 76 ConstPetscScalarKokkosViewType<MemorySpace> kv; 77 78 PetscFunctionBegin; 79 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 80 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 81 PetscValidPointer(ov,3); 82 kv = ov->view(); 83 PetscCall(VecRestoreKokkosView(vec,&kv)); 84 PetscFunctionReturn(0); 85 } 86 87 /* ============================== 2D ================================= */ 88 template<class MemorySpace> 89 PetscErrorCode DMDAVecGetKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView2DType<MemorySpace> *ov,PetscBool overwrite) 90 { 91 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 92 PetscScalarKokkosViewType<MemorySpace> kv; 93 94 PetscFunctionBegin; 95 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 96 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 97 PetscValidPointer(ov,3); 98 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 99 PetscCheck(dim == 2,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 2D but DMDA is %dD",(int)dim); 100 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 101 else PetscCall(VecGetKokkosView(vec,&kv)); 102 *ov = PetscScalarKokkosOffsetView2DType<MemorySpace>(kv.data(),{gys*dof,gxs*dof},{(gys+gym)*dof,(gxs+gxm)*dof}); 103 PetscFunctionReturn(0); 104 } 105 106 template<class MemorySpace> 107 PetscErrorCode DMDAVecRestoreKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView2DType<MemorySpace> *ov,PetscBool overwrite) 108 { 109 PetscScalarKokkosViewType<MemorySpace> kv; 110 111 PetscFunctionBegin; 112 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 113 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 114 PetscValidPointer(ov,3); 115 // kv = ov->view(); /* 2D OffsetView => 2D View => 1D View. Why does it not work? */ 116 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)); 117 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 118 else PetscCall(VecRestoreKokkosView(vec,&kv)); 119 PetscFunctionReturn(0); 120 } 121 122 template<class MemorySpace> 123 PetscErrorCode DMDAVecGetKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView2DType<MemorySpace> *ov) 124 { 125 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 126 ConstPetscScalarKokkosViewType<MemorySpace> kv; 127 128 PetscFunctionBegin; 129 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 130 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 131 PetscValidPointer(ov,3); 132 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 133 PetscCheck(dim == 2,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 2D but DMDA is %dD",(int)dim); 134 PetscCall(VecGetKokkosView(vec,&kv)); 135 *ov = ConstPetscScalarKokkosOffsetView2DType<MemorySpace>(kv.data(),{gys*dof,gxs*dof},{(gys+gym)*dof,(gxs+gxm)*dof}); 136 PetscFunctionReturn(0); 137 } 138 139 template<class MemorySpace> 140 PetscErrorCode DMDAVecRestoreKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView2DType<MemorySpace> *ov) 141 { 142 ConstPetscScalarKokkosViewType<MemorySpace> kv; 143 144 PetscFunctionBegin; 145 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 146 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 147 PetscValidPointer(ov,3); 148 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)); 149 PetscCall(VecRestoreKokkosView(vec,&kv)); 150 PetscFunctionReturn(0); 151 } 152 153 /* ============================== 3D ================================= */ 154 template<class MemorySpace> 155 PetscErrorCode DMDAVecGetKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView3DType<MemorySpace> *ov,PetscBool overwrite) 156 { 157 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 158 PetscScalarKokkosViewType<MemorySpace> kv; 159 160 PetscFunctionBegin; 161 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 162 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 163 PetscValidPointer(ov,3); 164 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 165 PetscCheck(dim == 3,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 3D but DMDA is %dD",(int)dim); 166 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 167 else PetscCall(VecGetKokkosView(vec,&kv)); 168 *ov = PetscScalarKokkosOffsetView3DType<MemorySpace>(kv.data(),{gzs*dof,gys*dof,gxs*dof},{(gzs+gzm)*dof,(gys+gym)*dof,(gxs+gxm)*dof}); 169 PetscFunctionReturn(0); 170 } 171 172 template<class MemorySpace> 173 PetscErrorCode DMDAVecRestoreKokkosOffsetView_Private(DM da,Vec vec,PetscScalarKokkosOffsetView3DType<MemorySpace> *ov,PetscBool overwrite) 174 { 175 PetscScalarKokkosViewType<MemorySpace> kv; 176 177 PetscFunctionBegin; 178 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 179 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 180 PetscValidPointer(ov,3); 181 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)); 182 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 183 else PetscCall(VecRestoreKokkosView(vec,&kv)); 184 PetscFunctionReturn(0); 185 } 186 187 template<class MemorySpace> 188 PetscErrorCode DMDAVecGetKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView3DType<MemorySpace> *ov) 189 { 190 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 191 ConstPetscScalarKokkosViewType<MemorySpace> kv; 192 193 PetscFunctionBegin; 194 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 195 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 196 PetscValidPointer(ov,3); 197 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 198 PetscCheck(dim == 3,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 3D but DMDA is %dD",(int)dim); 199 PetscCall(VecGetKokkosView(vec,&kv)); 200 *ov = ConstPetscScalarKokkosOffsetView3DType<MemorySpace>(kv.data(),{gzs*dof,gys*dof,gxs*dof},{(gzs+gzm)*dof,(gys+gym)*dof,(gxs+gxm)*dof}); 201 PetscFunctionReturn(0); 202 } 203 204 template<class MemorySpace> 205 PetscErrorCode DMDAVecRestoreKokkosOffsetView(DM da,Vec vec,ConstPetscScalarKokkosOffsetView3DType<MemorySpace> *ov) 206 { 207 ConstPetscScalarKokkosViewType<MemorySpace> kv; 208 209 PetscFunctionBegin; 210 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 211 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 212 PetscValidPointer(ov,3); 213 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)); 214 PetscCall(VecRestoreKokkosView(vec,&kv)); 215 PetscFunctionReturn(0); 216 } 217 218 /* Function template explicit instantiation */ 219 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView1D*); 220 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView1D*); 221 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView1D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 222 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView1D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 223 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView1D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 224 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView1D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 225 226 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView2D*); 227 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView2D*); 228 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 229 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 230 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 231 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 232 233 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView3D*); 234 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView3D*); 235 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 236 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 237 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 238 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 239 240 #if !defined(KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_HOST) /* Get host views if the default memory space is not host space */ 241 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView1DHost*); 242 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView1DHost*); 243 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView1DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 244 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView1DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 245 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView1DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 246 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView1DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 247 248 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView2DHost*); 249 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView2DHost*); 250 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 251 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 252 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 253 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 254 255 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView3DHost*); 256 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM,Vec, ConstPetscScalarKokkosOffsetView3DHost*); 257 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 258 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetView (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_FALSE);} 259 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewWrite (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecGetKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 260 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewWrite(DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecRestoreKokkosOffsetView_Private(da,vec,ov,PETSC_TRUE);} 261 #endif 262 263 /* ============================== 2D including DOF ================================= */ 264 template<class MemorySpace> 265 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView2DType<MemorySpace> *ov,PetscBool overwrite) 266 { 267 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 268 PetscScalarKokkosViewType<MemorySpace> kv; 269 270 PetscFunctionBegin; 271 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 272 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 273 PetscValidPointer(ov,3); 274 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 275 PetscCheck(dim == 1,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 2D but DMDA is %dD",(int)dim); 276 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 277 else PetscCall(VecGetKokkosView(vec,&kv)); 278 *ov = PetscScalarKokkosOffsetView2DType<MemorySpace>(kv.data(),{gxs,0},{gxs+gxm,dof}); 279 PetscFunctionReturn(0); 280 } 281 282 template<class MemorySpace> 283 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView2DType<MemorySpace> *ov,PetscBool overwrite) 284 { 285 PetscScalarKokkosViewType<MemorySpace> kv; 286 287 PetscFunctionBegin; 288 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 289 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 290 PetscValidPointer(ov,3); 291 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)); 292 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 293 else PetscCall(VecRestoreKokkosView(vec,&kv)); 294 PetscFunctionReturn(0); 295 } 296 297 template<class MemorySpace> 298 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView2DType<MemorySpace> *ov) 299 { 300 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 301 ConstPetscScalarKokkosViewType<MemorySpace> kv; 302 303 PetscFunctionBegin; 304 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 305 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 306 PetscValidPointer(ov,3); 307 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 308 PetscCheck(dim == 1,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 2D but DMDA is %dD",(int)dim); 309 PetscCall(VecGetKokkosView(vec,&kv)); 310 *ov = ConstPetscScalarKokkosOffsetView2DType<MemorySpace>(kv.data(),{gxs,0},{gxs+gxm,dof}); 311 PetscFunctionReturn(0); 312 } 313 314 template<class MemorySpace> 315 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView2DType<MemorySpace> *ov) 316 { 317 ConstPetscScalarKokkosViewType<MemorySpace> kv; 318 319 PetscFunctionBegin; 320 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 321 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 322 PetscValidPointer(ov,3); 323 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)); 324 PetscCall(VecRestoreKokkosView(vec,&kv)); 325 PetscFunctionReturn(0); 326 } 327 328 /* ============================== 3D including DOF ================================= */ 329 template<class MemorySpace> 330 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView3DType<MemorySpace> *ov,PetscBool overwrite) 331 { 332 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 333 PetscScalarKokkosViewType<MemorySpace> kv; 334 335 PetscFunctionBegin; 336 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 337 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 338 PetscValidPointer(ov,3); 339 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 340 PetscCheck(dim == 2,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 3D but DMDA is %dD",(int)dim); 341 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 342 else PetscCall(VecGetKokkosView(vec,&kv)); 343 *ov = PetscScalarKokkosOffsetView3DType<MemorySpace>(kv.data(),{gys,gxs,0},{gys+gym,gxs+gxm,dof}); 344 PetscFunctionReturn(0); 345 } 346 347 template<class MemorySpace> 348 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView3DType<MemorySpace> *ov,PetscBool overwrite) 349 { 350 PetscScalarKokkosViewType<MemorySpace> kv; 351 352 PetscFunctionBegin; 353 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 354 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 355 PetscValidPointer(ov,3); 356 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)); 357 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 358 else PetscCall(VecRestoreKokkosView(vec,&kv)); 359 PetscFunctionReturn(0); 360 } 361 362 template<class MemorySpace> 363 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView3DType<MemorySpace> *ov) 364 { 365 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 366 ConstPetscScalarKokkosViewType<MemorySpace> kv; 367 368 PetscFunctionBegin; 369 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 370 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 371 PetscValidPointer(ov,3); 372 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 373 PetscCheck(dim == 2,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 3D but DMDA is %dD",(int)dim); 374 PetscCall(VecGetKokkosView(vec,&kv)); 375 *ov = ConstPetscScalarKokkosOffsetView3DType<MemorySpace>(kv.data(),{gys,gxs,0},{gys+gym,gxs+gxm,dof}); 376 PetscFunctionReturn(0); 377 } 378 379 template<class MemorySpace> 380 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView3DType<MemorySpace> *ov) 381 { 382 ConstPetscScalarKokkosViewType<MemorySpace> kv; 383 384 PetscFunctionBegin; 385 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 386 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 387 PetscValidPointer(ov,3); 388 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)); 389 PetscCall(VecRestoreKokkosView(vec,&kv)); 390 PetscFunctionReturn(0); 391 } 392 393 /* ============================== 4D including DOF ================================= */ 394 template<class MemorySpace> 395 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView4DType<MemorySpace> *ov,PetscBool overwrite) 396 { 397 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 398 PetscScalarKokkosViewType<MemorySpace> kv; 399 400 PetscFunctionBegin; 401 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 402 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 403 PetscValidPointer(ov,3); 404 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 405 PetscCheck(dim == 3,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 4D but DMDA is %dD",(int)dim); 406 if (overwrite) PetscCall(VecGetKokkosViewWrite(vec,&kv)); 407 else PetscCall(VecGetKokkosView(vec,&kv)); 408 *ov = PetscScalarKokkosOffsetView4DType<MemorySpace>(kv.data(),{gzs,gys,gxs,0},{gzs+gzm,gys+gym,gxs+gxm,dof}); 409 PetscFunctionReturn(0); 410 } 411 412 template<class MemorySpace> 413 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF_Private(DM da,Vec vec,PetscScalarKokkosOffsetView4DType<MemorySpace> *ov,PetscBool overwrite) 414 { 415 PetscScalarKokkosViewType<MemorySpace> kv; 416 417 PetscFunctionBegin; 418 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 419 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 420 PetscValidPointer(ov,3); 421 kv = PetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)*ov->extent(3)); 422 if (overwrite) PetscCall(VecRestoreKokkosViewWrite(vec,&kv)); 423 else PetscCall(VecRestoreKokkosView(vec,&kv)); 424 PetscFunctionReturn(0); 425 } 426 427 template<class MemorySpace> 428 PetscErrorCode DMDAVecGetKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView4DType<MemorySpace> *ov) 429 { 430 PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof; 431 ConstPetscScalarKokkosViewType<MemorySpace> kv; 432 433 PetscFunctionBegin; 434 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 435 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 436 PetscValidPointer(ov,3); 437 DMDA_VEC_GET_SHAPE(da,vec,xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof); 438 PetscCheck(dim == 3,PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"KokkosOffsetView is 4D but DMDA is %dD",(int)dim); 439 PetscCall(VecGetKokkosView(vec,&kv)); 440 *ov = ConstPetscScalarKokkosOffsetView4DType<MemorySpace>(kv.data(),{gzs,gys,gxs,0},{gzs+gzm,gys+gym,gxs+gxm,dof}); 441 PetscFunctionReturn(0); 442 } 443 444 template<class MemorySpace> 445 PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF(DM da,Vec vec,ConstPetscScalarKokkosOffsetView4DType<MemorySpace> *ov) 446 { 447 ConstPetscScalarKokkosViewType<MemorySpace> kv; 448 449 PetscFunctionBegin; 450 PetscValidHeaderSpecificType(da,DM_CLASSID,1,DMDA); 451 PetscValidHeaderSpecific(vec,VEC_CLASSID,2); 452 PetscValidPointer(ov,3); 453 kv = ConstPetscScalarKokkosViewType<MemorySpace>(ov->data(),ov->extent(0)*ov->extent(1)*ov->extent(2)*ov->extent(3)); 454 PetscCall(VecRestoreKokkosView(vec,&kv)); 455 PetscFunctionReturn(0); 456 } 457 458 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView2D*); 459 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView2D*); 460 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 461 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 462 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 463 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView2D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 464 465 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView3D*); 466 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView3D*); 467 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 468 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 469 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 470 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView3D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 471 472 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView4D*); 473 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView4D*); 474 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView4D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 475 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView4D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 476 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView4D* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 477 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView4D* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 478 479 #if !defined(KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_HOST) /* Get host views if the default memory space is not host space */ 480 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView2DHost*); 481 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView2DHost*); 482 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 483 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 484 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 485 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView2DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 486 487 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView3DHost*); 488 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView3DHost*); 489 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 490 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 491 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 492 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView3DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 493 494 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView4DHost*); 495 template PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM,Vec, ConstPetscScalarKokkosOffsetView4DHost*); 496 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView4DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 497 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOF (DM da,Vec vec,PetscScalarKokkosOffsetView4DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_FALSE);} 498 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecGetKokkosOffsetViewDOFWrite (DM da,Vec vec,PetscScalarKokkosOffsetView4DHost* ov) {return DMDAVecGetKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 499 template<> PETSC_VISIBILITY_PUBLIC PetscErrorCode DMDAVecRestoreKokkosOffsetViewDOFWrite(DM da,Vec vec,PetscScalarKokkosOffsetView4DHost* ov) {return DMDAVecRestoreKokkosOffsetViewDOF_Private(da,vec,ov,PETSC_TRUE);} 500 #endif 501