220 static inline PetscErrorCode PetscSFLinkGetPack(PetscSFLink link, PetscMemType mtype, PetscErrorCod…  in PetscSFLinkGetPack()  argument
223   if (PetscMemTypeHost(mtype)) *Pack = link->h_Pack;  in PetscSFLinkGetPack()
225   else *Pack = link->d_Pack;  in PetscSFLinkGetPack()
266 static inline PetscErrorCode PetscSFLinkGetMPIBuffersAndRequests(PetscSF sf, PetscSFLink link, Pets…  in PetscSFLinkGetMPIBuffersAndRequests()  argument
268 …const PetscMemType rootmtype_mpi = link->rootmtype_mpi, leafmtype_mpi = link->leafmtype_mpi; /* me…  in PetscSFLinkGetMPIBuffersAndRequests()
269   const PetscInt     rootdirect_mpi = link->rootdirect_mpi, leafdirect_mpi = link->leafdirect_mpi;  in PetscSFLinkGetMPIBuffersAndRequests()
272 …if (link->InitMPIRequests) PetscCall((*link->InitMPIRequests)(sf, link, direction)); // init (pers…  in PetscSFLinkGetMPIBuffersAndRequests()
274   if (rootbuf) *rootbuf = link->rootbuf[PETSCSF_REMOTE][rootmtype_mpi];  in PetscSFLinkGetMPIBuffersAndRequests()
275   if (leafbuf) *leafbuf = link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi];  in PetscSFLinkGetMPIBuffersAndRequests()
276   if (rootreqs) *rootreqs = link->rootreqs[direction][rootmtype_mpi][rootdirect_mpi];  in PetscSFLinkGetMPIBuffersAndRequests()
277   if (leafreqs) *leafreqs = link->leafreqs[direction][leafmtype_mpi][leafdirect_mpi];  in PetscSFLinkGetMPIBuffersAndRequests()
281 static inline PetscErrorCode PetscSFLinkStartCommunication(PetscSF sf, PetscSFLink link, PetscSFDir…  in PetscSFLinkStartCommunication()  argument
284   if (link->StartCommunication) PetscCall((*link->StartCommunication)(sf, link, direction));  in PetscSFLinkStartCommunication()
288 static inline PetscErrorCode PetscSFLinkFinishCommunication(PetscSF sf, PetscSFLink link, PetscSFDi…  in PetscSFLinkFinishCommunication()  argument
291   if (link->FinishCommunication) PetscCall((*link->FinishCommunication)(sf, link, direction));  in PetscSFLinkFinishCommunication()
300 … PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI(PetscSF sf, PetscSFLink link, PetscBool device2h…  in PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI()  argument
306 …if (PetscMemTypeDevice(link->rootmtype) && PetscMemTypeHost(link->rootmtype_mpi) && bas->rootbufle…  in PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI()
307     void  *h_buf = link->rootbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_HOST];  in PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI()
308     void  *d_buf = link->rootbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_DEVICE];  in PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI()
309     size_t count = bas->rootbuflen[PETSCSF_REMOTE] * link->unitbytes;  in PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI()
311 …   PetscCall((*link->Memcpy)(link, PETSC_MEMTYPE_HOST, h_buf, PETSC_MEMTYPE_DEVICE, d_buf, count));  in PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI()
314 …   PetscCall((*link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, d_buf, PETSC_MEMTYPE_HOST, h_buf, count));  in PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI()
321 … PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI(PetscSF sf, PetscSFLink link, PetscBool device2h…  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()  argument
324 …if (PetscMemTypeDevice(link->leafmtype) && PetscMemTypeHost(link->leafmtype_mpi) && sf->leafbuflen…  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
325     void  *h_buf = link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_HOST];  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
326     void  *d_buf = link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_DEVICE];  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
327     size_t count = sf->leafbuflen[PETSCSF_REMOTE] * link->unitbytes;  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
329 …   PetscCall((*link->Memcpy)(link, PETSC_MEMTYPE_HOST, h_buf, PETSC_MEMTYPE_DEVICE, d_buf, count));  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
332 …   PetscCall((*link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, d_buf, PETSC_MEMTYPE_HOST, h_buf, count));  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
340 static inline PetscErrorCode PetscSFLinkSyncStreamBeforeCallMPI(PetscSF sf, PetscSFLink link)  in PetscSFLinkSyncStreamBeforeCallMPI()  argument
346 …link->rootmtype) && bas->rootbuflen[PETSCSF_REMOTE]) || (PetscMemTypeDevice(link->leafmtype) && sf…  in PetscSFLinkSyncStreamBeforeCallMPI()
369 static inline PetscErrorCode PetscSFLinkGetRootPackOptAndIndices(PetscSF sf, PetscSFLink link, Pets…  in PetscSFLinkGetRootPackOptAndIndices()  argument
398 …PetscCall((*link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, bas->rootpackopt_d[scope]->array, PETSC_MEMT…  in PetscSFLinkGetRootPackOptAndIndices()
405 …PetscCall((*link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, bas->irootloc_d[scope], PETSC_MEMTYPE_HOST, …  in PetscSFLinkGetRootPackOptAndIndices()
418 static inline PetscErrorCode PetscSFLinkGetLeafPackOptAndIndices(PetscSF sf, PetscSFLink link, Pets…  in PetscSFLinkGetLeafPackOptAndIndices()  argument
440 …PetscCall((*link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, sf->leafpackopt_d[scope]->array, PETSC_MEMTY…  in PetscSFLinkGetLeafPackOptAndIndices()
447 …PetscCall((*link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, sf->rmine_d[scope], PETSC_MEMTYPE_HOST, sf->…  in PetscSFLinkGetLeafPackOptAndIndices()