1a4963045SJacob Faibussowitsch #pragma once 20c312b8eSJed Brown 3*ce78bad3SBarry Smith /* MANSEC = Vec */ 4ac09b921SBarry Smith /* SUBMANSEC = PetscSF */ 5ac09b921SBarry Smith 60c312b8eSJed Brown /*S 787497f52SBarry Smith PetscSF - PETSc object for setting up and managing the communication of certain entries of arrays and `Vec` between MPI ranks. 80c312b8eSJed Brown 90c312b8eSJed Brown Level: intermediate 100c312b8eSJed Brown 1187497f52SBarry Smith `PetscSF` uses the concept of star forests to indicate and determine the communication patterns concisely and efficiently. 12af27ebaaSBarry Smith A star <https://en.wikipedia.org/wiki/Star_(graph_theory)> forest is simply a collection of trees of height 1. The leave nodes represent 130c312b8eSJed Brown "ghost locations" for the root nodes. 140c312b8eSJed Brown 15db781477SPatrick Sanan .seealso: `PetscSFCreate()`, `VecScatter`, `VecScatterCreate()` 160c312b8eSJed Brown S*/ 170c312b8eSJed Brown typedef struct _p_PetscSF *PetscSF; 180c312b8eSJed Brown 1997929ea7SJunchao Zhang /*J 2087497f52SBarry Smith PetscSFType - String with the name of a `PetscSF` type 2197929ea7SJunchao Zhang 2297929ea7SJunchao Zhang Level: beginner 2397929ea7SJunchao Zhang 24db781477SPatrick Sanan .seealso: `PetscSFSetType()`, `PetscSF` 2597929ea7SJunchao Zhang J*/ 2697929ea7SJunchao Zhang typedef const char *PetscSFType; 2720662ed9SBarry Smith #define PETSCSFBASIC "basic" 2820662ed9SBarry Smith #define PETSCSFNEIGHBOR "neighbor" 2920662ed9SBarry Smith #define PETSCSFALLGATHERV "allgatherv" 3020662ed9SBarry Smith #define PETSCSFALLGATHER "allgather" 3120662ed9SBarry Smith #define PETSCSFGATHERV "gatherv" 3220662ed9SBarry Smith #define PETSCSFGATHER "gather" 3320662ed9SBarry Smith #define PETSCSFALLTOALL "alltoall" 3420662ed9SBarry Smith #define PETSCSFWINDOW "window" 3597929ea7SJunchao Zhang 36799f573fSMatthew G. Knepley /*S 37799f573fSMatthew G. Knepley PetscSFNode - specifier of owner and index 38799f573fSMatthew G. Knepley 39799f573fSMatthew G. Knepley Level: beginner 40799f573fSMatthew G. Knepley 4138ab3f8aSBarry Smith Sample Usage: 4216a05f60SBarry Smith .vb 4316a05f60SBarry Smith PetscSFNode *remote; 4416a05f60SBarry Smith PetscCall(PetscMalloc1(nleaves,&remote)); 4516a05f60SBarry Smith for (i=0; i<size; i++) { 4616a05f60SBarry Smith remote[i].rank = i; 4716a05f60SBarry Smith remote[i].index = rank; 4816a05f60SBarry Smith } 4916a05f60SBarry Smith .ve 5038ab3f8aSBarry Smith 5138ab3f8aSBarry Smith Sample Fortran Usage: 5216a05f60SBarry Smith .vb 5316a05f60SBarry Smith type(PetscSFNode) remote(6) 5416a05f60SBarry Smith remote(1)%rank = modulo(rank+size-1,size) 5516a05f60SBarry Smith remote(1)%index = 1 * stride 5616a05f60SBarry Smith .ve 5738ab3f8aSBarry Smith 586497c311SBarry Smith Notes: 596497c311SBarry Smith Use `MPIU_SF_NODE` when performing MPI operations on arrays of `PetscSFNode` 606497c311SBarry Smith 616497c311SBarry Smith Generally the values of `rank` should be in $[ 0,size)$ and the value of `index` greater than or equal to 0, but there are some situations that violate this. 626497c311SBarry Smith 6316a05f60SBarry Smith .seealso: `PetscSF`, `PetscSFSetGraph()` 64799f573fSMatthew G. Knepley S*/ 65799f573fSMatthew G. Knepley typedef struct { 66799f573fSMatthew G. Knepley PetscInt rank; /* Rank of owner */ 67799f573fSMatthew G. Knepley PetscInt index; /* Index of node on rank */ 68799f573fSMatthew G. Knepley } PetscSFNode; 69799f573fSMatthew G. Knepley 706497c311SBarry Smith #define MPIU_SF_NODE MPIU_2INT 716497c311SBarry Smith 72*ce78bad3SBarry Smith typedef enum { 73*ce78bad3SBarry Smith PETSCSF_ROOT2LEAF = 0, 74*ce78bad3SBarry Smith PETSCSF_LEAF2ROOT = 1 75*ce78bad3SBarry Smith } PetscSFDirection; 76*ce78bad3SBarry Smith typedef enum { 77*ce78bad3SBarry Smith PETSCSF_BCAST = 0, 78*ce78bad3SBarry Smith PETSCSF_REDUCE = 1, 79*ce78bad3SBarry Smith PETSCSF_FETCH = 2 80*ce78bad3SBarry Smith } PetscSFOperation; 81*ce78bad3SBarry Smith /* When doing device-aware MPI, a backend refers to the SF/device interface */ 82*ce78bad3SBarry Smith typedef enum { 83*ce78bad3SBarry Smith PETSCSF_BACKEND_INVALID = 0, 84*ce78bad3SBarry Smith PETSCSF_BACKEND_CUDA = 1, 85*ce78bad3SBarry Smith PETSCSF_BACKEND_HIP = 2, 86*ce78bad3SBarry Smith PETSCSF_BACKEND_KOKKOS = 3 87*ce78bad3SBarry Smith } PetscSFBackend; 88*ce78bad3SBarry Smith typedef struct _n_PetscSFLink *PetscSFLink; 89*ce78bad3SBarry Smith 9097929ea7SJunchao Zhang /*S 9197929ea7SJunchao Zhang VecScatter - Object used to manage communication of data 920b4b7b1cSBarry Smith between vectors in parallel or between parallel and sequential vectors. Manages both scatters and gathers 9397929ea7SJunchao Zhang 9497929ea7SJunchao Zhang Level: beginner 9597929ea7SJunchao Zhang 96*ce78bad3SBarry Smith Note: 97*ce78bad3SBarry Smith This is an alias for `PetscSF` 98*ce78bad3SBarry Smith 9987497f52SBarry Smith .seealso: `Vec`, `PetscSF`, `VecScatterCreate()`, `VecScatterBegin()`, `VecScatterEnd()` 10097929ea7SJunchao Zhang S*/ 10197929ea7SJunchao Zhang typedef PetscSF VecScatter; 10297929ea7SJunchao Zhang 10397929ea7SJunchao Zhang /*J 10497929ea7SJunchao Zhang VecScatterType - String with the name of a PETSc vector scatter type 10597929ea7SJunchao Zhang 10697929ea7SJunchao Zhang Level: beginner 10797929ea7SJunchao Zhang 108*ce78bad3SBarry Smith Note: 109*ce78bad3SBarry Smith This is an alias for `PetscSFType` 110*ce78bad3SBarry Smith 11187497f52SBarry Smith .seealso: `PetscSFType`, `VecScatterSetType()`, `VecScatter`, `VecScatterCreate()`, `VecScatterDestroy()` 11297929ea7SJunchao Zhang J*/ 11397929ea7SJunchao Zhang typedef PetscSFType VecScatterType; 114