1 /* 2 A star forest (SF) describes a communication pattern 3 */ 4 #if !defined(__PETSCSF_H) 5 #define __PETSCSF_H 6 #include <petscsys.h> 7 8 PETSC_EXTERN PetscClassId PETSCSF_CLASSID; 9 10 /*S 11 PetscSF - PETSc object for setting up and managing the communication of certain entries of arrays and Vecs between MPI processes. 12 It uses the concept of star forests to indicate and determine the communication patterns concisely and efficiently. 13 14 Level: intermediate 15 16 Concepts: star forest 17 18 .seealso: PetscSFCreate(), VecScatter, VecScatterCreate() 19 S*/ 20 typedef struct _p_PetscSF* PetscSF; 21 22 23 /*J 24 PetscSFType - String with the name of a PetscSF method or the creation function 25 with an optional dynamic library name, for example 26 http://www.mcs.anl.gov/petsc/lib.so:mysfcreate() 27 28 Level: beginner 29 30 Notes: The two approaches provided are 31 $ PETSCSFBASIC which uses MPI 1 message passing to perform the communication and 32 $ PETSCSFWINDOW which uses MPI 2 one-sided operations to perform the communication, this may be more efficient, 33 $ but may not be available for all MPI distributions. In particular OpenMPI has bugs in its one-sided 34 $ operations that prevent its use. 35 36 .seealso: PetscSFSetType(), PetscSF 37 J*/ 38 typedef const char *PetscSFType; 39 #define PETSCSFBASIC "basic" 40 #define PETSCSFWINDOW "window" 41 42 /*S 43 PetscSFNode - specifier of owner and index 44 45 Level: beginner 46 47 Concepts: indexing, stride, distribution 48 49 .seealso: PetscSFSetGraph() 50 S*/ 51 typedef struct { 52 PetscInt rank; /* Rank of owner */ 53 PetscInt index; /* Index of node on rank */ 54 } PetscSFNode; 55 56 /*E 57 PetscSFWindowSyncType - Type of synchronization for PETSCSFWINDOW 58 59 $ PETSCSF_WINDOW_SYNC_FENCE - simplest model, synchronizing across communicator 60 $ PETSCSF_WINDOW_SYNC_LOCK - passive model, less synchronous, requires less setup than PETSCSF_WINDOW_SYNC_ACTIVE, but may require more handshakes 61 $ PETSCSF_WINDOW_SYNC_ACTIVE - active model, provides most information to MPI implementation, needs to construct 2-way process groups (more setup than PETSCSF_WINDOW_SYNC_LOCK) 62 63 Level: advanced 64 65 .seealso: PetscSFWindowSetSyncType(), PetscSFWindowGetSyncType() 66 E*/ 67 typedef enum {PETSCSF_WINDOW_SYNC_FENCE,PETSCSF_WINDOW_SYNC_LOCK,PETSCSF_WINDOW_SYNC_ACTIVE} PetscSFWindowSyncType; 68 PETSC_EXTERN const char *const PetscSFWindowSyncTypes[]; 69 70 /*E 71 PetscSFDuplicateOption - Aspects to preserve when duplicating a PetscSF 72 73 $ PETSCSF_DUPLICATE_CONFONLY - configuration only, user must call PetscSFSetGraph() 74 $ PETSCSF_DUPLICATE_RANKS - communication ranks preserved, but different graph (allows simpler setup after calling PetscSFSetGraph()) 75 $ PETSCSF_DUPLICATE_GRAPH - entire graph duplicated 76 77 Level: beginner 78 79 .seealso: PetscSFDuplicate() 80 E*/ 81 typedef enum {PETSCSF_DUPLICATE_CONFONLY,PETSCSF_DUPLICATE_RANKS,PETSCSF_DUPLICATE_GRAPH} PetscSFDuplicateOption; 82 PETSC_EXTERN const char *const PetscSFDuplicateOptions[]; 83 84 PETSC_EXTERN PetscFunctionList PetscSFunctionList; 85 PETSC_EXTERN PetscErrorCode PetscSFRegisterDestroy(void); 86 PETSC_EXTERN PetscErrorCode PetscSFRegisterAll(const char[]); 87 PETSC_EXTERN PetscErrorCode PetscSFRegister(const char[],const char[],const char[],PetscErrorCode (*)(PetscSF)); 88 89 /*MC 90 PetscSFRegisterDynamic - Adds an implementation of the PetscSF communication protocol. 91 92 Synopsis: 93 #include "petscsf.h" 94 PetscErrorCode PetscSFRegisterDynamic(const char *name_method,const char *path,const char *name_create,PetscErrorCode (*routine_create)(PetscSF)) 95 96 Not collective 97 98 Input Parameters: 99 + name_impl - name of a new user-defined implementation 100 . path - path (either absolute or relative) the library containing this solver 101 . name_create - name of routine to create method context 102 - routine_create - routine to create method context 103 104 Notes: 105 PetscSFRegisterDynamic() may be called multiple times to add several user-defined implementations. 106 107 If dynamic libraries are used, then the fourth input argument (routine_create) 108 is ignored. 109 110 Environmental variables such as ${PETSC_ARCH}, ${PETSC_DIR}, ${PETSC_LIB_DIR}, 111 and others of the form ${any_environmental_variable} occuring in pathname will be 112 replaced with appropriate values. 113 114 Sample usage: 115 .vb 116 PetscSFRegisterDynamic("my_impl",/home/username/my_lib/lib/libg/solaris/mylib.a, 117 "MyImplCreate",MyImplCreate); 118 .ve 119 120 Then, this implementation can be chosen with the procedural interface via 121 $ PetscSFSetType(sf,"my_impl") 122 or at runtime via the option 123 $ -snes_type my_solver 124 125 Level: advanced 126 127 Note: If your function is not being put into a shared library then use PetscSFRegister() instead 128 129 .keywords: PetscSF, register 130 131 .seealso: PetscSFRegisterAll(), PetscSFRegisterDestroy() 132 M*/ 133 #if defined(PETSC_USE_DYNAMIC_LIBRARIES) 134 #define PetscSFRegisterDynamic(a,b,c,d) PetscSFRegister(a,b,c,0) 135 #else 136 #define PetscSFRegisterDynamic(a,b,c,d) PetscSFRegister(a,b,c,d) 137 #endif 138 139 PETSC_EXTERN PetscErrorCode PetscSFInitializePackage(const char*); 140 PETSC_EXTERN PetscErrorCode PetscSFFinalizePackage(void); 141 PETSC_EXTERN PetscErrorCode PetscSFCreate(MPI_Comm comm,PetscSF*); 142 PETSC_EXTERN PetscErrorCode PetscSFDestroy(PetscSF*); 143 PETSC_EXTERN PetscErrorCode PetscSFSetType(PetscSF,PetscSFType); 144 PETSC_EXTERN PetscErrorCode PetscSFView(PetscSF,PetscViewer); 145 PETSC_EXTERN PetscErrorCode PetscSFSetUp(PetscSF); 146 PETSC_EXTERN PetscErrorCode PetscSFSetFromOptions(PetscSF); 147 PETSC_EXTERN PetscErrorCode PetscSFDuplicate(PetscSF,PetscSFDuplicateOption,PetscSF*); 148 PETSC_EXTERN PetscErrorCode PetscSFWindowSetSyncType(PetscSF,PetscSFWindowSyncType); 149 PETSC_EXTERN PetscErrorCode PetscSFWindowGetSyncType(PetscSF,PetscSFWindowSyncType*); 150 PETSC_EXTERN PetscErrorCode PetscSFSetRankOrder(PetscSF,PetscBool); 151 PETSC_EXTERN PetscErrorCode PetscSFSetGraph(PetscSF,PetscInt,PetscInt,const PetscInt*,PetscCopyMode,const PetscSFNode*,PetscCopyMode); 152 PETSC_EXTERN PetscErrorCode PetscSFGetGraph(PetscSF,PetscInt *nroots,PetscInt *nleaves,const PetscInt **ilocal,const PetscSFNode **iremote); 153 PETSC_EXTERN PetscErrorCode PetscSFGetLeafRange(PetscSF,PetscInt*,PetscInt*); 154 PETSC_EXTERN PetscErrorCode PetscSFCreateEmbeddedSF(PetscSF,PetscInt nroots,const PetscInt *selected,PetscSF *newsf); 155 PETSC_EXTERN PetscErrorCode PetscSFReset(PetscSF); 156 PETSC_EXTERN PetscErrorCode PetscSFGetRanks(PetscSF,PetscInt*,const PetscMPIInt**,const PetscInt**,const PetscInt**,const PetscInt**); 157 PETSC_EXTERN PetscErrorCode PetscSFGetGroups(PetscSF,MPI_Group*,MPI_Group*); 158 PETSC_EXTERN PetscErrorCode PetscSFGetMultiSF(PetscSF,PetscSF*); 159 PETSC_EXTERN PetscErrorCode PetscSFCreateInverseSF(PetscSF,PetscSF*); 160 161 /* broadcasts rootdata to leafdata */ 162 PETSC_EXTERN PetscErrorCode PetscSFBcastBegin(PetscSF,MPI_Datatype,const void *rootdata,void *leafdata) 163 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2); 164 PETSC_EXTERN PetscErrorCode PetscSFBcastEnd(PetscSF,MPI_Datatype,const void *rootdata,void *leafdata) 165 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2); 166 /* Reduce leafdata into rootdata using provided operation */ 167 PETSC_EXTERN PetscErrorCode PetscSFReduceBegin(PetscSF,MPI_Datatype,const void *leafdata,void *rootdata,MPI_Op) 168 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2); 169 PETSC_EXTERN PetscErrorCode PetscSFReduceEnd(PetscSF,MPI_Datatype,const void *leafdata,void *rootdata,MPI_Op) 170 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2); 171 /* Atomically modifies (using provided operation) rootdata using leafdata from each leaf, value at root at time of modification is returned in leafupdate. */ 172 PETSC_EXTERN PetscErrorCode PetscSFFetchAndOpBegin(PetscSF,MPI_Datatype,void *rootdata,const void *leafdata,void *leafupdate,MPI_Op) 173 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2) PetscAttrMPIPointerWithType(5,2); 174 PETSC_EXTERN PetscErrorCode PetscSFFetchAndOpEnd(PetscSF,MPI_Datatype,void *rootdata,const void *leafdata,void *leafupdate,MPI_Op) 175 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2) PetscAttrMPIPointerWithType(5,2); 176 /* Compute the degree of every root vertex (number of leaves in its star) */ 177 PETSC_EXTERN PetscErrorCode PetscSFComputeDegreeBegin(PetscSF,const PetscInt **degree); 178 PETSC_EXTERN PetscErrorCode PetscSFComputeDegreeEnd(PetscSF,const PetscInt **degree); 179 /* Concatenate data from all leaves into roots */ 180 PETSC_EXTERN PetscErrorCode PetscSFGatherBegin(PetscSF,MPI_Datatype,const void *leafdata,void *multirootdata) 181 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2); 182 PETSC_EXTERN PetscErrorCode PetscSFGatherEnd(PetscSF,MPI_Datatype,const void *leafdata,void *multirootdata) 183 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2); 184 /* Distribute distinct values to each leaf from roots */ 185 PETSC_EXTERN PetscErrorCode PetscSFScatterBegin(PetscSF,MPI_Datatype,const void *multirootdata,void *leafdata) 186 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2); 187 PETSC_EXTERN PetscErrorCode PetscSFScatterEnd(PetscSF,MPI_Datatype,const void *multirootdata,void *leafdata) 188 PetscAttrMPIPointerWithType(3,2) PetscAttrMPIPointerWithType(4,2); 189 190 #endif 191