1 #define PETSCVEC_DLL 2 /* 3 This file contains routines for Parallel vector operations that use shared memory 4 */ 5 #include "../src/vec/vec/impls/mpi/pvecimpl.h" /*I "petscvec.h" I*/ 6 7 #if defined(PETSC_USE_SHARED_MEMORY) 8 9 EXTERN PetscErrorCode PetscSharedMalloc(MPI_Comm,PetscInt,PetscInt,void**); 10 11 #undef __FUNCT__ 12 #define __FUNCT__ "VecDuplicate_Shared" 13 PetscErrorCode VecDuplicate_Shared(Vec win,Vec *v) 14 { 15 PetscErrorCode ierr; 16 Vec_MPI *w = (Vec_MPI *)win->data; 17 PetscScalar *array; 18 19 PetscFunctionBegin; 20 21 /* first processor allocates entire array and sends it's address to the others */ 22 ierr = PetscSharedMalloc(((PetscObject)win)->comm,win->map->n*sizeof(PetscScalar),win->map->N*sizeof(PetscScalar),(void**)&array);CHKERRQ(ierr); 23 24 ierr = VecCreate(((PetscObject)win)->comm,v);CHKERRQ(ierr); 25 ierr = VecSetSizes(*v,win->map->n,win->map->N);CHKERRQ(ierr); 26 ierr = VecCreate_MPI_Private(*v,PETSC_FALSE,w->nghost,array);CHKERRQ(ierr); 27 28 /* New vector should inherit stashing property of parent */ 29 (*v)->stash.donotstash = win->stash.donotstash; 30 (*v)->stash.ignorenegidx = win->stash.ignorenegidx; 31 32 ierr = PetscOListDuplicate(((PetscObject)win)->olist,&((PetscObject)*v)->olist);CHKERRQ(ierr); 33 ierr = PetscFListDuplicate(((PetscObject)win)->qlist,&((PetscObject)*v)->qlist);CHKERRQ(ierr); 34 35 if (win->mapping) { 36 ierr = PetscObjectReference((PetscObject)win->mapping);CHKERRQ(ierr); 37 (*v)->mapping = win->mapping; 38 } 39 if (win->bmapping) { 40 ierr = PetscObjectReference((PetscObject)win->bmapping);CHKERRQ(ierr); 41 (*v)->bmapping = win->bmapping; 42 } 43 (*v)->ops->duplicate = VecDuplicate_Shared; 44 (*v)->map->bs = win->map->bs; 45 (*v)->bstash.bs = win->bstash.bs; 46 PetscFunctionReturn(0); 47 } 48 49 50 EXTERN_C_BEGIN 51 #undef __FUNCT__ 52 #define __FUNCT__ "VecCreate_Shared" 53 PetscErrorCode PETSCVEC_DLLEXPORT VecCreate_Shared(Vec vv) 54 { 55 PetscErrorCode ierr; 56 PetscScalar *array; 57 58 PetscFunctionBegin; 59 ierr = PetscSplitOwnership(((PetscObject)vv)->comm,&vv->map->n,&vv->map->N);CHKERRQ(ierr); 60 ierr = PetscSharedMalloc(((PetscObject)vv)->comm,vv->map->n*sizeof(PetscScalar),vv->map->N*sizeof(PetscScalar),(void**)&array);CHKERRQ(ierr); 61 62 ierr = VecCreate_MPI_Private(vv,PETSC_FALSE,0,array);CHKERRQ(ierr); 63 vv->ops->duplicate = VecDuplicate_Shared; 64 65 PetscFunctionReturn(0); 66 } 67 EXTERN_C_END 68 69 70 /* ---------------------------------------------------------------------------------------- 71 Code to manage shared memory allocation using standard Unix shared memory 72 */ 73 #include "petscsys.h" 74 #if defined(PETSC_HAVE_PWD_H) 75 #include <pwd.h> 76 #endif 77 #include <ctype.h> 78 #include <sys/types.h> 79 #include <sys/stat.h> 80 #if defined(PETSC_HAVE_UNISTD_H) 81 #include <unistd.h> 82 #endif 83 #if defined(PETSC_HAVE_STDLIB_H) 84 #include <stdlib.h> 85 #endif 86 #if defined(PETSC_HAVE_SYS_PARAM_H) 87 #include <sys/param.h> 88 #endif 89 #if defined(PETSC_HAVE_SYS_UTSNAME_H) 90 #include <sys/utsname.h> 91 #endif 92 #include <fcntl.h> 93 #include <time.h> 94 #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H) 95 #include <sys/systeminfo.h> 96 #endif 97 #include <sys/shm.h> 98 #include <sys/mman.h> 99 100 101 static PetscMPIInt Petsc_Shared_keyval = MPI_KEYVAL_INVALID; 102 103 #undef __FUNCT__ 104 #define __FUNCT__ "Petsc_DeleteShared" 105 /* 106 Private routine to delete internal storage when a communicator is freed. 107 This is called by MPI, not by users. 108 109 The binding for the first argument changed from MPI 1.0 to 1.1; in 1.0 110 it was MPI_Comm *comm. 111 */ 112 static PetscErrorCode Petsc_DeleteShared(MPI_Comm comm,PetscInt keyval,void* attr_val,void* extra_state) 113 { 114 PetscErrorCode ierr; 115 116 PetscFunctionBegin; 117 ierr = PetscFree(attr_val);CHKERRQ(ierr); 118 PetscFunctionReturn(MPI_SUCCESS); 119 } 120 121 #undef __FUNCT__ 122 #define __FUNCT__ "PetscSharedMalloc" 123 /* 124 125 This routine is still incomplete and needs work. 126 127 For this to work on the Apple Mac OS X you will likely need to add something line the following to the file /etc/sysctl.conf 128 cat /etc/sysctl.conf 129 kern.sysv.shmmax=67108864 130 kern.sysv.shmmin=1 131 kern.sysv.shmmni=32 132 kern.sysv.shmseg=512 133 kern.sysv.shmall=1024 134 135 This does not currently free the shared memory after the program runs. Use the Unix command ipcs to see the shared memory in use and 136 ipcrm to remove the shared memory in use. 137 138 */ 139 PetscErrorCode PetscSharedMalloc(MPI_Comm comm,PetscInt llen,PetscInt len,void **result) 140 { 141 PetscErrorCode ierr; 142 PetscInt shift; 143 PetscMPIInt rank,flag; 144 int *arena,id,key = 0; 145 char *value; 146 147 PetscFunctionBegin; 148 *result = 0; 149 150 ierr = MPI_Scan(&llen,&shift,1,MPI_INT,MPI_SUM,comm);CHKERRQ(ierr); 151 shift -= llen; 152 153 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 154 if (!rank) { 155 id = shmget(key,len, 0666 |IPC_CREAT); 156 if (id == -1) { 157 perror("Unable to malloc shared memory"); 158 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to malloc shared memory"); 159 } 160 } else { 161 id = shmget(key,len, 0666); 162 if (id == -1) { 163 perror("Unable to malloc shared memory"); 164 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to malloc shared memory"); 165 } 166 } 167 value = shmat(id,(void*)0,0); 168 if (value == (char*)-1) { 169 perror("Unable to access shared memory allocated"); 170 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to access shared memory allocated"); 171 } 172 *result = (void*) (value + shift); 173 174 PetscFunctionReturn(0); 175 } 176 177 #else 178 179 EXTERN_C_BEGIN 180 extern PetscErrorCode PETSCVEC_DLLEXPORT VecCreate_Seq(Vec); 181 EXTERN_C_END 182 183 EXTERN_C_BEGIN 184 #undef __FUNCT__ 185 #define __FUNCT__ "VecCreate_Shared" 186 PetscErrorCode PETSCVEC_DLLEXPORT VecCreate_Shared(Vec vv) 187 { 188 PetscErrorCode ierr; 189 PetscMPIInt size; 190 191 PetscFunctionBegin; 192 ierr = MPI_Comm_size(((PetscObject)vv)->comm,&size);CHKERRQ(ierr); 193 if (size > 1) { 194 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"No supported for shared memory vector objects on this machine"); 195 } 196 ierr = VecCreate_Seq(vv);CHKERRQ(ierr); 197 PetscFunctionReturn(0); 198 } 199 EXTERN_C_END 200 201 #endif 202 203 #undef __FUNCT__ 204 #define __FUNCT__ "VecCreateShared" 205 /*@ 206 VecCreateShared - Creates a parallel vector that uses shared memory. 207 208 Input Parameters: 209 . comm - the MPI communicator to use 210 . n - local vector length (or PETSC_DECIDE to have calculated if N is given) 211 . N - global vector length (or PETSC_DECIDE to have calculated if n is given) 212 213 Output Parameter: 214 . vv - the vector 215 216 Collective on MPI_Comm 217 218 Notes: 219 Currently VecCreateShared() is available only on the SGI; otherwise, 220 this routine is the same as VecCreateMPI(). 221 222 Use VecDuplicate() or VecDuplicateVecs() to form additional vectors of the 223 same type as an existing vector. 224 225 Level: advanced 226 227 Concepts: vectors^creating with shared memory 228 229 .seealso: VecCreateSeq(), VecCreate(), VecCreateMPI(), VecDuplicate(), VecDuplicateVecs(), 230 VecCreateGhost(), VecCreateMPIWithArray(), VecCreateGhostWithArray() 231 232 @*/ 233 PetscErrorCode PETSCVEC_DLLEXPORT VecCreateShared(MPI_Comm comm,PetscInt n,PetscInt N,Vec *v) 234 { 235 PetscErrorCode ierr; 236 237 PetscFunctionBegin; 238 ierr = VecCreate(comm,v);CHKERRQ(ierr); 239 ierr = VecSetSizes(*v,n,N);CHKERRQ(ierr); 240 ierr = VecSetType(*v,VECSHARED);CHKERRQ(ierr); 241 PetscFunctionReturn(0); 242 } 243 244 245 246 247 248