1 2 /* 3 This file contains routines for Parallel vector operations that use shared memory 4 */ 5 #include <../src/vec/vec/impls/mpi/pvecimpl.h> /*I "petscvec.h" I*/ 6 7 #if defined(PETSC_USE_SHARED_MEMORY) 8 9 extern PetscErrorCode PetscSharedMalloc(MPI_Comm,PetscInt,PetscInt,void**); 10 11 #undef __FUNCT__ 12 #define __FUNCT__ "VecDuplicate_Shared" 13 PetscErrorCode VecDuplicate_Shared(Vec win,Vec *v) 14 { 15 PetscErrorCode ierr; 16 Vec_MPI *w = (Vec_MPI *)win->data; 17 PetscScalar *array; 18 19 PetscFunctionBegin; 20 /* first processor allocates entire array and sends it's address to the others */ 21 ierr = PetscSharedMalloc(((PetscObject)win)->comm,win->map->n*sizeof(PetscScalar),win->map->N*sizeof(PetscScalar),(void**)&array);CHKERRQ(ierr); 22 23 ierr = VecCreate(((PetscObject)win)->comm,v);CHKERRQ(ierr); 24 ierr = VecSetSizes(*v,win->map->n,win->map->N);CHKERRQ(ierr); 25 ierr = VecCreate_MPI_Private(*v,PETSC_FALSE,w->nghost,array);CHKERRQ(ierr); 26 ierr = PetscLayoutReference(win->map,&(*v)->map);CHKERRQ(ierr); 27 28 /* New vector should inherit stashing property of parent */ 29 (*v)->stash.donotstash = win->stash.donotstash; 30 (*v)->stash.ignorenegidx = win->stash.ignorenegidx; 31 32 ierr = PetscObjectListDuplicate(((PetscObject)win)->olist,&((PetscObject)*v)->olist);CHKERRQ(ierr); 33 ierr = PetscFunctionListDuplicate(((PetscObject)win)->qlist,&((PetscObject)*v)->qlist);CHKERRQ(ierr); 34 35 (*v)->ops->duplicate = VecDuplicate_Shared; 36 (*v)->bstash.bs = win->bstash.bs; 37 PetscFunctionReturn(0); 38 } 39 40 41 EXTERN_C_BEGIN 42 #undef __FUNCT__ 43 #define __FUNCT__ "VecCreate_Shared" 44 PetscErrorCode VecCreate_Shared(Vec vv) 45 { 46 PetscErrorCode ierr; 47 PetscScalar *array; 48 49 PetscFunctionBegin; 50 ierr = PetscSplitOwnership(((PetscObject)vv)->comm,&vv->map->n,&vv->map->N);CHKERRQ(ierr); 51 ierr = PetscSharedMalloc(((PetscObject)vv)->comm,vv->map->n*sizeof(PetscScalar),vv->map->N*sizeof(PetscScalar),(void**)&array);CHKERRQ(ierr); 52 53 ierr = VecCreate_MPI_Private(vv,PETSC_FALSE,0,array);CHKERRQ(ierr); 54 vv->ops->duplicate = VecDuplicate_Shared; 55 PetscFunctionReturn(0); 56 } 57 EXTERN_C_END 58 59 60 /* ---------------------------------------------------------------------------------------- 61 Code to manage shared memory allocation using standard Unix shared memory 62 */ 63 #include <petscsys.h> 64 #if defined(PETSC_HAVE_PWD_H) 65 #include <pwd.h> 66 #endif 67 #include <ctype.h> 68 #include <sys/types.h> 69 #include <sys/stat.h> 70 #if defined(PETSC_HAVE_UNISTD_H) 71 #include <unistd.h> 72 #endif 73 #if defined(PETSC_HAVE_STDLIB_H) 74 #include <stdlib.h> 75 #endif 76 #if defined(PETSC_HAVE_SYS_PARAM_H) 77 #include <sys/param.h> 78 #endif 79 #if defined(PETSC_HAVE_SYS_UTSNAME_H) 80 #include <sys/utsname.h> 81 #endif 82 #include <fcntl.h> 83 #include <time.h> 84 #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H) 85 #include <sys/systeminfo.h> 86 #endif 87 #include <sys/shm.h> 88 #include <sys/mman.h> 89 90 91 static PetscMPIInt Petsc_Shared_keyval = MPI_KEYVAL_INVALID; 92 93 #undef __FUNCT__ 94 #define __FUNCT__ "Petsc_DeleteShared" 95 /* 96 Private routine to delete internal storage when a communicator is freed. 97 This is called by MPI, not by users. 98 99 The binding for the first argument changed from MPI 1.0 to 1.1; in 1.0 100 it was MPI_Comm *comm. 101 */ 102 static PetscErrorCode Petsc_DeleteShared(MPI_Comm comm,PetscInt keyval,void* attr_val,void* extra_state) 103 { 104 PetscErrorCode ierr; 105 106 PetscFunctionBegin; 107 ierr = PetscFree(attr_val);CHKERRQ(ierr); 108 PetscFunctionReturn(MPI_SUCCESS); 109 } 110 111 #undef __FUNCT__ 112 #define __FUNCT__ "PetscSharedMalloc" 113 /* 114 115 This routine is still incomplete and needs work. 116 117 For this to work on the Apple Mac OS X you will likely need to add something line the following to the file /etc/sysctl.conf 118 cat /etc/sysctl.conf 119 kern.sysv.shmmax=67108864 120 kern.sysv.shmmin=1 121 kern.sysv.shmmni=32 122 kern.sysv.shmseg=512 123 kern.sysv.shmall=1024 124 125 This does not currently free the shared memory after the program runs. Use the Unix command ipcs to see the shared memory in use and 126 ipcrm to remove the shared memory in use. 127 128 */ 129 PetscErrorCode PetscSharedMalloc(MPI_Comm comm,PetscInt llen,PetscInt len,void **result) 130 { 131 PetscErrorCode ierr; 132 PetscInt shift; 133 PetscMPIInt rank,flag; 134 int *arena,id,key = 0; 135 char *value; 136 137 PetscFunctionBegin; 138 *result = 0; 139 140 ierr = MPI_Scan(&llen,&shift,1,MPI_INT,MPI_SUM,comm);CHKERRQ(ierr); 141 shift -= llen; 142 143 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 144 if (!rank) { 145 id = shmget(key,len, 0666 |IPC_CREAT); 146 if (id == -1) { 147 perror("Unable to malloc shared memory"); 148 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to malloc shared memory"); 149 } 150 } else { 151 id = shmget(key,len, 0666); 152 if (id == -1) { 153 perror("Unable to malloc shared memory"); 154 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to malloc shared memory"); 155 } 156 } 157 value = shmat(id,(void*)0,0); 158 if (value == (char*)-1) { 159 perror("Unable to access shared memory allocated"); 160 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to access shared memory allocated"); 161 } 162 *result = (void*) (value + shift); 163 PetscFunctionReturn(0); 164 } 165 166 #else 167 168 EXTERN_C_BEGIN 169 extern PetscErrorCode VecCreate_Seq(Vec); 170 EXTERN_C_END 171 172 EXTERN_C_BEGIN 173 #undef __FUNCT__ 174 #define __FUNCT__ "VecCreate_Shared" 175 PetscErrorCode VecCreate_Shared(Vec vv) 176 { 177 PetscErrorCode ierr; 178 PetscMPIInt size; 179 180 PetscFunctionBegin; 181 ierr = MPI_Comm_size(((PetscObject)vv)->comm,&size);CHKERRQ(ierr); 182 if (size > 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"No supported for shared memory vector objects on this machine"); 183 ierr = VecCreate_Seq(vv);CHKERRQ(ierr); 184 PetscFunctionReturn(0); 185 } 186 EXTERN_C_END 187 188 #endif 189 190 #undef __FUNCT__ 191 #define __FUNCT__ "VecCreateShared" 192 /*@ 193 VecCreateShared - Creates a parallel vector that uses shared memory. 194 195 Input Parameters: 196 . comm - the MPI communicator to use 197 . n - local vector length (or PETSC_DECIDE to have calculated if N is given) 198 . N - global vector length (or PETSC_DECIDE to have calculated if n is given) 199 200 Output Parameter: 201 . vv - the vector 202 203 Collective on MPI_Comm 204 205 Notes: 206 Currently VecCreateShared() is available only on the SGI; otherwise, 207 this routine is the same as VecCreateMPI(). 208 209 Use VecDuplicate() or VecDuplicateVecs() to form additional vectors of the 210 same type as an existing vector. 211 212 Level: advanced 213 214 Concepts: vectors^creating with shared memory 215 216 .seealso: VecCreateSeq(), VecCreate(), VecCreateMPI(), VecDuplicate(), VecDuplicateVecs(), 217 VecCreateGhost(), VecCreateMPIWithArray(), VecCreateGhostWithArray() 218 219 @*/ 220 PetscErrorCode VecCreateShared(MPI_Comm comm,PetscInt n,PetscInt N,Vec *v) 221 { 222 PetscErrorCode ierr; 223 224 PetscFunctionBegin; 225 ierr = VecCreate(comm,v);CHKERRQ(ierr); 226 ierr = VecSetSizes(*v,n,N);CHKERRQ(ierr); 227 ierr = VecSetType(*v,VECSHARED);CHKERRQ(ierr); 228 PetscFunctionReturn(0); 229 } 230 231 232 233 234 235