1 #define PETSCVEC_DLL 2 /* 3 This file contains routines for Parallel vector operations that use shared memory 4 */ 5 #include "../src/vec/vec/impls/mpi/pvecimpl.h" /*I "petscvec.h" I*/ 6 7 #if defined(PETSC_USE_SHARED_MEMORY) 8 9 extern PetscErrorCode PetscSharedMalloc(MPI_Comm,PetscInt,PetscInt,void**); 10 11 #undef __FUNCT__ 12 #define __FUNCT__ "VecDuplicate_Shared" 13 PetscErrorCode VecDuplicate_Shared(Vec win,Vec *v) 14 { 15 PetscErrorCode ierr; 16 Vec_MPI *w = (Vec_MPI *)win->data; 17 PetscScalar *array; 18 19 PetscFunctionBegin; 20 21 /* first processor allocates entire array and sends it's address to the others */ 22 ierr = PetscSharedMalloc(((PetscObject)win)->comm,win->map->n*sizeof(PetscScalar),win->map->N*sizeof(PetscScalar),(void**)&array);CHKERRQ(ierr); 23 24 ierr = VecCreate(((PetscObject)win)->comm,v);CHKERRQ(ierr); 25 ierr = VecSetSizes(*v,win->map->n,win->map->N);CHKERRQ(ierr); 26 ierr = VecCreate_MPI_Private(*v,PETSC_FALSE,w->nghost,array);CHKERRQ(ierr); 27 28 /* New vector should inherit stashing property of parent */ 29 (*v)->stash.donotstash = win->stash.donotstash; 30 (*v)->stash.ignorenegidx = win->stash.ignorenegidx; 31 32 ierr = PetscOListDuplicate(((PetscObject)win)->olist,&((PetscObject)*v)->olist);CHKERRQ(ierr); 33 ierr = PetscFListDuplicate(((PetscObject)win)->qlist,&((PetscObject)*v)->qlist);CHKERRQ(ierr); 34 35 if (win->mapping) { 36 ierr = PetscObjectReference((PetscObject)win->mapping);CHKERRQ(ierr); 37 (*v)->mapping = win->mapping; 38 } 39 if (win->bmapping) { 40 ierr = PetscObjectReference((PetscObject)win->bmapping);CHKERRQ(ierr); 41 (*v)->bmapping = win->bmapping; 42 } 43 (*v)->ops->duplicate = VecDuplicate_Shared; 44 (*v)->map->bs = win->map->bs; 45 (*v)->bstash.bs = win->bstash.bs; 46 PetscFunctionReturn(0); 47 } 48 49 50 EXTERN_C_BEGIN 51 #undef __FUNCT__ 52 #define __FUNCT__ "VecCreate_Shared" 53 PetscErrorCode VecCreate_Shared(Vec vv) 54 { 55 PetscErrorCode ierr; 56 PetscScalar *array; 57 58 PetscFunctionBegin; 59 ierr = PetscSplitOwnership(((PetscObject)vv)->comm,&vv->map->n,&vv->map->N);CHKERRQ(ierr); 60 ierr = PetscSharedMalloc(((PetscObject)vv)->comm,vv->map->n*sizeof(PetscScalar),vv->map->N*sizeof(PetscScalar),(void**)&array);CHKERRQ(ierr); 61 62 ierr = VecCreate_MPI_Private(vv,PETSC_FALSE,0,array);CHKERRQ(ierr); 63 vv->ops->duplicate = VecDuplicate_Shared; 64 65 PetscFunctionReturn(0); 66 } 67 EXTERN_C_END 68 69 70 /* ---------------------------------------------------------------------------------------- 71 Code to manage shared memory allocation using standard Unix shared memory 72 */ 73 #include "petscsys.h" 74 #if defined(PETSC_HAVE_PWD_H) 75 #include <pwd.h> 76 #endif 77 #include <ctype.h> 78 #include <sys/types.h> 79 #include <sys/stat.h> 80 #if defined(PETSC_HAVE_UNISTD_H) 81 #include <unistd.h> 82 #endif 83 #if defined(PETSC_HAVE_STDLIB_H) 84 #include <stdlib.h> 85 #endif 86 #if defined(PETSC_HAVE_SYS_PARAM_H) 87 #include <sys/param.h> 88 #endif 89 #if defined(PETSC_HAVE_SYS_UTSNAME_H) 90 #include <sys/utsname.h> 91 #endif 92 #include <fcntl.h> 93 #include <time.h> 94 #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H) 95 #include <sys/systeminfo.h> 96 #endif 97 #include <sys/shm.h> 98 #include <sys/mman.h> 99 100 101 static PetscMPIInt Petsc_Shared_keyval = MPI_KEYVAL_INVALID; 102 103 #undef __FUNCT__ 104 #define __FUNCT__ "Petsc_DeleteShared" 105 /* 106 Private routine to delete internal storage when a communicator is freed. 107 This is called by MPI, not by users. 108 109 The binding for the first argument changed from MPI 1.0 to 1.1; in 1.0 110 it was MPI_Comm *comm. 111 */ 112 static PetscErrorCode Petsc_DeleteShared(MPI_Comm comm,PetscInt keyval,void* attr_val,void* extra_state) 113 { 114 PetscErrorCode ierr; 115 116 PetscFunctionBegin; 117 ierr = PetscFree(attr_val);CHKERRQ(ierr); 118 PetscFunctionReturn(MPI_SUCCESS); 119 } 120 121 #undef __FUNCT__ 122 #define __FUNCT__ "PetscSharedMalloc" 123 /* 124 125 This routine is still incomplete and needs work. 126 127 For this to work on the Apple Mac OS X you will likely need to add something line the following to the file /etc/sysctl.conf 128 cat /etc/sysctl.conf 129 kern.sysv.shmmax=67108864 130 kern.sysv.shmmin=1 131 kern.sysv.shmmni=32 132 kern.sysv.shmseg=512 133 kern.sysv.shmall=1024 134 135 This does not currently free the shared memory after the program runs. Use the Unix command ipcs to see the shared memory in use and 136 ipcrm to remove the shared memory in use. 137 138 */ 139 PetscErrorCode PetscSharedMalloc(MPI_Comm comm,PetscInt llen,PetscInt len,void **result) 140 { 141 PetscErrorCode ierr; 142 PetscInt shift; 143 PetscMPIInt rank,flag; 144 int *arena,id,key = 0; 145 char *value; 146 147 PetscFunctionBegin; 148 *result = 0; 149 150 ierr = MPI_Scan(&llen,&shift,1,MPI_INT,MPI_SUM,comm);CHKERRQ(ierr); 151 shift -= llen; 152 153 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 154 if (!rank) { 155 id = shmget(key,len, 0666 |IPC_CREAT); 156 if (id == -1) { 157 perror("Unable to malloc shared memory"); 158 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to malloc shared memory"); 159 } 160 } else { 161 id = shmget(key,len, 0666); 162 if (id == -1) { 163 perror("Unable to malloc shared memory"); 164 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to malloc shared memory"); 165 } 166 } 167 value = shmat(id,(void*)0,0); 168 if (value == (char*)-1) { 169 perror("Unable to access shared memory allocated"); 170 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to access shared memory allocated"); 171 } 172 *result = (void*) (value + shift); 173 174 PetscFunctionReturn(0); 175 } 176 177 #else 178 179 EXTERN_C_BEGIN 180 extern PetscErrorCode VecCreate_Seq(Vec); 181 EXTERN_C_END 182 183 EXTERN_C_BEGIN 184 #undef __FUNCT__ 185 #define __FUNCT__ "VecCreate_Shared" 186 PetscErrorCode VecCreate_Shared(Vec vv) 187 { 188 PetscErrorCode ierr; 189 PetscMPIInt size; 190 191 PetscFunctionBegin; 192 ierr = MPI_Comm_size(((PetscObject)vv)->comm,&size);CHKERRQ(ierr); 193 if (size > 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"No supported for shared memory vector objects on this machine"); 194 ierr = VecCreate_Seq(vv);CHKERRQ(ierr); 195 PetscFunctionReturn(0); 196 } 197 EXTERN_C_END 198 199 #endif 200 201 #undef __FUNCT__ 202 #define __FUNCT__ "VecCreateShared" 203 /*@ 204 VecCreateShared - Creates a parallel vector that uses shared memory. 205 206 Input Parameters: 207 . comm - the MPI communicator to use 208 . n - local vector length (or PETSC_DECIDE to have calculated if N is given) 209 . N - global vector length (or PETSC_DECIDE to have calculated if n is given) 210 211 Output Parameter: 212 . vv - the vector 213 214 Collective on MPI_Comm 215 216 Notes: 217 Currently VecCreateShared() is available only on the SGI; otherwise, 218 this routine is the same as VecCreateMPI(). 219 220 Use VecDuplicate() or VecDuplicateVecs() to form additional vectors of the 221 same type as an existing vector. 222 223 Level: advanced 224 225 Concepts: vectors^creating with shared memory 226 227 .seealso: VecCreateSeq(), VecCreate(), VecCreateMPI(), VecDuplicate(), VecDuplicateVecs(), 228 VecCreateGhost(), VecCreateMPIWithArray(), VecCreateGhostWithArray() 229 230 @*/ 231 PetscErrorCode VecCreateShared(MPI_Comm comm,PetscInt n,PetscInt N,Vec *v) 232 { 233 PetscErrorCode ierr; 234 235 PetscFunctionBegin; 236 ierr = VecCreate(comm,v);CHKERRQ(ierr); 237 ierr = VecSetSizes(*v,n,N);CHKERRQ(ierr); 238 ierr = VecSetType(*v,VECSHARED);CHKERRQ(ierr); 239 PetscFunctionReturn(0); 240 } 241 242 243 244 245 246