1 #define PETSCVEC_DLL 2 /* 3 This file contains routines for Parallel vector operations that use shared memory 4 */ 5 #include "../src/vec/vec/impls/mpi/pvecimpl.h" /*I "petscvec.h" I*/ 6 7 #if defined(PETSC_USE_SHARED_MEMORY) 8 9 EXTERN PetscErrorCode PetscSharedMalloc(MPI_Comm,PetscInt,PetscInt,void**); 10 11 #undef __FUNCT__ 12 #define __FUNCT__ "VecDuplicate_Shared" 13 PetscErrorCode VecDuplicate_Shared(Vec win,Vec *v) 14 { 15 PetscErrorCode ierr; 16 Vec_MPI *w = (Vec_MPI *)win->data; 17 PetscScalar *array; 18 19 PetscFunctionBegin; 20 21 /* first processor allocates entire array and sends it's address to the others */ 22 ierr = PetscSharedMalloc(((PetscObject)win)->comm,win->map->n*sizeof(PetscScalar),win->map->N*sizeof(PetscScalar),(void**)&array);CHKERRQ(ierr); 23 24 ierr = VecCreate(((PetscObject)win)->comm,v);CHKERRQ(ierr); 25 ierr = VecSetSizes(*v,win->map->n,win->map->N);CHKERRQ(ierr); 26 ierr = VecCreate_MPI_Private(*v,PETSC_FALSE,w->nghost,array);CHKERRQ(ierr); 27 28 /* New vector should inherit stashing property of parent */ 29 (*v)->stash.donotstash = win->stash.donotstash; 30 (*v)->stash.ignorenegidx = win->stash.ignorenegidx; 31 32 ierr = PetscOListDuplicate(((PetscObject)win)->olist,&((PetscObject)*v)->olist);CHKERRQ(ierr); 33 ierr = PetscFListDuplicate(((PetscObject)win)->qlist,&((PetscObject)*v)->qlist);CHKERRQ(ierr); 34 35 if (win->mapping) { 36 ierr = PetscObjectReference((PetscObject)win->mapping);CHKERRQ(ierr); 37 (*v)->mapping = win->mapping; 38 } 39 if (win->bmapping) { 40 ierr = PetscObjectReference((PetscObject)win->bmapping);CHKERRQ(ierr); 41 (*v)->bmapping = win->bmapping; 42 } 43 (*v)->ops->duplicate = VecDuplicate_Shared; 44 (*v)->map->bs = win->map->bs; 45 (*v)->bstash.bs = win->bstash.bs; 46 PetscFunctionReturn(0); 47 } 48 49 50 EXTERN_C_BEGIN 51 #undef __FUNCT__ 52 #define __FUNCT__ "VecCreate_Shared" 53 PetscErrorCode PETSCVEC_DLLEXPORT VecCreate_Shared(Vec vv) 54 { 55 PetscErrorCode ierr; 56 PetscScalar *array; 57 58 PetscFunctionBegin; 59 ierr = PetscSplitOwnership(((PetscObject)vv)->comm,&vv->map->n,&vv->map->N);CHKERRQ(ierr); 60 ierr = PetscSharedMalloc(((PetscObject)vv)->comm,vv->map->n*sizeof(PetscScalar),vv->map->N*sizeof(PetscScalar),(void**)&array);CHKERRQ(ierr); 61 62 ierr = VecCreate_MPI_Private(vv,PETSC_FALSE,0,array);CHKERRQ(ierr); 63 vv->ops->duplicate = VecDuplicate_Shared; 64 65 PetscFunctionReturn(0); 66 } 67 EXTERN_C_END 68 69 70 /* ---------------------------------------------------------------------------------------- 71 Code to manage shared memory allocation using standard Unix shared memory 72 */ 73 #include "petscsys.h" 74 #include "petscfix.h" 75 #if defined(PETSC_HAVE_PWD_H) 76 #include <pwd.h> 77 #endif 78 #include <ctype.h> 79 #include <sys/types.h> 80 #include <sys/stat.h> 81 #if defined(PETSC_HAVE_UNISTD_H) 82 #include <unistd.h> 83 #endif 84 #if defined(PETSC_HAVE_STDLIB_H) 85 #include <stdlib.h> 86 #endif 87 #if defined(PETSC_HAVE_SYS_PARAM_H) 88 #include <sys/param.h> 89 #endif 90 #if defined(PETSC_HAVE_SYS_UTSNAME_H) 91 #include <sys/utsname.h> 92 #endif 93 #include <fcntl.h> 94 #include <time.h> 95 #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H) 96 #include <sys/systeminfo.h> 97 #endif 98 #include <sys/shm.h> 99 #include <sys/mman.h> 100 101 102 static PetscMPIInt Petsc_Shared_keyval = MPI_KEYVAL_INVALID; 103 104 #undef __FUNCT__ 105 #define __FUNCT__ "Petsc_DeleteShared" 106 /* 107 Private routine to delete internal storage when a communicator is freed. 108 This is called by MPI, not by users. 109 110 The binding for the first argument changed from MPI 1.0 to 1.1; in 1.0 111 it was MPI_Comm *comm. 112 */ 113 static PetscErrorCode Petsc_DeleteShared(MPI_Comm comm,PetscInt keyval,void* attr_val,void* extra_state) 114 { 115 PetscErrorCode ierr; 116 117 PetscFunctionBegin; 118 ierr = PetscFree(attr_val);CHKERRQ(ierr); 119 PetscFunctionReturn(MPI_SUCCESS); 120 } 121 122 #include "petscfix.h" 123 124 #undef __FUNCT__ 125 #define __FUNCT__ "PetscSharedMalloc" 126 /* 127 128 This routine is still incomplete and needs work. 129 130 For this to work on the Apple Mac OS X you will likely need to add something line the following to the file /etc/sysctl.conf 131 cat /etc/sysctl.conf 132 kern.sysv.shmmax=67108864 133 kern.sysv.shmmin=1 134 kern.sysv.shmmni=32 135 kern.sysv.shmseg=512 136 kern.sysv.shmall=1024 137 138 This does not currently free the shared memory after the program runs. Use the Unix command ipcs to see the shared memory in use and 139 ipcrm to remove the shared memory in use. 140 141 */ 142 PetscErrorCode PetscSharedMalloc(MPI_Comm comm,PetscInt llen,PetscInt len,void **result) 143 { 144 PetscErrorCode ierr; 145 PetscInt shift; 146 PetscMPIInt rank,flag; 147 int *arena,id,key = 0; 148 char *value; 149 150 PetscFunctionBegin; 151 *result = 0; 152 153 ierr = MPI_Scan(&llen,&shift,1,MPI_INT,MPI_SUM,comm);CHKERRQ(ierr); 154 shift -= llen; 155 156 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 157 if (!rank) { 158 id = shmget(key,len, 0666 |IPC_CREAT); 159 if (id == -1) { 160 perror("Unable to malloc shared memory"); 161 SETERRQ(PETSC_ERR_LIB,"Unable to malloc shared memory"); 162 } 163 } else { 164 id = shmget(key,len, 0666); 165 if (id == -1) { 166 perror("Unable to malloc shared memory"); 167 SETERRQ(PETSC_ERR_LIB,"Unable to malloc shared memory"); 168 } 169 } 170 value = shmat(id,(void*)0,0); 171 if (value == (char*)-1) { 172 perror("Unable to access shared memory allocated"); 173 SETERRQ(PETSC_ERR_LIB,"Unable to access shared memory allocated"); 174 } 175 *result = (void*) (value + shift); 176 177 PetscFunctionReturn(0); 178 } 179 180 #else 181 182 EXTERN_C_BEGIN 183 extern PetscErrorCode PETSCVEC_DLLEXPORT VecCreate_Seq(Vec); 184 EXTERN_C_END 185 186 EXTERN_C_BEGIN 187 #undef __FUNCT__ 188 #define __FUNCT__ "VecCreate_Shared" 189 PetscErrorCode PETSCVEC_DLLEXPORT VecCreate_Shared(Vec vv) 190 { 191 PetscErrorCode ierr; 192 PetscMPIInt size; 193 194 PetscFunctionBegin; 195 ierr = MPI_Comm_size(((PetscObject)vv)->comm,&size);CHKERRQ(ierr); 196 if (size > 1) { 197 SETERRQ(PETSC_ERR_SUP_SYS,"No supported for shared memory vector objects on this machine"); 198 } 199 ierr = VecCreate_Seq(vv);CHKERRQ(ierr); 200 PetscFunctionReturn(0); 201 } 202 EXTERN_C_END 203 204 #endif 205 206 #undef __FUNCT__ 207 #define __FUNCT__ "VecCreateShared" 208 /*@ 209 VecCreateShared - Creates a parallel vector that uses shared memory. 210 211 Input Parameters: 212 . comm - the MPI communicator to use 213 . n - local vector length (or PETSC_DECIDE to have calculated if N is given) 214 . N - global vector length (or PETSC_DECIDE to have calculated if n is given) 215 216 Output Parameter: 217 . vv - the vector 218 219 Collective on MPI_Comm 220 221 Notes: 222 Currently VecCreateShared() is available only on the SGI; otherwise, 223 this routine is the same as VecCreateMPI(). 224 225 Use VecDuplicate() or VecDuplicateVecs() to form additional vectors of the 226 same type as an existing vector. 227 228 Level: advanced 229 230 Concepts: vectors^creating with shared memory 231 232 .seealso: VecCreateSeq(), VecCreate(), VecCreateMPI(), VecDuplicate(), VecDuplicateVecs(), 233 VecCreateGhost(), VecCreateMPIWithArray(), VecCreateGhostWithArray() 234 235 @*/ 236 PetscErrorCode PETSCVEC_DLLEXPORT VecCreateShared(MPI_Comm comm,PetscInt n,PetscInt N,Vec *v) 237 { 238 PetscErrorCode ierr; 239 240 PetscFunctionBegin; 241 ierr = VecCreate(comm,v);CHKERRQ(ierr); 242 ierr = VecSetSizes(*v,n,N);CHKERRQ(ierr); 243 ierr = VecSetType(*v,VECSHARED);CHKERRQ(ierr); 244 PetscFunctionReturn(0); 245 } 246 247 248 249 250 251