1 #include <stdio.h> 2 #include <stdlib.h> 3 #include <string.h> 4 #include <assert.h> 5 6 #include <mpi.h> 7 8 #include "common_c.h" 9 10 #include "FCMangle.h" 11 12 //commu_int.f 13 #define commu_int FortranCInterface_GLOBAL_(commu_int, COMMU_INT) 14 void commu_int(int* global, int* ilwork, int* n, char* code); 15 16 #define gen_ncorp FortranCInterface_GLOBAL_(gen_ncorp, GEN_NCORP) 17 18 // KEJ changed ncorp_t to lcorp_t (used for on proc numbering 19 // and introduced gcorp_t for global numbering 20 typedef int lcorp_t; 21 #define NCORP_MPI_T MPI_INTEGER 22 typedef long long int gcorp_t; 23 24 static lcorp_t count_owned(int* ilwork, int nlwork,gcorp_t* ncorp_tmp, int* num_nodes); 25 static lcorp_t count_local(int* ilwork, int nlwork,gcorp_t* ncorp_tmp, int* num_nodes); 26 27 28 void gen_ncorp(gcorp_t* ncorp, int* ilwork, lcorp_t* nilwork_f, int* num_nodes) 29 { 30 int part; 31 int num_parts; 32 int i; 33 lcorp_t nilwork = *nilwork_f; 34 lcorp_t owned; 35 lcorp_t local; 36 lcorp_t* owner_counts; 37 gcorp_t local_start_id; 38 gcorp_t gid; 39 40 MPI_Comm_rank(MPI_COMM_WORLD, &part); 41 MPI_Comm_size(MPI_COMM_WORLD, &num_parts); 42 43 memset(ncorp, 0, sizeof(gcorp_t)*(*num_nodes)); 44 owned = count_owned(ilwork, nilwork, ncorp, num_nodes); 45 local = count_local(ilwork, nilwork, ncorp, num_nodes); 46 conpar.iownnodes = owned+local; 47 #ifdef PRINT_EVERYTHING 48 printf("%d: %d local only nodes\n", part, local); 49 printf("%d: %d owned nodes\n", part, owned); 50 #endif 51 assert( owned <= *num_nodes ); 52 assert( owned+local <= *num_nodes ); 53 54 owner_counts = (lcorp_t*) malloc(sizeof(lcorp_t)*num_parts); 55 memset(owner_counts, 0, sizeof(lcorp_t)*num_parts); 56 owner_counts[part] = owned+local; 57 #ifdef PRINT_EVERYTHING 58 for(i=0;i<num_parts;i++) 59 { 60 printf("%d,", owner_counts[i]); 61 } 62 printf("\n"); 63 #endif 64 MPI_Allgather(MPI_IN_PLACE, 1, NCORP_MPI_T, owner_counts, 65 1, NCORP_MPI_T, MPI_COMM_WORLD); 66 #ifdef PRINT_EVERYTHING 67 for(i=0;i<num_parts;i++) 68 { 69 printf("%d,", owner_counts[i]); 70 } 71 printf("\n"); 72 #endif 73 local_start_id=0; 74 for(i=0;i<part;i++) //TODO: MPI_Exscan()? 75 { 76 // global so needs long long 77 local_start_id += owner_counts[i]; 78 } 79 local_start_id++; //Fortran numbering 80 #ifdef PRINT_EVERYTHING 81 printf("%d: %d\n", part, local_start_id); 82 #endif 83 // global so needs long long 84 gid = local_start_id; 85 if(gid<0) printf("part,gid, %d %ld",part,gid); 86 assert(gid>=0); 87 for(i=0;i<*num_nodes;i++) //assign owned node's numbers 88 { 89 //if shared, owned 1 90 //if shared, slave -1 91 //if local only, 0 92 if(ncorp[i] == 1) 93 { 94 // global so needs long long 95 ncorp[i]=gid; 96 assert(ncorp[i]>=0); 97 98 // global so needs long long 99 gid++; 100 continue; 101 } 102 if(ncorp[i] == 0) 103 { 104 ncorp[i] = gid; 105 assert(ncorp[i]>=0); 106 gid++; 107 continue; 108 } 109 if(ncorp[i] == -1) 110 { 111 ncorp[i] = 0; //commu() adds, so zero slaves 112 } 113 114 } 115 //char code[] = "out"; 116 //int ione = 1; 117 //commu_int(ncorp, ilwork, &ione, code); 118 119 } 120 121 static lcorp_t count_local(int* ilwork, int nlwork,gcorp_t* ncorp_tmp, int* num_nodes) 122 { 123 int i; 124 lcorp_t num_local = 0; 125 for(i=0;i<*num_nodes;i++) 126 { 127 if(ncorp_tmp[i] == 0) 128 num_local++; //nodes away from part boundary 129 assert(!(ncorp_tmp[i] < -1 || ncorp_tmp[i] > 1)); 130 } 131 return(num_local); 132 } 133 static lcorp_t count_owned(int* ilwork, int nlwork,gcorp_t* ncorp_tmp, int* num_nodes) 134 { 135 int numtask = ilwork[0]; 136 int itkbeg = 0; //task offset 137 int owned = 0; 138 int i,j,k; 139 for(i=0;i<numtask;i++) 140 { 141 int itag = ilwork[itkbeg+1]; //mpi tag 142 int iacc = ilwork[itkbeg+2]; //0 for slave, 1 for master 143 assert(iacc >= 0 && iacc <= 1); 144 int iother = ilwork[itkbeg+3]-1; //other rank (see ctypes.f for off by one) 145 int numseg = ilwork[itkbeg+4]; //number of segments 146 for(j=0;j<numseg;j++) 147 { 148 int isgbeg = ilwork[itkbeg+5+(j*2)]; //first idx of seg 149 int lenseg = ilwork[itkbeg+6+(j*2)]; //length of seg 150 assert(iacc == 0 || iacc == 1); 151 if(iacc) 152 { 153 for(k=0;k<lenseg;k++) 154 { 155 if(ncorp_tmp[isgbeg-1+k] == 0) 156 owned++; 157 //make sure we're not both master and slave 158 assert(ncorp_tmp[isgbeg-1+k] != -1); 159 ncorp_tmp[isgbeg-1+k] = 1; 160 assert(isgbeg-1+k < *num_nodes); 161 } 162 assert(owned <= *num_nodes); 163 } 164 else 165 { 166 for(k=0;k<lenseg;k++) 167 { 168 ncorp_tmp[isgbeg-1+k] = -1; 169 assert(isgbeg-1+k < *num_nodes); 170 } 171 } 172 //ncorp_tmp init'd to 0 173 //if shared, owned 1 174 //if shared, slave -1 175 //if local only, 0 176 177 assert(itkbeg+6+(j*2) < nlwork); 178 } 179 itkbeg+= 4+2*numseg; 180 } 181 return(owned); 182 } 183