1 #include <stdio.h> 2 #include <iostream> 3 #include <string.h> 4 #include <stdlib.h> 5 //#define OMPI_SKIP_MPICXX 1 //Added in the CMakeList.txt file 6 #include <mpi.h> 7 #include <math.h> 8 #include <sys/stat.h> 9 #include <sys/types.h> 10 #include "phastaIO.h" 11 12 inline int 13 cscompare( const char teststring[], 14 const char targetstring[] ) 15 { 16 char* s1 = const_cast<char*>(teststring); 17 char* s2 = const_cast<char*>(targetstring); 18 19 while( *s1 == ' ') s1++; 20 while( *s2 == ' ') s2++; 21 while( ( *s1 ) 22 && ( *s2 ) 23 && ( *s2 != '?') 24 && ( tolower( *s1 )==tolower( *s2 ) ) ) { 25 s1++; 26 s2++; 27 while( *s1 == ' ') s1++; 28 while( *s2 == ' ') s2++; 29 } 30 if ( !( *s1 ) || ( *s1 == '?') ) return 1; 31 else return 0; 32 } 33 34 int main(int argc, char *argv[]) { 35 36 MPI_Init(&argc,&argv); 37 38 int myrank, N_procs; 39 MPI_Comm_rank(MPI_COMM_WORLD, &myrank); 40 MPI_Comm_size(MPI_COMM_WORLD, &N_procs); 41 42 FILE * pFile; 43 char target[1024], pool[256]; 44 char * temp, * token; 45 int i, j, k, N_restart_integer, N_restart_double; 46 int N_geombc_double, N_geombc_integer; 47 int N_steps, N_parts, N_files; 48 49 pFile = fopen("./IO.N2O.input","r"); 50 if (pFile == NULL) 51 printf("Error openning\n"); 52 53 fgets( target, 1024, pFile ); 54 token = strtok ( target, ";" );strcpy(pool,token); 55 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 56 N_geombc_double = atoi(temp); 57 58 fgets( target, 1024, pFile ); 59 token = strtok ( target, ";" );strcpy(pool,token); 60 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 61 N_geombc_integer = atoi(temp); 62 63 fgets( target, 1024, pFile ); 64 token = strtok ( target, ";" );strcpy(pool,token); 65 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 66 N_restart_double = atoi(temp); 67 68 fgets( target, 1024, pFile ); 69 token = strtok ( target, ";" );strcpy(pool,token); 70 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 71 N_restart_integer = atoi(temp); 72 73 fgets( target, 1024, pFile ); 74 token = strtok ( target, ";" );strcpy(pool,token); 75 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 76 N_steps = atoi(temp); 77 78 fgets( target, 1024, pFile ); 79 token = strtok ( target, ";" );strcpy(pool,token); 80 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 81 N_parts = atoi(temp); 82 83 if(myrank==0){ 84 printf("numpe is %d and start is %d\n",N_parts,N_steps); 85 } 86 87 fgets( target, 1024, pFile ); 88 token = strtok ( target, ";" );strcpy(pool,token); 89 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 90 N_files = atoi(temp); 91 92 double ***Dfield; int ***Ifield; 93 int ***paraD, ***paraI, *expectD, *expectI; 94 char **fieldNameD, **fileTypeD, **dataTypeD, **headerTypeD; 95 char **fieldNameI, **fileTypeI, **dataTypeI, **headerTypeI; 96 97 int WriteLockD[N_restart_double]; 98 int WriteLockI[N_restart_integer]; 99 100 int nppp = N_parts/N_procs; 101 int startpart = myrank * nppp +1; 102 int endpart = startpart + nppp - 1; 103 char gfname[64], numTemp[128]; 104 int iarray[10], igeom, isize; 105 106 107 if (N_parts != N_procs) { 108 printf("Input error: number of parts should be equal to the number of procs!\n"); 109 printf("Please modify the IO.O2N.input file!\n"); 110 return 0; 111 } 112 113 114 115 ///////////////////// reading /////////////////////////////// 116 117 int nppf = N_parts/N_files; 118 int N_geombc = N_geombc_double + N_geombc_integer; 119 int readHandle, GPID; 120 char fname[255],fieldtag[255]; 121 122 int irestart; 123 124 Dfield = new double**[N_restart_double]; 125 Ifield = new int**[N_restart_integer]; 126 127 paraD = new int**[N_restart_double]; 128 paraI = new int**[N_restart_integer]; 129 130 expectD = new int[N_restart_double]; 131 expectI = new int[N_restart_integer]; 132 133 fieldNameD = new char*[N_restart_double]; 134 fileTypeD = new char*[N_restart_double]; 135 dataTypeD = new char*[N_restart_double]; 136 headerTypeD = new char*[N_restart_double]; 137 138 fieldNameI = new char*[N_restart_integer]; 139 fileTypeI = new char*[N_restart_integer]; 140 dataTypeI = new char*[N_restart_integer]; 141 headerTypeI = new char*[N_restart_integer]; 142 143 if (N_restart_double>0) 144 for ( i = 0; i < N_restart_double; i++ ) 145 { 146 WriteLockD[i]=0; 147 Dfield[i] = new double*[nppp]; 148 149 paraD[i] = new int*[nppp]; 150 151 fieldNameD[i] = new char[128]; 152 fileTypeD[i] = new char[128]; 153 dataTypeD[i] = new char[128]; 154 headerTypeD[i] = new char[128]; 155 } 156 157 if (N_restart_integer>0) 158 for ( i = 0; i < N_restart_integer; i++ ) 159 { 160 WriteLockI[i]=0; 161 Ifield[i] = new int*[nppp]; 162 163 paraI[i] = new int*[nppp]; 164 165 fieldNameI[i] = new char[128]; 166 fileTypeI[i] = new char[128]; 167 dataTypeI[i] = new char[128]; 168 headerTypeI[i] = new char[128]; 169 } 170 171 172 ///////////////////// reading /////////////////////////////// 173 174 int N_restart = N_restart_double + N_restart_integer; 175 int readHandle1; 176 177 bzero((void*)fname,255); 178 sprintf(fname,"./%d-procs_case/restart-dat.%d.%d",N_parts,N_steps,((int)(myrank/(N_procs/N_files))+1)); 179 180 //if(myrank==0){ 181 // printf("Myrank is %d - Filename is %s \n",myrank,fname); 182 //} 183 184 int nfields; 185 queryphmpiio(fname, &nfields, &nppf); 186 //initphmpiio(&N_restart, &nppf, &N_files,&readHandle1, "write") ;//WRONG 187 initphmpiio(&nfields, &nppf, &N_files, &readHandle1, "read"); 188 openfile(fname, "read", &readHandle1); 189 190 for ( i = 0; i < N_restart_double; i++ ) 191 { 192 fgets( target, 1024, pFile ); 193 temp = strtok( target, ";" ); 194 token = strtok( temp, "," ); 195 strcpy( fileTypeD[i], token ); 196 token = strtok ( NULL, "," ); 197 strcpy( fieldNameD[i], token ); 198 token = strtok ( NULL, "," ); 199 strcpy( dataTypeD[i], token ); 200 token = strtok ( NULL, "," ); 201 strcpy( headerTypeD[i], token ); 202 token = strtok ( NULL, "," ); 203 strcpy( numTemp, token ); 204 expectD[i] = atoi (numTemp); 205 206 for ( j = 0; j < nppp; j++ ) 207 { 208 paraD[i][j] = new int[expectD[i]]; 209 210 for ( k = 0; k < 10; k++ ) 211 iarray[k]=0; 212 213 GPID = startpart + j; 214 bzero((void*)fieldtag,255); 215 sprintf(fieldtag,"%s@%d",fieldNameD[i],GPID); 216 217 //printf("myrank %d - filedtag %s\n",myrank,fieldtag); 218 219 iarray[0]=-1; 220 readheader( &readHandle1, 221 fieldtag, 222 (void*)iarray, 223 &expectD[i], 224 "double", 225 "binary" ); 226 227 if ( iarray[0]==-1 ) 228 WriteLockD[i]=1; 229 if ( WriteLockD[i]==0 ) 230 { 231 for ( k = 0; k < expectD[i]; k++ ) 232 paraD[i][j][k] = iarray[k]; 233 234 if ( cscompare("block",headerTypeD[i]) ) 235 { 236 if ( expectD[i]==1) 237 isize = paraD[i][j][0]; 238 else 239 isize = paraD[i][j][0] * paraD[i][j][1]; 240 241 Dfield[i][j] = new double[isize]; 242 readdatablock( &readHandle1, 243 fieldtag, 244 (void*)Dfield[i][j], 245 &isize, 246 "double", 247 "binary" ); 248 } 249 250 } 251 } 252 } 253 254 for ( i = 0; i < N_restart_integer; i++ ) 255 { 256 fgets( target, 1024, pFile ); 257 temp = strtok( target, ";" ); 258 token = strtok( temp, "," ); 259 strcpy( fileTypeI[i], token ); 260 token = strtok ( NULL, "," ); 261 strcpy( fieldNameI[i], token ); 262 token = strtok ( NULL, "," ); 263 strcpy( dataTypeI[i], token ); 264 token = strtok ( NULL, "," ); 265 strcpy( headerTypeI[i], token ); 266 token = strtok ( NULL, "," ); 267 strcpy( numTemp, token ); 268 expectI[i] = atoi (numTemp); 269 270 for ( j = 0; j < nppp; j++ ) 271 { 272 paraI[i][j] = new int[expectI[i]]; 273 274 for ( k = 0; k < 10; k++ ) 275 iarray[k]=0; 276 277 GPID = startpart + j; 278 bzero((void*)fieldtag,255); 279 sprintf(fieldtag,"%s@%d",fieldNameI[i],GPID); 280 iarray[0]=-1; 281 282 //printf("Rank %d, fieldname is %s \n",myrank,fieldtag); 283 284 readheader( &readHandle1, 285 fieldtag, 286 (void*)iarray, 287 &expectI[i], 288 "integer", 289 "binary" ); 290 291 if ( iarray[0]==-1) 292 WriteLockI[i]=1; 293 if ( WriteLockI[i]==0 ) 294 { 295 for ( k = 0; k < expectI[i]; k++ ) 296 paraI[i][j][k] = iarray[k]; 297 298 if ( cscompare("block",headerTypeI[i]) ) 299 { 300 if ( expectI[i]==1) 301 isize = paraI[i][j][0]; 302 else 303 isize = paraI[i][j][0] * paraI[i][j][1]; 304 305 Ifield[i][j] = new int[isize]; 306 readdatablock( &readHandle1, 307 fieldtag, 308 (void*)Ifield[i][j], 309 &isize, 310 "integer", 311 "binary" ); 312 } 313 } 314 } 315 316 } 317 318 closefile(&readHandle1, "write"); 319 finalizephmpiio(&readHandle1); 320 321 //////////////////////////writing//////////////////////////// 322 323 int irstou; 324 int magic_number = 362436; 325 int* mptr = &magic_number; 326 int nitems = 1; 327 328 //MR CHANGE 329 bzero((void*)fname,255); 330 sprintf(fname,"./%d-procs_case-1PPP",N_parts); 331 if(0<mkdir(fname,0777)) { printf("ERROR - Could not create procs_case-1PPP directory\n"); return 1; } 332 //MR CHANGE END 333 334 bzero((void*)fname,255); 335 sprintf(fname,"./%d-procs_case-1PPP/restart.%d.%d",N_parts,N_steps,myrank+1); 336 openfile(fname,"write", &irstou); 337 338 /* writing the top ascii header for the restart file */ 339 340 writestring( &irstou,"# PHASTA Input File Version 2.0\n"); 341 writestring( &irstou, 342 "# format \"keyphrase : sizeofnextblock usual headers\"\n"); 343 344 bzero( (void*)fname, 255 ); 345 writestring( &irstou, fname ); 346 347 writestring( &irstou, fname ); 348 writestring( &irstou,"\n"); 349 350 351 isize = 1; 352 nitems = 1; 353 iarray[ 0 ] = 1; 354 writeheader( &irstou, "byteorder magic number ", 355 (void*)iarray, &nitems, &isize, "integer", "binary" ); 356 357 nitems = 1; 358 writedatablock( &irstou, "byteorder magic number ", 359 (void*)mptr, &nitems, "integer", "binary" ); 360 361 for ( i = 0; i < N_restart_double; i++ ) 362 { 363 for ( j = 0; j < nppp; j++ ) 364 { 365 if ( WriteLockD[i] == 0 ) 366 { 367 if ( cscompare("header",headerTypeD[i]) ) 368 { 369 bzero( (void*)fname, 255 ); 370 sprintf(fname,"%s : < 0 > %d\n", fieldNameD[i],paraD[i][j][0]); 371 writestring( &irstou, fname ); 372 } 373 374 if ( cscompare("block",headerTypeD[i]) ) 375 { 376 if ( expectD[i]==1 ) 377 isize = paraD[i][j][0]; 378 else 379 isize = paraD[i][j][0] * paraD[i][j][1]; 380 381 for ( k = 0; k < expectD[i]; k++ ) 382 iarray[k] = paraD[i][j][k]; 383 384 if ( cscompare("header",headerTypeD[i]) ) 385 isize = 0; 386 387 writeheader( &irstou, 388 fieldNameD[i], 389 (void*)iarray, 390 &expectD[i], 391 &isize, 392 "double", 393 "binary"); 394 writedatablock( &irstou, 395 fieldNameD[i], 396 (void*)Dfield[i][j], 397 &isize, 398 "double", 399 "binary"); 400 } 401 402 403 if ( cscompare("block",headerTypeD[i]) ) 404 delete [] Dfield[i][j]; 405 } 406 delete [] paraD[i][j]; 407 } 408 } 409 410 for ( i = 0; i < N_restart_integer; i++ ) 411 { 412 for ( j = 0; j < nppp; j++ ) 413 { 414 415 if ( WriteLockI[i] == 0 ) 416 { 417 418 if ( cscompare("header",headerTypeI[i]) ) 419 { 420 bzero( (void*)fname, 255 ); 421 sprintf(fname,"%s : < 0 > %d\n", fieldNameI[i],paraI[i][j][0]); 422 writestring( &irstou, fname ); 423 } 424 425 if ( cscompare("block",headerTypeI[i]) ) 426 { 427 if ( expectI[i]==1 ) 428 isize = paraI[i][j][0]; 429 else 430 isize = paraI[i][j][0] * paraI[i][j][1]; 431 432 for ( k = 0; k < expectI[i]; k++ ) 433 iarray[k] = paraI[i][j][k]; 434 435 writeheader( &irstou, 436 fieldNameI[i], 437 (void*)iarray, 438 &expectI[i], 439 &isize, 440 "integer", 441 "binary"); 442 writedatablock( &irstou, 443 fieldNameI[i], 444 (void*)Ifield[i][j], 445 &isize, 446 "integer", 447 "binary"); 448 } 449 450 if ( cscompare("block",headerTypeI[i]) ) 451 delete [] Ifield[i][j]; 452 } 453 delete [] paraI[i][j]; 454 } 455 } 456 457 458 closefile( &irstou, "write" ); 459 MPI_Barrier(MPI_COMM_WORLD); 460 461 462 if (N_restart_double>0) 463 for ( i = 0; i < N_restart_double; i++ ) 464 { 465 delete [] Dfield[i]; 466 delete [] paraD[i]; 467 468 delete [] fieldNameD[i]; 469 delete [] fileTypeD[i]; 470 delete [] dataTypeD[i]; 471 delete [] headerTypeD[i]; 472 } 473 474 if (N_restart_integer>0) 475 for ( i = 0; i < N_restart_integer; i++ ) 476 { 477 delete [] Ifield[i]; 478 delete [] paraI[i]; 479 480 delete [] fieldNameI[i]; 481 delete [] fileTypeI[i]; 482 delete [] dataTypeI[i]; 483 delete [] headerTypeI[i]; 484 } 485 486 delete [] Dfield; 487 delete [] Ifield; 488 489 delete [] paraD; 490 delete [] paraI; 491 492 delete [] expectD; 493 delete [] expectI; 494 495 delete [] fieldNameD; 496 delete [] fileTypeD; 497 delete [] dataTypeD; 498 delete [] headerTypeD; 499 500 delete [] fieldNameI; 501 delete [] fileTypeI; 502 delete [] dataTypeI; 503 delete [] headerTypeI; 504 505 fclose(pFile); 506 507 if (myrank==0) 508 { 509 printf("\nFinished transfer, please check data using:\n"); 510 printf(" grep -a ': <' filename \n\n"); 511 } 512 513 MPI_Finalize(); 514 515 } 516 517 518