1 /* 2 * 3 * This is the SyncIO library that uses MPI-IO collective functions to 4 * implement a flexible I/O checkpoint solution for a large number of 5 * processors. 6 * 7 * Previous developer: Ning Liu (liun2@cs.rpi.edu) 8 * Jing Fu (fuj@cs.rpi.edu) 9 * Current developers: Michel Rasquin (Michel.Rasquin@colorado.edu), 10 * Ben Matthews (benjamin.a.matthews@colorado.edu) 11 * 12 */ 13 14 #include <map> 15 #include <vector> 16 #include <string> 17 #include <string.h> 18 #include <ctype.h> 19 #include <stdlib.h> 20 #include <stdio.h> 21 #include <math.h> 22 #include <sstream> 23 #include "phastaIO.h" 24 #include "mpi.h" 25 #include "phiotmrc.h" 26 27 #define VERSION_INFO_HEADER_SIZE 8192 28 #define DB_HEADER_SIZE 1024 29 #define ONE_MEGABYTE 1048576 30 #define TWO_MEGABYTE 2097152 31 #define ENDIAN_TEST_NUMBER 12180 // Troy's Zip Code!! 32 #define MAX_PHASTA_FILES 64 33 #define MAX_PHASTA_FILE_NAME_LENGTH 1024 34 #define MAX_FIELDS_NUMBER ((VERSION_INFO_HEADER_SIZE/MAX_FIELDS_NAME_LENGTH)-4) // The meta data include - MPI_IO_Tag, nFields, nFields*names of the fields, nppf 35 // -3 for MPI_IO_Tag, nFields and nppf, -4 for extra security (former nFiles) 36 #define MAX_FIELDS_NAME_LENGTH 128 37 #define DefaultMHSize (4*ONE_MEGABYTE) 38 //#define DefaultMHSize (8350) //For test 39 #define LATEST_WRITE_VERSION 1 40 #define inv1024sq 953.674316406e-9 // = 1/1024/1024 41 int MasterHeaderSize = -1; 42 43 bool PRINT_PERF = false; // default print no perf results 44 int irank = -1; // global rank, should never be manually manipulated 45 int mysize = -1; 46 47 // Static variables are bad but used here to store the subcommunicator and associated variables 48 // Prevent MPI_Comm_split to be called more than once, especially on BGQ with the V1R2M1 driver (leak detected in MPI_Comm_split - IBM working on it) 49 static int s_assign_local_comm = 0; 50 static MPI_Comm s_local_comm; 51 static int s_local_size = -1; 52 static int s_local_rank = -1; 53 54 // the following defines are for debug printf 55 #define PHASTAIO_PREFIX "phastaIO debug: " 56 #define PHASTAIO_DEBUG 0 //default to not print any debugging info 57 58 #if PHASTAIO_DEBUG 59 #define phprintf( s, arg...) printf(PHASTAIO_PREFIX s "\n", ##arg) 60 #define phprintf_0( s, arg...) do{ \ 61 MPI_Comm_rank(MPI_COMM_WORLD, &irank); \ 62 if(irank == 0){ \ 63 printf(PHASTAIO_PREFIX "irank=0: " s "\n", ##arg); \ 64 } \ 65 } while(0) 66 #else 67 #define phprintf( s, arg...) 68 #define phprintf_0( s, arg...) 69 #endif 70 71 enum PhastaIO_Errors 72 { 73 MAX_PHASTA_FILES_EXCEEDED = -1, 74 UNABLE_TO_OPEN_FILE = -2, 75 NOT_A_MPI_FILE = -3, 76 GPID_EXCEEDED = -4, 77 DATA_TYPE_ILLEGAL = -5, 78 }; 79 80 using namespace std; 81 82 namespace{ 83 84 map<int, std::string> LastHeaderKey; 85 vector< FILE* > fileArray; 86 vector< bool > byte_order; 87 vector< int > header_type; 88 int DataSize=0; 89 bool LastHeaderNotFound = false; 90 bool Wrong_Endian = false ; 91 bool Strict_Error = false ; 92 bool binary_format = true; 93 94 /***********************************************************************/ 95 /***************** NEW PHASTA IO CODE STARTS HERE **********************/ 96 /***********************************************************************/ 97 98 typedef struct 99 { 100 char filename[MAX_PHASTA_FILE_NAME_LENGTH]; /* defafults to 1024 */ 101 unsigned long my_offset; 102 unsigned long next_start_address; 103 unsigned long **my_offset_table; 104 unsigned long **my_read_table; 105 106 double * double_chunk; 107 double * read_double_chunk; 108 109 int field_count; 110 int part_count; 111 int read_field_count; 112 int read_part_count; 113 int GPid; 114 int start_id; 115 116 int mhsize; 117 118 int myrank; 119 int numprocs; 120 int local_myrank; 121 int local_numprocs; 122 123 int nppp; 124 int nPPF; 125 int nFiles; 126 int nFields; 127 128 int * int_chunk; 129 int * read_int_chunk; 130 131 int Wrong_Endian; /* default to false */ 132 char * master_header; 133 MPI_File file_handle; 134 MPI_Comm local_comm; 135 } phastaio_file_t; 136 137 typedef struct 138 { 139 int nppf, nfields; 140 char * masterHeader; 141 }serial_file; 142 143 serial_file *SerialFile; 144 phastaio_file_t *PhastaIOActiveFiles[MAX_PHASTA_FILES]; 145 int PhastaIONextActiveIndex = 0; /* indicates next index to allocate */ 146 147 // the caller has the responsibility to delete the returned string 148 // TODO: StringStipper("nbc value? ") returns NULL? 149 char* 150 StringStripper( const char istring[] ) { 151 152 int length = strlen( istring ); 153 154 char* dest = (char *)malloc( length + 1 ); 155 156 strcpy( dest, istring ); 157 dest[ length ] = '\0'; 158 159 if ( char* p = strpbrk( dest, " ") ) 160 *p = '\0'; 161 162 return dest; 163 } 164 165 inline int 166 cscompare( const char teststring[], 167 const char targetstring[] ) { 168 169 char* s1 = const_cast<char*>(teststring); 170 char* s2 = const_cast<char*>(targetstring); 171 172 while( *s1 == ' ') s1++; 173 while( *s2 == ' ') s2++; 174 while( ( *s1 ) 175 && ( *s2 ) 176 && ( *s2 != '?') 177 && ( tolower( *s1 )==tolower( *s2 ) ) ) { 178 s1++; 179 s2++; 180 while( *s1 == ' ') s1++; 181 while( *s2 == ' ') s2++; 182 } 183 if ( !( *s1 ) || ( *s1 == '?') ) return 1; 184 else return 0; 185 } 186 187 inline void 188 isBinary( const char iotype[] ) { 189 190 char* fname = StringStripper( iotype ); 191 if ( cscompare( fname, "binary" ) ) binary_format = true; 192 else binary_format = false; 193 free (fname); 194 195 } 196 197 inline size_t 198 typeSize( const char typestring[] ) { 199 200 char* ts1 = StringStripper( typestring ); 201 202 if ( cscompare( "integer", ts1 ) ) { 203 free (ts1); 204 return sizeof(int); 205 } else if ( cscompare( "double", ts1 ) ) { 206 free (ts1); 207 return sizeof( double ); 208 } else { 209 free (ts1); 210 fprintf(stderr,"unknown type : %s\n",ts1); 211 return 0; 212 } 213 } 214 215 int 216 readHeader( FILE* fileObject, 217 const char phrase[], 218 int* params, 219 int expect ) { 220 221 char* text_header; 222 char* token; 223 char Line[1024]; 224 char junk; 225 bool FOUND = false ; 226 int real_length; 227 int skip_size, integer_value; 228 int rewind_count=0; 229 230 if( !fgets( Line, 1024, fileObject ) && feof( fileObject ) ) { 231 rewind( fileObject ); 232 clearerr( fileObject ); 233 rewind_count++; 234 fgets( Line, 1024, fileObject ); 235 } 236 237 while( !FOUND && ( rewind_count < 2 ) ) { 238 if ( ( Line[0] != '\n' ) && ( real_length = strcspn( Line, "#" )) ){ 239 text_header = (char *)malloc( real_length + 1 ); 240 241 strncpy( text_header, Line, real_length ); 242 text_header[ real_length ] =static_cast<char>(NULL); 243 token = strtok ( text_header, ":" ); 244 //if( cscompare( phrase , token ) ) { 245 // Double comparison required because different fields can still start 246 // with the same name for mixed meshes (nbc code, nbc values, etc). 247 // Would be easy to fix cscompare instead but would it break sth else? 248 if( cscompare( phrase , token ) && cscompare( token , phrase ) ) { 249 FOUND = true ; 250 token = strtok( NULL, " ,;<>" ); 251 skip_size = atoi( token ); 252 int i; 253 for( i=0; i < expect && ( token = strtok( NULL," ,;<>") ); i++) { 254 params[i] = atoi( token ); 255 } 256 if ( i < expect ) { 257 fprintf(stderr,"Aloha Expected # of ints not found for: %s\n",phrase ); 258 } 259 } else if ( cscompare(token,"byteorder magic number") ) { 260 if ( binary_format ) { 261 fread((void*)&integer_value,sizeof(int),1,fileObject); 262 fread( &junk, sizeof(char), 1 , fileObject ); 263 if ( 362436 != integer_value ) Wrong_Endian = true; 264 } else{ 265 fscanf(fileObject, "%d\n", &integer_value ); 266 } 267 } else { 268 /* some other header, so just skip over */ 269 token = strtok( NULL, " ,;<>" ); 270 skip_size = atoi( token ); 271 if ( binary_format) 272 fseek( fileObject, skip_size, SEEK_CUR ); 273 else 274 for( int gama=0; gama < skip_size; gama++ ) 275 fgets( Line, 1024, fileObject ); 276 } 277 free (text_header); 278 } // end of if before while loop 279 280 if ( !FOUND ) 281 if( !fgets( Line, 1024, fileObject ) && feof( fileObject ) ) { 282 rewind( fileObject ); 283 clearerr( fileObject ); 284 rewind_count++; 285 fgets( Line, 1024, fileObject ); 286 } 287 } 288 289 if ( !FOUND ) { 290 //fprintf(stderr, "Error: Could not find: %s\n", phrase); 291 if(irank == 0) printf("WARNING: Could not find: %s\n", phrase); 292 return 1; 293 } 294 return 0; 295 } 296 297 } // end unnamed namespace 298 299 300 // begin of publicly visible functions 301 302 /** 303 * This function takes a long long pointer and assign (start) phiotmrc value to it 304 */ 305 void startTimer(double* start) { 306 307 if( !PRINT_PERF ) return; 308 309 MPI_Barrier(MPI_COMM_WORLD); 310 *start = phiotmrc(); 311 } 312 313 /** 314 * This function takes a long long pointer and assign (end) phiotmrc value to it 315 */ 316 void endTimer(double* end) { 317 318 if( !PRINT_PERF ) return; 319 320 *end = phiotmrc(); 321 MPI_Barrier(MPI_COMM_WORLD); 322 } 323 324 /** 325 * choose to print some performance results (or not) according to 326 * the PRINT_PERF macro 327 */ 328 void printPerf( 329 const char* func_name, 330 double start, 331 double end, 332 unsigned long datasize, 333 int printdatainfo, 334 const char* extra_msg) { 335 336 if( !PRINT_PERF ) return; 337 338 unsigned long data_size = datasize; 339 340 double time = end - start; 341 342 unsigned long isizemin,isizemax,isizetot; 343 double sizemin,sizemax,sizeavg,sizetot,rate; 344 double tmin, tmax, tavg, ttot; 345 346 MPI_Allreduce(&time, &tmin,1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); 347 MPI_Allreduce(&time, &tmax,1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); 348 MPI_Allreduce(&time, &ttot,1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); 349 tavg = ttot/mysize; 350 351 if(irank == 0) { 352 if ( PhastaIONextActiveIndex == 0 ) printf("** 1PFPP "); 353 else printf("** syncIO "); 354 printf("%s(): Tmax = %f sec, Tmin = %f sec, Tavg = %f sec", func_name, tmax, tmin, tavg); 355 } 356 357 if(printdatainfo == 1) { // if printdatainfo ==1, compute I/O rate and block size 358 MPI_Allreduce(&data_size,&isizemin,1,MPI_LONG_LONG_INT,MPI_MIN,MPI_COMM_WORLD); 359 MPI_Allreduce(&data_size,&isizemax,1,MPI_LONG_LONG_INT,MPI_MAX,MPI_COMM_WORLD); 360 MPI_Allreduce(&data_size,&isizetot,1,MPI_LONG_LONG_INT,MPI_SUM,MPI_COMM_WORLD); 361 362 sizemin=(double)(isizemin*inv1024sq); 363 sizemax=(double)(isizemax*inv1024sq); 364 sizetot=(double)(isizetot*inv1024sq); 365 sizeavg=(double)(1.0*sizetot/mysize); 366 rate=(double)(1.0*sizetot/tmax); 367 368 if( irank == 0) { 369 printf(", Rate = %f MB/s [%s] \n \t\t\t block size: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB\n", rate, extra_msg, sizemin,sizemax,sizeavg,sizetot); 370 } 371 } 372 else { 373 if(irank == 0) { 374 printf(" \n"); 375 //printf(" (%s) \n", extra_msg); 376 } 377 } 378 } 379 380 /** 381 * This function is normally called at the beginning of a read operation, before 382 * init function. 383 * This function (uses rank 0) reads out nfields, nppf, master header size, 384 * endianess and allocates for masterHeader string. 385 * These values are essential for following read operations. Rank 0 will bcast 386 * these values to other ranks in the commm world 387 * 388 * If the file set is of old POSIX format, it would throw error and exit 389 */ 390 void queryphmpiio(const char filename[],int *nfields, int *nppf) 391 { 392 MPI_Comm_rank(MPI_COMM_WORLD, &irank); 393 MPI_Comm_size(MPI_COMM_WORLD, &mysize); 394 395 if(irank == 0) { 396 FILE * fileHandle; 397 char* fname = StringStripper( filename ); 398 399 fileHandle = fopen (fname,"rb"); 400 if (fileHandle == NULL ) { 401 printf("\nError: File %s doesn't exist! Please check!\n",fname); 402 } 403 else { 404 SerialFile =(serial_file *)calloc( 1, sizeof( serial_file) ); 405 int meta_size_limit = VERSION_INFO_HEADER_SIZE; 406 SerialFile->masterHeader = (char *)malloc( meta_size_limit ); 407 fread(SerialFile->masterHeader, 1, meta_size_limit, fileHandle); 408 409 char read_out_tag[MAX_FIELDS_NAME_LENGTH]; 410 char version[MAX_FIELDS_NAME_LENGTH/4]; 411 int mhsize; 412 char * token; 413 int magic_number; 414 415 memcpy( read_out_tag, 416 SerialFile->masterHeader, 417 MAX_FIELDS_NAME_LENGTH-1 ); 418 419 if ( cscompare ("MPI_IO_Tag",read_out_tag) ) { 420 // Test endianess ... 421 memcpy (&magic_number, 422 SerialFile->masterHeader + sizeof("MPI_IO_Tag : ")-1, //-1 sizeof returns the size of the string+1 for "\0" 423 sizeof(int) ); // masterheader should look like "MPI_IO_Tag : 12180 " with 12180 in binary format 424 425 if ( magic_number != ENDIAN_TEST_NUMBER ) { 426 printf("Endian is different!\n"); 427 // Will do swap later 428 } 429 430 // test version, old version, default masterheader size is 4M 431 // newer version masterheader size is read from first line 432 memcpy(version, 433 SerialFile->masterHeader + MAX_FIELDS_NAME_LENGTH/2, 434 MAX_FIELDS_NAME_LENGTH/4 - 1); //TODO: why -1? 435 436 if( cscompare ("version",version) ) { 437 // if there is "version" tag in the file, then it is newer format 438 // read master header size from here, otherwise use default 439 // Note: if version is "1", we know mhsize is at 3/4 place... 440 441 token = strtok(version, ":"); 442 token = strtok(NULL, " ,;<>" ); 443 int iversion = atoi(token); 444 445 if( iversion == 1) { 446 memcpy( &mhsize, 447 SerialFile->masterHeader + MAX_FIELDS_NAME_LENGTH/4*3 + sizeof("mhsize : ")-1, 448 sizeof(int)); 449 if ( magic_number != ENDIAN_TEST_NUMBER ) 450 SwapArrayByteOrder(&mhsize, sizeof(int), 1); 451 452 if( mhsize > DefaultMHSize ) { 453 //if actual headersize is larger than default, let's re-read 454 free(SerialFile->masterHeader); 455 SerialFile->masterHeader = (char *)malloc(mhsize); 456 fseek(fileHandle, 0, SEEK_SET); // reset the file stream position 457 fread(SerialFile->masterHeader,1,mhsize,fileHandle); 458 } 459 } 460 //TODO: check if this is a valid int?? 461 MasterHeaderSize = mhsize; 462 } 463 else { // else it's version 0's format w/o version tag, implicating MHSize=4M 464 MasterHeaderSize = DefaultMHSize; 465 } 466 467 memcpy( read_out_tag, 468 SerialFile->masterHeader+MAX_FIELDS_NAME_LENGTH+1, 469 MAX_FIELDS_NAME_LENGTH ); //TODO: why +1 470 471 // Read in # fields ... 472 token = strtok( read_out_tag, ":" ); 473 token = strtok( NULL," ,;<>" ); 474 *nfields = atoi( token ); 475 if ( *nfields > MAX_FIELDS_NUMBER) { 476 printf("Error queryphmpiio: nfields is larger than MAX_FIELDS_NUMBER!\n"); 477 } 478 SerialFile->nfields=*nfields; //TODO: sanity check of this int? 479 480 memcpy( read_out_tag, 481 SerialFile->masterHeader + MAX_FIELDS_NAME_LENGTH * 2 482 + *nfields * MAX_FIELDS_NAME_LENGTH, 483 MAX_FIELDS_NAME_LENGTH); 484 485 token = strtok( read_out_tag, ":" ); 486 token = strtok( NULL," ,;<>" ); 487 *nppf = atoi( token ); 488 SerialFile->nppf=*nppf; //TODO: sanity check of int 489 } // end of if("MPI_IO_TAG") 490 else { 491 printf("Error queryphmpiio: The file you opened is not of syncIO new format, please check! read_out_tag = %s\n",read_out_tag); 492 exit(1); 493 } 494 fclose(fileHandle); 495 free(SerialFile->masterHeader); 496 free(SerialFile); 497 } //end of else 498 free(fname); 499 } 500 501 // Bcast value to every one 502 MPI_Bcast( nfields, 1, MPI_INT, 0, MPI_COMM_WORLD ); 503 MPI_Bcast( nppf, 1, MPI_INT, 0, MPI_COMM_WORLD ); 504 MPI_Bcast( &MasterHeaderSize, 1, MPI_INT, 0, MPI_COMM_WORLD ); 505 phprintf("Info queryphmpiio: myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 506 } 507 508 /** 509 * This function computes the right master header size (round to size of 2^n). 510 * This is only needed for file format version 1 in "write" mode. 511 */ 512 int computeMHSize(int nfields, int nppf, int version) { 513 int mhsize; 514 if(version == 1) { 515 //int meta_info_size = (2+nfields+1) * MAX_FIELDS_NAME_LENGTH; // 2 is MPI_IO_TAG and nFields, the others 1 is nppf 516 int meta_info_size = VERSION_INFO_HEADER_SIZE; 517 int actual_size = nfields * nppf * sizeof(unsigned long) + meta_info_size; 518 //printf("actual_size = %d, offset table size = %d\n", actual_size, nfields * nppf * sizeof(long long)); 519 if (actual_size > DefaultMHSize) { 520 mhsize = (int) ceil( (double) actual_size/DefaultMHSize); // it's rounded to ceiling of this value 521 mhsize *= DefaultMHSize; 522 } 523 else { 524 mhsize = DefaultMHSize; 525 } 526 } 527 return mhsize; 528 } 529 530 /** 531 * Computes correct color of a rank according to number of files. 532 */ 533 extern "C" int computeColor( int myrank, int numprocs, int nfiles) { 534 int color = 535 (int)(myrank / (numprocs / nfiles)); 536 return color; 537 } 538 539 540 /** 541 * Check the file descriptor. 542 */ 543 void checkFileDescriptor(const char fctname[], 544 int* fileDescriptor ) { 545 if ( *fileDescriptor < 0 ) { 546 printf("Error: File descriptor = %d in %s\n",*fileDescriptor,fctname); 547 exit(1); 548 } 549 } 550 551 /** 552 * Initialize the file struct members and allocate space for file struct 553 * buffers. 554 * 555 * Note: this function is only called when we are using new format. Old POSIX 556 * format should skip this routine and call openfile() directly instead. 557 */ 558 int initphmpiio( int *nfields, int *nppf, int *nfiles, int *filehandle, const char mode[]) 559 { 560 // we init irank again in case query not called (e.g. syncIO write case) 561 MPI_Comm_rank(MPI_COMM_WORLD, &irank); 562 MPI_Comm_size(MPI_COMM_WORLD, &mysize); 563 564 phprintf("Info initphmpiio: entering function, myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 565 566 double timer_start, timer_end; 567 startTimer(&timer_start); 568 569 char* imode = StringStripper( mode ); 570 571 // Note: if it's read, we presume query was called prior to init and 572 // MasterHeaderSize is already set to correct value from parsing header 573 // otherwise it's write then it needs some computation to be set 574 if ( cscompare( "read", imode ) ) { 575 // do nothing 576 } 577 else if( cscompare( "write", imode ) ) { 578 MasterHeaderSize = computeMHSize(*nfields, *nppf, LATEST_WRITE_VERSION); 579 } 580 else { 581 printf("Error initphmpiio: can't recognize the mode %s", imode); 582 exit(1); 583 } 584 free ( imode ); 585 586 phprintf("Info initphmpiio: myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 587 588 int i, j; 589 590 if( PhastaIONextActiveIndex == MAX_PHASTA_FILES ) { 591 printf("Error initphmpiio: PhastaIONextActiveIndex = MAX_PHASTA_FILES"); 592 endTimer(&timer_end); 593 printPerf("initphmpiio", timer_start, timer_end, 0, 0, ""); 594 return MAX_PHASTA_FILES_EXCEEDED; 595 } 596 // else if( PhastaIONextActiveIndex == 0 ) //Hang in debug mode on Intrepid 597 // { 598 // for( i = 0; i < MAX_PHASTA_FILES; i++ ); 599 // { 600 // PhastaIOActiveFiles[i] = NULL; 601 // } 602 // } 603 604 605 PhastaIOActiveFiles[PhastaIONextActiveIndex] = (phastaio_file_t *)calloc( 1, sizeof( phastaio_file_t) ); 606 607 i = PhastaIONextActiveIndex; 608 PhastaIONextActiveIndex++; 609 610 //PhastaIOActiveFiles[i]->next_start_address = 2*TWO_MEGABYTE; 611 612 PhastaIOActiveFiles[i]->next_start_address = MasterHeaderSize; // what does this mean??? TODO 613 614 PhastaIOActiveFiles[i]->Wrong_Endian = false; 615 616 PhastaIOActiveFiles[i]->nFields = *nfields; 617 PhastaIOActiveFiles[i]->nPPF = *nppf; 618 PhastaIOActiveFiles[i]->nFiles = *nfiles; 619 MPI_Comm_rank(MPI_COMM_WORLD, &(PhastaIOActiveFiles[i]->myrank)); 620 MPI_Comm_size(MPI_COMM_WORLD, &(PhastaIOActiveFiles[i]->numprocs)); 621 622 623 if( *nfiles > 1 ) { // split the ranks according to each mpiio file 624 625 if ( s_assign_local_comm == 0) { // call mpi_comm_split for the first (and only) time 626 627 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Building subcommunicator\n"); 628 629 int color = computeColor(PhastaIOActiveFiles[i]->myrank, PhastaIOActiveFiles[i]->numprocs, PhastaIOActiveFiles[i]->nFiles); 630 MPI_Comm_split(MPI_COMM_WORLD, 631 color, 632 PhastaIOActiveFiles[i]->myrank, 633 &(PhastaIOActiveFiles[i]->local_comm)); 634 MPI_Comm_size(PhastaIOActiveFiles[i]->local_comm, 635 &(PhastaIOActiveFiles[i]->local_numprocs)); 636 MPI_Comm_rank(PhastaIOActiveFiles[i]->local_comm, 637 &(PhastaIOActiveFiles[i]->local_myrank)); 638 639 // back up now these variables so that we do not need to call comm_split again 640 s_local_comm = PhastaIOActiveFiles[i]->local_comm; 641 s_local_size = PhastaIOActiveFiles[i]->local_numprocs; 642 s_local_rank = PhastaIOActiveFiles[i]->local_myrank; 643 s_assign_local_comm = 1; 644 } 645 else { // recycle the subcommunicator 646 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Recycling subcommunicator\n"); 647 PhastaIOActiveFiles[i]->local_comm = s_local_comm; 648 PhastaIOActiveFiles[i]->local_numprocs = s_local_size; 649 PhastaIOActiveFiles[i]->local_myrank = s_local_rank; 650 } 651 } 652 else { // *nfiles == 1 here - no need to call mpi_comm_split here 653 654 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Bypassing subcommunicator\n"); 655 PhastaIOActiveFiles[i]->local_comm = MPI_COMM_WORLD; 656 PhastaIOActiveFiles[i]->local_numprocs = PhastaIOActiveFiles[i]->numprocs; 657 PhastaIOActiveFiles[i]->local_myrank = PhastaIOActiveFiles[i]->myrank; 658 659 } 660 661 PhastaIOActiveFiles[i]->nppp = 662 PhastaIOActiveFiles[i]->nPPF/PhastaIOActiveFiles[i]->local_numprocs; 663 664 PhastaIOActiveFiles[i]->start_id = PhastaIOActiveFiles[i]->nPPF * 665 (int)(PhastaIOActiveFiles[i]->myrank/PhastaIOActiveFiles[i]->local_numprocs) + 666 (PhastaIOActiveFiles[i]->local_myrank * PhastaIOActiveFiles[i]->nppp); 667 668 PhastaIOActiveFiles[i]->my_offset_table = 669 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 670 671 PhastaIOActiveFiles[i]->my_read_table = 672 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 673 674 for (j=0; j<*nfields; j++) 675 { 676 PhastaIOActiveFiles[i]->my_offset_table[j] = 677 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 678 679 PhastaIOActiveFiles[i]->my_read_table[j] = 680 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 681 } 682 *filehandle = i; 683 684 PhastaIOActiveFiles[i]->master_header = (char *)calloc(MasterHeaderSize,sizeof( char )); 685 PhastaIOActiveFiles[i]->double_chunk = (double *)calloc(1,sizeof( double )); 686 PhastaIOActiveFiles[i]->int_chunk = (int *)calloc(1,sizeof( int )); 687 PhastaIOActiveFiles[i]->read_double_chunk = (double *)calloc(1,sizeof( double )); 688 PhastaIOActiveFiles[i]->read_int_chunk = (int *)calloc(1,sizeof( int )); 689 690 // Time monitoring 691 endTimer(&timer_end); 692 printPerf("initphmpiio", timer_start, timer_end, 0, 0, ""); 693 694 phprintf_0("Info initphmpiio: quiting function"); 695 696 return i; 697 } 698 699 /** 700 * Destruct the file struct and free buffers allocated in init function. 701 */ 702 void finalizephmpiio( int *fileDescriptor ) 703 { 704 double timer_start, timer_end; 705 startTimer(&timer_start); 706 707 int i, j; 708 i = *fileDescriptor; 709 //PhastaIONextActiveIndex--; 710 711 /* //free the offset table for this phasta file */ 712 //for(j=0; j<MAX_FIELDS_NUMBER; j++) //Danger: undefined behavior for my_*_table.[j] not allocated or not initialized to NULL 713 for(j=0; j<PhastaIOActiveFiles[i]->nFields; j++) 714 { 715 free( PhastaIOActiveFiles[i]->my_offset_table[j]); 716 free( PhastaIOActiveFiles[i]->my_read_table[j]); 717 } 718 free ( PhastaIOActiveFiles[i]->my_offset_table ); 719 free ( PhastaIOActiveFiles[i]->my_read_table ); 720 free ( PhastaIOActiveFiles[i]->master_header ); 721 free ( PhastaIOActiveFiles[i]->double_chunk ); 722 free ( PhastaIOActiveFiles[i]->int_chunk ); 723 free ( PhastaIOActiveFiles[i]->read_double_chunk ); 724 free ( PhastaIOActiveFiles[i]->read_int_chunk ); 725 726 if( PhastaIOActiveFiles[i]->nFiles > 1 && s_assign_local_comm ) { // the comm was split 727 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Freeing subcommunicator\n"); 728 s_assign_local_comm = 0; 729 MPI_Comm_free(&(PhastaIOActiveFiles[i]->local_comm)); 730 } 731 732 free( PhastaIOActiveFiles[i]); 733 734 endTimer(&timer_end); 735 printPerf("finalizempiio", timer_start, timer_end, 0, 0, ""); 736 737 PhastaIONextActiveIndex--; 738 } 739 740 741 /** 742 * Special init for M2N in order to create a subcommunicator for the reduced solution (requires PRINT_PERF to be false for now) 743 * Initialize the file struct members and allocate space for file struct buffers. 744 * 745 * Note: this function is only called when we are using new format. Old POSIX 746 * format should skip this routine and call openfile() directly instead. 747 */ 748 int initphmpiiosub( int *nfields, int *nppf, int *nfiles, int *filehandle, const char mode[],MPI_Comm my_local_comm) 749 { 750 // we init irank again in case query not called (e.g. syncIO write case) 751 752 MPI_Comm_rank(my_local_comm, &irank); 753 MPI_Comm_size(my_local_comm, &mysize); 754 755 phprintf("Info initphmpiio: entering function, myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 756 757 double timer_start, timer_end; 758 startTimer(&timer_start); 759 760 char* imode = StringStripper( mode ); 761 762 // Note: if it's read, we presume query was called prior to init and 763 // MasterHeaderSize is already set to correct value from parsing header 764 // otherwise it's write then it needs some computation to be set 765 if ( cscompare( "read", imode ) ) { 766 // do nothing 767 } 768 else if( cscompare( "write", imode ) ) { 769 MasterHeaderSize = computeMHSize(*nfields, *nppf, LATEST_WRITE_VERSION); 770 } 771 else { 772 printf("Error initphmpiio: can't recognize the mode %s", imode); 773 exit(1); 774 } 775 free ( imode ); 776 777 phprintf("Info initphmpiio: myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 778 779 int i, j; 780 781 if( PhastaIONextActiveIndex == MAX_PHASTA_FILES ) { 782 printf("Error initphmpiio: PhastaIONextActiveIndex = MAX_PHASTA_FILES"); 783 endTimer(&timer_end); 784 printPerf("initphmpiio", timer_start, timer_end, 0, 0, ""); 785 return MAX_PHASTA_FILES_EXCEEDED; 786 } 787 // else if( PhastaIONextActiveIndex == 0 ) //Hang in debug mode on Intrepid 788 // { 789 // for( i = 0; i < MAX_PHASTA_FILES; i++ ); 790 // { 791 // PhastaIOActiveFiles[i] = NULL; 792 // } 793 // } 794 795 796 PhastaIOActiveFiles[PhastaIONextActiveIndex] = (phastaio_file_t *)calloc( 1, sizeof( phastaio_file_t) ); 797 798 i = PhastaIONextActiveIndex; 799 PhastaIONextActiveIndex++; 800 801 //PhastaIOActiveFiles[i]->next_start_address = 2*TWO_MEGABYTE; 802 803 PhastaIOActiveFiles[i]->next_start_address = MasterHeaderSize; // what does this mean??? TODO 804 805 PhastaIOActiveFiles[i]->Wrong_Endian = false; 806 807 PhastaIOActiveFiles[i]->nFields = *nfields; 808 PhastaIOActiveFiles[i]->nPPF = *nppf; 809 PhastaIOActiveFiles[i]->nFiles = *nfiles; 810 MPI_Comm_rank(my_local_comm, &(PhastaIOActiveFiles[i]->myrank)); 811 MPI_Comm_size(my_local_comm, &(PhastaIOActiveFiles[i]->numprocs)); 812 813 int color = computeColor(PhastaIOActiveFiles[i]->myrank, PhastaIOActiveFiles[i]->numprocs, PhastaIOActiveFiles[i]->nFiles); 814 MPI_Comm_split(my_local_comm, 815 color, 816 PhastaIOActiveFiles[i]->myrank, 817 &(PhastaIOActiveFiles[i]->local_comm)); 818 MPI_Comm_size(PhastaIOActiveFiles[i]->local_comm, 819 &(PhastaIOActiveFiles[i]->local_numprocs)); 820 MPI_Comm_rank(PhastaIOActiveFiles[i]->local_comm, 821 &(PhastaIOActiveFiles[i]->local_myrank)); 822 PhastaIOActiveFiles[i]->nppp = 823 PhastaIOActiveFiles[i]->nPPF/PhastaIOActiveFiles[i]->local_numprocs; 824 825 PhastaIOActiveFiles[i]->start_id = PhastaIOActiveFiles[i]->nPPF * 826 (int)(PhastaIOActiveFiles[i]->myrank/PhastaIOActiveFiles[i]->local_numprocs) + 827 (PhastaIOActiveFiles[i]->local_myrank * PhastaIOActiveFiles[i]->nppp); 828 829 PhastaIOActiveFiles[i]->my_offset_table = 830 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 831 832 PhastaIOActiveFiles[i]->my_read_table = 833 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 834 835 for (j=0; j<*nfields; j++) 836 { 837 PhastaIOActiveFiles[i]->my_offset_table[j] = 838 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 839 840 PhastaIOActiveFiles[i]->my_read_table[j] = 841 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 842 } 843 *filehandle = i; 844 845 PhastaIOActiveFiles[i]->master_header = (char *)calloc(MasterHeaderSize,sizeof( char )); 846 PhastaIOActiveFiles[i]->double_chunk = (double *)calloc(1,sizeof( double )); 847 PhastaIOActiveFiles[i]->int_chunk = (int *)calloc(1,sizeof( int )); 848 PhastaIOActiveFiles[i]->read_double_chunk = (double *)calloc(1,sizeof( double )); 849 PhastaIOActiveFiles[i]->read_int_chunk = (int *)calloc(1,sizeof( int )); 850 851 // Time monitoring 852 endTimer(&timer_end); 853 printPerf("initphmpiiosub", timer_start, timer_end, 0, 0, ""); 854 855 phprintf_0("Info initphmpiiosub: quiting function"); 856 857 return i; 858 } 859 860 861 862 /** open file for both POSIX and MPI-IO syncIO format. 863 * 864 * If it's old POSIX format, simply call posix fopen(). 865 * 866 * If it's MPI-IO foramt: 867 * in "read" mode, it builds the header table that points to the offset of 868 * fields for parts; 869 * in "write" mode, it opens the file with MPI-IO open routine. 870 */ 871 void openfile(const char filename[], 872 const char mode[], 873 int* fileDescriptor ) 874 { 875 phprintf_0("Info: entering openfile"); 876 877 double timer_start, timer_end; 878 startTimer(&timer_start); 879 880 if ( PhastaIONextActiveIndex == 0 ) 881 { 882 FILE* file=NULL ; 883 *fileDescriptor = 0; 884 char* fname = StringStripper( filename ); 885 char* imode = StringStripper( mode ); 886 887 if ( cscompare( "read", imode ) ) file = fopen(fname, "rb" ); 888 else if( cscompare( "write", imode ) ) file = fopen(fname, "wb" ); 889 else if( cscompare( "append", imode ) ) file = fopen(fname, "ab" ); 890 891 if ( !file ){ 892 fprintf(stderr,"Error openfile: unable to open file %s",fname ) ; 893 } else { 894 fileArray.push_back( file ); 895 byte_order.push_back( false ); 896 header_type.push_back( sizeof(int) ); 897 *fileDescriptor = fileArray.size(); 898 } 899 free (fname); 900 free (imode); 901 } 902 else // else it would be parallel I/O, opposed to posix io 903 { 904 char* fname = StringStripper( filename ); 905 char* imode = StringStripper( mode ); 906 int rc; 907 int i = *fileDescriptor; 908 checkFileDescriptor("openfile",&i); 909 char* token; 910 911 if ( cscompare( "read", imode ) ) 912 { 913 // if (PhastaIOActiveFiles[i]->myrank == 0) 914 // printf("\n **********\nRead open ... ... regular version\n"); 915 916 rc = MPI_File_open( PhastaIOActiveFiles[i]->local_comm, 917 fname, 918 MPI_MODE_RDONLY, 919 MPI_INFO_NULL, 920 &(PhastaIOActiveFiles[i]->file_handle) ); 921 922 if(rc) 923 { 924 *fileDescriptor = UNABLE_TO_OPEN_FILE; 925 printf("Error openfile: Unable to open file %s! File descriptor = %d\n",fname,*fileDescriptor); 926 endTimer(&timer_end); 927 printPerf("openfile", timer_start, timer_end, 0, 0, ""); 928 return; 929 } 930 931 MPI_Status read_tag_status; 932 char read_out_tag[MAX_FIELDS_NAME_LENGTH]; 933 int j; 934 int magic_number; 935 936 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 937 MPI_File_read_at( PhastaIOActiveFiles[i]->file_handle, 938 0, 939 PhastaIOActiveFiles[i]->master_header, 940 MasterHeaderSize, 941 MPI_CHAR, 942 &read_tag_status ); 943 } 944 945 MPI_Bcast( PhastaIOActiveFiles[i]->master_header, 946 MasterHeaderSize, 947 MPI_CHAR, 948 0, 949 PhastaIOActiveFiles[i]->local_comm ); 950 951 memcpy( read_out_tag, 952 PhastaIOActiveFiles[i]->master_header, 953 MAX_FIELDS_NAME_LENGTH-1 ); 954 955 if ( cscompare ("MPI_IO_Tag",read_out_tag) ) 956 { 957 // Test endianess ... 958 memcpy ( &magic_number, 959 PhastaIOActiveFiles[i]->master_header+sizeof("MPI_IO_Tag : ")-1, //-1 sizeof returns the size of the string+1 for "\0" 960 sizeof(int) ); // masterheader should look like "MPI_IO_Tag : 12180 " with 12180 in binary format 961 962 if ( magic_number != ENDIAN_TEST_NUMBER ) 963 { 964 PhastaIOActiveFiles[i]->Wrong_Endian = true; 965 } 966 967 memcpy( read_out_tag, 968 PhastaIOActiveFiles[i]->master_header+MAX_FIELDS_NAME_LENGTH+1, // TODO: WHY +1??? 969 MAX_FIELDS_NAME_LENGTH ); 970 971 // Read in # fields ... 972 token = strtok ( read_out_tag, ":" ); 973 token = strtok( NULL," ,;<>" ); 974 PhastaIOActiveFiles[i]->nFields = atoi( token ); 975 976 unsigned long **header_table; 977 header_table = ( unsigned long ** )calloc(PhastaIOActiveFiles[i]->nFields, sizeof(unsigned long *)); 978 979 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) 980 { 981 header_table[j]=( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nPPF , sizeof( unsigned long)); 982 } 983 984 // Read in the offset table ... 985 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) 986 { 987 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 988 memcpy( header_table[j], 989 PhastaIOActiveFiles[i]->master_header + 990 VERSION_INFO_HEADER_SIZE + 991 j * PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long), 992 PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long) ); 993 } 994 995 MPI_Scatter( header_table[j], 996 PhastaIOActiveFiles[i]->nppp, 997 MPI_LONG_LONG_INT, 998 PhastaIOActiveFiles[i]->my_read_table[j], 999 PhastaIOActiveFiles[i]->nppp, 1000 MPI_LONG_LONG_INT, 1001 0, 1002 PhastaIOActiveFiles[i]->local_comm ); 1003 1004 // Swap byte order if endianess is different ... 1005 if ( PhastaIOActiveFiles[i]->Wrong_Endian ) { 1006 SwapArrayByteOrder( PhastaIOActiveFiles[i]->my_read_table[j], 1007 sizeof(unsigned long), 1008 PhastaIOActiveFiles[i]->nppp ); 1009 } 1010 } 1011 1012 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1013 free ( header_table[j] ); 1014 } 1015 free (header_table); 1016 1017 } // end of if MPI_IO_TAG 1018 else //else not valid MPI file 1019 { 1020 *fileDescriptor = NOT_A_MPI_FILE; 1021 printf("Error openfile: The file %s you opened is not in syncIO new format, please check again! File descriptor = %d, MasterHeaderSize = %d, read_out_tag = %s\n",fname,*fileDescriptor,MasterHeaderSize,read_out_tag); 1022 //Printing MasterHeaderSize is useful to test a compiler bug on Intrepid BGP 1023 endTimer(&timer_end); 1024 printPerf("openfile", timer_start, timer_end, 0, 0, ""); 1025 return; 1026 } 1027 } // end of if "read" 1028 else if( cscompare( "write", imode ) ) 1029 { 1030 rc = MPI_File_open( PhastaIOActiveFiles[i]->local_comm, 1031 fname, 1032 MPI_MODE_WRONLY | MPI_MODE_CREATE, 1033 MPI_INFO_NULL, 1034 &(PhastaIOActiveFiles[i]->file_handle) ); 1035 if(rc) 1036 { 1037 *fileDescriptor = UNABLE_TO_OPEN_FILE; 1038 return; 1039 } 1040 } // end of if "write" 1041 free (fname); 1042 free (imode); 1043 } // end of if FileIndex != 0 1044 1045 endTimer(&timer_end); 1046 printPerf("openfile", timer_start, timer_end, 0, 0, ""); 1047 } 1048 1049 /** close file for both POSIX and MPI-IO syncIO format. 1050 * 1051 * If it's old POSIX format, simply call posix fclose(). 1052 * 1053 * If it's MPI-IO foramt: 1054 * in "read" mode, it simply close file with MPI-IO close routine. 1055 * in "write" mode, rank 0 in each group will re-assemble the master header and 1056 * offset table and write to the beginning of file, then close the file. 1057 */ 1058 void closefile( int* fileDescriptor, 1059 const char mode[] ) 1060 { 1061 double timer_start, timer_end; 1062 startTimer(&timer_start); 1063 1064 int i = *fileDescriptor; 1065 checkFileDescriptor("closefile",&i); 1066 1067 if ( PhastaIONextActiveIndex == 0 ) { 1068 char* imode = StringStripper( mode ); 1069 1070 if( cscompare( "write", imode ) 1071 || cscompare( "append", imode ) ) { 1072 fflush( fileArray[ *fileDescriptor - 1 ] ); 1073 } 1074 1075 fclose( fileArray[ *fileDescriptor - 1 ] ); 1076 free (imode); 1077 } 1078 else { 1079 char* imode = StringStripper( mode ); 1080 1081 //write master header here: 1082 if ( cscompare( "write", imode ) ) { 1083 // if ( PhastaIOActiveFiles[i]->nPPF * PhastaIOActiveFiles[i]->nFields < 2*ONE_MEGABYTE/8 ) //SHOULD BE CHECKED 1084 // MasterHeaderSize = 4*ONE_MEGABYTE; 1085 // else 1086 // MasterHeaderSize = 4*ONE_MEGABYTE + PhastaIOActiveFiles[i]->nPPF * PhastaIOActiveFiles[i]->nFields * 8 - 2*ONE_MEGABYTE; 1087 1088 MasterHeaderSize = computeMHSize( PhastaIOActiveFiles[i]->nFields, PhastaIOActiveFiles[i]->nPPF, LATEST_WRITE_VERSION); 1089 phprintf_0("Info closefile: myrank = %d, MasterHeaderSize = %d\n", PhastaIOActiveFiles[i]->myrank, MasterHeaderSize); 1090 1091 MPI_Status write_header_status; 1092 char mpi_tag[MAX_FIELDS_NAME_LENGTH]; 1093 char version[MAX_FIELDS_NAME_LENGTH/4]; 1094 char mhsize[MAX_FIELDS_NAME_LENGTH/4]; 1095 int magic_number = ENDIAN_TEST_NUMBER; 1096 1097 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) 1098 { 1099 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1100 sprintf(mpi_tag, "MPI_IO_Tag : "); 1101 memcpy(PhastaIOActiveFiles[i]->master_header, 1102 mpi_tag, 1103 MAX_FIELDS_NAME_LENGTH); 1104 1105 bzero((void*)version,MAX_FIELDS_NAME_LENGTH/4); 1106 // this version is "1", print version in ASCII 1107 sprintf(version, "version : %d",1); 1108 memcpy(PhastaIOActiveFiles[i]->master_header + MAX_FIELDS_NAME_LENGTH/2, 1109 version, 1110 MAX_FIELDS_NAME_LENGTH/4); 1111 1112 // master header size is computed using the formula above 1113 bzero((void*)mhsize,MAX_FIELDS_NAME_LENGTH/4); 1114 sprintf(mhsize, "mhsize : "); 1115 memcpy(PhastaIOActiveFiles[i]->master_header + MAX_FIELDS_NAME_LENGTH/4*3, 1116 mhsize, 1117 MAX_FIELDS_NAME_LENGTH/4); 1118 1119 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1120 sprintf(mpi_tag, 1121 "\nnFields : %d\n", 1122 PhastaIOActiveFiles[i]->nFields); 1123 memcpy(PhastaIOActiveFiles[i]->master_header+MAX_FIELDS_NAME_LENGTH, 1124 mpi_tag, 1125 MAX_FIELDS_NAME_LENGTH); 1126 1127 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1128 sprintf(mpi_tag, "\nnPPF : %d\n", PhastaIOActiveFiles[i]->nPPF); 1129 memcpy( PhastaIOActiveFiles[i]->master_header+ 1130 PhastaIOActiveFiles[i]->nFields * 1131 MAX_FIELDS_NAME_LENGTH + 1132 MAX_FIELDS_NAME_LENGTH * 2, 1133 mpi_tag, 1134 MAX_FIELDS_NAME_LENGTH); 1135 1136 memcpy( PhastaIOActiveFiles[i]->master_header+sizeof("MPI_IO_Tag : ")-1, //-1 sizeof returns the size of the string+1 for "\0" 1137 &magic_number, 1138 sizeof(int)); 1139 1140 memcpy( PhastaIOActiveFiles[i]->master_header+sizeof("mhsize : ") -1 + MAX_FIELDS_NAME_LENGTH/4*3, 1141 &MasterHeaderSize, 1142 sizeof(int)); 1143 } 1144 1145 int j = 0; 1146 unsigned long **header_table; 1147 header_table = ( unsigned long ** )calloc(PhastaIOActiveFiles[i]->nFields, sizeof(unsigned long *)); 1148 1149 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1150 header_table[j]=( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nPPF , sizeof( unsigned long)); 1151 } 1152 1153 //if( irank == 0 ) printf("gonna mpi_gather, myrank = %d\n", irank); 1154 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1155 MPI_Gather( PhastaIOActiveFiles[i]->my_offset_table[j], 1156 PhastaIOActiveFiles[i]->nppp, 1157 MPI_LONG_LONG_INT, 1158 header_table[j], 1159 PhastaIOActiveFiles[i]->nppp, 1160 MPI_LONG_LONG_INT, 1161 0, 1162 PhastaIOActiveFiles[i]->local_comm ); 1163 } 1164 1165 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 1166 1167 //if( irank == 0 ) printf("gonna memcpy for every procs, myrank = %d\n", irank); 1168 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1169 memcpy ( PhastaIOActiveFiles[i]->master_header + 1170 VERSION_INFO_HEADER_SIZE + 1171 j * PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long), 1172 header_table[j], 1173 PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long) ); 1174 } 1175 1176 //if( irank == 0 ) printf("gonna file_write_at(), myrank = %d\n", irank); 1177 MPI_File_write_at( PhastaIOActiveFiles[i]->file_handle, 1178 0, 1179 PhastaIOActiveFiles[i]->master_header, 1180 MasterHeaderSize, 1181 MPI_CHAR, 1182 &write_header_status ); 1183 } 1184 1185 ////free(PhastaIOActiveFiles[i]->master_header); 1186 1187 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1188 free ( header_table[j] ); 1189 } 1190 free (header_table); 1191 } 1192 1193 //if( irank == 0 ) printf("gonna file_close(), myrank = %d\n", irank); 1194 MPI_File_close( &( PhastaIOActiveFiles[i]->file_handle ) ); 1195 free ( imode ); 1196 } 1197 1198 endTimer(&timer_end); 1199 printPerf("closefile_", timer_start, timer_end, 0, 0, ""); 1200 } 1201 1202 int readHeader( FILE* f, const char phrase[], 1203 int* params, int numParams, const char* iotype) { 1204 isBinary(iotype); 1205 return readHeader(f,phrase,params,numParams); 1206 } 1207 1208 void readheader( int* fileDescriptor, 1209 const char keyphrase[], 1210 void* valueArray, 1211 int* nItems, 1212 const char datatype[], 1213 const char iotype[] ) 1214 { 1215 double timer_start, timer_end; 1216 1217 startTimer(&timer_start); 1218 1219 int i = *fileDescriptor; 1220 checkFileDescriptor("readheader",&i); 1221 1222 if ( PhastaIONextActiveIndex == 0 ) { 1223 int filePtr = *fileDescriptor - 1; 1224 FILE* fileObject; 1225 int* valueListInt; 1226 1227 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1228 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1229 fprintf(stderr,"openfile_ function has to be called before \n") ; 1230 fprintf(stderr,"acessing the file\n ") ; 1231 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1232 endTimer(&timer_end); 1233 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1234 return; 1235 } 1236 1237 LastHeaderKey[filePtr] = std::string(keyphrase); 1238 LastHeaderNotFound = false; 1239 1240 fileObject = fileArray[ filePtr ] ; 1241 Wrong_Endian = byte_order[ filePtr ]; 1242 1243 isBinary( iotype ); 1244 typeSize( datatype ); //redundant call, just avoid a compiler warning. 1245 1246 // right now we are making the assumption that we will only write integers 1247 // on the header line. 1248 1249 valueListInt = static_cast< int* >( valueArray ); 1250 int ierr = readHeader( fileObject , 1251 keyphrase, 1252 valueListInt, 1253 *nItems ) ; 1254 1255 byte_order[ filePtr ] = Wrong_Endian ; 1256 1257 if ( ierr ) LastHeaderNotFound = true; 1258 1259 //return ; // don't return, go to the end to print perf 1260 } 1261 else { 1262 unsigned int skip_size; 1263 int* valueListInt; 1264 valueListInt = static_cast <int*>(valueArray); 1265 char* token = NULL; 1266 bool FOUND = false ; 1267 isBinary( iotype ); 1268 1269 MPI_Status read_offset_status; 1270 char read_out_tag[MAX_FIELDS_NAME_LENGTH]; 1271 memset(read_out_tag, '\0', MAX_FIELDS_NAME_LENGTH); 1272 char readouttag[MAX_FIELDS_NUMBER][MAX_FIELDS_NAME_LENGTH]; 1273 int j; 1274 1275 int string_length = strlen( keyphrase ); 1276 char* buffer = (char*) malloc ( string_length+1 ); 1277 1278 strcpy ( buffer, keyphrase ); 1279 buffer[ string_length ] = '\0'; 1280 1281 char* st2 = strtok ( buffer, "@" ); 1282 st2 = strtok (NULL, "@"); 1283 PhastaIOActiveFiles[i]->GPid = atoi(st2); 1284 if ( char* p = strpbrk(buffer, "@") ) 1285 *p = '\0'; 1286 1287 // Check if the user has input the right GPid 1288 if ( ( PhastaIOActiveFiles[i]->GPid <= 1289 PhastaIOActiveFiles[i]->myrank * 1290 PhastaIOActiveFiles[i]->nppp )|| 1291 ( PhastaIOActiveFiles[i]->GPid > 1292 ( PhastaIOActiveFiles[i]->myrank + 1 ) * 1293 PhastaIOActiveFiles[i]->nppp ) ) 1294 { 1295 *fileDescriptor = NOT_A_MPI_FILE; 1296 printf("Error readheader: The file is not in syncIO new format, please check! myrank = %d, GPid = %d, nppp = %d, keyphrase = %s\n", PhastaIOActiveFiles[i]->myrank, PhastaIOActiveFiles[i]->GPid, PhastaIOActiveFiles[i]->nppp, keyphrase); 1297 // It is possible atoi could not generate a clear integer from st2 because of additional garbage character in keyphrase 1298 endTimer(&timer_end); 1299 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1300 return; 1301 } 1302 1303 // Find the field we want ... 1304 //for ( j = 0; j<MAX_FIELDS_NUMBER; j++ ) 1305 for ( j = 0; j<PhastaIOActiveFiles[i]->nFields; j++ ) 1306 { 1307 memcpy( readouttag[j], 1308 PhastaIOActiveFiles[i]->master_header + j*MAX_FIELDS_NAME_LENGTH+MAX_FIELDS_NAME_LENGTH*2+1, 1309 MAX_FIELDS_NAME_LENGTH-1 ); 1310 } 1311 1312 for ( j = 0; j<PhastaIOActiveFiles[i]->nFields; j++ ) 1313 { 1314 token = strtok ( readouttag[j], ":" ); 1315 1316 //if ( cscompare( buffer, token ) ) 1317 if ( cscompare( token , buffer ) && cscompare( buffer, token ) ) 1318 // This double comparison is required for the field "number of nodes" and all the other fields that start with "number of nodes" (i.g. number of nodes in the mesh"). 1319 // Would be safer to rename "number of nodes" by "number of nodes in the part" so that the name are completely unique. But much more work to do that (Nspre, phParAdapt, etc). 1320 // Since the field name are unique in SyncIO (as it includes part ID), this should be safe and there should be no issue with the "?" trailing character. 1321 { 1322 PhastaIOActiveFiles[i]->read_field_count = j; 1323 FOUND = true; 1324 //printf("buffer: %s | token: %s | j: %d\n",buffer,token,j); 1325 break; 1326 } 1327 } 1328 free(buffer); 1329 1330 if (!FOUND) 1331 { 1332 //if(irank==0) printf("Warning readheader: Not found %s \n",keyphrase); //PhastaIOActiveFiles[i]->myrank is certainly initialized here. 1333 if(PhastaIOActiveFiles[i]->myrank == 0) printf("WARNING readheader: Not found %s\n",keyphrase); 1334 endTimer(&timer_end); 1335 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1336 return; 1337 } 1338 1339 // Find the part we want ... 1340 PhastaIOActiveFiles[i]->read_part_count = PhastaIOActiveFiles[i]->GPid - 1341 PhastaIOActiveFiles[i]->myrank * PhastaIOActiveFiles[i]->nppp - 1; 1342 1343 PhastaIOActiveFiles[i]->my_offset = 1344 PhastaIOActiveFiles[i]->my_read_table[PhastaIOActiveFiles[i]->read_field_count][PhastaIOActiveFiles[i]->read_part_count]; 1345 1346 // printf("****Rank %d offset is %d\n",PhastaIOActiveFiles[i]->myrank,PhastaIOActiveFiles[i]->my_offset); 1347 1348 // Read each datablock header here ... 1349 1350 MPI_File_read_at_all( PhastaIOActiveFiles[i]->file_handle, 1351 PhastaIOActiveFiles[i]->my_offset+1, 1352 read_out_tag, 1353 MAX_FIELDS_NAME_LENGTH-1, 1354 MPI_CHAR, 1355 &read_offset_status ); 1356 token = strtok ( read_out_tag, ":" ); 1357 1358 // printf("&&&&Rank %d read_out_tag is %s\n",PhastaIOActiveFiles[i]->myrank,read_out_tag); 1359 1360 if( cscompare( keyphrase , token ) ) //No need to compare also token with keyphrase like above. We should already have the right one. Otherwise there is a problem. 1361 { 1362 FOUND = true ; 1363 token = strtok( NULL, " ,;<>" ); 1364 skip_size = atoi( token ); 1365 for( j=0; j < *nItems && ( token = strtok( NULL," ,;<>") ); j++ ) 1366 valueListInt[j] = atoi( token ); 1367 1368 if ( j < *nItems ) 1369 { 1370 fprintf( stderr, "Expected # of ints not found for: %s\n", keyphrase ); 1371 } 1372 } 1373 else { 1374 //if(irank==0) 1375 if(PhastaIOActiveFiles[i]->myrank == 0) 1376 // If we enter this if, there is a problem with the name of some fields 1377 { 1378 printf("Error readheader: Unexpected mismatch between keyphrase = %s and token = %s\n",keyphrase,token); 1379 } 1380 } 1381 } 1382 1383 endTimer(&timer_end); 1384 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1385 1386 } 1387 1388 void readDataBlock( 1389 FILE* fileObject, 1390 void* valueArray, 1391 int nItems, 1392 const char datatype[], 1393 const char iotype[] ) 1394 { 1395 isBinary(iotype); 1396 size_t type_size = typeSize( datatype ); 1397 if ( binary_format ) { 1398 char junk = '\0'; 1399 fread( valueArray, type_size, nItems, fileObject ); 1400 fread( &junk, sizeof(char), 1 , fileObject ); 1401 if ( Wrong_Endian ) SwapArrayByteOrder( valueArray, type_size, nItems ); 1402 } else { 1403 char* ts1 = StringStripper( datatype ); 1404 if ( cscompare( "integer", ts1 ) ) { 1405 for( int n=0; n < nItems ; n++ ) 1406 fscanf(fileObject, "%d\n",(int*)((int*)valueArray+n) ); 1407 } else if ( cscompare( "double", ts1 ) ) { 1408 for( int n=0; n < nItems ; n++ ) 1409 fscanf(fileObject, "%lf\n",(double*)((double*)valueArray+n) ); 1410 } 1411 free (ts1); 1412 } 1413 } 1414 1415 void readdatablock( int* fileDescriptor, 1416 const char keyphrase[], 1417 void* valueArray, 1418 int* nItems, 1419 const char datatype[], 1420 const char iotype[] ) 1421 { 1422 //if(irank == 0) printf("entering readdatablock()\n"); 1423 unsigned long data_size = 0; 1424 double timer_start, timer_end; 1425 startTimer(&timer_start); 1426 1427 int i = *fileDescriptor; 1428 checkFileDescriptor("readdatablock",&i); 1429 1430 if ( PhastaIONextActiveIndex == 0 ) { 1431 int filePtr = *fileDescriptor - 1; 1432 FILE* fileObject; 1433 char junk; 1434 1435 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1436 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1437 fprintf(stderr,"openfile_ function has to be called before\n") ; 1438 fprintf(stderr,"acessing the file\n ") ; 1439 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1440 endTimer(&timer_end); 1441 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1442 return; 1443 } 1444 1445 // error check.. 1446 // since we require that a consistant header always preceed the data block 1447 // let us check to see that it is actually the case. 1448 1449 if ( ! cscompare( LastHeaderKey[ filePtr ].c_str(), keyphrase ) ) { 1450 fprintf(stderr, "Header not consistant with data block\n"); 1451 fprintf(stderr, "Header: %s\n", LastHeaderKey[ filePtr ].c_str() ); 1452 fprintf(stderr, "DataBlock: %s\n ", keyphrase ); 1453 fprintf(stderr, "Please recheck read sequence \n"); 1454 if( Strict_Error ) { 1455 fprintf(stderr, "fatal error: cannot continue, returning out of call\n"); 1456 endTimer(&timer_end); 1457 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1458 return; 1459 } 1460 } 1461 1462 if ( LastHeaderNotFound ) { 1463 endTimer(&timer_end); 1464 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1465 return; 1466 } 1467 fileObject = fileArray[ filePtr ]; 1468 Wrong_Endian = byte_order[ filePtr ]; 1469 LastHeaderKey.erase(filePtr); 1470 readDataBlock(fileObject,valueArray,*nItems,datatype,iotype); 1471 1472 //return; 1473 } 1474 else { 1475 // printf("read data block\n"); 1476 MPI_Status read_data_status; 1477 size_t type_size = typeSize( datatype ); 1478 int nUnits = *nItems; 1479 isBinary( iotype ); 1480 1481 // read datablock then 1482 //MR CHANGE 1483 // if ( cscompare ( datatype, "double")) 1484 char* ts2 = StringStripper( datatype ); 1485 if ( cscompare ( "double" , ts2)) 1486 //MR CHANGE END 1487 { 1488 1489 MPI_File_read_at_all_begin( PhastaIOActiveFiles[i]->file_handle, 1490 PhastaIOActiveFiles[i]->my_offset + DB_HEADER_SIZE, 1491 valueArray, 1492 nUnits, 1493 MPI_DOUBLE ); 1494 MPI_File_read_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1495 valueArray, 1496 &read_data_status ); 1497 data_size=8*nUnits; 1498 1499 } 1500 //MR CHANGE 1501 // else if ( cscompare ( datatype, "integer")) 1502 else if ( cscompare ( "integer" , ts2)) 1503 //MR CHANGE END 1504 { 1505 MPI_File_read_at_all_begin(PhastaIOActiveFiles[i]->file_handle, 1506 PhastaIOActiveFiles[i]->my_offset + DB_HEADER_SIZE, 1507 valueArray, 1508 nUnits, 1509 MPI_INT ); 1510 MPI_File_read_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1511 valueArray, 1512 &read_data_status ); 1513 data_size=4*nUnits; 1514 } 1515 else 1516 { 1517 *fileDescriptor = DATA_TYPE_ILLEGAL; 1518 printf("readdatablock - DATA_TYPE_ILLEGAL - %s\n",datatype); 1519 endTimer(&timer_end); 1520 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1521 return; 1522 } 1523 free(ts2); 1524 1525 1526 // printf("%d Read finishe\n",PhastaIOActiveFiles[i]->myrank); 1527 1528 // Swap data byte order if endianess is different ... 1529 if ( PhastaIOActiveFiles[i]->Wrong_Endian ) 1530 { 1531 SwapArrayByteOrder( valueArray, type_size, nUnits ); 1532 } 1533 } 1534 1535 endTimer(&timer_end); 1536 char extra_msg[1024]; 1537 memset(extra_msg, '\0', 1024); 1538 char* key = StringStripper(keyphrase); 1539 sprintf(extra_msg, " field is %s ", key); 1540 printPerf("readdatablock", timer_start, timer_end, data_size, 1, extra_msg); 1541 free(key); 1542 1543 } 1544 1545 void writeHeader(FILE* f, 1546 const char keyphrase[], 1547 const void* valueArray, 1548 const int nItems, 1549 const int ndataItems, 1550 const char datatype[], 1551 const char iotype[]) 1552 { 1553 isBinary( iotype ); 1554 1555 const int _newline = 1556 ( ndataItems > 0 ) ? sizeof( char ) : 0; 1557 int size_of_nextblock = 1558 ( binary_format ) ? typeSize(datatype) * ndataItems + _newline : ndataItems; 1559 1560 fprintf( f, "%s : < %d > ", keyphrase, size_of_nextblock ); 1561 for( int i = 0; i < nItems; i++ ) 1562 fprintf( f, "%d ", *((int*)((int*)valueArray+i))); 1563 fprintf( f, "\n"); 1564 } 1565 1566 void writeheader( const int* fileDescriptor, 1567 const char keyphrase[], 1568 const void* valueArray, 1569 const int* nItems, 1570 const int* ndataItems, 1571 const char datatype[], 1572 const char iotype[]) 1573 { 1574 1575 //if(irank == 0) printf("entering writeheader()\n"); 1576 1577 double timer_start, timer_end; 1578 startTimer(&timer_start); 1579 1580 int i = *fileDescriptor; 1581 checkFileDescriptor("writeheader",&i); 1582 1583 if ( PhastaIONextActiveIndex == 0 ) { 1584 int filePtr = *fileDescriptor - 1; 1585 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1586 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1587 fprintf(stderr,"openfile_ function has to be called before \n") ; 1588 fprintf(stderr,"acessing the file\n ") ; 1589 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1590 endTimer(&timer_end); 1591 printPerf("writeheader", timer_start, timer_end, 0, 0, ""); 1592 return; 1593 } 1594 1595 LastHeaderKey[filePtr] = std::string(keyphrase); 1596 DataSize = *ndataItems; 1597 FILE* fileObject = fileArray[ filePtr ] ; 1598 header_type[ filePtr ] = typeSize( datatype ); 1599 writeHeader(fileObject,keyphrase,valueArray,*nItems, 1600 *ndataItems,datatype,iotype); 1601 } 1602 else { // else it's parallel I/O 1603 DataSize = *ndataItems; 1604 size_t type_size = typeSize( datatype ); 1605 isBinary( iotype ); 1606 char mpi_tag[MAX_FIELDS_NAME_LENGTH]; 1607 1608 int string_length = strlen( keyphrase ); 1609 char* buffer = (char*) malloc ( string_length+1 ); 1610 1611 strcpy ( buffer, keyphrase); 1612 buffer[ string_length ] = '\0'; 1613 1614 char* st2 = strtok ( buffer, "@" ); 1615 st2 = strtok (NULL, "@"); 1616 PhastaIOActiveFiles[i]->GPid = atoi(st2); 1617 1618 if ( char* p = strpbrk(buffer, "@") ) 1619 *p = '\0'; 1620 1621 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1622 sprintf(mpi_tag, "\n%s : %d\n", buffer, PhastaIOActiveFiles[i]->field_count); 1623 unsigned long offset_value; 1624 1625 int temp = *ndataItems; 1626 unsigned long number_of_items = (unsigned long)temp; 1627 MPI_Barrier(PhastaIOActiveFiles[i]->local_comm); 1628 1629 MPI_Scan( &number_of_items, 1630 &offset_value, 1631 1, 1632 MPI_LONG_LONG_INT, 1633 MPI_SUM, 1634 PhastaIOActiveFiles[i]->local_comm ); 1635 1636 offset_value = (offset_value - number_of_items) * type_size; 1637 1638 offset_value += PhastaIOActiveFiles[i]->local_myrank * 1639 DB_HEADER_SIZE + 1640 PhastaIOActiveFiles[i]->next_start_address; 1641 // This offset is the starting address of each datablock header... 1642 PhastaIOActiveFiles[i]->my_offset = offset_value; 1643 1644 // Write in my offset table ... 1645 PhastaIOActiveFiles[i]->my_offset_table[PhastaIOActiveFiles[i]->field_count][PhastaIOActiveFiles[i]->part_count] = 1646 PhastaIOActiveFiles[i]->my_offset; 1647 1648 // Update the next-start-address ... 1649 PhastaIOActiveFiles[i]->next_start_address = offset_value + 1650 number_of_items * type_size + 1651 DB_HEADER_SIZE; 1652 MPI_Bcast( &(PhastaIOActiveFiles[i]->next_start_address), 1653 1, 1654 MPI_LONG_LONG_INT, 1655 PhastaIOActiveFiles[i]->local_numprocs-1, 1656 PhastaIOActiveFiles[i]->local_comm ); 1657 1658 // Prepare datablock header ... 1659 int _newline = (*ndataItems>0)?sizeof(char):0; 1660 unsigned int size_of_nextblock = type_size * (*ndataItems) + _newline; 1661 1662 //char datablock_header[255]; 1663 //bzero((void*)datablock_header,255); 1664 char datablock_header[DB_HEADER_SIZE]; 1665 bzero((void*)datablock_header,DB_HEADER_SIZE); 1666 1667 PhastaIOActiveFiles[i]->GPid = PhastaIOActiveFiles[i]->nppp*PhastaIOActiveFiles[i]->myrank+PhastaIOActiveFiles[i]->part_count; 1668 sprintf( datablock_header, 1669 "\n%s : < %u >", 1670 keyphrase, 1671 size_of_nextblock ); 1672 1673 for ( int j = 0; j < *nItems; j++ ) 1674 { 1675 sprintf( datablock_header, 1676 "%s %d ", 1677 datablock_header, 1678 *((int*)((int*)valueArray+j))); 1679 } 1680 sprintf( datablock_header, 1681 "%s\n ", 1682 datablock_header ); 1683 1684 // Write datablock header ... 1685 //MR CHANGE 1686 // if ( cscompare(datatype,"double") ) 1687 char* ts1 = StringStripper( datatype ); 1688 if ( cscompare("double",ts1) ) 1689 //MR CHANGE END 1690 { 1691 free ( PhastaIOActiveFiles[i]->double_chunk ); 1692 PhastaIOActiveFiles[i]->double_chunk = ( double * )malloc( (sizeof( double )*number_of_items+ DB_HEADER_SIZE)); 1693 1694 double * aa = ( double * )datablock_header; 1695 memcpy(PhastaIOActiveFiles[i]->double_chunk, aa, DB_HEADER_SIZE); 1696 } 1697 //MR CHANGE 1698 // if ( cscompare(datatype,"integer") ) 1699 else if ( cscompare("integer",ts1) ) 1700 //MR CHANGE END 1701 { 1702 free ( PhastaIOActiveFiles[i]->int_chunk ); 1703 PhastaIOActiveFiles[i]->int_chunk = ( int * )malloc( (sizeof( int )*number_of_items+ DB_HEADER_SIZE)); 1704 1705 int * aa = ( int * )datablock_header; 1706 memcpy(PhastaIOActiveFiles[i]->int_chunk, aa, DB_HEADER_SIZE); 1707 } 1708 else { 1709 // *fileDescriptor = DATA_TYPE_ILLEGAL; 1710 printf("writeheader - DATA_TYPE_ILLEGAL - %s\n",datatype); 1711 endTimer(&timer_end); 1712 printPerf("writeheader", timer_start, timer_end, 0, 0, ""); 1713 return; 1714 } 1715 free(ts1); 1716 1717 PhastaIOActiveFiles[i]->part_count++; 1718 if ( PhastaIOActiveFiles[i]->part_count == PhastaIOActiveFiles[i]->nppp ) { 1719 //A new field will be written 1720 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 1721 memcpy( PhastaIOActiveFiles[i]->master_header + 1722 PhastaIOActiveFiles[i]->field_count * 1723 MAX_FIELDS_NAME_LENGTH + 1724 MAX_FIELDS_NAME_LENGTH * 2, 1725 mpi_tag, 1726 MAX_FIELDS_NAME_LENGTH-1); 1727 } 1728 PhastaIOActiveFiles[i]->field_count++; 1729 PhastaIOActiveFiles[i]->part_count=0; 1730 } 1731 free(buffer); 1732 } 1733 1734 endTimer(&timer_end); 1735 printPerf("writeheader", timer_start, timer_end, 0, 0, ""); 1736 } 1737 1738 void writeDataBlock( FILE* f, 1739 const void* valueArray, 1740 const int nItems, 1741 const char datatype[], 1742 const char iotype[] ) 1743 { 1744 isBinary( iotype ); 1745 size_t type_size = typeSize( datatype ); 1746 if ( binary_format ) { 1747 fwrite( valueArray, type_size, nItems, f ); 1748 fprintf( f,"\n"); 1749 } else { 1750 char* ts1 = StringStripper( datatype ); 1751 if ( cscompare( "integer", ts1 ) ) { 1752 for( int n=0; n < nItems ; n++ ) 1753 fprintf(f,"%d\n",*((int*)((int*)valueArray+n))); 1754 } else if ( cscompare( "double", ts1 ) ) { 1755 for( int n=0; n < nItems ; n++ ) 1756 fprintf(f,"%lf\n",*((double*)((double*)valueArray+n))); 1757 } 1758 free (ts1); 1759 } 1760 } 1761 1762 void writedatablock( const int* fileDescriptor, 1763 const char keyphrase[], 1764 const void* valueArray, 1765 const int* nItems, 1766 const char datatype[], 1767 const char iotype[] ) 1768 { 1769 //if(irank == 0) printf("entering writedatablock()\n"); 1770 1771 unsigned long data_size = 0; 1772 double timer_start, timer_end; 1773 startTimer(&timer_start); 1774 1775 int i = *fileDescriptor; 1776 checkFileDescriptor("writedatablock",&i); 1777 1778 if ( PhastaIONextActiveIndex == 0 ) { 1779 int filePtr = *fileDescriptor - 1; 1780 1781 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1782 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1783 fprintf(stderr,"openfile_ function has to be called before \n") ; 1784 fprintf(stderr,"acessing the file\n ") ; 1785 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1786 endTimer(&timer_end); 1787 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1788 return; 1789 } 1790 // since we require that a consistant header always preceed the data block 1791 // let us check to see that it is actually the case. 1792 1793 if ( ! cscompare( LastHeaderKey[ filePtr ].c_str(), keyphrase ) ) { 1794 fprintf(stderr, "Header not consistant with data block\n"); 1795 fprintf(stderr, "Header: %s\n", LastHeaderKey[ filePtr ].c_str() ); 1796 fprintf(stderr, "DataBlock: %s\n ", keyphrase ); 1797 fprintf(stderr, "Please recheck write sequence \n"); 1798 if( Strict_Error ) { 1799 fprintf(stderr, "fatal error: cannot continue, returning out of call\n"); 1800 endTimer(&timer_end); 1801 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1802 return; 1803 } 1804 } 1805 1806 FILE* fileObject = fileArray[ filePtr ] ; 1807 size_t type_size=typeSize( datatype ); 1808 isBinary( iotype ); 1809 1810 LastHeaderKey.erase(filePtr); 1811 1812 if ( header_type[filePtr] != (int)type_size ) { 1813 fprintf(stderr,"header and datablock differ on typeof data in the block for\n"); 1814 fprintf(stderr,"keyphrase : %s\n", keyphrase); 1815 if( Strict_Error ) { 1816 fprintf(stderr,"fatal error: cannot continue, returning out of call\n" ); 1817 endTimer(&timer_end); 1818 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1819 return; 1820 } 1821 } 1822 1823 int nUnits = *nItems; 1824 1825 if ( nUnits != DataSize ) { 1826 fprintf(stderr,"header and datablock differ on number of data items for\n"); 1827 fprintf(stderr,"keyphrase : %s\n", keyphrase); 1828 if( Strict_Error ) { 1829 fprintf(stderr,"fatal error: cannot continue, returning out of call\n" ); 1830 endTimer(&timer_end); 1831 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1832 return; 1833 } 1834 } 1835 writeDataBlock(fileObject,valueArray,*nItems,datatype,iotype); 1836 } 1837 else { // syncIO case 1838 MPI_Status write_data_status; 1839 isBinary( iotype ); 1840 int nUnits = *nItems; 1841 1842 //MR CHANGE 1843 // if ( cscompare(datatype,"double") ) 1844 char* ts1 = StringStripper( datatype ); 1845 if ( cscompare("double",ts1) ) 1846 //MR CHANGE END 1847 { 1848 memcpy((PhastaIOActiveFiles[i]->double_chunk+DB_HEADER_SIZE/sizeof(double)), valueArray, nUnits*sizeof(double)); 1849 MPI_File_write_at_all_begin( PhastaIOActiveFiles[i]->file_handle, 1850 PhastaIOActiveFiles[i]->my_offset, 1851 PhastaIOActiveFiles[i]->double_chunk, 1852 //BLOCK_SIZE/sizeof(double), 1853 nUnits+DB_HEADER_SIZE/sizeof(double), 1854 MPI_DOUBLE ); 1855 MPI_File_write_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1856 PhastaIOActiveFiles[i]->double_chunk, 1857 &write_data_status ); 1858 data_size=8*nUnits; 1859 } 1860 //MR CHANGE 1861 // else if ( cscompare ( datatype, "integer")) 1862 else if ( cscompare("integer",ts1) ) 1863 //MR CHANGE END 1864 { 1865 memcpy((PhastaIOActiveFiles[i]->int_chunk+DB_HEADER_SIZE/sizeof(int)), valueArray, nUnits*sizeof(int)); 1866 MPI_File_write_at_all_begin( PhastaIOActiveFiles[i]->file_handle, 1867 PhastaIOActiveFiles[i]->my_offset, 1868 PhastaIOActiveFiles[i]->int_chunk, 1869 nUnits+DB_HEADER_SIZE/sizeof(int), 1870 MPI_INT ); 1871 MPI_File_write_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1872 PhastaIOActiveFiles[i]->int_chunk, 1873 &write_data_status ); 1874 data_size=4*nUnits; 1875 } 1876 else { 1877 printf("Error: writedatablock - DATA_TYPE_ILLEGAL - %s\n",datatype); 1878 endTimer(&timer_end); 1879 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1880 return; 1881 } 1882 free(ts1); 1883 } 1884 1885 endTimer(&timer_end); 1886 char extra_msg[1024]; 1887 memset(extra_msg, '\0', 1024); 1888 char* key = StringStripper(keyphrase); 1889 sprintf(extra_msg, " field is %s ", key); 1890 printPerf("writedatablock", timer_start, timer_end, data_size, 1, extra_msg); 1891 free(key); 1892 1893 } 1894 1895 void 1896 SwapArrayByteOrder( void* array, 1897 int nbytes, 1898 int nItems ) 1899 { 1900 /* This swaps the byte order for the array of nItems each 1901 of size nbytes , This will be called only locally */ 1902 int i,j; 1903 unsigned char* ucDst = (unsigned char*)array; 1904 1905 for(i=0; i < nItems; i++) { 1906 for(j=0; j < (nbytes/2); j++) 1907 std::swap( ucDst[j] , ucDst[(nbytes - 1) - j] ); 1908 ucDst += nbytes; 1909 } 1910 } 1911 1912 void 1913 writestring( int* fileDescriptor, 1914 const char inString[] ) 1915 { 1916 1917 int filePtr = *fileDescriptor - 1; 1918 FILE* fileObject = fileArray[filePtr] ; 1919 fprintf(fileObject,"%s",inString ); 1920 return; 1921 } 1922 1923 void 1924 Gather_Headers( int* fileDescriptor, 1925 vector< string >& headers ) 1926 { 1927 1928 FILE* fileObject; 1929 char Line[1024]; 1930 1931 fileObject = fileArray[ (*fileDescriptor)-1 ]; 1932 1933 while( !feof(fileObject) ) { 1934 fgets( Line, 1024, fileObject); 1935 if ( Line[0] == '#' ) { 1936 headers.push_back( Line ); 1937 } else { 1938 break; 1939 } 1940 } 1941 rewind( fileObject ); 1942 clearerr( fileObject ); 1943 } 1944 void 1945 isWrong( void ) { (Wrong_Endian) ? fprintf(stdout,"YES\n"): fprintf(stdout,"NO\n") ; } 1946 1947 void 1948 togglestrictmode( void ) { Strict_Error = !Strict_Error; } 1949 1950 int 1951 isLittleEndian( void ) 1952 { 1953 // this function returns a 1 if the current running architecture is 1954 // LittleEndian Byte Ordered, else it returns a zero 1955 1956 union { 1957 long a; 1958 char c[sizeof( long )]; 1959 } endianUnion; 1960 1961 endianUnion.a = 1 ; 1962 1963 if ( endianUnion.c[sizeof(long)-1] != 1 ) return 1 ; 1964 else return 0; 1965 } 1966 1967 namespace PHASTA { 1968 const char* const PhastaIO_traits<int>::type_string = "integer"; 1969 const char* const PhastaIO_traits<double>::type_string = "double"; 1970 } 1971