1 /* 2 * 3 * This is the SyncIO library that uses MPI-IO collective functions to 4 * implement a flexible I/O checkpoint solution for a large number of 5 * processors. 6 * 7 * Previous developer: Ning Liu (liun2@cs.rpi.edu) 8 * Jing Fu (fuj@cs.rpi.edu) 9 * Current developers: Michel Rasquin (Michel.Rasquin@colorado.edu), 10 * Ben Matthews (benjamin.a.matthews@colorado.edu) 11 * 12 */ 13 14 #include <map> 15 #include <vector> 16 #include <string> 17 #include <string.h> 18 #include <ctype.h> 19 #include <stdlib.h> 20 #include <stdio.h> 21 #include <math.h> 22 #include <sstream> 23 #include "phastaIO.h" 24 #include "mpi.h" 25 #include "phiotmrc.h" 26 #include <assert.h> 27 28 #define VERSION_INFO_HEADER_SIZE 8192 29 #define DB_HEADER_SIZE 1024 30 #define ONE_MEGABYTE 1048576 31 #define TWO_MEGABYTE 2097152 32 #define ENDIAN_TEST_NUMBER 12180 // Troy's Zip Code!! 33 #define MAX_PHASTA_FILES 64 34 #define MAX_PHASTA_FILE_NAME_LENGTH 1024 35 #define MAX_FIELDS_NUMBER ((VERSION_INFO_HEADER_SIZE/MAX_FIELDS_NAME_LENGTH)-4) // The meta data include - MPI_IO_Tag, nFields, nFields*names of the fields, nppf 36 // -3 for MPI_IO_Tag, nFields and nppf, -4 for extra security (former nFiles) 37 #define MAX_FIELDS_NAME_LENGTH 128 38 #define DefaultMHSize (4*ONE_MEGABYTE) 39 //#define DefaultMHSize (8350) //For test 40 #define LATEST_WRITE_VERSION 1 41 #define inv1024sq 953.674316406e-9 // = 1/1024/1024 42 int MasterHeaderSize = -1; 43 44 bool PRINT_PERF = false; // default print no perf results 45 int irank = -1; // global rank, should never be manually manipulated 46 int mysize = -1; 47 48 // Static variables are bad but used here to store the subcommunicator and associated variables 49 // Prevent MPI_Comm_split to be called more than once, especially on BGQ with the V1R2M1 driver (leak detected in MPI_Comm_split - IBM working on it) 50 static int s_assign_local_comm = 0; 51 static MPI_Comm s_local_comm; 52 static int s_local_size = -1; 53 static int s_local_rank = -1; 54 55 // the following defines are for debug printf 56 #define PHASTAIO_DEBUG 0 //default to not print any debugging info 57 58 void phprintf(const char* fmt, ...) { 59 #if PHASTAIO_DEBUG 60 char format[1024]; 61 snprintf(format, sizeof(format), "phastaIO debug: %s", fmt); 62 va_list ap; 63 va_start(ap,fmt); 64 vprintf(format,ap) 65 va_end(ap); 66 #endif 67 } 68 69 void phprintf_0(const char* fmt, ...) { 70 #if PHASTAIO_DEBUG 71 int rank = 0; 72 MPI_Comm_rank(MPI_COMM_WORLD, &rank); 73 if(rank == 0){ 74 char format[1024]; 75 snprintf(format, sizeof(format), "phastaIO debug: irank=0 %s", fmt); 76 va_list ap; 77 va_start(ap,s); 78 vprintf(format, ap); 79 va_end(ap); 80 } 81 #endif 82 } 83 84 enum PhastaIO_Errors 85 { 86 MAX_PHASTA_FILES_EXCEEDED = -1, 87 UNABLE_TO_OPEN_FILE = -2, 88 NOT_A_MPI_FILE = -3, 89 GPID_EXCEEDED = -4, 90 DATA_TYPE_ILLEGAL = -5 91 }; 92 93 using namespace std; 94 95 namespace{ 96 97 map<int, std::string> LastHeaderKey; 98 vector< FILE* > fileArray; 99 vector< bool > byte_order; 100 vector< int > header_type; 101 int DataSize=0; 102 bool LastHeaderNotFound = false; 103 bool Wrong_Endian = false ; 104 bool Strict_Error = false ; 105 bool binary_format = true; 106 107 /***********************************************************************/ 108 /***************** NEW PHASTA IO CODE STARTS HERE **********************/ 109 /***********************************************************************/ 110 111 typedef struct 112 { 113 char filename[MAX_PHASTA_FILE_NAME_LENGTH]; /* defafults to 1024 */ 114 unsigned long my_offset; 115 unsigned long next_start_address; 116 unsigned long **my_offset_table; 117 unsigned long **my_read_table; 118 119 double * double_chunk; 120 double * read_double_chunk; 121 122 int field_count; 123 int part_count; 124 int read_field_count; 125 int read_part_count; 126 int GPid; 127 int start_id; 128 129 int mhsize; 130 131 int myrank; 132 int numprocs; 133 int local_myrank; 134 int local_numprocs; 135 136 int nppp; 137 int nPPF; 138 int nFiles; 139 int nFields; 140 141 int * int_chunk; 142 int * read_int_chunk; 143 144 int Wrong_Endian; /* default to false */ 145 char * master_header; 146 MPI_File file_handle; 147 MPI_Comm local_comm; 148 } phastaio_file_t; 149 150 typedef struct 151 { 152 int nppf, nfields; 153 char * masterHeader; 154 }serial_file; 155 156 serial_file *SerialFile; 157 phastaio_file_t *PhastaIOActiveFiles[MAX_PHASTA_FILES]; 158 int PhastaIONextActiveIndex = 0; /* indicates next index to allocate */ 159 160 // the caller has the responsibility to delete the returned string 161 // TODO: StringStipper("nbc value? ") returns NULL? 162 char* 163 StringStripper( const char istring[] ) { 164 165 int length = strlen( istring ); 166 167 char* dest = (char *)malloc( length + 1 ); 168 169 strcpy( dest, istring ); 170 dest[ length ] = '\0'; 171 172 if ( char* p = strpbrk( dest, " ") ) 173 *p = '\0'; 174 175 return dest; 176 } 177 178 inline int 179 cscompare( const char teststring[], 180 const char targetstring[] ) { 181 182 char* s1 = const_cast<char*>(teststring); 183 char* s2 = const_cast<char*>(targetstring); 184 185 while( *s1 == ' ') s1++; 186 while( *s2 == ' ') s2++; 187 while( ( *s1 ) 188 && ( *s2 ) 189 && ( *s2 != '?') 190 && ( tolower( *s1 )==tolower( *s2 ) ) ) { 191 s1++; 192 s2++; 193 while( *s1 == ' ') s1++; 194 while( *s2 == ' ') s2++; 195 } 196 if ( !( *s1 ) || ( *s1 == '?') ) return 1; 197 else return 0; 198 } 199 200 inline void 201 isBinary( const char iotype[] ) { 202 203 char* fname = StringStripper( iotype ); 204 if ( cscompare( fname, "binary" ) ) binary_format = true; 205 else binary_format = false; 206 free (fname); 207 208 } 209 210 inline size_t 211 typeSize( const char typestring[] ) { 212 213 char* ts1 = StringStripper( typestring ); 214 215 if ( cscompare( "integer", ts1 ) ) { 216 free (ts1); 217 return sizeof(int); 218 } else if ( cscompare( "double", ts1 ) ) { 219 free (ts1); 220 return sizeof( double ); 221 } else { 222 free (ts1); 223 fprintf(stderr,"unknown type : %s\n",ts1); 224 return 0; 225 } 226 } 227 228 int 229 readHeader( FILE* fileObject, 230 const char phrase[], 231 int* params, 232 int expect ) { 233 234 char* text_header; 235 char* token; 236 char Line[1024] = "\0"; 237 char junk; 238 bool FOUND = false ; 239 int real_length; 240 int skip_size, integer_value; 241 int rewind_count=0; 242 243 if( !fgets( Line, 1024, fileObject ) && feof( fileObject ) ) { 244 rewind( fileObject ); 245 clearerr( fileObject ); 246 rewind_count++; 247 fgets( Line, 1024, fileObject ); 248 } 249 250 while( !FOUND && ( rewind_count < 2 ) ) { 251 if ( ( Line[0] != '\n' ) && ( real_length = strcspn( Line, "#" )) ){ 252 text_header = (char *)malloc( real_length + 1 ); 253 254 strncpy( text_header, Line, real_length ); 255 text_header[ real_length ] =static_cast<char>(NULL); 256 token = strtok ( text_header, ":" ); 257 assert(token); 258 if( cscompare( phrase , token ) ) { 259 FOUND = true ; 260 token = strtok( NULL, " ,;<>" ); 261 assert(token); 262 skip_size = atoi( token ); 263 int i; 264 for( i=0; i < expect && ( token = strtok( NULL," ,;<>") ); i++) { 265 assert(token); 266 params[i] = atoi( token ); 267 } 268 if ( i < expect ) { 269 fprintf(stderr,"Aloha Expected # of ints not found for: %s\n",phrase ); 270 } 271 } else if ( cscompare(token,"byteorder magic number") ) { 272 if ( binary_format ) { 273 fread((void*)&integer_value,sizeof(int),1,fileObject); 274 fread( &junk, sizeof(char), 1 , fileObject ); 275 if ( 362436 != integer_value ) Wrong_Endian = true; 276 } else{ 277 fscanf(fileObject, "%d\n", &integer_value ); 278 } 279 } else { 280 /* some other header, so just skip over */ 281 token = strtok( NULL, " ,;<>" ); 282 assert(token); 283 skip_size = atoi( token ); 284 if ( binary_format) 285 fseek( fileObject, skip_size, SEEK_CUR ); 286 else 287 for( int gama=0; gama < skip_size; gama++ ) 288 fgets( Line, 1024, fileObject ); 289 } 290 free (text_header); 291 } // end of if before while loop 292 293 if ( !FOUND ) 294 if( !fgets( Line, 1024, fileObject ) && feof( fileObject ) ) { 295 rewind( fileObject ); 296 clearerr( fileObject ); 297 rewind_count++; 298 fgets( Line, 1024, fileObject ); 299 } 300 } 301 302 if ( !FOUND ) { 303 //fprintf(stderr, "Error: Could not find: %s\n", phrase); 304 if(irank == 0) printf("WARNING: Could not find: %s\n", phrase); 305 return 1; 306 } 307 return 0; 308 } 309 310 } // end unnamed namespace 311 312 313 // begin of publicly visible functions 314 315 /** 316 * This function takes a long long pointer and assign (start) phiotmrc value to it 317 */ 318 void startTimer(double* start) { 319 320 if( !PRINT_PERF ) return; 321 322 MPI_Barrier(MPI_COMM_WORLD); 323 *start = phiotmrc(); 324 } 325 326 /** 327 * This function takes a long long pointer and assign (end) phiotmrc value to it 328 */ 329 void endTimer(double* end) { 330 331 if( !PRINT_PERF ) return; 332 333 *end = phiotmrc(); 334 MPI_Barrier(MPI_COMM_WORLD); 335 } 336 337 /** 338 * choose to print some performance results (or not) according to 339 * the PRINT_PERF macro 340 */ 341 void printPerf( 342 const char* func_name, 343 double start, 344 double end, 345 unsigned long datasize, 346 int printdatainfo, 347 const char* extra_msg) { 348 349 if( !PRINT_PERF ) return; 350 351 unsigned long data_size = datasize; 352 353 double time = end - start; 354 355 unsigned long isizemin,isizemax,isizetot; 356 double sizemin,sizemax,sizeavg,sizetot,rate; 357 double tmin, tmax, tavg, ttot; 358 359 MPI_Allreduce(&time, &tmin,1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); 360 MPI_Allreduce(&time, &tmax,1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); 361 MPI_Allreduce(&time, &ttot,1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); 362 tavg = ttot/mysize; 363 364 if(irank == 0) { 365 if ( PhastaIONextActiveIndex == 0 ) printf("** 1PFPP "); 366 else printf("** syncIO "); 367 printf("%s(): Tmax = %f sec, Tmin = %f sec, Tavg = %f sec", func_name, tmax, tmin, tavg); 368 } 369 370 if(printdatainfo == 1) { // if printdatainfo ==1, compute I/O rate and block size 371 MPI_Allreduce(&data_size,&isizemin,1,MPI_LONG_LONG_INT,MPI_MIN,MPI_COMM_WORLD); 372 MPI_Allreduce(&data_size,&isizemax,1,MPI_LONG_LONG_INT,MPI_MAX,MPI_COMM_WORLD); 373 MPI_Allreduce(&data_size,&isizetot,1,MPI_LONG_LONG_INT,MPI_SUM,MPI_COMM_WORLD); 374 375 sizemin=(double)(isizemin*inv1024sq); 376 sizemax=(double)(isizemax*inv1024sq); 377 sizetot=(double)(isizetot*inv1024sq); 378 sizeavg=(double)(1.0*sizetot/mysize); 379 rate=(double)(1.0*sizetot/tmax); 380 381 if( irank == 0) { 382 printf(", Rate = %f MB/s [%s] \n \t\t\t block size: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB\n", rate, extra_msg, sizemin,sizemax,sizeavg,sizetot); 383 } 384 } 385 else { 386 if(irank == 0) { 387 printf(" \n"); 388 //printf(" (%s) \n", extra_msg); 389 } 390 } 391 } 392 393 /** 394 * This function is normally called at the beginning of a read operation, before 395 * init function. 396 * This function (uses rank 0) reads out nfields, nppf, master header size, 397 * endianess and allocates for masterHeader string. 398 * These values are essential for following read operations. Rank 0 will bcast 399 * these values to other ranks in the commm world 400 * 401 * If the file set is of old POSIX format, it would throw error and exit 402 */ 403 void queryphmpiio(const char filename[],int *nfields, int *nppf) 404 { 405 MPI_Comm_rank(MPI_COMM_WORLD, &irank); 406 MPI_Comm_size(MPI_COMM_WORLD, &mysize); 407 408 if(irank == 0) { 409 FILE * fileHandle; 410 char* fname = StringStripper( filename ); 411 412 fileHandle = fopen (fname,"rb"); 413 if (fileHandle == NULL ) { 414 printf("\nError: File %s doesn't exist! Please check!\n",fname); 415 } 416 else { 417 SerialFile =(serial_file *)calloc( 1, sizeof( serial_file) ); 418 int meta_size_limit = VERSION_INFO_HEADER_SIZE; 419 SerialFile->masterHeader = (char *)malloc( meta_size_limit ); 420 fread(SerialFile->masterHeader, 1, meta_size_limit, fileHandle); 421 422 char read_out_tag[MAX_FIELDS_NAME_LENGTH]; 423 char version[MAX_FIELDS_NAME_LENGTH/4]; 424 int mhsize; 425 char * token; 426 int magic_number; 427 428 memcpy( read_out_tag, 429 SerialFile->masterHeader, 430 MAX_FIELDS_NAME_LENGTH-1 ); 431 432 if ( cscompare ("MPI_IO_Tag",read_out_tag) ) { 433 // Test endianess ... 434 memcpy (&magic_number, 435 SerialFile->masterHeader + sizeof("MPI_IO_Tag : ")-1, //-1 sizeof returns the size of the string+1 for "\0" 436 sizeof(int) ); // masterheader should look like "MPI_IO_Tag : 12180 " with 12180 in binary format 437 438 if ( magic_number != ENDIAN_TEST_NUMBER ) { 439 printf("Endian is different!\n"); 440 // Will do swap later 441 } 442 443 // test version, old version, default masterheader size is 4M 444 // newer version masterheader size is read from first line 445 memcpy(version, 446 SerialFile->masterHeader + MAX_FIELDS_NAME_LENGTH/2, 447 MAX_FIELDS_NAME_LENGTH/4 - 1); //TODO: why -1? 448 449 if( cscompare ("version",version) ) { 450 // if there is "version" tag in the file, then it is newer format 451 // read master header size from here, otherwise use default 452 // Note: if version is "1", we know mhsize is at 3/4 place... 453 454 token = strtok(version, ":"); 455 token = strtok(NULL, " ,;<>" ); 456 int iversion = atoi(token); 457 458 if( iversion == 1) { 459 memcpy( &mhsize, 460 SerialFile->masterHeader + MAX_FIELDS_NAME_LENGTH/4*3 + sizeof("mhsize : ")-1, 461 sizeof(int)); 462 if ( magic_number != ENDIAN_TEST_NUMBER ) 463 SwapArrayByteOrder(&mhsize, sizeof(int), 1); 464 465 if( mhsize > DefaultMHSize ) { 466 //if actual headersize is larger than default, let's re-read 467 free(SerialFile->masterHeader); 468 SerialFile->masterHeader = (char *)malloc(mhsize); 469 fseek(fileHandle, 0, SEEK_SET); // reset the file stream position 470 fread(SerialFile->masterHeader,1,mhsize,fileHandle); 471 } 472 } 473 //TODO: check if this is a valid int?? 474 MasterHeaderSize = mhsize; 475 } 476 else { // else it's version 0's format w/o version tag, implicating MHSize=4M 477 MasterHeaderSize = DefaultMHSize; 478 } 479 480 memcpy( read_out_tag, 481 SerialFile->masterHeader+MAX_FIELDS_NAME_LENGTH+1, 482 MAX_FIELDS_NAME_LENGTH ); //TODO: why +1 483 484 // Read in # fields ... 485 token = strtok( read_out_tag, ":" ); 486 token = strtok( NULL," ,;<>" ); 487 *nfields = atoi( token ); 488 if ( *nfields > MAX_FIELDS_NUMBER) { 489 printf("Error queryphmpiio: nfields is larger than MAX_FIELDS_NUMBER!\n"); 490 } 491 SerialFile->nfields=*nfields; //TODO: sanity check of this int? 492 493 memcpy( read_out_tag, 494 SerialFile->masterHeader + MAX_FIELDS_NAME_LENGTH * 2 495 + *nfields * MAX_FIELDS_NAME_LENGTH, 496 MAX_FIELDS_NAME_LENGTH); 497 498 token = strtok( read_out_tag, ":" ); 499 token = strtok( NULL," ,;<>" ); 500 *nppf = atoi( token ); 501 SerialFile->nppf=*nppf; //TODO: sanity check of int 502 } // end of if("MPI_IO_TAG") 503 else { 504 printf("Error queryphmpiio: The file you opened is not of syncIO new format, please check! read_out_tag = %s\n",read_out_tag); 505 exit(1); 506 } 507 fclose(fileHandle); 508 free(SerialFile->masterHeader); 509 free(SerialFile); 510 } //end of else 511 free(fname); 512 } 513 514 // Bcast value to every one 515 MPI_Bcast( nfields, 1, MPI_INT, 0, MPI_COMM_WORLD ); 516 MPI_Bcast( nppf, 1, MPI_INT, 0, MPI_COMM_WORLD ); 517 MPI_Bcast( &MasterHeaderSize, 1, MPI_INT, 0, MPI_COMM_WORLD ); 518 phprintf("Info queryphmpiio: myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 519 } 520 521 /** 522 * This function computes the right master header size (round to size of 2^n). 523 * This is only needed for file format version 1 in "write" mode. 524 */ 525 int computeMHSize(int nfields, int nppf, int version) { 526 int mhsize=0; 527 if(version == 1) { 528 //int meta_info_size = (2+nfields+1) * MAX_FIELDS_NAME_LENGTH; // 2 is MPI_IO_TAG and nFields, the others 1 is nppf 529 int meta_info_size = VERSION_INFO_HEADER_SIZE; 530 int actual_size = nfields * nppf * sizeof(unsigned long) + meta_info_size; 531 //printf("actual_size = %d, offset table size = %d\n", actual_size, nfields * nppf * sizeof(long long)); 532 if (actual_size > DefaultMHSize) { 533 mhsize = (int) ceil( (double) actual_size/DefaultMHSize); // it's rounded to ceiling of this value 534 mhsize *= DefaultMHSize; 535 } 536 else { 537 mhsize = DefaultMHSize; 538 } 539 } else { 540 int rank = 0; 541 MPI_Comm_rank(MPI_COMM_WORLD, &rank); 542 if(!rank) { 543 fprintf(stderr, 544 "ERROR invalid version passed to %s... exiting\n", __func__); 545 exit(EXIT_FAILURE); 546 } 547 } 548 return mhsize; 549 } 550 551 /** 552 * Computes correct color of a rank according to number of files. 553 */ 554 extern "C" int computeColor( int myrank, int numprocs, int nfiles) { 555 int color = 556 (int)(myrank / (numprocs / nfiles)); 557 return color; 558 } 559 560 561 /** 562 * Check the file descriptor. 563 */ 564 void checkFileDescriptor(const char fctname[], 565 int* fileDescriptor ) { 566 if ( *fileDescriptor < 0 ) { 567 printf("Error: File descriptor = %d in %s\n",*fileDescriptor,fctname); 568 exit(1); 569 } 570 } 571 572 /** 573 * Initialize the file struct members and allocate space for file struct 574 * buffers. 575 * 576 * Note: this function is only called when we are using new format. Old POSIX 577 * format should skip this routine and call openfile() directly instead. 578 */ 579 int initphmpiio( int *nfields, int *nppf, int *nfiles, int *filehandle, const char mode[]) 580 { 581 // we init irank again in case query not called (e.g. syncIO write case) 582 MPI_Comm_rank(MPI_COMM_WORLD, &irank); 583 MPI_Comm_size(MPI_COMM_WORLD, &mysize); 584 585 phprintf("Info initphmpiio: entering function, myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 586 587 double timer_start, timer_end; 588 startTimer(&timer_start); 589 590 char* imode = StringStripper( mode ); 591 592 // Note: if it's read, we presume query was called prior to init and 593 // MasterHeaderSize is already set to correct value from parsing header 594 // otherwise it's write then it needs some computation to be set 595 if ( cscompare( "read", imode ) ) { 596 // do nothing 597 } 598 else if( cscompare( "write", imode ) ) { 599 MasterHeaderSize = computeMHSize(*nfields, *nppf, LATEST_WRITE_VERSION); 600 } 601 else { 602 printf("Error initphmpiio: can't recognize the mode %s", imode); 603 exit(1); 604 } 605 free ( imode ); 606 607 phprintf("Info initphmpiio: myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 608 609 int i, j; 610 611 if( PhastaIONextActiveIndex == MAX_PHASTA_FILES ) { 612 printf("Error initphmpiio: PhastaIONextActiveIndex = MAX_PHASTA_FILES"); 613 endTimer(&timer_end); 614 printPerf("initphmpiio", timer_start, timer_end, 0, 0, ""); 615 return MAX_PHASTA_FILES_EXCEEDED; 616 } 617 // else if( PhastaIONextActiveIndex == 0 ) //Hang in debug mode on Intrepid 618 // { 619 // for( i = 0; i < MAX_PHASTA_FILES; i++ ); 620 // { 621 // PhastaIOActiveFiles[i] = NULL; 622 // } 623 // } 624 625 626 PhastaIOActiveFiles[PhastaIONextActiveIndex] = (phastaio_file_t *)calloc( 1, sizeof( phastaio_file_t) ); 627 628 i = PhastaIONextActiveIndex; 629 PhastaIONextActiveIndex++; 630 631 //PhastaIOActiveFiles[i]->next_start_address = 2*TWO_MEGABYTE; 632 633 PhastaIOActiveFiles[i]->next_start_address = MasterHeaderSize; // what does this mean??? TODO 634 635 PhastaIOActiveFiles[i]->Wrong_Endian = false; 636 637 PhastaIOActiveFiles[i]->nFields = *nfields; 638 PhastaIOActiveFiles[i]->nPPF = *nppf; 639 PhastaIOActiveFiles[i]->nFiles = *nfiles; 640 MPI_Comm_rank(MPI_COMM_WORLD, &(PhastaIOActiveFiles[i]->myrank)); 641 MPI_Comm_size(MPI_COMM_WORLD, &(PhastaIOActiveFiles[i]->numprocs)); 642 643 644 if( *nfiles > 1 ) { // split the ranks according to each mpiio file 645 646 if ( s_assign_local_comm == 0) { // call mpi_comm_split for the first (and only) time 647 648 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Building subcommunicator\n"); 649 650 int color = computeColor(PhastaIOActiveFiles[i]->myrank, PhastaIOActiveFiles[i]->numprocs, PhastaIOActiveFiles[i]->nFiles); 651 MPI_Comm_split(MPI_COMM_WORLD, 652 color, 653 PhastaIOActiveFiles[i]->myrank, 654 &(PhastaIOActiveFiles[i]->local_comm)); 655 MPI_Comm_size(PhastaIOActiveFiles[i]->local_comm, 656 &(PhastaIOActiveFiles[i]->local_numprocs)); 657 MPI_Comm_rank(PhastaIOActiveFiles[i]->local_comm, 658 &(PhastaIOActiveFiles[i]->local_myrank)); 659 660 // back up now these variables so that we do not need to call comm_split again 661 s_local_comm = PhastaIOActiveFiles[i]->local_comm; 662 s_local_size = PhastaIOActiveFiles[i]->local_numprocs; 663 s_local_rank = PhastaIOActiveFiles[i]->local_myrank; 664 s_assign_local_comm = 1; 665 } 666 else { // recycle the subcommunicator 667 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Recycling subcommunicator\n"); 668 PhastaIOActiveFiles[i]->local_comm = s_local_comm; 669 PhastaIOActiveFiles[i]->local_numprocs = s_local_size; 670 PhastaIOActiveFiles[i]->local_myrank = s_local_rank; 671 } 672 } 673 else { // *nfiles == 1 here - no need to call mpi_comm_split here 674 675 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Bypassing subcommunicator\n"); 676 PhastaIOActiveFiles[i]->local_comm = MPI_COMM_WORLD; 677 PhastaIOActiveFiles[i]->local_numprocs = PhastaIOActiveFiles[i]->numprocs; 678 PhastaIOActiveFiles[i]->local_myrank = PhastaIOActiveFiles[i]->myrank; 679 680 } 681 682 PhastaIOActiveFiles[i]->nppp = 683 PhastaIOActiveFiles[i]->nPPF/PhastaIOActiveFiles[i]->local_numprocs; 684 685 PhastaIOActiveFiles[i]->start_id = PhastaIOActiveFiles[i]->nPPF * 686 (int)(PhastaIOActiveFiles[i]->myrank/PhastaIOActiveFiles[i]->local_numprocs) + 687 (PhastaIOActiveFiles[i]->local_myrank * PhastaIOActiveFiles[i]->nppp); 688 689 PhastaIOActiveFiles[i]->my_offset_table = 690 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 691 692 PhastaIOActiveFiles[i]->my_read_table = 693 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 694 695 for (j=0; j<*nfields; j++) 696 { 697 PhastaIOActiveFiles[i]->my_offset_table[j] = 698 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 699 700 PhastaIOActiveFiles[i]->my_read_table[j] = 701 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 702 } 703 *filehandle = i; 704 705 PhastaIOActiveFiles[i]->master_header = (char *)calloc(MasterHeaderSize,sizeof( char )); 706 PhastaIOActiveFiles[i]->double_chunk = (double *)calloc(1,sizeof( double )); 707 PhastaIOActiveFiles[i]->int_chunk = (int *)calloc(1,sizeof( int )); 708 PhastaIOActiveFiles[i]->read_double_chunk = (double *)calloc(1,sizeof( double )); 709 PhastaIOActiveFiles[i]->read_int_chunk = (int *)calloc(1,sizeof( int )); 710 711 // Time monitoring 712 endTimer(&timer_end); 713 printPerf("initphmpiio", timer_start, timer_end, 0, 0, ""); 714 715 phprintf_0("Info initphmpiio: quiting function"); 716 717 return i; 718 } 719 720 /** 721 * Destruct the file struct and free buffers allocated in init function. 722 */ 723 void finalizephmpiio( int *fileDescriptor ) 724 { 725 double timer_start, timer_end; 726 startTimer(&timer_start); 727 728 int i, j; 729 i = *fileDescriptor; 730 //PhastaIONextActiveIndex--; 731 732 /* //free the offset table for this phasta file */ 733 //for(j=0; j<MAX_FIELDS_NUMBER; j++) //Danger: undefined behavior for my_*_table.[j] not allocated or not initialized to NULL 734 for(j=0; j<PhastaIOActiveFiles[i]->nFields; j++) 735 { 736 free( PhastaIOActiveFiles[i]->my_offset_table[j]); 737 free( PhastaIOActiveFiles[i]->my_read_table[j]); 738 } 739 free ( PhastaIOActiveFiles[i]->my_offset_table ); 740 free ( PhastaIOActiveFiles[i]->my_read_table ); 741 free ( PhastaIOActiveFiles[i]->master_header ); 742 free ( PhastaIOActiveFiles[i]->double_chunk ); 743 free ( PhastaIOActiveFiles[i]->int_chunk ); 744 free ( PhastaIOActiveFiles[i]->read_double_chunk ); 745 free ( PhastaIOActiveFiles[i]->read_int_chunk ); 746 747 if( PhastaIOActiveFiles[i]->nFiles > 1 && s_assign_local_comm ) { // the comm was split 748 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Freeing subcommunicator\n"); 749 s_assign_local_comm = 0; 750 MPI_Comm_free(&(PhastaIOActiveFiles[i]->local_comm)); 751 } 752 753 free( PhastaIOActiveFiles[i]); 754 755 endTimer(&timer_end); 756 printPerf("finalizempiio", timer_start, timer_end, 0, 0, ""); 757 758 PhastaIONextActiveIndex--; 759 } 760 761 762 /** 763 * Special init for M2N in order to create a subcommunicator for the reduced solution (requires PRINT_PERF to be false for now) 764 * Initialize the file struct members and allocate space for file struct buffers. 765 * 766 * Note: this function is only called when we are using new format. Old POSIX 767 * format should skip this routine and call openfile() directly instead. 768 */ 769 int initphmpiiosub( int *nfields, int *nppf, int *nfiles, int *filehandle, const char mode[],MPI_Comm my_local_comm) 770 { 771 // we init irank again in case query not called (e.g. syncIO write case) 772 773 MPI_Comm_rank(my_local_comm, &irank); 774 MPI_Comm_size(my_local_comm, &mysize); 775 776 phprintf("Info initphmpiio: entering function, myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 777 778 double timer_start, timer_end; 779 startTimer(&timer_start); 780 781 char* imode = StringStripper( mode ); 782 783 // Note: if it's read, we presume query was called prior to init and 784 // MasterHeaderSize is already set to correct value from parsing header 785 // otherwise it's write then it needs some computation to be set 786 if ( cscompare( "read", imode ) ) { 787 // do nothing 788 } 789 else if( cscompare( "write", imode ) ) { 790 MasterHeaderSize = computeMHSize(*nfields, *nppf, LATEST_WRITE_VERSION); 791 } 792 else { 793 printf("Error initphmpiio: can't recognize the mode %s", imode); 794 exit(1); 795 } 796 free ( imode ); 797 798 phprintf("Info initphmpiio: myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 799 800 int i, j; 801 802 if( PhastaIONextActiveIndex == MAX_PHASTA_FILES ) { 803 printf("Error initphmpiio: PhastaIONextActiveIndex = MAX_PHASTA_FILES"); 804 endTimer(&timer_end); 805 printPerf("initphmpiio", timer_start, timer_end, 0, 0, ""); 806 return MAX_PHASTA_FILES_EXCEEDED; 807 } 808 // else if( PhastaIONextActiveIndex == 0 ) //Hang in debug mode on Intrepid 809 // { 810 // for( i = 0; i < MAX_PHASTA_FILES; i++ ); 811 // { 812 // PhastaIOActiveFiles[i] = NULL; 813 // } 814 // } 815 816 817 PhastaIOActiveFiles[PhastaIONextActiveIndex] = (phastaio_file_t *)calloc( 1, sizeof( phastaio_file_t) ); 818 819 i = PhastaIONextActiveIndex; 820 PhastaIONextActiveIndex++; 821 822 //PhastaIOActiveFiles[i]->next_start_address = 2*TWO_MEGABYTE; 823 824 PhastaIOActiveFiles[i]->next_start_address = MasterHeaderSize; // what does this mean??? TODO 825 826 PhastaIOActiveFiles[i]->Wrong_Endian = false; 827 828 PhastaIOActiveFiles[i]->nFields = *nfields; 829 PhastaIOActiveFiles[i]->nPPF = *nppf; 830 PhastaIOActiveFiles[i]->nFiles = *nfiles; 831 MPI_Comm_rank(my_local_comm, &(PhastaIOActiveFiles[i]->myrank)); 832 MPI_Comm_size(my_local_comm, &(PhastaIOActiveFiles[i]->numprocs)); 833 834 int color = computeColor(PhastaIOActiveFiles[i]->myrank, PhastaIOActiveFiles[i]->numprocs, PhastaIOActiveFiles[i]->nFiles); 835 MPI_Comm_split(my_local_comm, 836 color, 837 PhastaIOActiveFiles[i]->myrank, 838 &(PhastaIOActiveFiles[i]->local_comm)); 839 MPI_Comm_size(PhastaIOActiveFiles[i]->local_comm, 840 &(PhastaIOActiveFiles[i]->local_numprocs)); 841 MPI_Comm_rank(PhastaIOActiveFiles[i]->local_comm, 842 &(PhastaIOActiveFiles[i]->local_myrank)); 843 PhastaIOActiveFiles[i]->nppp = 844 PhastaIOActiveFiles[i]->nPPF/PhastaIOActiveFiles[i]->local_numprocs; 845 846 PhastaIOActiveFiles[i]->start_id = PhastaIOActiveFiles[i]->nPPF * 847 (int)(PhastaIOActiveFiles[i]->myrank/PhastaIOActiveFiles[i]->local_numprocs) + 848 (PhastaIOActiveFiles[i]->local_myrank * PhastaIOActiveFiles[i]->nppp); 849 850 PhastaIOActiveFiles[i]->my_offset_table = 851 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 852 853 PhastaIOActiveFiles[i]->my_read_table = 854 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 855 856 for (j=0; j<*nfields; j++) 857 { 858 PhastaIOActiveFiles[i]->my_offset_table[j] = 859 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 860 861 PhastaIOActiveFiles[i]->my_read_table[j] = 862 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 863 } 864 *filehandle = i; 865 866 PhastaIOActiveFiles[i]->master_header = (char *)calloc(MasterHeaderSize,sizeof( char )); 867 PhastaIOActiveFiles[i]->double_chunk = (double *)calloc(1,sizeof( double )); 868 PhastaIOActiveFiles[i]->int_chunk = (int *)calloc(1,sizeof( int )); 869 PhastaIOActiveFiles[i]->read_double_chunk = (double *)calloc(1,sizeof( double )); 870 PhastaIOActiveFiles[i]->read_int_chunk = (int *)calloc(1,sizeof( int )); 871 872 // Time monitoring 873 endTimer(&timer_end); 874 printPerf("initphmpiiosub", timer_start, timer_end, 0, 0, ""); 875 876 phprintf_0("Info initphmpiiosub: quiting function"); 877 878 return i; 879 } 880 881 882 883 /** open file for both POSIX and MPI-IO syncIO format. 884 * 885 * If it's old POSIX format, simply call posix fopen(). 886 * 887 * If it's MPI-IO foramt: 888 * in "read" mode, it builds the header table that points to the offset of 889 * fields for parts; 890 * in "write" mode, it opens the file with MPI-IO open routine. 891 */ 892 void openfile(const char filename[], 893 const char mode[], 894 int* fileDescriptor ) 895 { 896 phprintf_0("Info: entering openfile"); 897 898 double timer_start, timer_end; 899 startTimer(&timer_start); 900 901 if ( PhastaIONextActiveIndex == 0 ) 902 { 903 FILE* file=NULL ; 904 *fileDescriptor = 0; 905 char* fname = StringStripper( filename ); 906 char* imode = StringStripper( mode ); 907 908 if ( cscompare( "read", imode ) ) file = fopen(fname, "rb" ); 909 else if( cscompare( "write", imode ) ) file = fopen(fname, "wb" ); 910 else if( cscompare( "append", imode ) ) file = fopen(fname, "ab" ); 911 912 if ( !file ){ 913 fprintf(stderr,"Error openfile: unable to open file %s",fname ) ; 914 } else { 915 fileArray.push_back( file ); 916 byte_order.push_back( false ); 917 header_type.push_back( sizeof(int) ); 918 *fileDescriptor = fileArray.size(); 919 } 920 free (fname); 921 free (imode); 922 } 923 else // else it would be parallel I/O, opposed to posix io 924 { 925 char* fname = StringStripper( filename ); 926 char* imode = StringStripper( mode ); 927 int rc; 928 int i = *fileDescriptor; 929 checkFileDescriptor("openfile",&i); 930 char* token; 931 932 if ( cscompare( "read", imode ) ) 933 { 934 // if (PhastaIOActiveFiles[i]->myrank == 0) 935 // printf("\n **********\nRead open ... ... regular version\n"); 936 937 rc = MPI_File_open( PhastaIOActiveFiles[i]->local_comm, 938 fname, 939 MPI_MODE_RDONLY, 940 MPI_INFO_NULL, 941 &(PhastaIOActiveFiles[i]->file_handle) ); 942 943 if(rc) 944 { 945 *fileDescriptor = UNABLE_TO_OPEN_FILE; 946 printf("Error openfile: Unable to open file %s! File descriptor = %d\n",fname,*fileDescriptor); 947 endTimer(&timer_end); 948 printPerf("openfile", timer_start, timer_end, 0, 0, ""); 949 return; 950 } 951 952 MPI_Status read_tag_status; 953 char read_out_tag[MAX_FIELDS_NAME_LENGTH]; 954 int j; 955 int magic_number; 956 957 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 958 MPI_File_read_at( PhastaIOActiveFiles[i]->file_handle, 959 0, 960 PhastaIOActiveFiles[i]->master_header, 961 MasterHeaderSize, 962 MPI_CHAR, 963 &read_tag_status ); 964 } 965 966 MPI_Bcast( PhastaIOActiveFiles[i]->master_header, 967 MasterHeaderSize, 968 MPI_CHAR, 969 0, 970 PhastaIOActiveFiles[i]->local_comm ); 971 972 memcpy( read_out_tag, 973 PhastaIOActiveFiles[i]->master_header, 974 MAX_FIELDS_NAME_LENGTH-1 ); 975 976 if ( cscompare ("MPI_IO_Tag",read_out_tag) ) 977 { 978 // Test endianess ... 979 memcpy ( &magic_number, 980 PhastaIOActiveFiles[i]->master_header+sizeof("MPI_IO_Tag : ")-1, //-1 sizeof returns the size of the string+1 for "\0" 981 sizeof(int) ); // masterheader should look like "MPI_IO_Tag : 12180 " with 12180 in binary format 982 983 if ( magic_number != ENDIAN_TEST_NUMBER ) 984 { 985 PhastaIOActiveFiles[i]->Wrong_Endian = true; 986 } 987 988 memcpy( read_out_tag, 989 PhastaIOActiveFiles[i]->master_header+MAX_FIELDS_NAME_LENGTH+1, // TODO: WHY +1??? 990 MAX_FIELDS_NAME_LENGTH ); 991 992 // Read in # fields ... 993 token = strtok ( read_out_tag, ":" ); 994 token = strtok( NULL," ,;<>" ); 995 PhastaIOActiveFiles[i]->nFields = atoi( token ); 996 997 unsigned long **header_table; 998 header_table = ( unsigned long ** )calloc(PhastaIOActiveFiles[i]->nFields, sizeof(unsigned long *)); 999 1000 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) 1001 { 1002 header_table[j]=( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nPPF , sizeof( unsigned long)); 1003 } 1004 1005 // Read in the offset table ... 1006 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) 1007 { 1008 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 1009 memcpy( header_table[j], 1010 PhastaIOActiveFiles[i]->master_header + 1011 VERSION_INFO_HEADER_SIZE + 1012 j * PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long), 1013 PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long) ); 1014 } 1015 1016 MPI_Scatter( header_table[j], 1017 PhastaIOActiveFiles[i]->nppp, 1018 MPI_LONG_LONG_INT, 1019 PhastaIOActiveFiles[i]->my_read_table[j], 1020 PhastaIOActiveFiles[i]->nppp, 1021 MPI_LONG_LONG_INT, 1022 0, 1023 PhastaIOActiveFiles[i]->local_comm ); 1024 1025 // Swap byte order if endianess is different ... 1026 if ( PhastaIOActiveFiles[i]->Wrong_Endian ) { 1027 SwapArrayByteOrder( PhastaIOActiveFiles[i]->my_read_table[j], 1028 sizeof(unsigned long), 1029 PhastaIOActiveFiles[i]->nppp ); 1030 } 1031 } 1032 1033 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1034 free ( header_table[j] ); 1035 } 1036 free (header_table); 1037 1038 } // end of if MPI_IO_TAG 1039 else //else not valid MPI file 1040 { 1041 *fileDescriptor = NOT_A_MPI_FILE; 1042 printf("Error openfile: The file %s you opened is not in syncIO new format, please check again! File descriptor = %d, MasterHeaderSize = %d, read_out_tag = %s\n",fname,*fileDescriptor,MasterHeaderSize,read_out_tag); 1043 //Printing MasterHeaderSize is useful to test a compiler bug on Intrepid BGP 1044 endTimer(&timer_end); 1045 printPerf("openfile", timer_start, timer_end, 0, 0, ""); 1046 return; 1047 } 1048 } // end of if "read" 1049 else if( cscompare( "write", imode ) ) 1050 { 1051 rc = MPI_File_open( PhastaIOActiveFiles[i]->local_comm, 1052 fname, 1053 MPI_MODE_WRONLY | MPI_MODE_CREATE, 1054 MPI_INFO_NULL, 1055 &(PhastaIOActiveFiles[i]->file_handle) ); 1056 if(rc) 1057 { 1058 *fileDescriptor = UNABLE_TO_OPEN_FILE; 1059 return; 1060 } 1061 } // end of if "write" 1062 free (fname); 1063 free (imode); 1064 } // end of if FileIndex != 0 1065 1066 endTimer(&timer_end); 1067 printPerf("openfile", timer_start, timer_end, 0, 0, ""); 1068 } 1069 1070 /** close file for both POSIX and MPI-IO syncIO format. 1071 * 1072 * If it's old POSIX format, simply call posix fclose(). 1073 * 1074 * If it's MPI-IO foramt: 1075 * in "read" mode, it simply close file with MPI-IO close routine. 1076 * in "write" mode, rank 0 in each group will re-assemble the master header and 1077 * offset table and write to the beginning of file, then close the file. 1078 */ 1079 void closefile( int* fileDescriptor, 1080 const char mode[] ) 1081 { 1082 double timer_start, timer_end; 1083 startTimer(&timer_start); 1084 1085 int i = *fileDescriptor; 1086 checkFileDescriptor("closefile",&i); 1087 1088 if ( PhastaIONextActiveIndex == 0 ) { 1089 char* imode = StringStripper( mode ); 1090 1091 if( cscompare( "write", imode ) 1092 || cscompare( "append", imode ) ) { 1093 fflush( fileArray[ *fileDescriptor - 1 ] ); 1094 } 1095 1096 fclose( fileArray[ *fileDescriptor - 1 ] ); 1097 free (imode); 1098 } 1099 else { 1100 char* imode = StringStripper( mode ); 1101 1102 //write master header here: 1103 if ( cscompare( "write", imode ) ) { 1104 // if ( PhastaIOActiveFiles[i]->nPPF * PhastaIOActiveFiles[i]->nFields < 2*ONE_MEGABYTE/8 ) //SHOULD BE CHECKED 1105 // MasterHeaderSize = 4*ONE_MEGABYTE; 1106 // else 1107 // MasterHeaderSize = 4*ONE_MEGABYTE + PhastaIOActiveFiles[i]->nPPF * PhastaIOActiveFiles[i]->nFields * 8 - 2*ONE_MEGABYTE; 1108 1109 MasterHeaderSize = computeMHSize( PhastaIOActiveFiles[i]->nFields, PhastaIOActiveFiles[i]->nPPF, LATEST_WRITE_VERSION); 1110 phprintf_0("Info closefile: myrank = %d, MasterHeaderSize = %d\n", PhastaIOActiveFiles[i]->myrank, MasterHeaderSize); 1111 1112 MPI_Status write_header_status; 1113 char mpi_tag[MAX_FIELDS_NAME_LENGTH]; 1114 char version[MAX_FIELDS_NAME_LENGTH/4]; 1115 char mhsize[MAX_FIELDS_NAME_LENGTH/4]; 1116 int magic_number = ENDIAN_TEST_NUMBER; 1117 1118 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) 1119 { 1120 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1121 sprintf(mpi_tag, "MPI_IO_Tag : "); 1122 memcpy(PhastaIOActiveFiles[i]->master_header, 1123 mpi_tag, 1124 MAX_FIELDS_NAME_LENGTH); 1125 1126 bzero((void*)version,MAX_FIELDS_NAME_LENGTH/4); 1127 // this version is "1", print version in ASCII 1128 sprintf(version, "version : %d",1); 1129 memcpy(PhastaIOActiveFiles[i]->master_header + MAX_FIELDS_NAME_LENGTH/2, 1130 version, 1131 MAX_FIELDS_NAME_LENGTH/4); 1132 1133 // master header size is computed using the formula above 1134 bzero((void*)mhsize,MAX_FIELDS_NAME_LENGTH/4); 1135 sprintf(mhsize, "mhsize : "); 1136 memcpy(PhastaIOActiveFiles[i]->master_header + MAX_FIELDS_NAME_LENGTH/4*3, 1137 mhsize, 1138 MAX_FIELDS_NAME_LENGTH/4); 1139 1140 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1141 sprintf(mpi_tag, 1142 "\nnFields : %d\n", 1143 PhastaIOActiveFiles[i]->nFields); 1144 memcpy(PhastaIOActiveFiles[i]->master_header+MAX_FIELDS_NAME_LENGTH, 1145 mpi_tag, 1146 MAX_FIELDS_NAME_LENGTH); 1147 1148 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1149 sprintf(mpi_tag, "\nnPPF : %d\n", PhastaIOActiveFiles[i]->nPPF); 1150 memcpy( PhastaIOActiveFiles[i]->master_header+ 1151 PhastaIOActiveFiles[i]->nFields * 1152 MAX_FIELDS_NAME_LENGTH + 1153 MAX_FIELDS_NAME_LENGTH * 2, 1154 mpi_tag, 1155 MAX_FIELDS_NAME_LENGTH); 1156 1157 memcpy( PhastaIOActiveFiles[i]->master_header+sizeof("MPI_IO_Tag : ")-1, //-1 sizeof returns the size of the string+1 for "\0" 1158 &magic_number, 1159 sizeof(int)); 1160 1161 memcpy( PhastaIOActiveFiles[i]->master_header+sizeof("mhsize : ") -1 + MAX_FIELDS_NAME_LENGTH/4*3, 1162 &MasterHeaderSize, 1163 sizeof(int)); 1164 } 1165 1166 int j = 0; 1167 unsigned long **header_table; 1168 header_table = ( unsigned long ** )calloc(PhastaIOActiveFiles[i]->nFields, sizeof(unsigned long *)); 1169 1170 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1171 header_table[j]=( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nPPF , sizeof( unsigned long)); 1172 } 1173 1174 //if( irank == 0 ) printf("gonna mpi_gather, myrank = %d\n", irank); 1175 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1176 MPI_Gather( PhastaIOActiveFiles[i]->my_offset_table[j], 1177 PhastaIOActiveFiles[i]->nppp, 1178 MPI_LONG_LONG_INT, 1179 header_table[j], 1180 PhastaIOActiveFiles[i]->nppp, 1181 MPI_LONG_LONG_INT, 1182 0, 1183 PhastaIOActiveFiles[i]->local_comm ); 1184 } 1185 1186 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 1187 1188 //if( irank == 0 ) printf("gonna memcpy for every procs, myrank = %d\n", irank); 1189 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1190 memcpy ( PhastaIOActiveFiles[i]->master_header + 1191 VERSION_INFO_HEADER_SIZE + 1192 j * PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long), 1193 header_table[j], 1194 PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long) ); 1195 } 1196 1197 //if( irank == 0 ) printf("gonna file_write_at(), myrank = %d\n", irank); 1198 MPI_File_write_at( PhastaIOActiveFiles[i]->file_handle, 1199 0, 1200 PhastaIOActiveFiles[i]->master_header, 1201 MasterHeaderSize, 1202 MPI_CHAR, 1203 &write_header_status ); 1204 } 1205 1206 ////free(PhastaIOActiveFiles[i]->master_header); 1207 1208 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1209 free ( header_table[j] ); 1210 } 1211 free (header_table); 1212 } 1213 1214 //if( irank == 0 ) printf("gonna file_close(), myrank = %d\n", irank); 1215 MPI_File_close( &( PhastaIOActiveFiles[i]->file_handle ) ); 1216 free ( imode ); 1217 } 1218 1219 endTimer(&timer_end); 1220 printPerf("closefile_", timer_start, timer_end, 0, 0, ""); 1221 } 1222 1223 int readHeader( FILE* f, const char phrase[], 1224 int* params, int numParams, const char* iotype) { 1225 isBinary(iotype); 1226 return readHeader(f,phrase,params,numParams); 1227 } 1228 1229 void readheader( int* fileDescriptor, 1230 const char keyphrase[], 1231 void* valueArray, 1232 int* nItems, 1233 const char datatype[], 1234 const char iotype[] ) 1235 { 1236 double timer_start, timer_end; 1237 1238 startTimer(&timer_start); 1239 1240 int i = *fileDescriptor; 1241 checkFileDescriptor("readheader",&i); 1242 1243 if ( PhastaIONextActiveIndex == 0 ) { 1244 int filePtr = *fileDescriptor - 1; 1245 FILE* fileObject; 1246 int* valueListInt; 1247 1248 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1249 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1250 fprintf(stderr,"openfile_ function has to be called before \n") ; 1251 fprintf(stderr,"acessing the file\n ") ; 1252 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1253 endTimer(&timer_end); 1254 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1255 return; 1256 } 1257 1258 LastHeaderKey[filePtr] = std::string(keyphrase); 1259 LastHeaderNotFound = false; 1260 1261 fileObject = fileArray[ filePtr ] ; 1262 Wrong_Endian = byte_order[ filePtr ]; 1263 1264 isBinary( iotype ); 1265 typeSize( datatype ); //redundant call, just avoid a compiler warning. 1266 1267 // right now we are making the assumption that we will only write integers 1268 // on the header line. 1269 1270 valueListInt = static_cast< int* >( valueArray ); 1271 int ierr = readHeader( fileObject , 1272 keyphrase, 1273 valueListInt, 1274 *nItems ) ; 1275 1276 byte_order[ filePtr ] = Wrong_Endian ; 1277 1278 if ( ierr ) LastHeaderNotFound = true; 1279 1280 //return ; // don't return, go to the end to print perf 1281 } 1282 else { 1283 unsigned int skip_size; 1284 int* valueListInt; 1285 valueListInt = static_cast <int*>(valueArray); 1286 char* token = NULL; 1287 bool FOUND = false ; 1288 isBinary( iotype ); 1289 1290 MPI_Status read_offset_status; 1291 char read_out_tag[MAX_FIELDS_NAME_LENGTH]; 1292 memset(read_out_tag, '\0', MAX_FIELDS_NAME_LENGTH); 1293 char readouttag[MAX_FIELDS_NUMBER][MAX_FIELDS_NAME_LENGTH]; 1294 int j; 1295 1296 int string_length = strlen( keyphrase ); 1297 char* buffer = (char*) malloc ( string_length+1 ); 1298 1299 strcpy ( buffer, keyphrase ); 1300 buffer[ string_length ] = '\0'; 1301 1302 char* st2 = strtok ( buffer, "@" ); 1303 st2 = strtok (NULL, "@"); 1304 PhastaIOActiveFiles[i]->GPid = atoi(st2); 1305 if ( char* p = strpbrk(buffer, "@") ) 1306 *p = '\0'; 1307 1308 // Check if the user has input the right GPid 1309 if ( ( PhastaIOActiveFiles[i]->GPid <= 1310 PhastaIOActiveFiles[i]->myrank * 1311 PhastaIOActiveFiles[i]->nppp )|| 1312 ( PhastaIOActiveFiles[i]->GPid > 1313 ( PhastaIOActiveFiles[i]->myrank + 1 ) * 1314 PhastaIOActiveFiles[i]->nppp ) ) 1315 { 1316 *fileDescriptor = NOT_A_MPI_FILE; 1317 printf("Error readheader: The file is not in syncIO new format, please check! myrank = %d, GPid = %d, nppp = %d, keyphrase = %s\n", PhastaIOActiveFiles[i]->myrank, PhastaIOActiveFiles[i]->GPid, PhastaIOActiveFiles[i]->nppp, keyphrase); 1318 // It is possible atoi could not generate a clear integer from st2 because of additional garbage character in keyphrase 1319 endTimer(&timer_end); 1320 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1321 return; 1322 } 1323 1324 // Find the field we want ... 1325 //for ( j = 0; j<MAX_FIELDS_NUMBER; j++ ) 1326 for ( j = 0; j<PhastaIOActiveFiles[i]->nFields; j++ ) 1327 { 1328 memcpy( readouttag[j], 1329 PhastaIOActiveFiles[i]->master_header + j*MAX_FIELDS_NAME_LENGTH+MAX_FIELDS_NAME_LENGTH*2+1, 1330 MAX_FIELDS_NAME_LENGTH-1 ); 1331 } 1332 1333 for ( j = 0; j<PhastaIOActiveFiles[i]->nFields; j++ ) 1334 { 1335 token = strtok ( readouttag[j], ":" ); 1336 1337 //if ( cscompare( buffer, token ) ) 1338 if ( cscompare( token , buffer ) && cscompare( buffer, token ) ) 1339 // This double comparison is required for the field "number of nodes" and all the other fields that start with "number of nodes" (i.g. number of nodes in the mesh"). 1340 // Would be safer to rename "number of nodes" by "number of nodes in the part" so that the name are completely unique. But much more work to do that (Nspre, phParAdapt, etc). 1341 // Since the field name are unique in SyncIO (as it includes part ID), this should be safe and there should be no issue with the "?" trailing character. 1342 { 1343 PhastaIOActiveFiles[i]->read_field_count = j; 1344 FOUND = true; 1345 //printf("buffer: %s | token: %s | j: %d\n",buffer,token,j); 1346 break; 1347 } 1348 } 1349 free(buffer); 1350 1351 if (!FOUND) 1352 { 1353 //if(irank==0) printf("Warning readheader: Not found %s \n",keyphrase); //PhastaIOActiveFiles[i]->myrank is certainly initialized here. 1354 if(PhastaIOActiveFiles[i]->myrank == 0) printf("WARNING readheader: Not found %s\n",keyphrase); 1355 endTimer(&timer_end); 1356 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1357 return; 1358 } 1359 1360 // Find the part we want ... 1361 PhastaIOActiveFiles[i]->read_part_count = PhastaIOActiveFiles[i]->GPid - 1362 PhastaIOActiveFiles[i]->myrank * PhastaIOActiveFiles[i]->nppp - 1; 1363 1364 PhastaIOActiveFiles[i]->my_offset = 1365 PhastaIOActiveFiles[i]->my_read_table[PhastaIOActiveFiles[i]->read_field_count][PhastaIOActiveFiles[i]->read_part_count]; 1366 1367 // printf("****Rank %d offset is %d\n",PhastaIOActiveFiles[i]->myrank,PhastaIOActiveFiles[i]->my_offset); 1368 1369 // Read each datablock header here ... 1370 1371 MPI_File_read_at_all( PhastaIOActiveFiles[i]->file_handle, 1372 PhastaIOActiveFiles[i]->my_offset+1, 1373 read_out_tag, 1374 MAX_FIELDS_NAME_LENGTH-1, 1375 MPI_CHAR, 1376 &read_offset_status ); 1377 token = strtok ( read_out_tag, ":" ); 1378 1379 // printf("&&&&Rank %d read_out_tag is %s\n",PhastaIOActiveFiles[i]->myrank,read_out_tag); 1380 1381 if( cscompare( keyphrase , token ) ) //No need to compare also token with keyphrase like above. We should already have the right one. Otherwise there is a problem. 1382 { 1383 FOUND = true ; 1384 token = strtok( NULL, " ,;<>" ); 1385 skip_size = atoi( token ); 1386 for( j=0; j < *nItems && ( token = strtok( NULL," ,;<>") ); j++ ) 1387 valueListInt[j] = atoi( token ); 1388 1389 if ( j < *nItems ) 1390 { 1391 fprintf( stderr, "Expected # of ints not found for: %s\n", keyphrase ); 1392 } 1393 } 1394 else { 1395 //if(irank==0) 1396 if(PhastaIOActiveFiles[i]->myrank == 0) 1397 // If we enter this if, there is a problem with the name of some fields 1398 { 1399 printf("Error readheader: Unexpected mismatch between keyphrase = %s and token = %s\n",keyphrase,token); 1400 } 1401 } 1402 } 1403 1404 endTimer(&timer_end); 1405 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1406 1407 } 1408 1409 void readDataBlock( 1410 FILE* fileObject, 1411 void* valueArray, 1412 int nItems, 1413 const char datatype[], 1414 const char iotype[] ) 1415 { 1416 isBinary(iotype); 1417 size_t type_size = typeSize( datatype ); 1418 if ( binary_format ) { 1419 char junk = '\0'; 1420 fread( valueArray, type_size, nItems, fileObject ); 1421 fread( &junk, sizeof(char), 1 , fileObject ); 1422 if ( Wrong_Endian ) SwapArrayByteOrder( valueArray, type_size, nItems ); 1423 } else { 1424 char* ts1 = StringStripper( datatype ); 1425 if ( cscompare( "integer", ts1 ) ) { 1426 for( int n=0; n < nItems ; n++ ) 1427 fscanf(fileObject, "%d\n",(int*)((int*)valueArray+n) ); 1428 } else if ( cscompare( "double", ts1 ) ) { 1429 for( int n=0; n < nItems ; n++ ) 1430 fscanf(fileObject, "%lf\n",(double*)((double*)valueArray+n) ); 1431 } 1432 free (ts1); 1433 } 1434 } 1435 1436 void readdatablock( int* fileDescriptor, 1437 const char keyphrase[], 1438 void* valueArray, 1439 int* nItems, 1440 const char datatype[], 1441 const char iotype[] ) 1442 { 1443 //if(irank == 0) printf("entering readdatablock()\n"); 1444 unsigned long data_size = 0; 1445 double timer_start, timer_end; 1446 startTimer(&timer_start); 1447 1448 int i = *fileDescriptor; 1449 checkFileDescriptor("readdatablock",&i); 1450 1451 if ( PhastaIONextActiveIndex == 0 ) { 1452 int filePtr = *fileDescriptor - 1; 1453 FILE* fileObject; 1454 char junk; 1455 1456 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1457 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1458 fprintf(stderr,"openfile_ function has to be called before\n") ; 1459 fprintf(stderr,"acessing the file\n ") ; 1460 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1461 endTimer(&timer_end); 1462 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1463 return; 1464 } 1465 1466 // error check.. 1467 // since we require that a consistant header always preceed the data block 1468 // let us check to see that it is actually the case. 1469 1470 if ( ! cscompare( LastHeaderKey[ filePtr ].c_str(), keyphrase ) ) { 1471 fprintf(stderr, "Header not consistant with data block\n"); 1472 fprintf(stderr, "Header: %s\n", LastHeaderKey[ filePtr ].c_str() ); 1473 fprintf(stderr, "DataBlock: %s\n ", keyphrase ); 1474 fprintf(stderr, "Please recheck read sequence \n"); 1475 if( Strict_Error ) { 1476 fprintf(stderr, "fatal error: cannot continue, returning out of call\n"); 1477 endTimer(&timer_end); 1478 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1479 return; 1480 } 1481 } 1482 1483 if ( LastHeaderNotFound ) { 1484 endTimer(&timer_end); 1485 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1486 return; 1487 } 1488 fileObject = fileArray[ filePtr ]; 1489 Wrong_Endian = byte_order[ filePtr ]; 1490 LastHeaderKey.erase(filePtr); 1491 readDataBlock(fileObject,valueArray,*nItems,datatype,iotype); 1492 1493 //return; 1494 } 1495 else { 1496 // printf("read data block\n"); 1497 MPI_Status read_data_status; 1498 size_t type_size = typeSize( datatype ); 1499 int nUnits = *nItems; 1500 isBinary( iotype ); 1501 1502 // read datablock then 1503 //MR CHANGE 1504 // if ( cscompare ( datatype, "double")) 1505 char* ts2 = StringStripper( datatype ); 1506 if ( cscompare ( "double" , ts2)) 1507 //MR CHANGE END 1508 { 1509 1510 MPI_File_read_at_all_begin( PhastaIOActiveFiles[i]->file_handle, 1511 PhastaIOActiveFiles[i]->my_offset + DB_HEADER_SIZE, 1512 valueArray, 1513 nUnits, 1514 MPI_DOUBLE ); 1515 MPI_File_read_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1516 valueArray, 1517 &read_data_status ); 1518 data_size=8*nUnits; 1519 1520 } 1521 //MR CHANGE 1522 // else if ( cscompare ( datatype, "integer")) 1523 else if ( cscompare ( "integer" , ts2)) 1524 //MR CHANGE END 1525 { 1526 MPI_File_read_at_all_begin(PhastaIOActiveFiles[i]->file_handle, 1527 PhastaIOActiveFiles[i]->my_offset + DB_HEADER_SIZE, 1528 valueArray, 1529 nUnits, 1530 MPI_INT ); 1531 MPI_File_read_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1532 valueArray, 1533 &read_data_status ); 1534 data_size=4*nUnits; 1535 } 1536 else 1537 { 1538 *fileDescriptor = DATA_TYPE_ILLEGAL; 1539 printf("readdatablock - DATA_TYPE_ILLEGAL - %s\n",datatype); 1540 endTimer(&timer_end); 1541 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1542 return; 1543 } 1544 free(ts2); 1545 1546 1547 // printf("%d Read finishe\n",PhastaIOActiveFiles[i]->myrank); 1548 1549 // Swap data byte order if endianess is different ... 1550 if ( PhastaIOActiveFiles[i]->Wrong_Endian ) 1551 { 1552 SwapArrayByteOrder( valueArray, type_size, nUnits ); 1553 } 1554 } 1555 1556 endTimer(&timer_end); 1557 char extra_msg[1024]; 1558 memset(extra_msg, '\0', 1024); 1559 char* key = StringStripper(keyphrase); 1560 sprintf(extra_msg, " field is %s ", key); 1561 printPerf("readdatablock", timer_start, timer_end, data_size, 1, extra_msg); 1562 free(key); 1563 1564 } 1565 1566 void writeHeader(FILE* f, 1567 const char keyphrase[], 1568 const void* valueArray, 1569 const int nItems, 1570 const int ndataItems, 1571 const char datatype[], 1572 const char iotype[]) 1573 { 1574 isBinary( iotype ); 1575 1576 const int _newline = 1577 ( ndataItems > 0 ) ? sizeof( char ) : 0; 1578 int size_of_nextblock = 1579 ( binary_format ) ? typeSize(datatype) * ndataItems + _newline : ndataItems; 1580 1581 fprintf( f, "%s : < %d > ", keyphrase, size_of_nextblock ); 1582 for( int i = 0; i < nItems; i++ ) 1583 fprintf( f, "%d ", *((int*)((int*)valueArray+i))); 1584 fprintf( f, "\n"); 1585 } 1586 1587 void writeheader( const int* fileDescriptor, 1588 const char keyphrase[], 1589 const void* valueArray, 1590 const int* nItems, 1591 const int* ndataItems, 1592 const char datatype[], 1593 const char iotype[]) 1594 { 1595 1596 //if(irank == 0) printf("entering writeheader()\n"); 1597 1598 double timer_start, timer_end; 1599 startTimer(&timer_start); 1600 1601 int i = *fileDescriptor; 1602 checkFileDescriptor("writeheader",&i); 1603 1604 if ( PhastaIONextActiveIndex == 0 ) { 1605 int filePtr = *fileDescriptor - 1; 1606 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1607 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1608 fprintf(stderr,"openfile_ function has to be called before \n") ; 1609 fprintf(stderr,"acessing the file\n ") ; 1610 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1611 endTimer(&timer_end); 1612 printPerf("writeheader", timer_start, timer_end, 0, 0, ""); 1613 return; 1614 } 1615 1616 LastHeaderKey[filePtr] = std::string(keyphrase); 1617 DataSize = *ndataItems; 1618 FILE* fileObject = fileArray[ filePtr ] ; 1619 header_type[ filePtr ] = typeSize( datatype ); 1620 writeHeader(fileObject,keyphrase,valueArray,*nItems, 1621 *ndataItems,datatype,iotype); 1622 } 1623 else { // else it's parallel I/O 1624 DataSize = *ndataItems; 1625 size_t type_size = typeSize( datatype ); 1626 isBinary( iotype ); 1627 char mpi_tag[MAX_FIELDS_NAME_LENGTH]; 1628 1629 int string_length = strlen( keyphrase ); 1630 char* buffer = (char*) malloc ( string_length+1 ); 1631 1632 strcpy ( buffer, keyphrase); 1633 buffer[ string_length ] = '\0'; 1634 1635 char* st2 = strtok ( buffer, "@" ); 1636 st2 = strtok (NULL, "@"); 1637 PhastaIOActiveFiles[i]->GPid = atoi(st2); 1638 1639 if ( char* p = strpbrk(buffer, "@") ) 1640 *p = '\0'; 1641 1642 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1643 sprintf(mpi_tag, "\n%s : %d\n", buffer, PhastaIOActiveFiles[i]->field_count); 1644 unsigned long offset_value; 1645 1646 int temp = *ndataItems; 1647 unsigned long number_of_items = (unsigned long)temp; 1648 MPI_Barrier(PhastaIOActiveFiles[i]->local_comm); 1649 1650 MPI_Scan( &number_of_items, 1651 &offset_value, 1652 1, 1653 MPI_LONG_LONG_INT, 1654 MPI_SUM, 1655 PhastaIOActiveFiles[i]->local_comm ); 1656 1657 offset_value = (offset_value - number_of_items) * type_size; 1658 1659 offset_value += PhastaIOActiveFiles[i]->local_myrank * 1660 DB_HEADER_SIZE + 1661 PhastaIOActiveFiles[i]->next_start_address; 1662 // This offset is the starting address of each datablock header... 1663 PhastaIOActiveFiles[i]->my_offset = offset_value; 1664 1665 // Write in my offset table ... 1666 PhastaIOActiveFiles[i]->my_offset_table[PhastaIOActiveFiles[i]->field_count][PhastaIOActiveFiles[i]->part_count] = 1667 PhastaIOActiveFiles[i]->my_offset; 1668 1669 // Update the next-start-address ... 1670 PhastaIOActiveFiles[i]->next_start_address = offset_value + 1671 number_of_items * type_size + 1672 DB_HEADER_SIZE; 1673 MPI_Bcast( &(PhastaIOActiveFiles[i]->next_start_address), 1674 1, 1675 MPI_LONG_LONG_INT, 1676 PhastaIOActiveFiles[i]->local_numprocs-1, 1677 PhastaIOActiveFiles[i]->local_comm ); 1678 1679 // Prepare datablock header ... 1680 int _newline = (*ndataItems>0)?sizeof(char):0; 1681 unsigned int size_of_nextblock = type_size * (*ndataItems) + _newline; 1682 1683 //char datablock_header[255]; 1684 //bzero((void*)datablock_header,255); 1685 char datablock_header[DB_HEADER_SIZE]; 1686 bzero((void*)datablock_header,DB_HEADER_SIZE); 1687 1688 PhastaIOActiveFiles[i]->GPid = PhastaIOActiveFiles[i]->nppp*PhastaIOActiveFiles[i]->myrank+PhastaIOActiveFiles[i]->part_count; 1689 sprintf( datablock_header, 1690 "\n%s : < %u >", 1691 keyphrase, 1692 size_of_nextblock ); 1693 1694 for ( int j = 0; j < *nItems; j++ ) 1695 { 1696 sprintf( datablock_header, 1697 "%s %d ", 1698 datablock_header, 1699 *((int*)((int*)valueArray+j))); 1700 } 1701 sprintf( datablock_header, 1702 "%s\n ", 1703 datablock_header ); 1704 1705 // Write datablock header ... 1706 //MR CHANGE 1707 // if ( cscompare(datatype,"double") ) 1708 char* ts1 = StringStripper( datatype ); 1709 if ( cscompare("double",ts1) ) 1710 //MR CHANGE END 1711 { 1712 free ( PhastaIOActiveFiles[i]->double_chunk ); 1713 PhastaIOActiveFiles[i]->double_chunk = ( double * )malloc( (sizeof( double )*number_of_items+ DB_HEADER_SIZE)); 1714 1715 double * aa = ( double * )datablock_header; 1716 memcpy(PhastaIOActiveFiles[i]->double_chunk, aa, DB_HEADER_SIZE); 1717 } 1718 //MR CHANGE 1719 // if ( cscompare(datatype,"integer") ) 1720 else if ( cscompare("integer",ts1) ) 1721 //MR CHANGE END 1722 { 1723 free ( PhastaIOActiveFiles[i]->int_chunk ); 1724 PhastaIOActiveFiles[i]->int_chunk = ( int * )malloc( (sizeof( int )*number_of_items+ DB_HEADER_SIZE)); 1725 1726 int * aa = ( int * )datablock_header; 1727 memcpy(PhastaIOActiveFiles[i]->int_chunk, aa, DB_HEADER_SIZE); 1728 } 1729 else { 1730 // *fileDescriptor = DATA_TYPE_ILLEGAL; 1731 printf("writeheader - DATA_TYPE_ILLEGAL - %s\n",datatype); 1732 endTimer(&timer_end); 1733 printPerf("writeheader", timer_start, timer_end, 0, 0, ""); 1734 return; 1735 } 1736 free(ts1); 1737 1738 PhastaIOActiveFiles[i]->part_count++; 1739 if ( PhastaIOActiveFiles[i]->part_count == PhastaIOActiveFiles[i]->nppp ) { 1740 //A new field will be written 1741 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 1742 memcpy( PhastaIOActiveFiles[i]->master_header + 1743 PhastaIOActiveFiles[i]->field_count * 1744 MAX_FIELDS_NAME_LENGTH + 1745 MAX_FIELDS_NAME_LENGTH * 2, 1746 mpi_tag, 1747 MAX_FIELDS_NAME_LENGTH-1); 1748 } 1749 PhastaIOActiveFiles[i]->field_count++; 1750 PhastaIOActiveFiles[i]->part_count=0; 1751 } 1752 free(buffer); 1753 } 1754 1755 endTimer(&timer_end); 1756 printPerf("writeheader", timer_start, timer_end, 0, 0, ""); 1757 } 1758 1759 void writeDataBlock( FILE* f, 1760 const void* valueArray, 1761 const int nItems, 1762 const char datatype[], 1763 const char iotype[] ) 1764 { 1765 isBinary( iotype ); 1766 size_t type_size = typeSize( datatype ); 1767 if ( binary_format ) { 1768 fwrite( valueArray, type_size, nItems, f ); 1769 fprintf( f,"\n"); 1770 } else { 1771 char* ts1 = StringStripper( datatype ); 1772 if ( cscompare( "integer", ts1 ) ) { 1773 for( int n=0; n < nItems ; n++ ) 1774 fprintf(f,"%d\n",*((int*)((int*)valueArray+n))); 1775 } else if ( cscompare( "double", ts1 ) ) { 1776 for( int n=0; n < nItems ; n++ ) 1777 fprintf(f,"%lf\n",*((double*)((double*)valueArray+n))); 1778 } 1779 free (ts1); 1780 } 1781 } 1782 1783 void writedatablock( const int* fileDescriptor, 1784 const char keyphrase[], 1785 const void* valueArray, 1786 const int* nItems, 1787 const char datatype[], 1788 const char iotype[] ) 1789 { 1790 //if(irank == 0) printf("entering writedatablock()\n"); 1791 1792 unsigned long data_size = 0; 1793 double timer_start, timer_end; 1794 startTimer(&timer_start); 1795 1796 int i = *fileDescriptor; 1797 checkFileDescriptor("writedatablock",&i); 1798 1799 if ( PhastaIONextActiveIndex == 0 ) { 1800 int filePtr = *fileDescriptor - 1; 1801 1802 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1803 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1804 fprintf(stderr,"openfile_ function has to be called before \n") ; 1805 fprintf(stderr,"acessing the file\n ") ; 1806 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1807 endTimer(&timer_end); 1808 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1809 return; 1810 } 1811 // since we require that a consistant header always preceed the data block 1812 // let us check to see that it is actually the case. 1813 1814 if ( ! cscompare( LastHeaderKey[ filePtr ].c_str(), keyphrase ) ) { 1815 fprintf(stderr, "Header not consistant with data block\n"); 1816 fprintf(stderr, "Header: %s\n", LastHeaderKey[ filePtr ].c_str() ); 1817 fprintf(stderr, "DataBlock: %s\n ", keyphrase ); 1818 fprintf(stderr, "Please recheck write sequence \n"); 1819 if( Strict_Error ) { 1820 fprintf(stderr, "fatal error: cannot continue, returning out of call\n"); 1821 endTimer(&timer_end); 1822 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1823 return; 1824 } 1825 } 1826 1827 FILE* fileObject = fileArray[ filePtr ] ; 1828 size_t type_size=typeSize( datatype ); 1829 isBinary( iotype ); 1830 1831 LastHeaderKey.erase(filePtr); 1832 1833 if ( header_type[filePtr] != (int)type_size ) { 1834 fprintf(stderr,"header and datablock differ on typeof data in the block for\n"); 1835 fprintf(stderr,"keyphrase : %s\n", keyphrase); 1836 if( Strict_Error ) { 1837 fprintf(stderr,"fatal error: cannot continue, returning out of call\n" ); 1838 endTimer(&timer_end); 1839 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1840 return; 1841 } 1842 } 1843 1844 int nUnits = *nItems; 1845 1846 if ( nUnits != DataSize ) { 1847 fprintf(stderr,"header and datablock differ on number of data items for\n"); 1848 fprintf(stderr,"keyphrase : %s\n", keyphrase); 1849 if( Strict_Error ) { 1850 fprintf(stderr,"fatal error: cannot continue, returning out of call\n" ); 1851 endTimer(&timer_end); 1852 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1853 return; 1854 } 1855 } 1856 writeDataBlock(fileObject,valueArray,*nItems,datatype,iotype); 1857 } 1858 else { // syncIO case 1859 MPI_Status write_data_status; 1860 isBinary( iotype ); 1861 int nUnits = *nItems; 1862 1863 //MR CHANGE 1864 // if ( cscompare(datatype,"double") ) 1865 char* ts1 = StringStripper( datatype ); 1866 if ( cscompare("double",ts1) ) 1867 //MR CHANGE END 1868 { 1869 memcpy((PhastaIOActiveFiles[i]->double_chunk+DB_HEADER_SIZE/sizeof(double)), valueArray, nUnits*sizeof(double)); 1870 MPI_File_write_at_all_begin( PhastaIOActiveFiles[i]->file_handle, 1871 PhastaIOActiveFiles[i]->my_offset, 1872 PhastaIOActiveFiles[i]->double_chunk, 1873 //BLOCK_SIZE/sizeof(double), 1874 nUnits+DB_HEADER_SIZE/sizeof(double), 1875 MPI_DOUBLE ); 1876 MPI_File_write_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1877 PhastaIOActiveFiles[i]->double_chunk, 1878 &write_data_status ); 1879 data_size=8*nUnits; 1880 } 1881 //MR CHANGE 1882 // else if ( cscompare ( datatype, "integer")) 1883 else if ( cscompare("integer",ts1) ) 1884 //MR CHANGE END 1885 { 1886 memcpy((PhastaIOActiveFiles[i]->int_chunk+DB_HEADER_SIZE/sizeof(int)), valueArray, nUnits*sizeof(int)); 1887 MPI_File_write_at_all_begin( PhastaIOActiveFiles[i]->file_handle, 1888 PhastaIOActiveFiles[i]->my_offset, 1889 PhastaIOActiveFiles[i]->int_chunk, 1890 nUnits+DB_HEADER_SIZE/sizeof(int), 1891 MPI_INT ); 1892 MPI_File_write_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1893 PhastaIOActiveFiles[i]->int_chunk, 1894 &write_data_status ); 1895 data_size=4*nUnits; 1896 } 1897 else { 1898 printf("Error: writedatablock - DATA_TYPE_ILLEGAL - %s\n",datatype); 1899 endTimer(&timer_end); 1900 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1901 return; 1902 } 1903 free(ts1); 1904 } 1905 1906 endTimer(&timer_end); 1907 char extra_msg[1024]; 1908 memset(extra_msg, '\0', 1024); 1909 char* key = StringStripper(keyphrase); 1910 sprintf(extra_msg, " field is %s ", key); 1911 printPerf("writedatablock", timer_start, timer_end, data_size, 1, extra_msg); 1912 free(key); 1913 1914 } 1915 1916 void 1917 SwapArrayByteOrder( void* array, 1918 int nbytes, 1919 int nItems ) 1920 { 1921 /* This swaps the byte order for the array of nItems each 1922 of size nbytes , This will be called only locally */ 1923 int i,j; 1924 unsigned char* ucDst = (unsigned char*)array; 1925 1926 for(i=0; i < nItems; i++) { 1927 for(j=0; j < (nbytes/2); j++) 1928 std::swap( ucDst[j] , ucDst[(nbytes - 1) - j] ); 1929 ucDst += nbytes; 1930 } 1931 } 1932 1933 void 1934 writestring( int* fileDescriptor, 1935 const char inString[] ) 1936 { 1937 1938 int filePtr = *fileDescriptor - 1; 1939 FILE* fileObject = fileArray[filePtr] ; 1940 fprintf(fileObject,"%s",inString ); 1941 return; 1942 } 1943 1944 void 1945 Gather_Headers( int* fileDescriptor, 1946 vector< string >& headers ) 1947 { 1948 1949 FILE* fileObject; 1950 char Line[1024]; 1951 1952 fileObject = fileArray[ (*fileDescriptor)-1 ]; 1953 1954 while( !feof(fileObject) ) { 1955 fgets( Line, 1024, fileObject); 1956 if ( Line[0] == '#' ) { 1957 headers.push_back( Line ); 1958 } else { 1959 break; 1960 } 1961 } 1962 rewind( fileObject ); 1963 clearerr( fileObject ); 1964 } 1965 void 1966 isWrong( void ) { (Wrong_Endian) ? fprintf(stdout,"YES\n"): fprintf(stdout,"NO\n") ; } 1967 1968 void 1969 togglestrictmode( void ) { Strict_Error = !Strict_Error; } 1970 1971 int 1972 isLittleEndian( void ) 1973 { 1974 // this function returns a 1 if the current running architecture is 1975 // LittleEndian Byte Ordered, else it returns a zero 1976 1977 union { 1978 long a; 1979 char c[sizeof( long )]; 1980 } endianUnion; 1981 1982 endianUnion.a = 1 ; 1983 1984 if ( endianUnion.c[sizeof(long)-1] != 1 ) return 1 ; 1985 else return 0; 1986 } 1987 1988 namespace PHASTA { 1989 const char* const PhastaIO_traits<int>::type_string = "integer"; 1990 const char* const PhastaIO_traits<double>::type_string = "double"; 1991 } 1992