1 /* 2 * 3 * This is the SyncIO library that uses MPI-IO collective functions to 4 * implement a flexible I/O checkpoint solution for a large number of 5 * processors. 6 * 7 * Previous developer: Ning Liu (liun2@cs.rpi.edu) 8 * Jing Fu (fuj@cs.rpi.edu) 9 * Current developers: Michel Rasquin (Michel.Rasquin@colorado.edu), 10 * Ben Matthews (benjamin.a.matthews@colorado.edu) 11 * 12 */ 13 14 #include <map> 15 #include <vector> 16 #include <string> 17 #include <string.h> 18 #include <ctype.h> 19 #include <stdlib.h> 20 #include <stdio.h> 21 #include <math.h> 22 #include <sstream> 23 #include "phastaIO.h" 24 #include "mpi.h" 25 #include "phiotmrc.h" 26 #include <assert.h> 27 28 #define VERSION_INFO_HEADER_SIZE 8192 29 #define DB_HEADER_SIZE 1024 30 #define ONE_MEGABYTE 1048576 31 #define TWO_MEGABYTE 2097152 32 #define ENDIAN_TEST_NUMBER 12180 // Troy's Zip Code!! 33 #define MAX_PHASTA_FILES 64 34 #define MAX_PHASTA_FILE_NAME_LENGTH 1024 35 #define MAX_FIELDS_NUMBER ((VERSION_INFO_HEADER_SIZE/MAX_FIELDS_NAME_LENGTH)-4) // The meta data include - MPI_IO_Tag, nFields, nFields*names of the fields, nppf 36 // -3 for MPI_IO_Tag, nFields and nppf, -4 for extra security (former nFiles) 37 #define MAX_FIELDS_NAME_LENGTH 128 38 #define DefaultMHSize (4*ONE_MEGABYTE) 39 //#define DefaultMHSize (8350) //For test 40 #define LATEST_WRITE_VERSION 1 41 #define inv1024sq 953.674316406e-9 // = 1/1024/1024 42 int MasterHeaderSize = -1; 43 44 bool PRINT_PERF = false; // default print no perf results 45 int irank = -1; // global rank, should never be manually manipulated 46 int mysize = -1; 47 48 // Static variables are bad but used here to store the subcommunicator and associated variables 49 // Prevent MPI_Comm_split to be called more than once, especially on BGQ with the V1R2M1 driver (leak detected in MPI_Comm_split - IBM working on it) 50 static int s_assign_local_comm = 0; 51 static MPI_Comm s_local_comm; 52 static int s_local_size = -1; 53 static int s_local_rank = -1; 54 55 // the following defines are for debug printf 56 #define PHASTAIO_DEBUG 0 //default to not print any debugging info 57 58 void phprintf(const char* fmt, ...) { 59 (void)fmt; 60 #if PHASTAIO_DEBUG 61 char format[1024]; 62 snprintf(format, sizeof(format), "phastaIO debug: %s", fmt); 63 va_list ap; 64 va_start(ap,fmt); 65 vprintf(format,ap) 66 va_end(ap); 67 #endif 68 } 69 70 void phprintf_0(const char* fmt, ...) { 71 (void)fmt; 72 #if PHASTAIO_DEBUG 73 int rank = 0; 74 MPI_Comm_rank(MPI_COMM_WORLD, &rank); 75 if(rank == 0){ 76 char format[1024]; 77 snprintf(format, sizeof(format), "phastaIO debug: irank=0 %s", fmt); 78 va_list ap; 79 va_start(ap,s); 80 vprintf(format, ap); 81 va_end(ap); 82 } 83 #endif 84 } 85 86 enum PhastaIO_Errors 87 { 88 MAX_PHASTA_FILES_EXCEEDED = -1, 89 UNABLE_TO_OPEN_FILE = -2, 90 NOT_A_MPI_FILE = -3, 91 GPID_EXCEEDED = -4, 92 DATA_TYPE_ILLEGAL = -5 93 }; 94 95 using namespace std; 96 97 namespace{ 98 99 map<int, std::string> LastHeaderKey; 100 vector< FILE* > fileArray; 101 vector< bool > byte_order; 102 vector< int > header_type; 103 int DataSize=0; 104 bool LastHeaderNotFound = false; 105 bool Wrong_Endian = false ; 106 bool Strict_Error = false ; 107 bool binary_format = true; 108 109 /***********************************************************************/ 110 /***************** NEW PHASTA IO CODE STARTS HERE **********************/ 111 /***********************************************************************/ 112 113 typedef struct 114 { 115 char filename[MAX_PHASTA_FILE_NAME_LENGTH]; /* defafults to 1024 */ 116 unsigned long my_offset; 117 unsigned long next_start_address; 118 unsigned long **my_offset_table; 119 unsigned long **my_read_table; 120 121 double * double_chunk; 122 double * read_double_chunk; 123 124 int field_count; 125 int part_count; 126 int read_field_count; 127 int read_part_count; 128 int GPid; 129 int start_id; 130 131 int mhsize; 132 133 int myrank; 134 int numprocs; 135 int local_myrank; 136 int local_numprocs; 137 138 int nppp; 139 int nPPF; 140 int nFiles; 141 int nFields; 142 143 int * int_chunk; 144 int * read_int_chunk; 145 146 int Wrong_Endian; /* default to false */ 147 char * master_header; 148 MPI_File file_handle; 149 MPI_Comm local_comm; 150 } phastaio_file_t; 151 152 typedef struct 153 { 154 int nppf, nfields; 155 char * masterHeader; 156 }serial_file; 157 158 serial_file *SerialFile; 159 phastaio_file_t *PhastaIOActiveFiles[MAX_PHASTA_FILES]; 160 int PhastaIONextActiveIndex = 0; /* indicates next index to allocate */ 161 162 // the caller has the responsibility to delete the returned string 163 // TODO: StringStipper("nbc value? ") returns NULL? 164 char* StringStripper( const char istring[] ) { 165 int length = strlen( istring ); 166 char* dest = (char *)malloc( length + 1 ); 167 strcpy( dest, istring ); 168 dest[ length ] = '\0'; 169 if ( char* p = strpbrk( dest, " ") ) 170 *p = '\0'; 171 return dest; 172 } 173 174 inline int cscompare( const char teststring[], const char targetstring[] ) { 175 char* s1 = const_cast<char*>(teststring); 176 char* s2 = const_cast<char*>(targetstring); 177 178 while( *s1 == ' ') s1++; 179 while( *s2 == ' ') s2++; 180 while( ( *s1 ) 181 && ( *s2 ) 182 && ( *s2 != '?') 183 && ( tolower( *s1 )==tolower( *s2 ) ) ) { 184 s1++; 185 s2++; 186 while( *s1 == ' ') s1++; 187 while( *s2 == ' ') s2++; 188 } 189 if ( !( *s1 ) || ( *s1 == '?') ) return 1; 190 else return 0; 191 } 192 193 inline void isBinary( const char iotype[] ) { 194 char* fname = StringStripper( iotype ); 195 if ( cscompare( fname, "binary" ) ) binary_format = true; 196 else binary_format = false; 197 free (fname); 198 199 } 200 201 inline size_t typeSize( const char typestring[] ) { 202 char* ts1 = StringStripper( typestring ); 203 if ( cscompare( "integer", ts1 ) ) { 204 free (ts1); 205 return sizeof(int); 206 } else if ( cscompare( "double", ts1 ) ) { 207 free (ts1); 208 return sizeof( double ); 209 } else { 210 free (ts1); 211 fprintf(stderr,"unknown type : %s\n",ts1); 212 return 0; 213 } 214 } 215 216 int readHeader( 217 FILE* fileObject, 218 const char phrase[], 219 int* params, 220 int expect ) { 221 char* text_header; 222 char* token; 223 char Line[1024] = "\0"; 224 char junk; 225 bool FOUND = false ; 226 int real_length; 227 int skip_size, integer_value; 228 int rewind_count=0; 229 230 if( !fgets( Line, 1024, fileObject ) && feof( fileObject ) ) { 231 rewind( fileObject ); 232 clearerr( fileObject ); 233 rewind_count++; 234 fgets( Line, 1024, fileObject ); 235 } 236 237 while( !FOUND && ( rewind_count < 2 ) ) { 238 if ( ( Line[0] != '\n' ) && ( real_length = strcspn( Line, "#" )) ) { 239 text_header = (char *)malloc( real_length + 1 ); 240 strncpy( text_header, Line, real_length ); 241 text_header[ real_length ] =static_cast<char>(NULL); 242 token = strtok ( text_header, ":" ); 243 assert(token); 244 if( cscompare( phrase , token ) ) { 245 FOUND = true ; 246 token = strtok( NULL, " ,;<>" ); 247 assert(token); 248 skip_size = atoi( token ); 249 int i; 250 for( i=0; i < expect && ( token = strtok( NULL," ,;<>") ); i++) { 251 assert(token); 252 params[i] = atoi( token ); 253 } 254 if ( i < expect ) { 255 fprintf(stderr, 256 "Aloha Expected # of ints not found for: %s\n",phrase ); 257 } 258 } else if ( cscompare(token,"byteorder magic number") ) { 259 if ( binary_format ) { 260 fread((void*)&integer_value,sizeof(int),1,fileObject); 261 fread( &junk, sizeof(char), 1 , fileObject ); 262 if ( 362436 != integer_value ) Wrong_Endian = true; 263 } else{ 264 fscanf(fileObject, "%d\n", &integer_value ); 265 } 266 } else { 267 /* some other header, so just skip over */ 268 token = strtok( NULL, " ,;<>" ); 269 assert(token); 270 skip_size = atoi( token ); 271 if ( binary_format) 272 fseek( fileObject, skip_size, SEEK_CUR ); 273 else 274 for( int gama=0; gama < skip_size; gama++ ) 275 fgets( Line, 1024, fileObject ); 276 } 277 free (text_header); 278 } // end of if before while loop 279 280 if ( !FOUND ) 281 if( !fgets( Line, 1024, fileObject ) && feof( fileObject ) ) { 282 rewind( fileObject ); 283 clearerr( fileObject ); 284 rewind_count++; 285 fgets( Line, 1024, fileObject ); 286 } 287 } 288 289 if ( !FOUND ) { 290 //fprintf(stderr, "Error: Could not find: %s\n", phrase); 291 if(irank == 0) printf("WARNING: Could not find: %s\n", phrase); 292 return 1; 293 } 294 return 0; 295 } 296 297 } // end unnamed namespace 298 299 300 // begin of publicly visible functions 301 302 /** 303 * This function takes a long long pointer and assign (start) phiotmrc value to it 304 */ 305 void startTimer(double* start) { 306 if( !PRINT_PERF ) return; 307 MPI_Barrier(MPI_COMM_WORLD); 308 *start = phiotmrc(); 309 } 310 311 /** 312 * This function takes a long long pointer and assign (end) phiotmrc value to it 313 */ 314 void endTimer(double* end) { 315 if( !PRINT_PERF ) return; 316 *end = phiotmrc(); 317 MPI_Barrier(MPI_COMM_WORLD); 318 } 319 320 /** 321 * choose to print some performance results (or not) according to 322 * the PRINT_PERF macro 323 */ 324 void printPerf( 325 const char* func_name, 326 double start, 327 double end, 328 unsigned long datasize, 329 int printdatainfo, 330 const char* extra_msg) { 331 if( !PRINT_PERF ) return; 332 unsigned long data_size = datasize; 333 double time = end - start; 334 unsigned long isizemin,isizemax,isizetot; 335 double sizemin,sizemax,sizeavg,sizetot,rate; 336 double tmin, tmax, tavg, ttot; 337 338 MPI_Allreduce(&time, &tmin,1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); 339 MPI_Allreduce(&time, &tmax,1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); 340 MPI_Allreduce(&time, &ttot,1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); 341 tavg = ttot/mysize; 342 343 if(irank == 0) { 344 if ( PhastaIONextActiveIndex == 0 ) printf("** 1PFPP "); 345 else printf("** syncIO "); 346 printf("%s(): Tmax = %f sec, Tmin = %f sec, Tavg = %f sec", func_name, tmax, tmin, tavg); 347 } 348 349 if(printdatainfo == 1) { // if printdatainfo ==1, compute I/O rate and block size 350 MPI_Allreduce(&data_size,&isizemin,1,MPI_LONG_LONG_INT,MPI_MIN,MPI_COMM_WORLD); 351 MPI_Allreduce(&data_size,&isizemax,1,MPI_LONG_LONG_INT,MPI_MAX,MPI_COMM_WORLD); 352 MPI_Allreduce(&data_size,&isizetot,1,MPI_LONG_LONG_INT,MPI_SUM,MPI_COMM_WORLD); 353 354 sizemin=(double)(isizemin*inv1024sq); 355 sizemax=(double)(isizemax*inv1024sq); 356 sizetot=(double)(isizetot*inv1024sq); 357 sizeavg=(double)(1.0*sizetot/mysize); 358 rate=(double)(1.0*sizetot/tmax); 359 360 if( irank == 0) { 361 printf(", Rate = %f MB/s [%s] \n \t\t\t" 362 " block size: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB\n", 363 rate, extra_msg, sizemin,sizemax,sizeavg,sizetot); 364 } 365 } 366 else { 367 if(irank == 0) { 368 printf(" \n"); 369 //printf(" (%s) \n", extra_msg); 370 } 371 } 372 } 373 374 /** 375 * This function is normally called at the beginning of a read operation, before 376 * init function. 377 * This function (uses rank 0) reads out nfields, nppf, master header size, 378 * endianess and allocates for masterHeader string. 379 * These values are essential for following read operations. Rank 0 will bcast 380 * these values to other ranks in the commm world 381 * 382 * If the file set is of old POSIX format, it would throw error and exit 383 */ 384 void queryphmpiio(const char filename[],int *nfields, int *nppf) 385 { 386 MPI_Comm_rank(MPI_COMM_WORLD, &irank); 387 MPI_Comm_size(MPI_COMM_WORLD, &mysize); 388 389 if(irank == 0) { 390 FILE * fileHandle; 391 char* fname = StringStripper( filename ); 392 393 fileHandle = fopen (fname,"rb"); 394 if (fileHandle == NULL ) { 395 printf("\nError: File %s doesn't exist! Please check!\n",fname); 396 } 397 else { 398 SerialFile =(serial_file *)calloc( 1, sizeof( serial_file) ); 399 int meta_size_limit = VERSION_INFO_HEADER_SIZE; 400 SerialFile->masterHeader = (char *)malloc( meta_size_limit ); 401 fread(SerialFile->masterHeader, 1, meta_size_limit, fileHandle); 402 403 char read_out_tag[MAX_FIELDS_NAME_LENGTH]; 404 char version[MAX_FIELDS_NAME_LENGTH/4]; 405 int mhsize; 406 char * token; 407 int magic_number; 408 409 memcpy( read_out_tag, 410 SerialFile->masterHeader, 411 MAX_FIELDS_NAME_LENGTH-1 ); 412 413 if ( cscompare ("MPI_IO_Tag",read_out_tag) ) { 414 // Test endianess ... 415 memcpy (&magic_number, 416 SerialFile->masterHeader + sizeof("MPI_IO_Tag : ")-1, //-1 sizeof returns the size of the string+1 for "\0" 417 sizeof(int) ); // masterheader should look like "MPI_IO_Tag : 12180 " with 12180 in binary format 418 419 if ( magic_number != ENDIAN_TEST_NUMBER ) { 420 printf("Endian is different!\n"); 421 // Will do swap later 422 } 423 424 // test version, old version, default masterheader size is 4M 425 // newer version masterheader size is read from first line 426 memcpy(version, 427 SerialFile->masterHeader + MAX_FIELDS_NAME_LENGTH/2, 428 MAX_FIELDS_NAME_LENGTH/4 - 1); //TODO: why -1? 429 430 if( cscompare ("version",version) ) { 431 // if there is "version" tag in the file, then it is newer format 432 // read master header size from here, otherwise use default 433 // Note: if version is "1", we know mhsize is at 3/4 place... 434 435 token = strtok(version, ":"); 436 token = strtok(NULL, " ,;<>" ); 437 int iversion = atoi(token); 438 439 if( iversion == 1) { 440 memcpy( &mhsize, 441 SerialFile->masterHeader + MAX_FIELDS_NAME_LENGTH/4*3 + sizeof("mhsize : ")-1, 442 sizeof(int)); 443 if ( magic_number != ENDIAN_TEST_NUMBER ) 444 SwapArrayByteOrder(&mhsize, sizeof(int), 1); 445 446 if( mhsize > DefaultMHSize ) { 447 //if actual headersize is larger than default, let's re-read 448 free(SerialFile->masterHeader); 449 SerialFile->masterHeader = (char *)malloc(mhsize); 450 fseek(fileHandle, 0, SEEK_SET); // reset the file stream position 451 fread(SerialFile->masterHeader,1,mhsize,fileHandle); 452 } 453 } 454 //TODO: check if this is a valid int?? 455 MasterHeaderSize = mhsize; 456 } 457 else { // else it's version 0's format w/o version tag, implicating MHSize=4M 458 MasterHeaderSize = DefaultMHSize; 459 } 460 461 memcpy( read_out_tag, 462 SerialFile->masterHeader+MAX_FIELDS_NAME_LENGTH+1, 463 MAX_FIELDS_NAME_LENGTH ); //TODO: why +1 464 465 // Read in # fields ... 466 token = strtok( read_out_tag, ":" ); 467 token = strtok( NULL," ,;<>" ); 468 *nfields = atoi( token ); 469 if ( *nfields > MAX_FIELDS_NUMBER) { 470 printf("Error queryphmpiio: nfields is larger than MAX_FIELDS_NUMBER!\n"); 471 } 472 SerialFile->nfields=*nfields; //TODO: sanity check of this int? 473 474 memcpy( read_out_tag, 475 SerialFile->masterHeader + MAX_FIELDS_NAME_LENGTH * 2 476 + *nfields * MAX_FIELDS_NAME_LENGTH, 477 MAX_FIELDS_NAME_LENGTH); 478 479 token = strtok( read_out_tag, ":" ); 480 token = strtok( NULL," ,;<>" ); 481 *nppf = atoi( token ); 482 SerialFile->nppf=*nppf; //TODO: sanity check of int 483 } // end of if("MPI_IO_TAG") 484 else { 485 printf("Error queryphmpiio: The file you opened is not of syncIO new format, please check! read_out_tag = %s\n",read_out_tag); 486 exit(1); 487 } 488 fclose(fileHandle); 489 free(SerialFile->masterHeader); 490 free(SerialFile); 491 } //end of else 492 free(fname); 493 } 494 495 // Bcast value to every one 496 MPI_Bcast( nfields, 1, MPI_INT, 0, MPI_COMM_WORLD ); 497 MPI_Bcast( nppf, 1, MPI_INT, 0, MPI_COMM_WORLD ); 498 MPI_Bcast( &MasterHeaderSize, 1, MPI_INT, 0, MPI_COMM_WORLD ); 499 phprintf("Info queryphmpiio: myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 500 } 501 502 /** 503 * This function computes the right master header size (round to size of 2^n). 504 * This is only needed for file format version 1 in "write" mode. 505 */ 506 int computeMHSize(int nfields, int nppf, int version) { 507 int mhsize=0; 508 if(version == 1) { 509 //int meta_info_size = (2+nfields+1) * MAX_FIELDS_NAME_LENGTH; // 2 is MPI_IO_TAG and nFields, the others 1 is nppf 510 int meta_info_size = VERSION_INFO_HEADER_SIZE; 511 int actual_size = nfields * nppf * sizeof(unsigned long) + meta_info_size; 512 //printf("actual_size = %d, offset table size = %d\n", actual_size, nfields * nppf * sizeof(long long)); 513 if (actual_size > DefaultMHSize) { 514 mhsize = (int) ceil( (double) actual_size/DefaultMHSize); // it's rounded to ceiling of this value 515 mhsize *= DefaultMHSize; 516 } 517 else { 518 mhsize = DefaultMHSize; 519 } 520 } else { 521 int rank = 0; 522 MPI_Comm_rank(MPI_COMM_WORLD, &rank); 523 if(!rank) { 524 fprintf(stderr, 525 "ERROR invalid version passed to %s... exiting\n", __func__); 526 exit(EXIT_FAILURE); 527 } 528 } 529 return mhsize; 530 } 531 532 /** 533 * Computes correct color of a rank according to number of files. 534 */ 535 extern "C" int computeColor( int myrank, int numprocs, int nfiles) { 536 int color = 537 (int)(myrank / (numprocs / nfiles)); 538 return color; 539 } 540 541 542 /** 543 * Check the file descriptor. 544 */ 545 void checkFileDescriptor(const char fctname[], int* fileDescriptor ) { 546 if ( *fileDescriptor < 0 ) { 547 printf("Error: File descriptor = %d in %s\n",*fileDescriptor,fctname); 548 exit(1); 549 } 550 } 551 552 /** 553 * Initialize the file struct members and allocate space for file struct 554 * buffers. 555 * 556 * Note: this function is only called when we are using new format. Old POSIX 557 * format should skip this routine and call openfile() directly instead. 558 */ 559 int initphmpiio( int *nfields, int *nppf, int *nfiles, int *filehandle, const char mode[]) 560 { 561 // we init irank again in case query not called (e.g. syncIO write case) 562 MPI_Comm_rank(MPI_COMM_WORLD, &irank); 563 MPI_Comm_size(MPI_COMM_WORLD, &mysize); 564 565 phprintf("Info initphmpiio: entering function, myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 566 567 double timer_start, timer_end; 568 startTimer(&timer_start); 569 570 char* imode = StringStripper( mode ); 571 572 // Note: if it's read, we presume query was called prior to init and 573 // MasterHeaderSize is already set to correct value from parsing header 574 // otherwise it's write then it needs some computation to be set 575 if ( cscompare( "read", imode ) ) { 576 // do nothing 577 } 578 else if( cscompare( "write", imode ) ) { 579 MasterHeaderSize = computeMHSize(*nfields, *nppf, LATEST_WRITE_VERSION); 580 } 581 else { 582 printf("Error initphmpiio: can't recognize the mode %s", imode); 583 exit(1); 584 } 585 free ( imode ); 586 587 phprintf("Info initphmpiio: myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 588 589 int i, j; 590 591 if( PhastaIONextActiveIndex == MAX_PHASTA_FILES ) { 592 printf("Error initphmpiio: PhastaIONextActiveIndex = MAX_PHASTA_FILES"); 593 endTimer(&timer_end); 594 printPerf("initphmpiio", timer_start, timer_end, 0, 0, ""); 595 return MAX_PHASTA_FILES_EXCEEDED; 596 } 597 // else if( PhastaIONextActiveIndex == 0 ) //Hang in debug mode on Intrepid 598 // { 599 // for( i = 0; i < MAX_PHASTA_FILES; i++ ); 600 // { 601 // PhastaIOActiveFiles[i] = NULL; 602 // } 603 // } 604 605 606 PhastaIOActiveFiles[PhastaIONextActiveIndex] = (phastaio_file_t *)calloc( 1, sizeof( phastaio_file_t) ); 607 608 i = PhastaIONextActiveIndex; 609 PhastaIONextActiveIndex++; 610 611 //PhastaIOActiveFiles[i]->next_start_address = 2*TWO_MEGABYTE; 612 613 PhastaIOActiveFiles[i]->next_start_address = MasterHeaderSize; // what does this mean??? TODO 614 615 PhastaIOActiveFiles[i]->Wrong_Endian = false; 616 617 PhastaIOActiveFiles[i]->nFields = *nfields; 618 PhastaIOActiveFiles[i]->nPPF = *nppf; 619 PhastaIOActiveFiles[i]->nFiles = *nfiles; 620 MPI_Comm_rank(MPI_COMM_WORLD, &(PhastaIOActiveFiles[i]->myrank)); 621 MPI_Comm_size(MPI_COMM_WORLD, &(PhastaIOActiveFiles[i]->numprocs)); 622 623 624 if( *nfiles > 1 ) { // split the ranks according to each mpiio file 625 626 if ( s_assign_local_comm == 0) { // call mpi_comm_split for the first (and only) time 627 628 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Building subcommunicator\n"); 629 630 int color = computeColor(PhastaIOActiveFiles[i]->myrank, PhastaIOActiveFiles[i]->numprocs, PhastaIOActiveFiles[i]->nFiles); 631 MPI_Comm_split(MPI_COMM_WORLD, 632 color, 633 PhastaIOActiveFiles[i]->myrank, 634 &(PhastaIOActiveFiles[i]->local_comm)); 635 MPI_Comm_size(PhastaIOActiveFiles[i]->local_comm, 636 &(PhastaIOActiveFiles[i]->local_numprocs)); 637 MPI_Comm_rank(PhastaIOActiveFiles[i]->local_comm, 638 &(PhastaIOActiveFiles[i]->local_myrank)); 639 640 // back up now these variables so that we do not need to call comm_split again 641 s_local_comm = PhastaIOActiveFiles[i]->local_comm; 642 s_local_size = PhastaIOActiveFiles[i]->local_numprocs; 643 s_local_rank = PhastaIOActiveFiles[i]->local_myrank; 644 s_assign_local_comm = 1; 645 } 646 else { // recycle the subcommunicator 647 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Recycling subcommunicator\n"); 648 PhastaIOActiveFiles[i]->local_comm = s_local_comm; 649 PhastaIOActiveFiles[i]->local_numprocs = s_local_size; 650 PhastaIOActiveFiles[i]->local_myrank = s_local_rank; 651 } 652 } 653 else { // *nfiles == 1 here - no need to call mpi_comm_split here 654 655 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Bypassing subcommunicator\n"); 656 PhastaIOActiveFiles[i]->local_comm = MPI_COMM_WORLD; 657 PhastaIOActiveFiles[i]->local_numprocs = PhastaIOActiveFiles[i]->numprocs; 658 PhastaIOActiveFiles[i]->local_myrank = PhastaIOActiveFiles[i]->myrank; 659 660 } 661 662 PhastaIOActiveFiles[i]->nppp = 663 PhastaIOActiveFiles[i]->nPPF/PhastaIOActiveFiles[i]->local_numprocs; 664 665 PhastaIOActiveFiles[i]->start_id = PhastaIOActiveFiles[i]->nPPF * 666 (int)(PhastaIOActiveFiles[i]->myrank/PhastaIOActiveFiles[i]->local_numprocs) + 667 (PhastaIOActiveFiles[i]->local_myrank * PhastaIOActiveFiles[i]->nppp); 668 669 PhastaIOActiveFiles[i]->my_offset_table = 670 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 671 672 PhastaIOActiveFiles[i]->my_read_table = 673 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 674 675 for (j=0; j<*nfields; j++) 676 { 677 PhastaIOActiveFiles[i]->my_offset_table[j] = 678 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 679 680 PhastaIOActiveFiles[i]->my_read_table[j] = 681 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 682 } 683 *filehandle = i; 684 685 PhastaIOActiveFiles[i]->master_header = (char *)calloc(MasterHeaderSize,sizeof( char )); 686 PhastaIOActiveFiles[i]->double_chunk = (double *)calloc(1,sizeof( double )); 687 PhastaIOActiveFiles[i]->int_chunk = (int *)calloc(1,sizeof( int )); 688 PhastaIOActiveFiles[i]->read_double_chunk = (double *)calloc(1,sizeof( double )); 689 PhastaIOActiveFiles[i]->read_int_chunk = (int *)calloc(1,sizeof( int )); 690 691 // Time monitoring 692 endTimer(&timer_end); 693 printPerf("initphmpiio", timer_start, timer_end, 0, 0, ""); 694 695 phprintf_0("Info initphmpiio: quiting function"); 696 697 return i; 698 } 699 700 /** 701 * Destruct the file struct and free buffers allocated in init function. 702 */ 703 void finalizephmpiio( int *fileDescriptor ) 704 { 705 double timer_start, timer_end; 706 startTimer(&timer_start); 707 708 int i, j; 709 i = *fileDescriptor; 710 //PhastaIONextActiveIndex--; 711 712 /* //free the offset table for this phasta file */ 713 //for(j=0; j<MAX_FIELDS_NUMBER; j++) //Danger: undefined behavior for my_*_table.[j] not allocated or not initialized to NULL 714 for(j=0; j<PhastaIOActiveFiles[i]->nFields; j++) 715 { 716 free( PhastaIOActiveFiles[i]->my_offset_table[j]); 717 free( PhastaIOActiveFiles[i]->my_read_table[j]); 718 } 719 free ( PhastaIOActiveFiles[i]->my_offset_table ); 720 free ( PhastaIOActiveFiles[i]->my_read_table ); 721 free ( PhastaIOActiveFiles[i]->master_header ); 722 free ( PhastaIOActiveFiles[i]->double_chunk ); 723 free ( PhastaIOActiveFiles[i]->int_chunk ); 724 free ( PhastaIOActiveFiles[i]->read_double_chunk ); 725 free ( PhastaIOActiveFiles[i]->read_int_chunk ); 726 727 if( PhastaIOActiveFiles[i]->nFiles > 1 && s_assign_local_comm ) { // the comm was split 728 if (PhastaIOActiveFiles[i]->myrank == 0) printf("Freeing subcommunicator\n"); 729 s_assign_local_comm = 0; 730 MPI_Comm_free(&(PhastaIOActiveFiles[i]->local_comm)); 731 } 732 733 free( PhastaIOActiveFiles[i]); 734 735 endTimer(&timer_end); 736 printPerf("finalizempiio", timer_start, timer_end, 0, 0, ""); 737 738 PhastaIONextActiveIndex--; 739 } 740 741 742 /** 743 * Special init for M2N in order to create a subcommunicator for the reduced solution (requires PRINT_PERF to be false for now) 744 * Initialize the file struct members and allocate space for file struct buffers. 745 * 746 * Note: this function is only called when we are using new format. Old POSIX 747 * format should skip this routine and call openfile() directly instead. 748 */ 749 int initphmpiiosub( int *nfields, int *nppf, int *nfiles, int *filehandle, const char mode[],MPI_Comm my_local_comm) 750 { 751 // we init irank again in case query not called (e.g. syncIO write case) 752 753 MPI_Comm_rank(my_local_comm, &irank); 754 MPI_Comm_size(my_local_comm, &mysize); 755 756 phprintf("Info initphmpiio: entering function, myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 757 758 double timer_start, timer_end; 759 startTimer(&timer_start); 760 761 char* imode = StringStripper( mode ); 762 763 // Note: if it's read, we presume query was called prior to init and 764 // MasterHeaderSize is already set to correct value from parsing header 765 // otherwise it's write then it needs some computation to be set 766 if ( cscompare( "read", imode ) ) { 767 // do nothing 768 } 769 else if( cscompare( "write", imode ) ) { 770 MasterHeaderSize = computeMHSize(*nfields, *nppf, LATEST_WRITE_VERSION); 771 } 772 else { 773 printf("Error initphmpiio: can't recognize the mode %s", imode); 774 exit(1); 775 } 776 free ( imode ); 777 778 phprintf("Info initphmpiio: myrank = %d, MasterHeaderSize = %d", irank, MasterHeaderSize); 779 780 int i, j; 781 782 if( PhastaIONextActiveIndex == MAX_PHASTA_FILES ) { 783 printf("Error initphmpiio: PhastaIONextActiveIndex = MAX_PHASTA_FILES"); 784 endTimer(&timer_end); 785 printPerf("initphmpiio", timer_start, timer_end, 0, 0, ""); 786 return MAX_PHASTA_FILES_EXCEEDED; 787 } 788 // else if( PhastaIONextActiveIndex == 0 ) //Hang in debug mode on Intrepid 789 // { 790 // for( i = 0; i < MAX_PHASTA_FILES; i++ ); 791 // { 792 // PhastaIOActiveFiles[i] = NULL; 793 // } 794 // } 795 796 797 PhastaIOActiveFiles[PhastaIONextActiveIndex] = (phastaio_file_t *)calloc( 1, sizeof( phastaio_file_t) ); 798 799 i = PhastaIONextActiveIndex; 800 PhastaIONextActiveIndex++; 801 802 //PhastaIOActiveFiles[i]->next_start_address = 2*TWO_MEGABYTE; 803 804 PhastaIOActiveFiles[i]->next_start_address = MasterHeaderSize; // what does this mean??? TODO 805 806 PhastaIOActiveFiles[i]->Wrong_Endian = false; 807 808 PhastaIOActiveFiles[i]->nFields = *nfields; 809 PhastaIOActiveFiles[i]->nPPF = *nppf; 810 PhastaIOActiveFiles[i]->nFiles = *nfiles; 811 MPI_Comm_rank(my_local_comm, &(PhastaIOActiveFiles[i]->myrank)); 812 MPI_Comm_size(my_local_comm, &(PhastaIOActiveFiles[i]->numprocs)); 813 814 int color = computeColor(PhastaIOActiveFiles[i]->myrank, PhastaIOActiveFiles[i]->numprocs, PhastaIOActiveFiles[i]->nFiles); 815 MPI_Comm_split(my_local_comm, 816 color, 817 PhastaIOActiveFiles[i]->myrank, 818 &(PhastaIOActiveFiles[i]->local_comm)); 819 MPI_Comm_size(PhastaIOActiveFiles[i]->local_comm, 820 &(PhastaIOActiveFiles[i]->local_numprocs)); 821 MPI_Comm_rank(PhastaIOActiveFiles[i]->local_comm, 822 &(PhastaIOActiveFiles[i]->local_myrank)); 823 PhastaIOActiveFiles[i]->nppp = 824 PhastaIOActiveFiles[i]->nPPF/PhastaIOActiveFiles[i]->local_numprocs; 825 826 PhastaIOActiveFiles[i]->start_id = PhastaIOActiveFiles[i]->nPPF * 827 (int)(PhastaIOActiveFiles[i]->myrank/PhastaIOActiveFiles[i]->local_numprocs) + 828 (PhastaIOActiveFiles[i]->local_myrank * PhastaIOActiveFiles[i]->nppp); 829 830 PhastaIOActiveFiles[i]->my_offset_table = 831 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 832 833 PhastaIOActiveFiles[i]->my_read_table = 834 ( unsigned long ** ) calloc( MAX_FIELDS_NUMBER , sizeof( unsigned long *) ); 835 836 for (j=0; j<*nfields; j++) 837 { 838 PhastaIOActiveFiles[i]->my_offset_table[j] = 839 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 840 841 PhastaIOActiveFiles[i]->my_read_table[j] = 842 ( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nppp , sizeof( unsigned long) ); 843 } 844 *filehandle = i; 845 846 PhastaIOActiveFiles[i]->master_header = (char *)calloc(MasterHeaderSize,sizeof( char )); 847 PhastaIOActiveFiles[i]->double_chunk = (double *)calloc(1,sizeof( double )); 848 PhastaIOActiveFiles[i]->int_chunk = (int *)calloc(1,sizeof( int )); 849 PhastaIOActiveFiles[i]->read_double_chunk = (double *)calloc(1,sizeof( double )); 850 PhastaIOActiveFiles[i]->read_int_chunk = (int *)calloc(1,sizeof( int )); 851 852 // Time monitoring 853 endTimer(&timer_end); 854 printPerf("initphmpiiosub", timer_start, timer_end, 0, 0, ""); 855 856 phprintf_0("Info initphmpiiosub: quiting function"); 857 858 return i; 859 } 860 861 862 863 /** open file for both POSIX and MPI-IO syncIO format. 864 * 865 * If it's old POSIX format, simply call posix fopen(). 866 * 867 * If it's MPI-IO foramt: 868 * in "read" mode, it builds the header table that points to the offset of 869 * fields for parts; 870 * in "write" mode, it opens the file with MPI-IO open routine. 871 */ 872 void openfile(const char filename[], const char mode[], int* fileDescriptor ) 873 { 874 phprintf_0("Info: entering openfile"); 875 876 double timer_start, timer_end; 877 startTimer(&timer_start); 878 879 if ( PhastaIONextActiveIndex == 0 ) 880 { 881 FILE* file=NULL ; 882 *fileDescriptor = 0; 883 char* fname = StringStripper( filename ); 884 char* imode = StringStripper( mode ); 885 886 if ( cscompare( "read", imode ) ) file = fopen(fname, "rb" ); 887 else if( cscompare( "write", imode ) ) file = fopen(fname, "wb" ); 888 else if( cscompare( "append", imode ) ) file = fopen(fname, "ab" ); 889 890 if ( !file ){ 891 fprintf(stderr,"Error openfile: unable to open file %s",fname ) ; 892 } else { 893 fileArray.push_back( file ); 894 byte_order.push_back( false ); 895 header_type.push_back( sizeof(int) ); 896 *fileDescriptor = fileArray.size(); 897 } 898 free (fname); 899 free (imode); 900 } 901 else // else it would be parallel I/O, opposed to posix io 902 { 903 char* fname = StringStripper( filename ); 904 char* imode = StringStripper( mode ); 905 int rc; 906 int i = *fileDescriptor; 907 checkFileDescriptor("openfile",&i); 908 char* token; 909 910 if ( cscompare( "read", imode ) ) 911 { 912 // if (PhastaIOActiveFiles[i]->myrank == 0) 913 // printf("\n **********\nRead open ... ... regular version\n"); 914 915 rc = MPI_File_open( PhastaIOActiveFiles[i]->local_comm, 916 fname, 917 MPI_MODE_RDONLY, 918 MPI_INFO_NULL, 919 &(PhastaIOActiveFiles[i]->file_handle) ); 920 921 if(rc) 922 { 923 *fileDescriptor = UNABLE_TO_OPEN_FILE; 924 printf("Error openfile: Unable to open file %s! File descriptor = %d\n",fname,*fileDescriptor); 925 endTimer(&timer_end); 926 printPerf("openfile", timer_start, timer_end, 0, 0, ""); 927 return; 928 } 929 930 MPI_Status read_tag_status; 931 char read_out_tag[MAX_FIELDS_NAME_LENGTH]; 932 int j; 933 int magic_number; 934 935 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 936 MPI_File_read_at( PhastaIOActiveFiles[i]->file_handle, 937 0, 938 PhastaIOActiveFiles[i]->master_header, 939 MasterHeaderSize, 940 MPI_CHAR, 941 &read_tag_status ); 942 } 943 944 MPI_Bcast( PhastaIOActiveFiles[i]->master_header, 945 MasterHeaderSize, 946 MPI_CHAR, 947 0, 948 PhastaIOActiveFiles[i]->local_comm ); 949 950 memcpy( read_out_tag, 951 PhastaIOActiveFiles[i]->master_header, 952 MAX_FIELDS_NAME_LENGTH-1 ); 953 954 if ( cscompare ("MPI_IO_Tag",read_out_tag) ) 955 { 956 // Test endianess ... 957 memcpy ( &magic_number, 958 PhastaIOActiveFiles[i]->master_header+sizeof("MPI_IO_Tag : ")-1, //-1 sizeof returns the size of the string+1 for "\0" 959 sizeof(int) ); // masterheader should look like "MPI_IO_Tag : 12180 " with 12180 in binary format 960 961 if ( magic_number != ENDIAN_TEST_NUMBER ) 962 { 963 PhastaIOActiveFiles[i]->Wrong_Endian = true; 964 } 965 966 memcpy( read_out_tag, 967 PhastaIOActiveFiles[i]->master_header+MAX_FIELDS_NAME_LENGTH+1, // TODO: WHY +1??? 968 MAX_FIELDS_NAME_LENGTH ); 969 970 // Read in # fields ... 971 token = strtok ( read_out_tag, ":" ); 972 token = strtok( NULL," ,;<>" ); 973 PhastaIOActiveFiles[i]->nFields = atoi( token ); 974 975 unsigned long **header_table; 976 header_table = ( unsigned long ** )calloc(PhastaIOActiveFiles[i]->nFields, sizeof(unsigned long *)); 977 978 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) 979 { 980 header_table[j]=( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nPPF , sizeof( unsigned long)); 981 } 982 983 // Read in the offset table ... 984 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) 985 { 986 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 987 memcpy( header_table[j], 988 PhastaIOActiveFiles[i]->master_header + 989 VERSION_INFO_HEADER_SIZE + 990 j * PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long), 991 PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long) ); 992 } 993 994 MPI_Scatter( header_table[j], 995 PhastaIOActiveFiles[i]->nppp, 996 MPI_LONG_LONG_INT, 997 PhastaIOActiveFiles[i]->my_read_table[j], 998 PhastaIOActiveFiles[i]->nppp, 999 MPI_LONG_LONG_INT, 1000 0, 1001 PhastaIOActiveFiles[i]->local_comm ); 1002 1003 // Swap byte order if endianess is different ... 1004 if ( PhastaIOActiveFiles[i]->Wrong_Endian ) { 1005 SwapArrayByteOrder( PhastaIOActiveFiles[i]->my_read_table[j], 1006 sizeof(unsigned long), 1007 PhastaIOActiveFiles[i]->nppp ); 1008 } 1009 } 1010 1011 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1012 free ( header_table[j] ); 1013 } 1014 free (header_table); 1015 1016 } // end of if MPI_IO_TAG 1017 else //else not valid MPI file 1018 { 1019 *fileDescriptor = NOT_A_MPI_FILE; 1020 printf("Error openfile: The file %s you opened is not in syncIO new format, please check again! File descriptor = %d, MasterHeaderSize = %d, read_out_tag = %s\n",fname,*fileDescriptor,MasterHeaderSize,read_out_tag); 1021 //Printing MasterHeaderSize is useful to test a compiler bug on Intrepid BGP 1022 endTimer(&timer_end); 1023 printPerf("openfile", timer_start, timer_end, 0, 0, ""); 1024 return; 1025 } 1026 } // end of if "read" 1027 else if( cscompare( "write", imode ) ) 1028 { 1029 rc = MPI_File_open( PhastaIOActiveFiles[i]->local_comm, 1030 fname, 1031 MPI_MODE_WRONLY | MPI_MODE_CREATE, 1032 MPI_INFO_NULL, 1033 &(PhastaIOActiveFiles[i]->file_handle) ); 1034 if(rc) 1035 { 1036 *fileDescriptor = UNABLE_TO_OPEN_FILE; 1037 return; 1038 } 1039 } // end of if "write" 1040 free (fname); 1041 free (imode); 1042 } // end of if FileIndex != 0 1043 1044 endTimer(&timer_end); 1045 printPerf("openfile", timer_start, timer_end, 0, 0, ""); 1046 } 1047 1048 /** close file for both POSIX and MPI-IO syncIO format. 1049 * 1050 * If it's old POSIX format, simply call posix fclose(). 1051 * 1052 * If it's MPI-IO foramt: 1053 * in "read" mode, it simply close file with MPI-IO close routine. 1054 * in "write" mode, rank 0 in each group will re-assemble the master header and 1055 * offset table and write to the beginning of file, then close the file. 1056 */ 1057 void closefile( int* fileDescriptor, const char mode[] ) 1058 { 1059 double timer_start, timer_end; 1060 startTimer(&timer_start); 1061 1062 int i = *fileDescriptor; 1063 checkFileDescriptor("closefile",&i); 1064 1065 if ( PhastaIONextActiveIndex == 0 ) { 1066 char* imode = StringStripper( mode ); 1067 1068 if( cscompare( "write", imode ) 1069 || cscompare( "append", imode ) ) { 1070 fflush( fileArray[ *fileDescriptor - 1 ] ); 1071 } 1072 1073 fclose( fileArray[ *fileDescriptor - 1 ] ); 1074 free (imode); 1075 } 1076 else { 1077 char* imode = StringStripper( mode ); 1078 1079 //write master header here: 1080 if ( cscompare( "write", imode ) ) { 1081 // if ( PhastaIOActiveFiles[i]->nPPF * PhastaIOActiveFiles[i]->nFields < 2*ONE_MEGABYTE/8 ) //SHOULD BE CHECKED 1082 // MasterHeaderSize = 4*ONE_MEGABYTE; 1083 // else 1084 // MasterHeaderSize = 4*ONE_MEGABYTE + PhastaIOActiveFiles[i]->nPPF * PhastaIOActiveFiles[i]->nFields * 8 - 2*ONE_MEGABYTE; 1085 1086 MasterHeaderSize = computeMHSize( PhastaIOActiveFiles[i]->nFields, PhastaIOActiveFiles[i]->nPPF, LATEST_WRITE_VERSION); 1087 phprintf_0("Info closefile: myrank = %d, MasterHeaderSize = %d\n", PhastaIOActiveFiles[i]->myrank, MasterHeaderSize); 1088 1089 MPI_Status write_header_status; 1090 char mpi_tag[MAX_FIELDS_NAME_LENGTH]; 1091 char version[MAX_FIELDS_NAME_LENGTH/4]; 1092 char mhsize[MAX_FIELDS_NAME_LENGTH/4]; 1093 int magic_number = ENDIAN_TEST_NUMBER; 1094 1095 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) 1096 { 1097 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1098 sprintf(mpi_tag, "MPI_IO_Tag : "); 1099 memcpy(PhastaIOActiveFiles[i]->master_header, 1100 mpi_tag, 1101 MAX_FIELDS_NAME_LENGTH); 1102 1103 bzero((void*)version,MAX_FIELDS_NAME_LENGTH/4); 1104 // this version is "1", print version in ASCII 1105 sprintf(version, "version : %d",1); 1106 memcpy(PhastaIOActiveFiles[i]->master_header + MAX_FIELDS_NAME_LENGTH/2, 1107 version, 1108 MAX_FIELDS_NAME_LENGTH/4); 1109 1110 // master header size is computed using the formula above 1111 bzero((void*)mhsize,MAX_FIELDS_NAME_LENGTH/4); 1112 sprintf(mhsize, "mhsize : "); 1113 memcpy(PhastaIOActiveFiles[i]->master_header + MAX_FIELDS_NAME_LENGTH/4*3, 1114 mhsize, 1115 MAX_FIELDS_NAME_LENGTH/4); 1116 1117 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1118 sprintf(mpi_tag, 1119 "\nnFields : %d\n", 1120 PhastaIOActiveFiles[i]->nFields); 1121 memcpy(PhastaIOActiveFiles[i]->master_header+MAX_FIELDS_NAME_LENGTH, 1122 mpi_tag, 1123 MAX_FIELDS_NAME_LENGTH); 1124 1125 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1126 sprintf(mpi_tag, "\nnPPF : %d\n", PhastaIOActiveFiles[i]->nPPF); 1127 memcpy( PhastaIOActiveFiles[i]->master_header+ 1128 PhastaIOActiveFiles[i]->nFields * 1129 MAX_FIELDS_NAME_LENGTH + 1130 MAX_FIELDS_NAME_LENGTH * 2, 1131 mpi_tag, 1132 MAX_FIELDS_NAME_LENGTH); 1133 1134 memcpy( PhastaIOActiveFiles[i]->master_header+sizeof("MPI_IO_Tag : ")-1, //-1 sizeof returns the size of the string+1 for "\0" 1135 &magic_number, 1136 sizeof(int)); 1137 1138 memcpy( PhastaIOActiveFiles[i]->master_header+sizeof("mhsize : ") -1 + MAX_FIELDS_NAME_LENGTH/4*3, 1139 &MasterHeaderSize, 1140 sizeof(int)); 1141 } 1142 1143 int j = 0; 1144 unsigned long **header_table; 1145 header_table = ( unsigned long ** )calloc(PhastaIOActiveFiles[i]->nFields, sizeof(unsigned long *)); 1146 1147 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1148 header_table[j]=( unsigned long * ) calloc( PhastaIOActiveFiles[i]->nPPF , sizeof( unsigned long)); 1149 } 1150 1151 //if( irank == 0 ) printf("gonna mpi_gather, myrank = %d\n", irank); 1152 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1153 MPI_Gather( PhastaIOActiveFiles[i]->my_offset_table[j], 1154 PhastaIOActiveFiles[i]->nppp, 1155 MPI_LONG_LONG_INT, 1156 header_table[j], 1157 PhastaIOActiveFiles[i]->nppp, 1158 MPI_LONG_LONG_INT, 1159 0, 1160 PhastaIOActiveFiles[i]->local_comm ); 1161 } 1162 1163 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 1164 1165 //if( irank == 0 ) printf("gonna memcpy for every procs, myrank = %d\n", irank); 1166 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1167 memcpy ( PhastaIOActiveFiles[i]->master_header + 1168 VERSION_INFO_HEADER_SIZE + 1169 j * PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long), 1170 header_table[j], 1171 PhastaIOActiveFiles[i]->nPPF * sizeof(unsigned long) ); 1172 } 1173 1174 //if( irank == 0 ) printf("gonna file_write_at(), myrank = %d\n", irank); 1175 MPI_File_write_at( PhastaIOActiveFiles[i]->file_handle, 1176 0, 1177 PhastaIOActiveFiles[i]->master_header, 1178 MasterHeaderSize, 1179 MPI_CHAR, 1180 &write_header_status ); 1181 } 1182 1183 ////free(PhastaIOActiveFiles[i]->master_header); 1184 1185 for ( j = 0; j < PhastaIOActiveFiles[i]->nFields; j++ ) { 1186 free ( header_table[j] ); 1187 } 1188 free (header_table); 1189 } 1190 1191 //if( irank == 0 ) printf("gonna file_close(), myrank = %d\n", irank); 1192 MPI_File_close( &( PhastaIOActiveFiles[i]->file_handle ) ); 1193 free ( imode ); 1194 } 1195 1196 endTimer(&timer_end); 1197 printPerf("closefile_", timer_start, timer_end, 0, 0, ""); 1198 } 1199 1200 int readHeader( FILE* f, const char phrase[], 1201 int* params, int numParams, const char* iotype) { 1202 isBinary(iotype); 1203 return readHeader(f,phrase,params,numParams); 1204 } 1205 1206 void readheader( 1207 int* fileDescriptor, 1208 const char keyphrase[], 1209 void* valueArray, 1210 int* nItems, 1211 const char datatype[], 1212 const char iotype[] ) 1213 { 1214 double timer_start, timer_end; 1215 1216 startTimer(&timer_start); 1217 1218 int i = *fileDescriptor; 1219 checkFileDescriptor("readheader",&i); 1220 1221 if ( PhastaIONextActiveIndex == 0 ) { 1222 int filePtr = *fileDescriptor - 1; 1223 FILE* fileObject; 1224 int* valueListInt; 1225 1226 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1227 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1228 fprintf(stderr,"openfile_ function has to be called before \n") ; 1229 fprintf(stderr,"acessing the file\n ") ; 1230 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1231 endTimer(&timer_end); 1232 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1233 return; 1234 } 1235 1236 LastHeaderKey[filePtr] = std::string(keyphrase); 1237 LastHeaderNotFound = false; 1238 1239 fileObject = fileArray[ filePtr ] ; 1240 Wrong_Endian = byte_order[ filePtr ]; 1241 1242 isBinary( iotype ); 1243 typeSize( datatype ); //redundant call, just avoid a compiler warning. 1244 1245 // right now we are making the assumption that we will only write integers 1246 // on the header line. 1247 1248 valueListInt = static_cast< int* >( valueArray ); 1249 int ierr = readHeader( fileObject , 1250 keyphrase, 1251 valueListInt, 1252 *nItems ) ; 1253 1254 byte_order[ filePtr ] = Wrong_Endian ; 1255 1256 if ( ierr ) LastHeaderNotFound = true; 1257 1258 //return ; // don't return, go to the end to print perf 1259 } 1260 else { 1261 int* valueListInt; 1262 valueListInt = static_cast <int*>(valueArray); 1263 char* token = NULL; 1264 bool FOUND = false ; 1265 isBinary( iotype ); 1266 1267 MPI_Status read_offset_status; 1268 char read_out_tag[MAX_FIELDS_NAME_LENGTH]; 1269 memset(read_out_tag, '\0', MAX_FIELDS_NAME_LENGTH); 1270 char readouttag[MAX_FIELDS_NUMBER][MAX_FIELDS_NAME_LENGTH]; 1271 int j; 1272 1273 int string_length = strlen( keyphrase ); 1274 char* buffer = (char*) malloc ( string_length+1 ); 1275 1276 strcpy ( buffer, keyphrase ); 1277 buffer[ string_length ] = '\0'; 1278 1279 char* st2 = strtok ( buffer, "@" ); 1280 st2 = strtok (NULL, "@"); 1281 PhastaIOActiveFiles[i]->GPid = atoi(st2); 1282 if ( char* p = strpbrk(buffer, "@") ) 1283 *p = '\0'; 1284 1285 // Check if the user has input the right GPid 1286 if ( ( PhastaIOActiveFiles[i]->GPid <= 1287 PhastaIOActiveFiles[i]->myrank * 1288 PhastaIOActiveFiles[i]->nppp )|| 1289 ( PhastaIOActiveFiles[i]->GPid > 1290 ( PhastaIOActiveFiles[i]->myrank + 1 ) * 1291 PhastaIOActiveFiles[i]->nppp ) ) 1292 { 1293 *fileDescriptor = NOT_A_MPI_FILE; 1294 printf("Error readheader: The file is not in syncIO new format, please check! myrank = %d, GPid = %d, nppp = %d, keyphrase = %s\n", PhastaIOActiveFiles[i]->myrank, PhastaIOActiveFiles[i]->GPid, PhastaIOActiveFiles[i]->nppp, keyphrase); 1295 // It is possible atoi could not generate a clear integer from st2 because of additional garbage character in keyphrase 1296 endTimer(&timer_end); 1297 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1298 return; 1299 } 1300 1301 // Find the field we want ... 1302 //for ( j = 0; j<MAX_FIELDS_NUMBER; j++ ) 1303 for ( j = 0; j<PhastaIOActiveFiles[i]->nFields; j++ ) 1304 { 1305 memcpy( readouttag[j], 1306 PhastaIOActiveFiles[i]->master_header + j*MAX_FIELDS_NAME_LENGTH+MAX_FIELDS_NAME_LENGTH*2+1, 1307 MAX_FIELDS_NAME_LENGTH-1 ); 1308 } 1309 1310 for ( j = 0; j<PhastaIOActiveFiles[i]->nFields; j++ ) 1311 { 1312 token = strtok ( readouttag[j], ":" ); 1313 1314 //if ( cscompare( buffer, token ) ) 1315 if ( cscompare( token , buffer ) && cscompare( buffer, token ) ) 1316 // This double comparison is required for the field "number of nodes" and all the other fields that start with "number of nodes" (i.g. number of nodes in the mesh"). 1317 // Would be safer to rename "number of nodes" by "number of nodes in the part" so that the name are completely unique. But much more work to do that (Nspre, phParAdapt, etc). 1318 // Since the field name are unique in SyncIO (as it includes part ID), this should be safe and there should be no issue with the "?" trailing character. 1319 { 1320 PhastaIOActiveFiles[i]->read_field_count = j; 1321 FOUND = true; 1322 //printf("buffer: %s | token: %s | j: %d\n",buffer,token,j); 1323 break; 1324 } 1325 } 1326 free(buffer); 1327 1328 if (!FOUND) 1329 { 1330 //if(irank==0) printf("Warning readheader: Not found %s \n",keyphrase); //PhastaIOActiveFiles[i]->myrank is certainly initialized here. 1331 if(PhastaIOActiveFiles[i]->myrank == 0) printf("WARNING readheader: Not found %s\n",keyphrase); 1332 endTimer(&timer_end); 1333 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1334 return; 1335 } 1336 1337 // Find the part we want ... 1338 PhastaIOActiveFiles[i]->read_part_count = PhastaIOActiveFiles[i]->GPid - 1339 PhastaIOActiveFiles[i]->myrank * PhastaIOActiveFiles[i]->nppp - 1; 1340 1341 PhastaIOActiveFiles[i]->my_offset = 1342 PhastaIOActiveFiles[i]->my_read_table[PhastaIOActiveFiles[i]->read_field_count][PhastaIOActiveFiles[i]->read_part_count]; 1343 1344 // printf("****Rank %d offset is %d\n",PhastaIOActiveFiles[i]->myrank,PhastaIOActiveFiles[i]->my_offset); 1345 1346 // Read each datablock header here ... 1347 1348 MPI_File_read_at_all( PhastaIOActiveFiles[i]->file_handle, 1349 PhastaIOActiveFiles[i]->my_offset+1, 1350 read_out_tag, 1351 MAX_FIELDS_NAME_LENGTH-1, 1352 MPI_CHAR, 1353 &read_offset_status ); 1354 token = strtok ( read_out_tag, ":" ); 1355 1356 // printf("&&&&Rank %d read_out_tag is %s\n",PhastaIOActiveFiles[i]->myrank,read_out_tag); 1357 1358 if( cscompare( keyphrase , token ) ) //No need to compare also token with keyphrase like above. We should already have the right one. Otherwise there is a problem. 1359 { 1360 FOUND = true ; 1361 token = strtok( NULL, " ,;<>" ); 1362 for( j=0; j < *nItems && ( token = strtok( NULL," ,;<>") ); j++ ) 1363 valueListInt[j] = atoi( token ); 1364 1365 if ( j < *nItems ) 1366 { 1367 fprintf( stderr, "Expected # of ints not found for: %s\n", keyphrase ); 1368 } 1369 } 1370 else { 1371 //if(irank==0) 1372 if(PhastaIOActiveFiles[i]->myrank == 0) 1373 // If we enter this if, there is a problem with the name of some fields 1374 { 1375 printf("Error readheader: Unexpected mismatch between keyphrase = %s and token = %s\n",keyphrase,token); 1376 } 1377 } 1378 } 1379 1380 endTimer(&timer_end); 1381 printPerf("readheader", timer_start, timer_end, 0, 0, ""); 1382 1383 } 1384 1385 void readDataBlock( 1386 FILE* fileObject, 1387 void* valueArray, 1388 int nItems, 1389 const char datatype[], 1390 const char iotype[] ) 1391 { 1392 isBinary(iotype); 1393 size_t type_size = typeSize( datatype ); 1394 if ( binary_format ) { 1395 char junk = '\0'; 1396 fread( valueArray, type_size, nItems, fileObject ); 1397 fread( &junk, sizeof(char), 1 , fileObject ); 1398 if ( Wrong_Endian ) SwapArrayByteOrder( valueArray, type_size, nItems ); 1399 } else { 1400 char* ts1 = StringStripper( datatype ); 1401 if ( cscompare( "integer", ts1 ) ) { 1402 for( int n=0; n < nItems ; n++ ) 1403 fscanf(fileObject, "%d\n",(int*)((int*)valueArray+n) ); 1404 } else if ( cscompare( "double", ts1 ) ) { 1405 for( int n=0; n < nItems ; n++ ) 1406 fscanf(fileObject, "%lf\n",(double*)((double*)valueArray+n) ); 1407 } 1408 free (ts1); 1409 } 1410 } 1411 1412 void readdatablock( 1413 int* fileDescriptor, 1414 const char keyphrase[], 1415 void* valueArray, 1416 int* nItems, 1417 const char datatype[], 1418 const char iotype[] ) 1419 { 1420 //if(irank == 0) printf("entering readdatablock()\n"); 1421 unsigned long data_size = 0; 1422 double timer_start, timer_end; 1423 startTimer(&timer_start); 1424 1425 int i = *fileDescriptor; 1426 checkFileDescriptor("readdatablock",&i); 1427 1428 if ( PhastaIONextActiveIndex == 0 ) { 1429 int filePtr = *fileDescriptor - 1; 1430 FILE* fileObject; 1431 1432 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1433 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1434 fprintf(stderr,"openfile_ function has to be called before\n") ; 1435 fprintf(stderr,"acessing the file\n ") ; 1436 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1437 endTimer(&timer_end); 1438 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1439 return; 1440 } 1441 1442 // error check.. 1443 // since we require that a consistant header always preceed the data block 1444 // let us check to see that it is actually the case. 1445 1446 if ( ! cscompare( LastHeaderKey[ filePtr ].c_str(), keyphrase ) ) { 1447 fprintf(stderr, "Header not consistant with data block\n"); 1448 fprintf(stderr, "Header: %s\n", LastHeaderKey[ filePtr ].c_str() ); 1449 fprintf(stderr, "DataBlock: %s\n ", keyphrase ); 1450 fprintf(stderr, "Please recheck read sequence \n"); 1451 if( Strict_Error ) { 1452 fprintf(stderr, "fatal error: cannot continue, returning out of call\n"); 1453 endTimer(&timer_end); 1454 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1455 return; 1456 } 1457 } 1458 1459 if ( LastHeaderNotFound ) { 1460 endTimer(&timer_end); 1461 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1462 return; 1463 } 1464 fileObject = fileArray[ filePtr ]; 1465 Wrong_Endian = byte_order[ filePtr ]; 1466 LastHeaderKey.erase(filePtr); 1467 readDataBlock(fileObject,valueArray,*nItems,datatype,iotype); 1468 1469 //return; 1470 } 1471 else { 1472 // printf("read data block\n"); 1473 MPI_Status read_data_status; 1474 size_t type_size = typeSize( datatype ); 1475 int nUnits = *nItems; 1476 isBinary( iotype ); 1477 1478 // read datablock then 1479 //MR CHANGE 1480 // if ( cscompare ( datatype, "double")) 1481 char* ts2 = StringStripper( datatype ); 1482 if ( cscompare ( "double" , ts2)) 1483 //MR CHANGE END 1484 { 1485 1486 MPI_File_read_at_all_begin( PhastaIOActiveFiles[i]->file_handle, 1487 PhastaIOActiveFiles[i]->my_offset + DB_HEADER_SIZE, 1488 valueArray, 1489 nUnits, 1490 MPI_DOUBLE ); 1491 MPI_File_read_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1492 valueArray, 1493 &read_data_status ); 1494 data_size=8*nUnits; 1495 1496 } 1497 //MR CHANGE 1498 // else if ( cscompare ( datatype, "integer")) 1499 else if ( cscompare ( "integer" , ts2)) 1500 //MR CHANGE END 1501 { 1502 MPI_File_read_at_all_begin(PhastaIOActiveFiles[i]->file_handle, 1503 PhastaIOActiveFiles[i]->my_offset + DB_HEADER_SIZE, 1504 valueArray, 1505 nUnits, 1506 MPI_INT ); 1507 MPI_File_read_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1508 valueArray, 1509 &read_data_status ); 1510 data_size=4*nUnits; 1511 } 1512 else 1513 { 1514 *fileDescriptor = DATA_TYPE_ILLEGAL; 1515 printf("readdatablock - DATA_TYPE_ILLEGAL - %s\n",datatype); 1516 endTimer(&timer_end); 1517 printPerf("readdatablock", timer_start, timer_end, 0, 0, ""); 1518 return; 1519 } 1520 free(ts2); 1521 1522 1523 // printf("%d Read finishe\n",PhastaIOActiveFiles[i]->myrank); 1524 1525 // Swap data byte order if endianess is different ... 1526 if ( PhastaIOActiveFiles[i]->Wrong_Endian ) 1527 { 1528 SwapArrayByteOrder( valueArray, type_size, nUnits ); 1529 } 1530 } 1531 1532 endTimer(&timer_end); 1533 char extra_msg[1024]; 1534 memset(extra_msg, '\0', 1024); 1535 char* key = StringStripper(keyphrase); 1536 sprintf(extra_msg, " field is %s ", key); 1537 printPerf("readdatablock", timer_start, timer_end, data_size, 1, extra_msg); 1538 free(key); 1539 1540 } 1541 1542 void writeHeader( 1543 FILE* f, 1544 const char keyphrase[], 1545 const void* valueArray, 1546 const int nItems, 1547 const int ndataItems, 1548 const char datatype[], 1549 const char iotype[]) 1550 { 1551 isBinary( iotype ); 1552 1553 const int _newline = 1554 ( ndataItems > 0 ) ? sizeof( char ) : 0; 1555 int size_of_nextblock = 1556 ( binary_format ) ? typeSize(datatype) * ndataItems + _newline : ndataItems; 1557 1558 fprintf( f, "%s : < %d > ", keyphrase, size_of_nextblock ); 1559 for( int i = 0; i < nItems; i++ ) 1560 fprintf( f, "%d ", *((int*)((int*)valueArray+i))); 1561 fprintf( f, "\n"); 1562 } 1563 1564 void writeheader( 1565 const int* fileDescriptor, 1566 const char keyphrase[], 1567 const void* valueArray, 1568 const int* nItems, 1569 const int* ndataItems, 1570 const char datatype[], 1571 const char iotype[]) 1572 { 1573 1574 //if(irank == 0) printf("entering writeheader()\n"); 1575 1576 double timer_start, timer_end; 1577 startTimer(&timer_start); 1578 1579 int i = *fileDescriptor; 1580 checkFileDescriptor("writeheader",&i); 1581 1582 if ( PhastaIONextActiveIndex == 0 ) { 1583 int filePtr = *fileDescriptor - 1; 1584 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1585 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1586 fprintf(stderr,"openfile_ function has to be called before \n") ; 1587 fprintf(stderr,"acessing the file\n ") ; 1588 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1589 endTimer(&timer_end); 1590 printPerf("writeheader", timer_start, timer_end, 0, 0, ""); 1591 return; 1592 } 1593 1594 LastHeaderKey[filePtr] = std::string(keyphrase); 1595 DataSize = *ndataItems; 1596 FILE* fileObject = fileArray[ filePtr ] ; 1597 header_type[ filePtr ] = typeSize( datatype ); 1598 writeHeader(fileObject,keyphrase,valueArray,*nItems, 1599 *ndataItems,datatype,iotype); 1600 } 1601 else { // else it's parallel I/O 1602 DataSize = *ndataItems; 1603 size_t type_size = typeSize( datatype ); 1604 isBinary( iotype ); 1605 char mpi_tag[MAX_FIELDS_NAME_LENGTH]; 1606 1607 int string_length = strlen( keyphrase ); 1608 char* buffer = (char*) malloc ( string_length+1 ); 1609 1610 strcpy ( buffer, keyphrase); 1611 buffer[ string_length ] = '\0'; 1612 1613 char* st2 = strtok ( buffer, "@" ); 1614 st2 = strtok (NULL, "@"); 1615 PhastaIOActiveFiles[i]->GPid = atoi(st2); 1616 1617 if ( char* p = strpbrk(buffer, "@") ) 1618 *p = '\0'; 1619 1620 bzero((void*)mpi_tag,MAX_FIELDS_NAME_LENGTH); 1621 sprintf(mpi_tag, "\n%s : %d\n", buffer, PhastaIOActiveFiles[i]->field_count); 1622 unsigned long offset_value; 1623 1624 int temp = *ndataItems; 1625 unsigned long number_of_items = (unsigned long)temp; 1626 MPI_Barrier(PhastaIOActiveFiles[i]->local_comm); 1627 1628 MPI_Scan( &number_of_items, 1629 &offset_value, 1630 1, 1631 MPI_LONG_LONG_INT, 1632 MPI_SUM, 1633 PhastaIOActiveFiles[i]->local_comm ); 1634 1635 offset_value = (offset_value - number_of_items) * type_size; 1636 1637 offset_value += PhastaIOActiveFiles[i]->local_myrank * 1638 DB_HEADER_SIZE + 1639 PhastaIOActiveFiles[i]->next_start_address; 1640 // This offset is the starting address of each datablock header... 1641 PhastaIOActiveFiles[i]->my_offset = offset_value; 1642 1643 // Write in my offset table ... 1644 PhastaIOActiveFiles[i]->my_offset_table[PhastaIOActiveFiles[i]->field_count][PhastaIOActiveFiles[i]->part_count] = 1645 PhastaIOActiveFiles[i]->my_offset; 1646 1647 // Update the next-start-address ... 1648 PhastaIOActiveFiles[i]->next_start_address = offset_value + 1649 number_of_items * type_size + 1650 DB_HEADER_SIZE; 1651 MPI_Bcast( &(PhastaIOActiveFiles[i]->next_start_address), 1652 1, 1653 MPI_LONG_LONG_INT, 1654 PhastaIOActiveFiles[i]->local_numprocs-1, 1655 PhastaIOActiveFiles[i]->local_comm ); 1656 1657 // Prepare datablock header ... 1658 int _newline = (*ndataItems>0)?sizeof(char):0; 1659 unsigned int size_of_nextblock = type_size * (*ndataItems) + _newline; 1660 1661 //char datablock_header[255]; 1662 //bzero((void*)datablock_header,255); 1663 char datablock_header[DB_HEADER_SIZE]; 1664 bzero((void*)datablock_header,DB_HEADER_SIZE); 1665 1666 PhastaIOActiveFiles[i]->GPid = PhastaIOActiveFiles[i]->nppp*PhastaIOActiveFiles[i]->myrank+PhastaIOActiveFiles[i]->part_count; 1667 sprintf( datablock_header, 1668 "\n%s : < %u >", 1669 keyphrase, 1670 size_of_nextblock ); 1671 1672 for ( int j = 0; j < *nItems; j++ ) 1673 { 1674 sprintf( datablock_header, 1675 "%s %d ", 1676 datablock_header, 1677 *((int*)((int*)valueArray+j))); 1678 } 1679 sprintf( datablock_header, 1680 "%s\n ", 1681 datablock_header ); 1682 1683 // Write datablock header ... 1684 //MR CHANGE 1685 // if ( cscompare(datatype,"double") ) 1686 char* ts1 = StringStripper( datatype ); 1687 if ( cscompare("double",ts1) ) 1688 //MR CHANGE END 1689 { 1690 free ( PhastaIOActiveFiles[i]->double_chunk ); 1691 PhastaIOActiveFiles[i]->double_chunk = ( double * )malloc( (sizeof( double )*number_of_items+ DB_HEADER_SIZE)); 1692 1693 double * aa = ( double * )datablock_header; 1694 memcpy(PhastaIOActiveFiles[i]->double_chunk, aa, DB_HEADER_SIZE); 1695 } 1696 //MR CHANGE 1697 // if ( cscompare(datatype,"integer") ) 1698 else if ( cscompare("integer",ts1) ) 1699 //MR CHANGE END 1700 { 1701 free ( PhastaIOActiveFiles[i]->int_chunk ); 1702 PhastaIOActiveFiles[i]->int_chunk = ( int * )malloc( (sizeof( int )*number_of_items+ DB_HEADER_SIZE)); 1703 1704 int * aa = ( int * )datablock_header; 1705 memcpy(PhastaIOActiveFiles[i]->int_chunk, aa, DB_HEADER_SIZE); 1706 } 1707 else { 1708 // *fileDescriptor = DATA_TYPE_ILLEGAL; 1709 printf("writeheader - DATA_TYPE_ILLEGAL - %s\n",datatype); 1710 endTimer(&timer_end); 1711 printPerf("writeheader", timer_start, timer_end, 0, 0, ""); 1712 return; 1713 } 1714 free(ts1); 1715 1716 PhastaIOActiveFiles[i]->part_count++; 1717 if ( PhastaIOActiveFiles[i]->part_count == PhastaIOActiveFiles[i]->nppp ) { 1718 //A new field will be written 1719 if ( PhastaIOActiveFiles[i]->local_myrank == 0 ) { 1720 memcpy( PhastaIOActiveFiles[i]->master_header + 1721 PhastaIOActiveFiles[i]->field_count * 1722 MAX_FIELDS_NAME_LENGTH + 1723 MAX_FIELDS_NAME_LENGTH * 2, 1724 mpi_tag, 1725 MAX_FIELDS_NAME_LENGTH-1); 1726 } 1727 PhastaIOActiveFiles[i]->field_count++; 1728 PhastaIOActiveFiles[i]->part_count=0; 1729 } 1730 free(buffer); 1731 } 1732 1733 endTimer(&timer_end); 1734 printPerf("writeheader", timer_start, timer_end, 0, 0, ""); 1735 } 1736 1737 void writeDataBlock( 1738 FILE* f, 1739 const void* valueArray, 1740 const int nItems, 1741 const char datatype[], 1742 const char iotype[] ) 1743 { 1744 isBinary( iotype ); 1745 size_t type_size = typeSize( datatype ); 1746 if ( binary_format ) { 1747 fwrite( valueArray, type_size, nItems, f ); 1748 fprintf( f,"\n"); 1749 } else { 1750 char* ts1 = StringStripper( datatype ); 1751 if ( cscompare( "integer", ts1 ) ) { 1752 const int* vals = (int*) valueArray; 1753 for( int n=0; n < nItems ; n++ ) 1754 fprintf(f,"%d\n",vals[n]); 1755 } else if ( cscompare( "double", ts1 ) ) { 1756 const double* vals = (double*) valueArray; 1757 for( int n=0; n < nItems ; n++ ) 1758 fprintf(f,"%f\n",vals[n]); 1759 } 1760 free (ts1); 1761 } 1762 } 1763 1764 void writedatablock( 1765 const int* fileDescriptor, 1766 const char keyphrase[], 1767 const void* valueArray, 1768 const int* nItems, 1769 const char datatype[], 1770 const char iotype[] ) 1771 { 1772 //if(irank == 0) printf("entering writedatablock()\n"); 1773 1774 unsigned long data_size = 0; 1775 double timer_start, timer_end; 1776 startTimer(&timer_start); 1777 1778 int i = *fileDescriptor; 1779 checkFileDescriptor("writedatablock",&i); 1780 1781 if ( PhastaIONextActiveIndex == 0 ) { 1782 int filePtr = *fileDescriptor - 1; 1783 1784 if ( *fileDescriptor < 1 || *fileDescriptor > (int)fileArray.size() ) { 1785 fprintf(stderr,"No file associated with Descriptor %d\n",*fileDescriptor); 1786 fprintf(stderr,"openfile_ function has to be called before \n") ; 1787 fprintf(stderr,"acessing the file\n ") ; 1788 fprintf(stderr,"fatal error: cannot continue, returning out of call\n"); 1789 endTimer(&timer_end); 1790 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1791 return; 1792 } 1793 // since we require that a consistant header always preceed the data block 1794 // let us check to see that it is actually the case. 1795 1796 if ( ! cscompare( LastHeaderKey[ filePtr ].c_str(), keyphrase ) ) { 1797 fprintf(stderr, "Header not consistant with data block\n"); 1798 fprintf(stderr, "Header: %s\n", LastHeaderKey[ filePtr ].c_str() ); 1799 fprintf(stderr, "DataBlock: %s\n ", keyphrase ); 1800 fprintf(stderr, "Please recheck write sequence \n"); 1801 if( Strict_Error ) { 1802 fprintf(stderr, "fatal error: cannot continue, returning out of call\n"); 1803 endTimer(&timer_end); 1804 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1805 return; 1806 } 1807 } 1808 1809 FILE* fileObject = fileArray[ filePtr ] ; 1810 size_t type_size=typeSize( datatype ); 1811 isBinary( iotype ); 1812 1813 LastHeaderKey.erase(filePtr); 1814 1815 if ( header_type[filePtr] != (int)type_size ) { 1816 fprintf(stderr,"header and datablock differ on typeof data in the block for\n"); 1817 fprintf(stderr,"keyphrase : %s\n", keyphrase); 1818 if( Strict_Error ) { 1819 fprintf(stderr,"fatal error: cannot continue, returning out of call\n" ); 1820 endTimer(&timer_end); 1821 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1822 return; 1823 } 1824 } 1825 1826 int nUnits = *nItems; 1827 1828 if ( nUnits != DataSize ) { 1829 fprintf(stderr,"header and datablock differ on number of data items for\n"); 1830 fprintf(stderr,"keyphrase : %s\n", keyphrase); 1831 if( Strict_Error ) { 1832 fprintf(stderr,"fatal error: cannot continue, returning out of call\n" ); 1833 endTimer(&timer_end); 1834 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1835 return; 1836 } 1837 } 1838 writeDataBlock(fileObject,valueArray,*nItems,datatype,iotype); 1839 } 1840 else { // syncIO case 1841 MPI_Status write_data_status; 1842 isBinary( iotype ); 1843 int nUnits = *nItems; 1844 1845 //MR CHANGE 1846 // if ( cscompare(datatype,"double") ) 1847 char* ts1 = StringStripper( datatype ); 1848 if ( cscompare("double",ts1) ) 1849 //MR CHANGE END 1850 { 1851 memcpy((PhastaIOActiveFiles[i]->double_chunk+DB_HEADER_SIZE/sizeof(double)), valueArray, nUnits*sizeof(double)); 1852 MPI_File_write_at_all_begin( PhastaIOActiveFiles[i]->file_handle, 1853 PhastaIOActiveFiles[i]->my_offset, 1854 PhastaIOActiveFiles[i]->double_chunk, 1855 //BLOCK_SIZE/sizeof(double), 1856 nUnits+DB_HEADER_SIZE/sizeof(double), 1857 MPI_DOUBLE ); 1858 MPI_File_write_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1859 PhastaIOActiveFiles[i]->double_chunk, 1860 &write_data_status ); 1861 data_size=8*nUnits; 1862 } 1863 //MR CHANGE 1864 // else if ( cscompare ( datatype, "integer")) 1865 else if ( cscompare("integer",ts1) ) 1866 //MR CHANGE END 1867 { 1868 memcpy((PhastaIOActiveFiles[i]->int_chunk+DB_HEADER_SIZE/sizeof(int)), valueArray, nUnits*sizeof(int)); 1869 MPI_File_write_at_all_begin( PhastaIOActiveFiles[i]->file_handle, 1870 PhastaIOActiveFiles[i]->my_offset, 1871 PhastaIOActiveFiles[i]->int_chunk, 1872 nUnits+DB_HEADER_SIZE/sizeof(int), 1873 MPI_INT ); 1874 MPI_File_write_at_all_end( PhastaIOActiveFiles[i]->file_handle, 1875 PhastaIOActiveFiles[i]->int_chunk, 1876 &write_data_status ); 1877 data_size=4*nUnits; 1878 } 1879 else { 1880 printf("Error: writedatablock - DATA_TYPE_ILLEGAL - %s\n",datatype); 1881 endTimer(&timer_end); 1882 printPerf("writedatablock", timer_start, timer_end, 0, 0, ""); 1883 return; 1884 } 1885 free(ts1); 1886 } 1887 1888 endTimer(&timer_end); 1889 char extra_msg[1024]; 1890 memset(extra_msg, '\0', 1024); 1891 char* key = StringStripper(keyphrase); 1892 sprintf(extra_msg, " field is %s ", key); 1893 printPerf("writedatablock", timer_start, timer_end, data_size, 1, extra_msg); 1894 free(key); 1895 1896 } 1897 1898 void SwapArrayByteOrder( void* array, int nbytes, int nItems ) 1899 { 1900 /* This swaps the byte order for the array of nItems each 1901 of size nbytes , This will be called only locally */ 1902 int i,j; 1903 unsigned char* ucDst = (unsigned char*)array; 1904 for(i=0; i < nItems; i++) { 1905 for(j=0; j < (nbytes/2); j++) 1906 std::swap( ucDst[j] , ucDst[(nbytes - 1) - j] ); 1907 ucDst += nbytes; 1908 } 1909 } 1910 1911 void writestring( int* fileDescriptor, const char inString[] ) 1912 { 1913 int filePtr = *fileDescriptor - 1; 1914 FILE* fileObject = fileArray[filePtr] ; 1915 fprintf(fileObject,"%s",inString ); 1916 return; 1917 } 1918 1919 void Gather_Headers( int* fileDescriptor, vector< string >& headers ) 1920 { 1921 FILE* fileObject; 1922 char Line[1024]; 1923 1924 fileObject = fileArray[ (*fileDescriptor)-1 ]; 1925 1926 while( !feof(fileObject) ) { 1927 fgets( Line, 1024, fileObject); 1928 if ( Line[0] == '#' ) { 1929 headers.push_back( Line ); 1930 } else { 1931 break; 1932 } 1933 } 1934 rewind( fileObject ); 1935 clearerr( fileObject ); 1936 } 1937 1938 void isWrong( void ) { 1939 (Wrong_Endian) ? fprintf(stdout,"YES\n") : fprintf(stdout,"NO\n"); 1940 } 1941 1942 void togglestrictmode( void ) { Strict_Error = !Strict_Error; } 1943 1944 int isLittleEndian( void ) 1945 { 1946 // this function returns a 1 if the current running architecture is 1947 // LittleEndian Byte Ordered, else it returns a zero 1948 union { 1949 long a; 1950 char c[sizeof( long )]; 1951 } endianUnion; 1952 endianUnion.a = 1 ; 1953 if ( endianUnion.c[sizeof(long)-1] != 1 ) return 1 ; 1954 else return 0; 1955 } 1956 1957 namespace PHASTA { 1958 const char* const PhastaIO_traits<int>::type_string = "integer"; 1959 const char* const PhastaIO_traits<double>::type_string = "double"; 1960 } 1961