/* This file provides interface functions for 'partial ' random access into the PHASTA input files Anil Karanam March 2001 */ #include #include #include #include #include #include #include "mpi.h" #include "phastaIO.h" #include "rdtsc.h" #include #include "new_interface.h" #include "phIO.h" //MR CHANGE #include "common_c.h" //MR CHANGE END #ifdef intel #include #else #include #include #endif //extern double cpu_speed = 2600000000.0; //for Jaguar XT5 //extern double cpu_speed = 2100000000.0; //for Jaguar xt4 //extern double cpu_speed = 850000000.0; //for Intrepid void igetMinMaxAvg(int *ivalue, double *stats, int *statRanks) { int isThisRank; double *value = (double*)malloc(sizeof(double)); *value = 1.0*(*ivalue); rgetMinMaxAvg(value,stats,statRanks); /* MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); isThisRank=workfc.numpe+1; if(*value==stats[0]) isThisRank=workfc.myrank; MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); isThisRank=workfc.numpe+1; if(*value==stats[1]) isThisRank=workfc.myrank; MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); stats[2] /= workfc.numpe; */ free(value); } void rgetMinMaxAvg(double *value, double *stats, int *statRanks) { int isThisRank; MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); isThisRank=workfc.numpe+1; if(*value==stats[0]) isThisRank=workfc.myrank; MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); isThisRank=workfc.numpe+1; if(*value==stats[1]) isThisRank=workfc.myrank; MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); stats[2] /= workfc.numpe; double sqValue = (*value)*(*value), sqValueAvg = 0.; MPI_Allreduce(&sqValue,&sqValueAvg,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); sqValueAvg /= workfc.numpe; // stats[3] = sqValueAvg; stats[3] = sqrt(sqValueAvg-stats[2]*stats[2]); } void print_mesh_stats(void) { int statRanks[2]; double iStats[4], rStats[4]; igetMinMaxAvg(&conpar.nshg,iStats,statRanks); if(workfc.myrank==workfc.master) printf("nshg : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); igetMinMaxAvg(&conpar.numel,iStats,statRanks); if(workfc.myrank==workfc.master) printf("numel : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); igetMinMaxAvg(&conpar.numelb,iStats,statRanks); if(workfc.myrank==workfc.master) printf("numelb : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); igetMinMaxAvg(&conpar.nnz_tot,iStats,statRanks); if(workfc.myrank==workfc.master) { printf("nnz_tot : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); printf("\n"); } } void print_mpi_stats(void) { int statRanks[2]; double iStats[4], rStats[4]; // NS equations igetMinMaxAvg(&mpistats.iISend,iStats,statRanks); if(workfc.myrank==workfc.master) printf("iISend : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); igetMinMaxAvg(&mpistats.iIRecv,iStats,statRanks); if(workfc.myrank==workfc.master) printf("iIRecv : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); igetMinMaxAvg(&mpistats.iWaitAll,iStats,statRanks); if(workfc.myrank==workfc.master) printf("iWtAll : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); igetMinMaxAvg(&mpistats.iAllR,iStats,statRanks); if(workfc.myrank==workfc.master) printf("iAllR : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); rgetMinMaxAvg(&mpistats.rISend,rStats,statRanks); if(workfc.myrank==workfc.master) printf("rISend : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&mpistats.rIRecv,rStats,statRanks); if(workfc.myrank==workfc.master) printf("rIRecv : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&mpistats.rWaitAll,rStats,statRanks); if(workfc.myrank==workfc.master) printf("rWtAll : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&mpistats.rCommu,rStats,statRanks); if(workfc.myrank==workfc.master) printf("rCommu : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&mpistats.rAllR,rStats,statRanks); if(workfc.myrank==workfc.master) { printf("rAllR : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); printf("\n"); } // Scalars igetMinMaxAvg(&mpistats.iISendScal,iStats,statRanks); if(workfc.myrank==workfc.master) printf("iISendScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); igetMinMaxAvg(&mpistats.iIRecvScal,iStats,statRanks); if(workfc.myrank==workfc.master) printf("iIRecvScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); igetMinMaxAvg(&mpistats.iWaitAllScal,iStats,statRanks); if(workfc.myrank==workfc.master) printf("iWtAllScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); igetMinMaxAvg(&mpistats.iAllRScal,iStats,statRanks); if(workfc.myrank==workfc.master) printf("iAllRScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); rgetMinMaxAvg(&mpistats.rISendScal,rStats,statRanks); if(workfc.myrank==workfc.master) printf("rISendScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&mpistats.rIRecvScal,rStats,statRanks); if(workfc.myrank==workfc.master) printf("rIRecvScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&mpistats.rWaitAllScal,rStats,statRanks); if(workfc.myrank==workfc.master) printf("rWtAllScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&mpistats.rCommuScal,rStats,statRanks); if(workfc.myrank==workfc.master) printf("rCommuScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&mpistats.rAllRScal,rStats,statRanks); if(workfc.myrank==workfc.master) printf("rAllRScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); } //void print_system_stats(double tcorecp[2]) { void print_system_stats(double *tcorecp, double *tcorecpscal) { int statRanks[2]; double iStats[4], rStats[4]; double syst_assembly, syst_solve; // NS equations syst_assembly = tcorecp[0]; syst_solve = tcorecp[1]; rgetMinMaxAvg(&syst_assembly,rStats,statRanks); if(workfc.myrank==workfc.master) printf("Elm. form. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&syst_solve,rStats,statRanks); if(workfc.myrank==workfc.master) printf("Lin. alg. sol : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); // Scalars syst_assembly = tcorecpscal[0]; syst_solve = tcorecpscal[1]; rgetMinMaxAvg(&syst_assembly,rStats,statRanks); if(workfc.myrank==workfc.master) printf("Elm. form. Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); rgetMinMaxAvg(&syst_solve,rStats,statRanks); if(workfc.myrank==workfc.master) { printf("Lin. alg. sol Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); printf("\n"); } //printf("rank %d - syst_assembly %f - syst_solve %f\n",workfc.myrank,syst_assembly,syst_solve); } void countfieldstowriterestart() { int nfields; // printf("TEST: %d %d %d %d %d\n",timdat.istep,timdat.itseq,inpdat.nstep[0],inpdat.nstep[1],timdat.lstep); nfields = 2; //solution, time derivatives if(outpar.ivort == 1){ nfields++; //vorticity } if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { nfields++; //instantaneous wss in bflux.f } // if(ideformwall.eq.1) not handled yet if(timdat.istep == inpdat.nstep[timdat.itseq-1]){ //Last time step of the computation //projection vectors and pressure projection vectors (call saveLesRestart in itrdrv) nfields = nfields +2; //if Print Error Indicators = true (call write_error in itrdrv) if(turbvar.ierrcalc == 1){ nfields++; } //if Print ybar = True (call write_field(myrank,'a','ybar',4,... in itrdrv) if(outpar.ioybar == 1){ nfields++; //ybar //phase average fields if(outpar.nphasesincycle >0) { nfields = nfields + outpar.nphasesincycle; } if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { nfields++; //wssbar } } if(turbvari.irans < 0) { nfields++; //dwal } } outpar.nsynciofieldswriterestart = nfields; if(workfc.myrank == 0) { printf("Number of fields to write in restart files: %d\n", nfields); } } void Write_Restart( int* pid, int* stepno, int* nshg, int* numVars, double* array1, double* array2 ) { char fname[255]; char rfile[60]; char existingfile[30], linkfile[30]; int irstou; int magic_number = 362436; int* mptr = &magic_number; // time_t timenow = time ( &timenow); double version=0.0; int isize, nitems; int iarray[10]; /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); openfile_(rfile,"write", &irstou); // writing the top ascii header for the restart file writestring_( &irstou,"# PHASTA Input File Version 2.0\n"); writestring_( &irstou, "# format \"keyphrase : sizeofnextblock usual headers\"\n"); bzero( (void*)fname, 255 ); sprintf(fname,"# Output generated by phasta version (NOT YET CURRENT): %lf \n", version); writestring_( &irstou, fname ); bzero( (void*)fname, 255 ); gethostname(fname,255); writestring_( &irstou,"# This result was produced on: "); writestring_( &irstou, fname ); writestring_( &irstou,"\n"); bzero( (void*)fname, 255 ); sprintf(fname,"# %s\n", ctime( &timenow )); writestring_( &irstou, fname ); isize = 1; nitems = 1; iarray[ 0 ] = 1; writeheader_( &irstou, "byteorder magic number ", (void*)iarray, &nitems, &isize, "integer", phasta_iotype ); nitems = 1; writedatablock_( &irstou, "byteorder magic number ", (void*)mptr, &nitems, "integer", phasta_iotype ); bzero( (void*)fname, 255 ); sprintf(fname,"number of modes : < 0 > %d\n", *nshg); writestring_( &irstou, fname ); bzero( (void*)fname, 255 ); sprintf(fname,"number of variables : < 0 > %d\n", *numVars); writestring_( &irstou, fname ); isize = (*nshg)*(*numVars); nitems = 3; iarray[ 0 ] = (*nshg); iarray[ 1 ] = (*numVars); iarray[ 2 ] = (*stepno); writeheader_( &irstou, "solution ", (void*)iarray, &nitems, &isize, "double", phasta_iotype ); nitems = (*nshg)*(*numVars); writedatablock_( &irstou, "solution ", (void*)(array1), &nitems, "double", phasta_iotype ); nitems = 3; writeheader_( &irstou, "time derivative of solution ", (void*)iarray, &nitems, &isize, "double", phasta_iotype ); nitems = (*nshg)*(*numVars); writedatablock_( &irstou, "time derivative of solution ", (void*)(array2), &nitems, "double", phasta_iotype ); closefile_( &irstou, "write" ); */ //MPI_Barrier(MPI_COMM_WORLD); /////////////////////////////// Start of writing using new-lib //////////////////////////// //MR CHANGE int nfiles; int nfields; int numparts; int irank; int nprocs; // First, count the number of fields to write and store the result in countfieldstowriterestart(); // Retrieve and compute the parameters required for SyncIO nfiles = outpar.nsynciofiles; nfields = outpar.nsynciofieldswriterestart; numparts = workfc.numpe; irank = *pid; //workfc.myrank; nprocs = workfc.numpe; //MR CHANGE END int nppf = numparts/nfiles; int GPID; // Calculate number of parts each proc deal with and where it start and end ... int nppp = numparts/nprocs;// nppp : Number of parts per proc ... int startpart = irank * nppp +1;// Part id from which I (myrank) start ... int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... int descriptor; char filename[255],path[255]; bzero((void*)filename,255); phio_restartname(stepno, filename); phio_openfile_write(filename, &nfiles, &nfields, &nppf, &f_descriptor); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("Time: 'openfile' of %s with %d fields and %d files is: %f s\n",filename,nfields,nfiles,time_span); // printf("*****************************\n"); // } //MR CHANGE END field_flag=0; int i; for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor // GPID : global part id, corresponds to rank ... // e.g : (in this example) // proc 0 : 1--4 // proc 1 : 5--8 ... GPID = startpart + i; // Write solution field ... isize = (*nshg)*(*numVars); nitems = 3; iarray[ 0 ] = (*nshg); iarray[ 1 ] = (*numVars); iarray[ 2 ] = (*stepno); //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writeheader( &f_descriptor, "solution", (void*)iarray, &nitems, &isize, "double", phasta_iotype); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("\n*****************************\n"); // printf("Time: header for 'Solution': %f s\n",time_span); // } //MR CHANGE END nitems = (*nshg)*(*numVars); //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writedatablock(&f_descriptor, "solution", (void*)(array1), &isize, "double", phasta_iotype ); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // int isizemin,isizemax,isizetot; // double sizemin,sizemax,sizeavg,sizetot,rate; // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); // sizemin=(double)(8.0*isizemin/1024.0/1024.0); // sizemax=(double)(8.0*isizemax/1024.0/1024.0); // sizetot=(double)(8.0*isizetot/1024.0/1024.0); // sizeavg=(double)(1.0*sizetot/workfc.numpe); // rate=(double)(1.0*sizetot/time_span); // if (*pid==0) { // printf("Time: block for 'Solution': %f s\n",time_span); // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); // } //MR CHANGE END } field_flag++; for ( i = 0; i < nppp; i++) { // GPID : global part id, corresponds to rank ... // e.g : (in this example) // proc 0 : 1--4 // proc 1 : 5--8 ... GPID = startpart + i; // Write solution field ... isize = (*nshg)*(*numVars); nitems = 3; iarray[ 0 ] = (*nshg); iarray[ 1 ] = (*numVars); iarray[ 2 ] = (*stepno); //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writeheader( &f_descriptor, "time derivative of solution", (void*)iarray, &nitems, &isize, "double", phasta_iotype); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("Time: header for 'Time Derivative of solution': %f s\n",time_span); // } //MR CHANGE END nitems = (*nshg)*(*numVars); //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writedatablock( &f_descriptor, "time derivative of solution", (void*)(array2), &isize, "double", phasta_iotype ); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // int isizemin,isizemax,isizetot; // double sizemin,sizemax,sizeavg,sizetot,rate; // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); // sizemin=(double)(8.0*isizemin/1024.0/1024.0); // sizemax=(double)(8.0*isizemax/1024.0/1024.0); // sizetot=(double)(8.0*isizetot/1024.0/1024.0); // sizeavg=sizetot/workfc.numpe; // rate=sizetot/time_span; // if (*pid==0) { // printf("Time: block for 'Time Derivative of Solution': %f s\n",time_span); // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); // printf("*****************************\n"); // } //MR CHANGE END } field_flag++; if (field_flag==nfields){ //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("\n*****************************\n"); // printf("Time: 'closefile' is: %f s\n",time_span); // } //MR CHANGE END //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_closefile_write(&f_descriptor); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); if (*pid==0) { // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); printf("\n"); // printf("*****************************\n"); } } //MR CHANGE END /////////////////////////////////////////////////////////////////////////////////////////// /* create a soft link of the restart we just wrote to restart.latest this is the file the next run will always try to start from */ /* sprintf( linkfile, "restart.latest.%d", *pid+1 ); unlink( linkfile ); sprintf( existingfile, "restart.%d.%d", *stepno, *pid+1 ); link( existingfile, linkfile ); */ } void Write_Error( int* pid, int* stepno, int* nshg, int* numVars, double* array1 ) { char fname[255]; char rfile[60]; int irstou; int magic_number = 362436; int* mptr = &magic_number; //printf("Time is commented\n"); //time_t timenow = time ( &timenow); //printf("Yes\n"); double version=0.0; int isize, nitems; int iarray[10]; /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); openfile_(rfile,"append", &irstou); isize = (*nshg)*(*numVars); nitems = 3; iarray[ 0 ] = (*nshg); iarray[ 1 ] = (*numVars); iarray[ 2 ] = (*stepno); writeheader_( &irstou, "errors", (void*)iarray, &nitems, &isize, "double", phasta_iotype ); nitems = (*nshg)*(*numVars); writedatablock_( &irstou, "errors ", (void*)(array1), &nitems, "double", phasta_iotype ); closefile_( &irstou, "append" );*/ /////////////////////////////// Start of writing using new-lib //////////////////////////// int nfiles; int nfields; int numparts; int irank; int nprocs; // unsigned long long timer_start; // unsigned long long timer_end; // double time_span; nfiles = outpar.nsynciofiles; nfields = outpar.nsynciofieldswriterestart; numparts = workfc.numpe; irank = *pid; //workfc.myrank; nprocs = workfc.numpe; int nppf = numparts/nfiles; int GPID; // Calculate number of parts each proc deal with and where it start and end ... int nppp = numparts/nprocs;// nppp : Number of parts per proc ... int startpart = irank * nppp +1;// Part id from which I (myrank) start ... int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... field_flag++; int i; for ( i = 0; i < nppp; i++ ) { GPID = startpart + i; if(*pid==0) { // printf("\n*****************************\n"); printf("\n"); printf("The %d/%d th field to be written is 'errors'\n",field_flag,nfields); } isize = (*nshg)*(*numVars); nitems = 3; iarray[ 0 ] = (*nshg); iarray[ 1 ] = (*numVars); iarray[ 2 ] = (*stepno); //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writeheader( &f_descriptor, "errors", (void*)iarray, &nitems, &isize, "double", phasta_iotype); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("Time: header for 'error': %f s\n",time_span); // } //MR CHANGE END //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writedatablock( &f_descriptor, "errors", (void*)array1, &isize, "double", phasta_iotype ); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // int isizemin,isizemax,isizetot; // double sizemin,sizemax,sizeavg,sizetot,rate; // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); // sizemin=(double)(8.0*isizemin/1024.0/1024.0); // sizemax=(double)(8.0*isizemax/1024.0/1024.0); // sizetot=(double)(8.0*isizetot/1024.0/1024.0); // sizeavg=sizetot/workfc.numpe; // rate=sizetot/time_span; // if (*pid==0) { // printf("Time: block for 'error': %f s\n",time_span); // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); // printf("*****************************\n"); // } //MR CHANGE END } // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)(timer_end-timer_start)/cpu_speed; // if (*pid==0) { // printf("Field 'error' written in: %f s\n",time_span); // printf("Write field '%s' finished! \n",fieldtag); // } if (field_flag==nfields){ //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("\n*****************************\n"); // printf("Time: 'closefile' is: %f s\n",time_span); // } //MR CHANGE END //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_closefile_write(&f_descriptor); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); if (*pid==0) { // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); printf("Last field %d 'errors' finished! \n",nfields); printf("\n"); // printf("*****************************\n"); } } //MR CHANGE END /////////////////////////////////////////////////////////////////////////////////////////// } void Write_Displ( int* pid, int* stepno, int* nshg, int* numVars, double* array1 ) { fprintf(stderr, "This function is dead...exiting\n"); exit(1); } void Write_Field( int *pid, char* filemode, char* fieldtag, int* tagsize, void* array, char* arraytype, int* nshg, int* numvars, int* stepno) { //printf("Rank is %d, field is %s, tagsize is %d, nshg is %d, numvars is %d\n",*pid,fieldtag,*tagsize,*nshg,*numvars); // char rfile[32]; // assuming restart.sn.(pid+1) // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); strncpy(fieldlabel, fieldtag, *tagsize); fieldlabel[*tagsize] = '\0'; int irstou; int magic_number = 362436; int* mptr = &magic_number; double version=0.0; int isize, nitems; int iarray[10]; char fmode[10]; if(!strncmp(filemode,"w",1)) strcpy(fmode,"write"); else // default is append strcpy(fmode,"append"); char datatype[10]; if(!strncmp(arraytype,"i",1)) strcpy(datatype,"int"); else // default is double strcpy(datatype,"double"); /* openfile_(rfile, fmode, &irstou); nitems = 3; // assuming field will write 3 items in iarray iarray[ 0 ] = (*nshg); iarray[ 1 ] = (*numvars); iarray[ 2 ] = (*stepno); isize = (*nshg)*(*numvars); writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); nitems = (*nshg)*(*numvars); writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); closefile_( &irstou, fmode); */ /////////////////////////////// Start of writing using new-lib //////////////////////////// int nfiles; int nfields; int numparts; int irank; int nprocs; // unsigned long long timer_start; // unsigned long long timer_end; // double time_span; nfiles = outpar.nsynciofiles; nfields = outpar.nsynciofieldswriterestart; numparts = workfc.numpe; irank = *pid; //workfc.myrank; nprocs = workfc.numpe; int nppf = numparts/nfiles; int GPID; // Calculate number of parts each proc deal with and where it start and end ... int nppp = numparts/nprocs;// nppp : Number of parts per proc ... int startpart = irank * nppp +1;// Part id from which I (myrank) start ... int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... char filename[255],path[255]; bzero((void*)filename,255); strncpy(fieldlabel, fieldtag, *tagsize); field_flag++; if(*pid==0) { // printf("\n*****************************\n"); printf("\n"); printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); } sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); int i; for ( i = 0; i < nppp; i++ ) { GPID = startpart + i; // Write solution field ... isize = (*nshg)*(*numvars); nitems = 3; iarray[ 0 ] = (*nshg); iarray[ 1 ] = (*numvars); iarray[ 2 ] = (*stepno); //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writeheader( &f_descriptor, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); // } //MR CHANGE END nitems = (*nshg)*(*numvars); //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writedatablock( &f_descriptor, fieldlabel, array, &isize, datatype, phasta_iotype ); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // int isizemin,isizemax,isizetot; // double sizemin,sizemax,sizeavg,sizetot,rate; // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); // sizemin=(double)(8.0*isizemin/1024.0/1024.0); // sizemax=(double)(8.0*isizemax/1024.0/1024.0); // sizetot=(double)(8.0*isizetot/1024.0/1024.0); // sizeavg=sizetot/workfc.numpe; // rate=sizetot/time_span; // if (*pid==0) { // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); // printf("*****************************\n"); // } //MR CHANGE END } // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)(timer_end-timer_start)/cpu_speed; // if (*pid==0) { // printf("Field '%s' written in: %f s\n",fieldtag,time_span); // printf("Write field '%s' finished! \n",fieldtag_s); // } // if (field_flag==nfields){ // closefile_(&f_descriptor, "write"); // finalizephmpiio_(&f_descriptor); // if(*pid==0) { // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); // printf("\n*****************************\n"); // } // } if (field_flag==nfields){ //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("\n*****************************\n"); // printf("Time: 'closefile' is: %f s\n",time_span); // } //MR CHANGE END //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_closefile_write(&f_descriptor); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); if (*pid==0) { // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); printf("Last field %d '%s' finished! \n",nfields, fieldtag); printf("\n"); // printf("*****************************\n"); } } //MR CHANGE END /////////////////////////////////////////////////////////////////////////////////////////// free(fieldlabel); } //MR CHANGE //MR CHANGE void Write_PhAvg2( int* pid, char* filemode, char* fieldtag, int* tagsize, int* iphase, int* nphasesincycle, void* array, char* arraytype, int* nshg, int* numvars, int* stepno) { // char rfile[32]; // assuming restart.sn.(pid+1) // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); int addtagsize=0; // phase number is added to the name of the field if(*iphase<10) addtagsize=1; else if(*iphase<100) addtagsize=2; else if(*iphase<1000) addtagsize=3; int tagsize2; tagsize2=*tagsize+addtagsize; // char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); // strncpy(fieldlabel, fieldtag, *tagsize); // fieldlabel[*tagsize] = '\0'; char *fieldlabel = (char *)malloc((tagsize2+1)*sizeof(char)); strncpy(fieldlabel, fieldtag, *tagsize); fieldlabel[tagsize2] = '\0'; char straddtagsize[10]; sprintf(straddtagsize,"%d",*iphase); if(*iphase<10) { fieldlabel[tagsize2-1]=straddtagsize[0]; } else if(*iphase<100) { fieldlabel[tagsize2-2]=straddtagsize[0]; fieldlabel[tagsize2-1]=straddtagsize[1]; } else if(*iphase<1000) { fieldlabel[tagsize2-3]=straddtagsize[0]; fieldlabel[tagsize2-2]=straddtagsize[1]; fieldlabel[tagsize2-1]=straddtagsize[2]; } int irstou; int magic_number = 362436; int* mptr = &magic_number; double version=0.0; int isize, nitems; int iarray[10]; char fmode[10]; if(!strncmp(filemode,"w",1)) strcpy(fmode,"write"); else // default is append strcpy(fmode,"append"); char datatype[10]; if(!strncmp(arraytype,"i",1)) strcpy(datatype,"int"); else // default is double strcpy(datatype,"double"); // // // if(*iphase==1) //open the file but then keep it open for the remaining cycles // openfile_(rfile, fmode, &irstou); // // // printf("iphase: %d - pid: %d - irstou %d\n",*iphase,*pid,irstou); // // // nitems = 3; // assuming field will write 3 items in iarray // iarray[ 0 ] = (*nshg); // iarray[ 1 ] = (*numvars); // iarray[ 2 ] = (*stepno); // // isize = (*nshg)*(*numvars); // writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); // // nitems = (*nshg)*(*numvars); // writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); // // // if(*iphase==*nphasesincycle) //close the file after nphasesincycle // closefile_( &irstou, fmode); // /////////////////////////////// Start of writing using new-lib //////////////////////////// int nfiles; int nfields; int numparts; int irank; int nprocs; // unsigned long long timer_start; // unsigned long long timer_end; // double time_span; nfiles = outpar.nsynciofiles; nfields = outpar.nsynciofieldswriterestart; numparts = workfc.numpe; irank = *pid; //workfc.myrank; nprocs = workfc.numpe; int nppf = numparts/nfiles; int GPID; // Calculate number of parts each proc deal with and where it start and end ... int nppp = numparts/nprocs;// nppp : Number of parts per proc ... int startpart = irank * nppp +1;// Part id from which I (myrank) start ... int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... //int descriptor; char filename[255],path[255]; bzero((void*)filename,255); // char * namer; // namer = strtok(fieldlabel," "); // strncpy(fieldlabel, fieldtag, *tagsize); field_flag++; if(*pid==0) { // printf("\n*****************************\n"); printf("\n"); printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); } sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); int i; for ( i = 0; i < nppp; i++ ) { GPID = startpart + i; // Write solution field ... //printf("This is %d and fieldtag_s is %s \n",myrank,fieldtag_s); isize = (*nshg)*(*numvars); nitems = 3; iarray[ 0 ] = (*nshg); iarray[ 1 ] = (*numvars); iarray[ 2 ] = (*stepno); //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writeheader( &f_descriptor, fieldlabel, (void*)iarray, &nitems, &isize, "double", phasta_iotype); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); // } //MR CHANGE END nitems = (*nshg)*(*numvars); //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_writedatablock( &f_descriptor, fieldlabel, array, &isize, "double", phasta_iotype ); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // int isizemin,isizemax,isizetot; // double sizemin,sizemax,sizeavg,sizetot,rate; // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); // sizemin=(double)(8.0*isizemin/1024.0/1024.0); // sizemax=(double)(8.0*isizemax/1024.0/1024.0); // sizetot=(double)(8.0*isizetot/1024.0/1024.0); // sizeavg=sizetot/workfc.numpe; // rate=sizetot/time_span; // if (*pid==0) { // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); // printf("*****************************\n"); // } //MR CHANGE END } // if (*pid==0) { // printf("Field '%s' written in: %f s\n",fieldtag,time_span); // printf("Write field '%s' finished! \n",fieldtag_s); // } // // if (field_flag==nfields){ // closefile_(&f_descriptor, "write"); // finalizephmpiio_(&f_descriptor); // if(*pid==0) { // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); // printf("\n*****************************\n"); // } // } if (field_flag==nfields){ //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); // if (*pid==0) { // printf("\n*****************************\n"); // printf("Time: 'closefile' is: %f s\n",time_span); // } //MR CHANGE END //MR CHANGE // Measure the time - Start the timer // MPI_Barrier(MPI_COMM_WORLD); // timer_start = rdtsc(); //MR CHANGE END phio_closefile_write(&f_descriptor); //MR CHANGE // Measure the time - End of timer // MPI_Barrier(MPI_COMM_WORLD); // timer_end = rdtsc(); // time_span=(double)((timer_end-timer_start)/cpu_speed); if (*pid==0) { // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); // printf("Last field %d '%s' finished! \n",nfields, fieldtag); printf("\n"); // printf("*****************************\n"); } } //MR CHANGE END /////////////////////////////////////////////////////////////////////////////////////////// free(fieldlabel); } void Write_d2wall( int* pid, int* numnp, double* array1 ) { // time_t timenow = time ( &timenow); int isize, nitems; int iarray[10]; // MPI_Barrier(MPI_COMM_WORLD); /////////////////////////////// Start of writing using new-lib //////////////////////////// int nfiles; int nfields; int numparts; int irank; int nprocs; // First, count the number of fields to write and store the result in //countfieldstowriterestart(); // Retrieve and compute the parameters required for SyncIO nfiles = outpar.nsynciofiles; nfields = 1; //outpar.nsynciofieldswriterestart; // Only the distance to the walls in d2wall numparts = workfc.numpe; irank = *pid; //workfc.myrank; nprocs = workfc.numpe; int nppf = numparts/nfiles; int GPID; // Calculate number of parts each proc deal with and where it start and end ... int nppp = numparts/nprocs;// nppp : Number of parts per proc ... int startpart = irank * nppp +1;// Part id from which I (myrank) start ... int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... int descriptor; char filename[255],path[255]; bzero((void*)filename,255); phio_openfile_write("d2wall.", &nfiles, &nfields, &nppf, &f_descriptor); field_flag=0; int i; for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor // GPID : global part id, corresponds to rank ... // e.g : (in this example) // proc 0 : 1--4 // proc 1 : 5--8 ... GPID = startpart + i; // Write solution field ... isize = (*numnp); nitems = 2; iarray[ 0 ] = (*numnp); iarray[ 1 ] = 1; //numVars = 1 phio_writeheader( &f_descriptor, "d2wall", (void*)iarray, &nitems, &isize, "double", phasta_iotype); //nitems = (*nshg)*(*numVars); //nitems = (*numnp); phio_writedatablock( &f_descriptor, "d2wall", (void*)(array1), &isize, "double", phasta_iotype ); } field_flag++; if (field_flag==nfields){ phio_closefile_write(&f_descriptor); if (irank==0) { printf("\n"); } } }