147c6ae99SBarry Smith 247c6ae99SBarry Smith /* 347c6ae99SBarry Smith Code for manipulating distributed regular arrays in parallel. 447c6ae99SBarry Smith */ 547c6ae99SBarry Smith 6b45d2f2cSJed Brown #include <petsc-private/daimpl.h> /*I "petscdmda.h" I*/ 747c6ae99SBarry Smith 847c6ae99SBarry Smith #if defined(PETSC_HAVE_MATLAB_ENGINE) 9c6db04a5SJed Brown #include <mat.h> /* MATLAB include file */ 1047c6ae99SBarry Smith 1147c6ae99SBarry Smith #undef __FUNCT__ 129a42bb27SBarry Smith #define __FUNCT__ "DMView_DA_Matlab" 139a42bb27SBarry Smith PetscErrorCode DMView_DA_Matlab(DM da,PetscViewer viewer) 1447c6ae99SBarry Smith { 1547c6ae99SBarry Smith PetscErrorCode ierr; 1647c6ae99SBarry Smith PetscMPIInt rank; 1747c6ae99SBarry Smith PetscInt dim,m,n,p,dof,swidth; 18aa219208SBarry Smith DMDAStencilType stencil; 191321219cSEthan Coon DMDABoundaryType bx,by,bz; 2047c6ae99SBarry Smith mxArray *mx; 211321219cSEthan Coon const char *fnames[] = {"dimension","m","n","p","dof","stencil_width","bx","by","bz","stencil_type"}; 2247c6ae99SBarry Smith 2347c6ae99SBarry Smith PetscFunctionBegin; 2447c6ae99SBarry Smith ierr = MPI_Comm_rank(((PetscObject)da)->comm,&rank);CHKERRQ(ierr); 2547c6ae99SBarry Smith if (!rank) { 261321219cSEthan Coon ierr = DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&dof,&swidth,&bx,&by,&bz,&stencil);CHKERRQ(ierr); 2747c6ae99SBarry Smith mx = mxCreateStructMatrix(1,1,8,(const char **)fnames); 28e3c5b3baSBarry Smith if (!mx) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Unable to generate MATLAB struct array to hold DMDA informations"); 2947c6ae99SBarry Smith mxSetFieldByNumber(mx,0,0,mxCreateDoubleScalar((double)dim)); 3047c6ae99SBarry Smith mxSetFieldByNumber(mx,0,1,mxCreateDoubleScalar((double)m)); 3147c6ae99SBarry Smith mxSetFieldByNumber(mx,0,2,mxCreateDoubleScalar((double)n)); 3247c6ae99SBarry Smith mxSetFieldByNumber(mx,0,3,mxCreateDoubleScalar((double)p)); 3347c6ae99SBarry Smith mxSetFieldByNumber(mx,0,4,mxCreateDoubleScalar((double)dof)); 3447c6ae99SBarry Smith mxSetFieldByNumber(mx,0,5,mxCreateDoubleScalar((double)swidth)); 351321219cSEthan Coon mxSetFieldByNumber(mx,0,6,mxCreateDoubleScalar((double)bx)); 361321219cSEthan Coon mxSetFieldByNumber(mx,0,7,mxCreateDoubleScalar((double)by)); 371321219cSEthan Coon mxSetFieldByNumber(mx,0,8,mxCreateDoubleScalar((double)bz)); 381321219cSEthan Coon mxSetFieldByNumber(mx,0,9,mxCreateDoubleScalar((double)stencil)); 3947c6ae99SBarry Smith ierr = PetscObjectName((PetscObject)da);CHKERRQ(ierr); 4047c6ae99SBarry Smith ierr = PetscViewerMatlabPutVariable(viewer,((PetscObject)da)->name,mx);CHKERRQ(ierr); 4147c6ae99SBarry Smith } 4247c6ae99SBarry Smith PetscFunctionReturn(0); 4347c6ae99SBarry Smith } 4447c6ae99SBarry Smith #endif 4547c6ae99SBarry Smith 4647c6ae99SBarry Smith #undef __FUNCT__ 479a42bb27SBarry Smith #define __FUNCT__ "DMView_DA_Binary" 489a42bb27SBarry Smith PetscErrorCode DMView_DA_Binary(DM da,PetscViewer viewer) 4947c6ae99SBarry Smith { 5047c6ae99SBarry Smith PetscErrorCode ierr; 5147c6ae99SBarry Smith PetscMPIInt rank; 52b859378eSBarry Smith PetscInt dim,m,n,p,dof,swidth,M,N,P; 53aa219208SBarry Smith DMDAStencilType stencil; 541321219cSEthan Coon DMDABoundaryType bx,by,bz; 5547c6ae99SBarry Smith MPI_Comm comm; 56bc2bf880SBarry Smith PetscBool coors = PETSC_FALSE; 5747c6ae99SBarry Smith 5847c6ae99SBarry Smith PetscFunctionBegin; 5947c6ae99SBarry Smith ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr); 6047c6ae99SBarry Smith 611321219cSEthan Coon ierr = DMDAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&dof,&swidth,&bx,&by,&bz,&stencil);CHKERRQ(ierr); 6247c6ae99SBarry Smith ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 6347c6ae99SBarry Smith if (!rank) { 64b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&dim,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); 65b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&m,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); 66b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&n,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); 67b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&p,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); 68b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&dof,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); 69b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&swidth,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); 70b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&bx,1,PETSC_ENUM,PETSC_FALSE);CHKERRQ(ierr); 71b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&by,1,PETSC_ENUM,PETSC_FALSE);CHKERRQ(ierr); 72b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&bz,1,PETSC_ENUM,PETSC_FALSE);CHKERRQ(ierr); 73b859378eSBarry Smith ierr = PetscViewerBinaryWrite(viewer,&stencil,1,PETSC_ENUM,PETSC_FALSE);CHKERRQ(ierr); 746636e97aSMatthew G Knepley if (da->coordinates) coors = PETSC_TRUE; 75bc2bf880SBarry Smith ierr = PetscViewerBinaryWrite(viewer,&coors,1,PETSC_BOOL,PETSC_FALSE);CHKERRQ(ierr); 7647c6ae99SBarry Smith } 7747c6ae99SBarry Smith 7847c6ae99SBarry Smith /* save the coordinates if they exist to disk (in the natural ordering) */ 796636e97aSMatthew G Knepley if (da->coordinates) { 806636e97aSMatthew G Knepley ierr = VecView(da->coordinates,viewer);CHKERRQ(ierr); 8147c6ae99SBarry Smith } 8247c6ae99SBarry Smith PetscFunctionReturn(0); 8347c6ae99SBarry Smith } 8447c6ae99SBarry Smith 8547c6ae99SBarry Smith #undef __FUNCT__ 869a42bb27SBarry Smith #define __FUNCT__ "DMView_DA_VTK" 879a42bb27SBarry Smith PetscErrorCode DMView_DA_VTK(DM da, PetscViewer viewer) 8847c6ae99SBarry Smith { 8947c6ae99SBarry Smith PetscInt dim, dof, M = 0, N = 0, P = 0; 9047c6ae99SBarry Smith PetscErrorCode ierr; 9147c6ae99SBarry Smith 9247c6ae99SBarry Smith PetscFunctionBegin; 931321219cSEthan Coon ierr = DMDAGetInfo(da, &dim, &M, &N, &P, PETSC_NULL, PETSC_NULL, PETSC_NULL, &dof, PETSC_NULL, PETSC_NULL, PETSC_NULL, PETSC_NULL, PETSC_NULL);CHKERRQ(ierr); 94aa219208SBarry Smith /* if (dim != 3) {SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP, "VTK output only works for three dimensional DMDAs.");} */ 956636e97aSMatthew G Knepley if (!da->coordinates) SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP, "VTK output requires DMDA coordinates."); 9647c6ae99SBarry Smith /* Write Header */ 9747c6ae99SBarry Smith ierr = PetscViewerASCIIPrintf(viewer,"# vtk DataFile Version 2.0\n");CHKERRQ(ierr); 9847c6ae99SBarry Smith ierr = PetscViewerASCIIPrintf(viewer,"Structured Mesh Example\n");CHKERRQ(ierr); 9947c6ae99SBarry Smith ierr = PetscViewerASCIIPrintf(viewer,"ASCII\n");CHKERRQ(ierr); 10047c6ae99SBarry Smith ierr = PetscViewerASCIIPrintf(viewer,"DATASET STRUCTURED_GRID\n");CHKERRQ(ierr); 10147c6ae99SBarry Smith ierr = PetscViewerASCIIPrintf(viewer,"DIMENSIONS %d %d %d\n", M, N, P);CHKERRQ(ierr); 10247c6ae99SBarry Smith ierr = PetscViewerASCIIPrintf(viewer,"POINTS %d double\n", M*N*P);CHKERRQ(ierr); 1036636e97aSMatthew G Knepley if (da->coordinates) { 1049a42bb27SBarry Smith DM dac; 10547c6ae99SBarry Smith Vec natural; 10647c6ae99SBarry Smith 1076636e97aSMatthew G Knepley ierr = DMGetCoordinateDM(da, &dac);CHKERRQ(ierr); 108aa219208SBarry Smith ierr = DMDACreateNaturalVector(dac, &natural);CHKERRQ(ierr); 10947c6ae99SBarry Smith ierr = PetscObjectSetOptionsPrefix((PetscObject) natural, "coor_");CHKERRQ(ierr); 1106636e97aSMatthew G Knepley ierr = DMDAGlobalToNaturalBegin(dac, da->coordinates, INSERT_VALUES, natural);CHKERRQ(ierr); 1116636e97aSMatthew G Knepley ierr = DMDAGlobalToNaturalEnd(dac, da->coordinates, INSERT_VALUES, natural);CHKERRQ(ierr); 11247c6ae99SBarry Smith ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_ASCII_VTK_COORDS);CHKERRQ(ierr); 11347c6ae99SBarry Smith ierr = VecView(natural, viewer);CHKERRQ(ierr); 11447c6ae99SBarry Smith ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr); 115fcfd50ebSBarry Smith ierr = VecDestroy(&natural);CHKERRQ(ierr); 11647c6ae99SBarry Smith } 11747c6ae99SBarry Smith PetscFunctionReturn(0); 11847c6ae99SBarry Smith } 11947c6ae99SBarry Smith 12047c6ae99SBarry Smith #undef __FUNCT__ 121aa219208SBarry Smith #define __FUNCT__ "DMDAGetInfo" 12247c6ae99SBarry Smith /*@C 123aa219208SBarry Smith DMDAGetInfo - Gets information about a given distributed array. 12447c6ae99SBarry Smith 12547c6ae99SBarry Smith Not Collective 12647c6ae99SBarry Smith 12747c6ae99SBarry Smith Input Parameter: 12847c6ae99SBarry Smith . da - the distributed array 12947c6ae99SBarry Smith 13047c6ae99SBarry Smith Output Parameters: 13147c6ae99SBarry Smith + dim - dimension of the distributed array (1, 2, or 3) 13247c6ae99SBarry Smith . M, N, P - global dimension in each direction of the array 13347c6ae99SBarry Smith . m, n, p - corresponding number of procs in each dimension 13447c6ae99SBarry Smith . dof - number of degrees of freedom per node 13547c6ae99SBarry Smith . s - stencil width 1361321219cSEthan Coon . bx,by,bz - type of ghost nodes at boundary, one of DMDA_BOUNDARY_NONE, DMDA_BOUNDARY_GHOSTED, 1371321219cSEthan Coon DMDA_BOUNDARY_MIRROR, DMDA_BOUNDARY_PERIODIC 138aa219208SBarry Smith - st - stencil type, either DMDA_STENCIL_STAR or DMDA_STENCIL_BOX 13947c6ae99SBarry Smith 14047c6ae99SBarry Smith Level: beginner 14147c6ae99SBarry Smith 14247c6ae99SBarry Smith Note: 14347c6ae99SBarry Smith Use PETSC_NULL (PETSC_NULL_INTEGER in Fortran) in place of any output parameter that is not of interest. 14447c6ae99SBarry Smith 14547c6ae99SBarry Smith .keywords: distributed array, get, information 14647c6ae99SBarry Smith 147aa219208SBarry Smith .seealso: DMView(), DMDAGetCorners(), DMDAGetLocalInfo() 14847c6ae99SBarry Smith @*/ 1491321219cSEthan Coon PetscErrorCode DMDAGetInfo(DM da,PetscInt *dim,PetscInt *M,PetscInt *N,PetscInt *P,PetscInt *m,PetscInt *n,PetscInt *p,PetscInt *dof,PetscInt *s,DMDABoundaryType *bx,DMDABoundaryType *by,DMDABoundaryType *bz,DMDAStencilType *st) 15047c6ae99SBarry Smith { 15147c6ae99SBarry Smith DM_DA *dd = (DM_DA*)da->data; 15247c6ae99SBarry Smith 15347c6ae99SBarry Smith PetscFunctionBegin; 15447c6ae99SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 15547c6ae99SBarry Smith if (dim) *dim = dd->dim; 15647c6ae99SBarry Smith if (M) *M = dd->M; 15747c6ae99SBarry Smith if (N) *N = dd->N; 15847c6ae99SBarry Smith if (P) *P = dd->P; 15947c6ae99SBarry Smith if (m) *m = dd->m; 16047c6ae99SBarry Smith if (n) *n = dd->n; 16147c6ae99SBarry Smith if (p) *p = dd->p; 16247c6ae99SBarry Smith if (dof) *dof = dd->w; 16347c6ae99SBarry Smith if (s) *s = dd->s; 1641321219cSEthan Coon if (bx) *bx = dd->bx; 1651321219cSEthan Coon if (by) *by = dd->by; 1661321219cSEthan Coon if (bz) *bz = dd->bz; 16747c6ae99SBarry Smith if (st) *st = dd->stencil_type; 16847c6ae99SBarry Smith PetscFunctionReturn(0); 16947c6ae99SBarry Smith } 17047c6ae99SBarry Smith 17147c6ae99SBarry Smith #undef __FUNCT__ 172aa219208SBarry Smith #define __FUNCT__ "DMDAGetLocalInfo" 17347c6ae99SBarry Smith /*@C 174aa219208SBarry Smith DMDAGetLocalInfo - Gets information about a given distributed array and this processors location in it 17547c6ae99SBarry Smith 17647c6ae99SBarry Smith Not Collective 17747c6ae99SBarry Smith 17847c6ae99SBarry Smith Input Parameter: 17947c6ae99SBarry Smith . da - the distributed array 18047c6ae99SBarry Smith 18147c6ae99SBarry Smith Output Parameters: 18247c6ae99SBarry Smith . dainfo - structure containing the information 18347c6ae99SBarry Smith 18447c6ae99SBarry Smith Level: beginner 18547c6ae99SBarry Smith 18647c6ae99SBarry Smith .keywords: distributed array, get, information 18747c6ae99SBarry Smith 188aa219208SBarry Smith .seealso: DMDAGetInfo(), DMDAGetCorners() 18947c6ae99SBarry Smith @*/ 1907087cfbeSBarry Smith PetscErrorCode DMDAGetLocalInfo(DM da,DMDALocalInfo *info) 19147c6ae99SBarry Smith { 19247c6ae99SBarry Smith PetscInt w; 19347c6ae99SBarry Smith DM_DA *dd = (DM_DA*)da->data; 19447c6ae99SBarry Smith 19547c6ae99SBarry Smith PetscFunctionBegin; 19647c6ae99SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 19747c6ae99SBarry Smith PetscValidPointer(info,2); 19847c6ae99SBarry Smith info->da = da; 19947c6ae99SBarry Smith info->dim = dd->dim; 20047c6ae99SBarry Smith info->mx = dd->M; 20147c6ae99SBarry Smith info->my = dd->N; 20247c6ae99SBarry Smith info->mz = dd->P; 20347c6ae99SBarry Smith info->dof = dd->w; 20447c6ae99SBarry Smith info->sw = dd->s; 2051321219cSEthan Coon info->bx = dd->bx; 2061321219cSEthan Coon info->by = dd->by; 2071321219cSEthan Coon info->bz = dd->bz; 20847c6ae99SBarry Smith info->st = dd->stencil_type; 20947c6ae99SBarry Smith 21047c6ae99SBarry Smith /* since the xs, xe ... have all been multiplied by the number of degrees 21147c6ae99SBarry Smith of freedom per cell, w = dd->w, we divide that out before returning.*/ 21247c6ae99SBarry Smith w = dd->w; 213*d886c4f4SPeter Brune info->xs = dd->xs/w + dd->xo; 21447c6ae99SBarry Smith info->xm = (dd->xe - dd->xs)/w; 21547c6ae99SBarry Smith /* the y and z have NOT been multiplied by w */ 216*d886c4f4SPeter Brune info->ys = dd->ys + dd->yo; 21747c6ae99SBarry Smith info->ym = (dd->ye - dd->ys); 218*d886c4f4SPeter Brune info->zs = dd->zs + dd->zo; 21947c6ae99SBarry Smith info->zm = (dd->ze - dd->zs); 22047c6ae99SBarry Smith 221*d886c4f4SPeter Brune info->gxs = dd->Xs/w + dd->xo; 22247c6ae99SBarry Smith info->gxm = (dd->Xe - dd->Xs)/w; 22347c6ae99SBarry Smith /* the y and z have NOT been multiplied by w */ 224*d886c4f4SPeter Brune info->gys = dd->Ys + dd->yo; 22547c6ae99SBarry Smith info->gym = (dd->Ye - dd->Ys); 226*d886c4f4SPeter Brune info->gzs = dd->Zs + dd->zo; 22747c6ae99SBarry Smith info->gzm = (dd->Ze - dd->Zs); 22847c6ae99SBarry Smith PetscFunctionReturn(0); 22947c6ae99SBarry Smith } 23047c6ae99SBarry Smith 2317aaf2d21SPeter Brune #undef __FUNCT__ 2327aaf2d21SPeter Brune #define __FUNCT__ "DMDAGetLocalBlockInfo" 2337aaf2d21SPeter Brune /*@C 2347aaf2d21SPeter Brune DMDAGetLocalBlockInfo - Gets information about a given distributed array and this processors location in it with overlap taken into account 2357aaf2d21SPeter Brune 2367aaf2d21SPeter Brune Not Collective 2377aaf2d21SPeter Brune 2387aaf2d21SPeter Brune Input Parameter: 2397aaf2d21SPeter Brune . da - the distributed array 2407aaf2d21SPeter Brune 2417aaf2d21SPeter Brune Output Parameters: 2427aaf2d21SPeter Brune . dainfo - structure containing the information 2437aaf2d21SPeter Brune 2447aaf2d21SPeter Brune Level: beginner 2457aaf2d21SPeter Brune 2467aaf2d21SPeter Brune .keywords: distributed array, get, information 2477aaf2d21SPeter Brune 2487aaf2d21SPeter Brune .seealso: DMDAGetLocalInfo(), DMDASetOverlap() 2497aaf2d21SPeter Brune @*/ 2507aaf2d21SPeter Brune PetscErrorCode DMDAGetLocalBlockInfo(DM da,DMDALocalInfo *info) 2517aaf2d21SPeter Brune { 2527aaf2d21SPeter Brune PetscErrorCode ierr; 2537aaf2d21SPeter Brune DM_DA *dd = (DM_DA*)da->data; 2547aaf2d21SPeter Brune PetscFunctionBegin; 2557aaf2d21SPeter Brune ierr = DMDAGetLocalInfo(da,info);CHKERRQ(ierr); 2567aaf2d21SPeter Brune 2577aaf2d21SPeter Brune if (dd->overlap > 0) { 2587aaf2d21SPeter Brune if (info->xs - dd->overlap > 0 || info->bx == DMDA_BOUNDARY_PERIODIC) { 2597aaf2d21SPeter Brune info->xs -= dd->overlap; 2607aaf2d21SPeter Brune info->xm += dd->overlap; 2617aaf2d21SPeter Brune } 2627aaf2d21SPeter Brune if (info->xs + info->xm + dd->overlap < info->mx || info->bx == DMDA_BOUNDARY_PERIODIC) { 2637aaf2d21SPeter Brune info->xm += dd->overlap; 2647aaf2d21SPeter Brune } 2657aaf2d21SPeter Brune if (info->ys - dd->overlap > 0 || info->by == DMDA_BOUNDARY_PERIODIC) { 2667aaf2d21SPeter Brune info->ys -= dd->overlap; 2677aaf2d21SPeter Brune info->ym += dd->overlap; 2687aaf2d21SPeter Brune } 2697aaf2d21SPeter Brune if (info->ys + info->ym + dd->overlap < info->my || info->by == DMDA_BOUNDARY_PERIODIC) { 2707aaf2d21SPeter Brune info->ym += dd->overlap; 2717aaf2d21SPeter Brune } 2727aaf2d21SPeter Brune if (info->zs - dd->overlap > 0 || info->bz == DMDA_BOUNDARY_PERIODIC) { 2737aaf2d21SPeter Brune info->zs -= dd->overlap; 2747aaf2d21SPeter Brune info->zm += dd->overlap; 2757aaf2d21SPeter Brune } 2767aaf2d21SPeter Brune if (info->zs + info->zm + dd->overlap < info->mz || info->bz == DMDA_BOUNDARY_PERIODIC) { 2777aaf2d21SPeter Brune info->zm += dd->overlap; 2787aaf2d21SPeter Brune } 2797aaf2d21SPeter Brune } 2807aaf2d21SPeter Brune PetscFunctionReturn(0); 2817aaf2d21SPeter Brune } 282