147c6ae99SBarry Smith #define PETSCDM_DLL 247c6ae99SBarry Smith 347c6ae99SBarry Smith /* 447c6ae99SBarry Smith Code for manipulating distributed regular arrays in parallel. 547c6ae99SBarry Smith */ 647c6ae99SBarry Smith 747c6ae99SBarry Smith #include "private/daimpl.h" /*I "petscda.h" I*/ 847c6ae99SBarry Smith 947c6ae99SBarry Smith #undef __FUNCT__ 109a42bb27SBarry Smith #define __FUNCT__ "DMGlobalToLocalBegin_DA" 119a42bb27SBarry Smith PetscErrorCode PETSCDM_DLLEXPORT DMGlobalToLocalBegin_DA(DM da,Vec g,InsertMode mode,Vec l) 1247c6ae99SBarry Smith { 1347c6ae99SBarry Smith PetscErrorCode ierr; 1447c6ae99SBarry Smith DM_DA *dd = (DM_DA*)da->data; 1547c6ae99SBarry Smith 1647c6ae99SBarry Smith PetscFunctionBegin; 1747c6ae99SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 1847c6ae99SBarry Smith PetscValidHeaderSpecific(g,VEC_CLASSID,2); 1947c6ae99SBarry Smith PetscValidHeaderSpecific(l,VEC_CLASSID,4); 2047c6ae99SBarry Smith ierr = VecScatterBegin(dd->gtol,g,l,mode,SCATTER_FORWARD);CHKERRQ(ierr); 2147c6ae99SBarry Smith PetscFunctionReturn(0); 2247c6ae99SBarry Smith } 2347c6ae99SBarry Smith 2447c6ae99SBarry Smith 2547c6ae99SBarry Smith #undef __FUNCT__ 269a42bb27SBarry Smith #define __FUNCT__ "DMGlobalToLocalEnd_DA" 279a42bb27SBarry Smith PetscErrorCode PETSCDM_DLLEXPORT DMGlobalToLocalEnd_DA(DM da,Vec g,InsertMode mode,Vec l) 2847c6ae99SBarry Smith { 2947c6ae99SBarry Smith PetscErrorCode ierr; 3047c6ae99SBarry Smith DM_DA *dd = (DM_DA*)da->data; 3147c6ae99SBarry Smith 3247c6ae99SBarry Smith PetscFunctionBegin; 3347c6ae99SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 3447c6ae99SBarry Smith PetscValidHeaderSpecific(g,VEC_CLASSID,2); 3547c6ae99SBarry Smith PetscValidHeaderSpecific(l,VEC_CLASSID,4); 3647c6ae99SBarry Smith ierr = VecScatterEnd(dd->gtol,g,l,mode,SCATTER_FORWARD);CHKERRQ(ierr); 3747c6ae99SBarry Smith PetscFunctionReturn(0); 3847c6ae99SBarry Smith } 3947c6ae99SBarry Smith 409a42bb27SBarry Smith #undef __FUNCT__ 419a42bb27SBarry Smith #define __FUNCT__ "DMLocalToGlobalBegin_DA" 429a42bb27SBarry Smith PetscErrorCode PETSCDM_DLLEXPORT DMLocalToGlobalBegin_DA(DM da,Vec l,InsertMode mode,Vec g) 439a42bb27SBarry Smith { 449a42bb27SBarry Smith PetscErrorCode ierr; 459a42bb27SBarry Smith DM_DA *dd = (DM_DA*)da->data; 469a42bb27SBarry Smith 479a42bb27SBarry Smith PetscFunctionBegin; 489a42bb27SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 499a42bb27SBarry Smith PetscValidHeaderSpecific(l,VEC_CLASSID,2); 509a42bb27SBarry Smith PetscValidHeaderSpecific(g,VEC_CLASSID,3); 519a42bb27SBarry Smith if (mode == ADD_VALUES) { 529a42bb27SBarry Smith ierr = VecScatterBegin(dd->gtol,l,g,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); 539a42bb27SBarry Smith } else if (mode == INSERT_VALUES) { 549a42bb27SBarry Smith ierr = VecScatterBegin(dd->ltog,l,g,mode,SCATTER_FORWARD);CHKERRQ(ierr); 559a42bb27SBarry Smith } else SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"Not yet implemented"); 569a42bb27SBarry Smith PetscFunctionReturn(0); 579a42bb27SBarry Smith } 589a42bb27SBarry Smith 599a42bb27SBarry Smith #undef __FUNCT__ 609a42bb27SBarry Smith #define __FUNCT__ "DMLocalToGlobalEnd_DA" 619a42bb27SBarry Smith PetscErrorCode PETSCDM_DLLEXPORT DMLocalToGlobalEnd_DA(DM da,Vec l,InsertMode mode,Vec g) 629a42bb27SBarry Smith { 639a42bb27SBarry Smith PetscErrorCode ierr; 649a42bb27SBarry Smith DM_DA *dd = (DM_DA*)da->data; 659a42bb27SBarry Smith 669a42bb27SBarry Smith PetscFunctionBegin; 679a42bb27SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 689a42bb27SBarry Smith PetscValidHeaderSpecific(l,VEC_CLASSID,2); 699a42bb27SBarry Smith PetscValidHeaderSpecific(g,VEC_CLASSID,3); 709a42bb27SBarry Smith if (mode == ADD_VALUES) { 719a42bb27SBarry Smith ierr = VecScatterEnd(dd->gtol,l,g,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); 729a42bb27SBarry Smith } else if (mode == INSERT_VALUES) { 739a42bb27SBarry Smith ierr = VecScatterEnd(dd->ltog,l,g,mode,SCATTER_FORWARD);CHKERRQ(ierr); 749a42bb27SBarry Smith } else SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"Not yet implemented"); 759a42bb27SBarry Smith PetscFunctionReturn(0); 769a42bb27SBarry Smith } 779a42bb27SBarry Smith 78*aa219208SBarry Smith EXTERN PetscErrorCode DMDAGetNatural_Private(DM,PetscInt*,IS*); 7947c6ae99SBarry Smith #undef __FUNCT__ 80*aa219208SBarry Smith #define __FUNCT__ "DMDAGlobalToNatural_Create" 8147c6ae99SBarry Smith /* 82*aa219208SBarry Smith DMDAGlobalToNatural_Create - Create the global to natural scatter object 8347c6ae99SBarry Smith 84*aa219208SBarry Smith Collective on DMDA 8547c6ae99SBarry Smith 8647c6ae99SBarry Smith Input Parameter: 8747c6ae99SBarry Smith . da - the distributed array context 8847c6ae99SBarry Smith 8947c6ae99SBarry Smith Level: developer 9047c6ae99SBarry Smith 91*aa219208SBarry Smith Notes: This is an internal routine called by DMDAGlobalToNatural() to 9247c6ae99SBarry Smith create the scatter context. 9347c6ae99SBarry Smith 9447c6ae99SBarry Smith .keywords: distributed array, global to local, begin 9547c6ae99SBarry Smith 96*aa219208SBarry Smith .seealso: DMDAGlobalToNaturalBegin(), DMDAGlobalToNaturalEnd(), DMLocalToGlobalBegin(), DMDACreate2d(), 97*aa219208SBarry Smith DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector() 9847c6ae99SBarry Smith */ 99*aa219208SBarry Smith PetscErrorCode DMDAGlobalToNatural_Create(DM da) 10047c6ae99SBarry Smith { 10147c6ae99SBarry Smith PetscErrorCode ierr; 10247c6ae99SBarry Smith PetscInt m,start,Nlocal; 10347c6ae99SBarry Smith IS from,to; 10447c6ae99SBarry Smith Vec global; 10547c6ae99SBarry Smith DM_DA *dd = (DM_DA*)da->data; 10647c6ae99SBarry Smith 10747c6ae99SBarry Smith PetscFunctionBegin; 10847c6ae99SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 10947c6ae99SBarry Smith if (!dd->natural) { 11047c6ae99SBarry Smith SETERRQ(((PetscObject)da)->comm,PETSC_ERR_ORDER,"Natural layout vector not yet created; cannot scatter into it"); 11147c6ae99SBarry Smith } 11247c6ae99SBarry Smith 11347c6ae99SBarry Smith /* create the scatter context */ 11447c6ae99SBarry Smith ierr = VecGetLocalSize(dd->natural,&m);CHKERRQ(ierr); 11547c6ae99SBarry Smith ierr = VecGetOwnershipRange(dd->natural,&start,PETSC_NULL);CHKERRQ(ierr); 11647c6ae99SBarry Smith 117*aa219208SBarry Smith ierr = DMDAGetNatural_Private(da,&Nlocal,&to);CHKERRQ(ierr); 11847c6ae99SBarry Smith if (Nlocal != m) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Internal error: Nlocal %D local vector size %D",Nlocal,m); 11947c6ae99SBarry Smith ierr = ISCreateStride(((PetscObject)da)->comm,m,start,1,&from);CHKERRQ(ierr); 12047c6ae99SBarry Smith ierr = VecCreateMPIWithArray(((PetscObject)da)->comm,dd->Nlocal,PETSC_DETERMINE,0,&global); 12147c6ae99SBarry Smith ierr = VecSetBlockSize(global,dd->w);CHKERRQ(ierr); 12247c6ae99SBarry Smith ierr = VecScatterCreate(global,from,dd->natural,to,&dd->gton);CHKERRQ(ierr); 12347c6ae99SBarry Smith ierr = VecDestroy(global);CHKERRQ(ierr); 12447c6ae99SBarry Smith ierr = ISDestroy(from);CHKERRQ(ierr); 12547c6ae99SBarry Smith ierr = ISDestroy(to);CHKERRQ(ierr); 12647c6ae99SBarry Smith PetscFunctionReturn(0); 12747c6ae99SBarry Smith } 12847c6ae99SBarry Smith 12947c6ae99SBarry Smith #undef __FUNCT__ 130*aa219208SBarry Smith #define __FUNCT__ "DMDAGlobalToNaturalBegin" 13147c6ae99SBarry Smith /*@ 132*aa219208SBarry Smith DMDAGlobalToNaturalBegin - Maps values from the global vector to a global vector 13347c6ae99SBarry Smith in the "natural" grid ordering. Must be followed by 134*aa219208SBarry Smith DMDAGlobalToNaturalEnd() to complete the exchange. 13547c6ae99SBarry Smith 136*aa219208SBarry Smith Neighbor-wise Collective on DMDA 13747c6ae99SBarry Smith 13847c6ae99SBarry Smith Input Parameters: 13947c6ae99SBarry Smith + da - the distributed array context 14047c6ae99SBarry Smith . g - the global vector 14147c6ae99SBarry Smith - mode - one of INSERT_VALUES or ADD_VALUES 14247c6ae99SBarry Smith 14347c6ae99SBarry Smith Output Parameter: 14447c6ae99SBarry Smith . l - the natural ordering values 14547c6ae99SBarry Smith 14647c6ae99SBarry Smith Level: advanced 14747c6ae99SBarry Smith 14847c6ae99SBarry Smith Notes: 14947c6ae99SBarry Smith The global and natrual vectors used here need not be the same as those 150*aa219208SBarry Smith obtained from DMCreateGlobalVector() and DMDACreateNaturalVector(), BUT they 15147c6ae99SBarry Smith must have the same parallel data layout; they could, for example, be 152*aa219208SBarry Smith obtained with VecDuplicate() from the DMDA originating vectors. 15347c6ae99SBarry Smith 154*aa219208SBarry Smith You must call DMDACreateNaturalVector() before using this routine 15547c6ae99SBarry Smith 15647c6ae99SBarry Smith .keywords: distributed array, global to local, begin 15747c6ae99SBarry Smith 158*aa219208SBarry Smith .seealso: DMDAGlobalToNaturalEnd(), DMLocalToGlobalBegin(), DMDACreate2d(), 159*aa219208SBarry Smith DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector() 1609a42bb27SBarry Smith 16147c6ae99SBarry Smith @*/ 162*aa219208SBarry Smith PetscErrorCode PETSCDM_DLLEXPORT DMDAGlobalToNaturalBegin(DM da,Vec g,InsertMode mode,Vec l) 16347c6ae99SBarry Smith { 16447c6ae99SBarry Smith PetscErrorCode ierr; 16547c6ae99SBarry Smith DM_DA *dd = (DM_DA*)da->data; 16647c6ae99SBarry Smith 16747c6ae99SBarry Smith PetscFunctionBegin; 16847c6ae99SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 16947c6ae99SBarry Smith PetscValidHeaderSpecific(l,VEC_CLASSID,2); 17047c6ae99SBarry Smith PetscValidHeaderSpecific(g,VEC_CLASSID,4); 17147c6ae99SBarry Smith if (!dd->gton) { 17247c6ae99SBarry Smith /* create the scatter context */ 173*aa219208SBarry Smith ierr = DMDAGlobalToNatural_Create(da);CHKERRQ(ierr); 17447c6ae99SBarry Smith } 17547c6ae99SBarry Smith ierr = VecScatterBegin(dd->gton,g,l,mode,SCATTER_FORWARD);CHKERRQ(ierr); 17647c6ae99SBarry Smith PetscFunctionReturn(0); 17747c6ae99SBarry Smith } 17847c6ae99SBarry Smith 17947c6ae99SBarry Smith #undef __FUNCT__ 180*aa219208SBarry Smith #define __FUNCT__ "DMDAGlobalToNaturalEnd" 18147c6ae99SBarry Smith /*@ 182*aa219208SBarry Smith DMDAGlobalToNaturalEnd - Maps values from the global vector to a global vector 183*aa219208SBarry Smith in the natural ordering. Must be preceeded by DMDAGlobalToNaturalBegin(). 18447c6ae99SBarry Smith 185*aa219208SBarry Smith Neighbor-wise Collective on DMDA 18647c6ae99SBarry Smith 18747c6ae99SBarry Smith Input Parameters: 18847c6ae99SBarry Smith + da - the distributed array context 18947c6ae99SBarry Smith . g - the global vector 19047c6ae99SBarry Smith - mode - one of INSERT_VALUES or ADD_VALUES 19147c6ae99SBarry Smith 19247c6ae99SBarry Smith Output Parameter: 19347c6ae99SBarry Smith . l - the global values in the natural ordering 19447c6ae99SBarry Smith 19547c6ae99SBarry Smith Level: advanced 19647c6ae99SBarry Smith 19747c6ae99SBarry Smith Notes: 19847c6ae99SBarry Smith The global and local vectors used here need not be the same as those 199*aa219208SBarry Smith obtained from DMCreateGlobalVector() and DMDACreateNaturalVector(), BUT they 20047c6ae99SBarry Smith must have the same parallel data layout; they could, for example, be 201*aa219208SBarry Smith obtained with VecDuplicate() from the DMDA originating vectors. 20247c6ae99SBarry Smith 20347c6ae99SBarry Smith .keywords: distributed array, global to local, end 20447c6ae99SBarry Smith 205*aa219208SBarry Smith .seealso: DMDAGlobalToNaturalBegin(), DMLocalToGlobalBegin(), DMDACreate2d(), 206*aa219208SBarry Smith DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector() 2079a42bb27SBarry Smith 20847c6ae99SBarry Smith @*/ 209*aa219208SBarry Smith PetscErrorCode PETSCDM_DLLEXPORT DMDAGlobalToNaturalEnd(DM da,Vec g,InsertMode mode,Vec l) 21047c6ae99SBarry Smith { 21147c6ae99SBarry Smith PetscErrorCode ierr; 21247c6ae99SBarry Smith DM_DA *dd = (DM_DA*)da->data; 21347c6ae99SBarry Smith 21447c6ae99SBarry Smith PetscFunctionBegin; 21547c6ae99SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 21647c6ae99SBarry Smith PetscValidHeaderSpecific(l,VEC_CLASSID,2); 21747c6ae99SBarry Smith PetscValidHeaderSpecific(g,VEC_CLASSID,4); 21847c6ae99SBarry Smith ierr = VecScatterEnd(dd->gton,g,l,mode,SCATTER_FORWARD);CHKERRQ(ierr); 21947c6ae99SBarry Smith PetscFunctionReturn(0); 22047c6ae99SBarry Smith } 22147c6ae99SBarry Smith 22247c6ae99SBarry Smith #undef __FUNCT__ 223*aa219208SBarry Smith #define __FUNCT__ "DMDANaturalToGlobalBegin" 22447c6ae99SBarry Smith /*@ 225*aa219208SBarry Smith DMDANaturalToGlobalBegin - Maps values from a global vector in the "natural" ordering 226*aa219208SBarry Smith to a global vector in the PETSc DMDA grid ordering. Must be followed by 227*aa219208SBarry Smith DMDANaturalToGlobalEnd() to complete the exchange. 22847c6ae99SBarry Smith 229*aa219208SBarry Smith Neighbor-wise Collective on DMDA 23047c6ae99SBarry Smith 23147c6ae99SBarry Smith Input Parameters: 23247c6ae99SBarry Smith + da - the distributed array context 23347c6ae99SBarry Smith . g - the global vector in a natural ordering 23447c6ae99SBarry Smith - mode - one of INSERT_VALUES or ADD_VALUES 23547c6ae99SBarry Smith 23647c6ae99SBarry Smith Output Parameter: 237*aa219208SBarry Smith . l - the values in the DMDA ordering 23847c6ae99SBarry Smith 23947c6ae99SBarry Smith Level: advanced 24047c6ae99SBarry Smith 24147c6ae99SBarry Smith Notes: 24247c6ae99SBarry Smith The global and natural vectors used here need not be the same as those 243*aa219208SBarry Smith obtained from DMCreateGlobalVector() and DMDACreateNaturalVector(), BUT they 24447c6ae99SBarry Smith must have the same parallel data layout; they could, for example, be 245*aa219208SBarry Smith obtained with VecDuplicate() from the DMDA originating vectors. 24647c6ae99SBarry Smith 24747c6ae99SBarry Smith .keywords: distributed array, global to local, begin 24847c6ae99SBarry Smith 249*aa219208SBarry Smith .seealso: DMDAGlobalToNaturalEnd(), DMDAGlobalToNaturalBegin(), DMLocalToGlobalBegin(), DMDACreate2d(), 250*aa219208SBarry Smith DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector() 25147c6ae99SBarry Smith 25247c6ae99SBarry Smith @*/ 253*aa219208SBarry Smith PetscErrorCode PETSCDM_DLLEXPORT DMDANaturalToGlobalBegin(DM da,Vec g,InsertMode mode,Vec l) 25447c6ae99SBarry Smith { 25547c6ae99SBarry Smith PetscErrorCode ierr; 25647c6ae99SBarry Smith DM_DA *dd = (DM_DA*)da->data; 25747c6ae99SBarry Smith 25847c6ae99SBarry Smith PetscFunctionBegin; 25947c6ae99SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 26047c6ae99SBarry Smith PetscValidHeaderSpecific(l,VEC_CLASSID,2); 26147c6ae99SBarry Smith PetscValidHeaderSpecific(g,VEC_CLASSID,4); 26247c6ae99SBarry Smith if (!dd->gton) { 26347c6ae99SBarry Smith /* create the scatter context */ 264*aa219208SBarry Smith ierr = DMDAGlobalToNatural_Create(da);CHKERRQ(ierr); 26547c6ae99SBarry Smith } 26647c6ae99SBarry Smith ierr = VecScatterBegin(dd->gton,g,l,mode,SCATTER_REVERSE);CHKERRQ(ierr); 26747c6ae99SBarry Smith PetscFunctionReturn(0); 26847c6ae99SBarry Smith } 26947c6ae99SBarry Smith 27047c6ae99SBarry Smith #undef __FUNCT__ 271*aa219208SBarry Smith #define __FUNCT__ "DMDANaturalToGlobalEnd" 27247c6ae99SBarry Smith /*@ 273*aa219208SBarry Smith DMDANaturalToGlobalEnd - Maps values from the natural ordering global vector 274*aa219208SBarry Smith to a global vector in the PETSc DMDA ordering. Must be preceeded by DMDANaturalToGlobalBegin(). 27547c6ae99SBarry Smith 276*aa219208SBarry Smith Neighbor-wise Collective on DMDA 27747c6ae99SBarry Smith 27847c6ae99SBarry Smith Input Parameters: 27947c6ae99SBarry Smith + da - the distributed array context 28047c6ae99SBarry Smith . g - the global vector in a natural ordering 28147c6ae99SBarry Smith - mode - one of INSERT_VALUES or ADD_VALUES 28247c6ae99SBarry Smith 28347c6ae99SBarry Smith Output Parameter: 284*aa219208SBarry Smith . l - the global values in the PETSc DMDA ordering 28547c6ae99SBarry Smith 28647c6ae99SBarry Smith Level: intermediate 28747c6ae99SBarry Smith 28847c6ae99SBarry Smith Notes: 28947c6ae99SBarry Smith The global and local vectors used here need not be the same as those 290*aa219208SBarry Smith obtained from DMCreateGlobalVector() and DMDACreateNaturalVector(), BUT they 29147c6ae99SBarry Smith must have the same parallel data layout; they could, for example, be 292*aa219208SBarry Smith obtained with VecDuplicate() from the DMDA originating vectors. 29347c6ae99SBarry Smith 29447c6ae99SBarry Smith .keywords: distributed array, global to local, end 29547c6ae99SBarry Smith 296*aa219208SBarry Smith .seealso: DMDAGlobalToNaturalBegin(), DMDAGlobalToNaturalEnd(), DMLocalToGlobalBegin(), DMDACreate2d(), 297*aa219208SBarry Smith DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector() 29847c6ae99SBarry Smith 29947c6ae99SBarry Smith @*/ 300*aa219208SBarry Smith PetscErrorCode PETSCDM_DLLEXPORT DMDANaturalToGlobalEnd(DM da,Vec g,InsertMode mode,Vec l) 30147c6ae99SBarry Smith { 30247c6ae99SBarry Smith PetscErrorCode ierr; 30347c6ae99SBarry Smith DM_DA *dd = (DM_DA*)da->data; 30447c6ae99SBarry Smith 30547c6ae99SBarry Smith PetscFunctionBegin; 30647c6ae99SBarry Smith PetscValidHeaderSpecific(da,DM_CLASSID,1); 30747c6ae99SBarry Smith PetscValidHeaderSpecific(l,VEC_CLASSID,2); 30847c6ae99SBarry Smith PetscValidHeaderSpecific(g,VEC_CLASSID,4); 30947c6ae99SBarry Smith ierr = VecScatterEnd(dd->gton,g,l,mode,SCATTER_REVERSE);CHKERRQ(ierr); 31047c6ae99SBarry Smith PetscFunctionReturn(0); 31147c6ae99SBarry Smith } 31247c6ae99SBarry Smith 313