10a96aa3bSJed Brown #include <petscds.h> 20a96aa3bSJed Brown #include <petsc/private/dmimpl.h> 30a96aa3bSJed Brown #include <petsc/private/dmforestimpl.h> 40a96aa3bSJed Brown #include <petsc/private/dmpleximpl.h> 50a96aa3bSJed Brown #include <petsc/private/dmlabelimpl.h> 60a96aa3bSJed Brown #include <petsc/private/viewerimpl.h> 70a96aa3bSJed Brown #include <../src/sys/classes/viewer/impls/vtk/vtkvimpl.h> 80a96aa3bSJed Brown #include "petsc_p4est_package.h" 90a96aa3bSJed Brown 100a96aa3bSJed Brown #if defined(PETSC_HAVE_P4EST) 110a96aa3bSJed Brown 120a96aa3bSJed Brown #if !defined(P4_TO_P8) 130a96aa3bSJed Brown #include <p4est.h> 140a96aa3bSJed Brown #include <p4est_extended.h> 150a96aa3bSJed Brown #include <p4est_geometry.h> 160a96aa3bSJed Brown #include <p4est_ghost.h> 170a96aa3bSJed Brown #include <p4est_lnodes.h> 180a96aa3bSJed Brown #include <p4est_vtk.h> 190a96aa3bSJed Brown #include <p4est_plex.h> 200a96aa3bSJed Brown #include <p4est_bits.h> 210a96aa3bSJed Brown #include <p4est_algorithms.h> 220a96aa3bSJed Brown #else 230a96aa3bSJed Brown #include <p8est.h> 240a96aa3bSJed Brown #include <p8est_extended.h> 250a96aa3bSJed Brown #include <p8est_geometry.h> 260a96aa3bSJed Brown #include <p8est_ghost.h> 270a96aa3bSJed Brown #include <p8est_lnodes.h> 280a96aa3bSJed Brown #include <p8est_vtk.h> 290a96aa3bSJed Brown #include <p8est_plex.h> 300a96aa3bSJed Brown #include <p8est_bits.h> 310a96aa3bSJed Brown #include <p8est_algorithms.h> 320a96aa3bSJed Brown #endif 330a96aa3bSJed Brown 340a96aa3bSJed Brown typedef enum {PATTERN_HASH,PATTERN_FRACTAL,PATTERN_CORNER,PATTERN_CENTER,PATTERN_COUNT} DMRefinePattern; 350a96aa3bSJed Brown static const char *DMRefinePatternName[PATTERN_COUNT] = {"hash","fractal","corner","center"}; 360a96aa3bSJed Brown 370a96aa3bSJed Brown typedef struct _DMRefinePatternCtx 380a96aa3bSJed Brown { 390a96aa3bSJed Brown PetscInt corner; 400a96aa3bSJed Brown PetscBool fractal[P4EST_CHILDREN]; 410a96aa3bSJed Brown PetscReal hashLikelihood; 420a96aa3bSJed Brown PetscInt maxLevel; 430a96aa3bSJed Brown p4est_refine_t refine_fn; 440a96aa3bSJed Brown } 450a96aa3bSJed Brown DMRefinePatternCtx; 460a96aa3bSJed Brown 470a96aa3bSJed Brown static int DMRefinePattern_Corner(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrant) 480a96aa3bSJed Brown { 490a96aa3bSJed Brown p4est_quadrant_t root, rootcorner; 500a96aa3bSJed Brown DMRefinePatternCtx *ctx; 510a96aa3bSJed Brown 520a96aa3bSJed Brown ctx = (DMRefinePatternCtx*) p4est->user_pointer; 530a96aa3bSJed Brown if (quadrant->level >= ctx->maxLevel) return 0; 540a96aa3bSJed Brown 550a96aa3bSJed Brown root.x = root.y = 0; 560a96aa3bSJed Brown #if defined(P4_TO_P8) 570a96aa3bSJed Brown root.z = 0; 580a96aa3bSJed Brown #endif 590a96aa3bSJed Brown root.level = 0; 600a96aa3bSJed Brown p4est_quadrant_corner_descendant(&root,&rootcorner,ctx->corner,quadrant->level); 610a96aa3bSJed Brown if (p4est_quadrant_is_equal(quadrant,&rootcorner)) return 1; 620a96aa3bSJed Brown return 0; 630a96aa3bSJed Brown } 640a96aa3bSJed Brown 650a96aa3bSJed Brown static int DMRefinePattern_Center(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrant) 660a96aa3bSJed Brown { 670a96aa3bSJed Brown int cid; 680a96aa3bSJed Brown p4est_quadrant_t ancestor, ancestorcorner; 690a96aa3bSJed Brown DMRefinePatternCtx *ctx; 700a96aa3bSJed Brown 710a96aa3bSJed Brown ctx = (DMRefinePatternCtx*) p4est->user_pointer; 720a96aa3bSJed Brown if (quadrant->level >= ctx->maxLevel) return 0; 730a96aa3bSJed Brown if (quadrant->level <= 1) return 1; 740a96aa3bSJed Brown 750a96aa3bSJed Brown p4est_quadrant_ancestor(quadrant,1,&ancestor); 760a96aa3bSJed Brown cid = p4est_quadrant_child_id(&ancestor); 770a96aa3bSJed Brown p4est_quadrant_corner_descendant(&ancestor,&ancestorcorner,P4EST_CHILDREN - 1 - cid,quadrant->level); 780a96aa3bSJed Brown if (p4est_quadrant_is_equal(quadrant,&ancestorcorner)) return 1; 790a96aa3bSJed Brown return 0; 800a96aa3bSJed Brown } 810a96aa3bSJed Brown 820a96aa3bSJed Brown static int DMRefinePattern_Fractal(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrant) 830a96aa3bSJed Brown { 840a96aa3bSJed Brown int cid; 850a96aa3bSJed Brown DMRefinePatternCtx *ctx; 860a96aa3bSJed Brown 870a96aa3bSJed Brown ctx = (DMRefinePatternCtx*) p4est->user_pointer; 880a96aa3bSJed Brown if (quadrant->level >= ctx->maxLevel) return 0; 890a96aa3bSJed Brown if (!quadrant->level) return 1; 900a96aa3bSJed Brown cid = p4est_quadrant_child_id(quadrant); 910a96aa3bSJed Brown if (ctx->fractal[cid ^ ((int) (quadrant->level % P4EST_CHILDREN))]) return 1; 920a96aa3bSJed Brown return 0; 930a96aa3bSJed Brown } 940a96aa3bSJed Brown 950a96aa3bSJed Brown /* simplified from MurmurHash3 by Austin Appleby */ 960a96aa3bSJed Brown #define DMPROT32(x, y) ((x << y) | (x >> (32 - y))) 970a96aa3bSJed Brown static uint32_t DMPforestHash(const uint32_t *blocks, uint32_t nblocks) 980a96aa3bSJed Brown { 990a96aa3bSJed Brown uint32_t c1 = 0xcc9e2d51; 1000a96aa3bSJed Brown uint32_t c2 = 0x1b873593; 1010a96aa3bSJed Brown uint32_t r1 = 15; 1020a96aa3bSJed Brown uint32_t r2 = 13; 1030a96aa3bSJed Brown uint32_t m = 5; 1040a96aa3bSJed Brown uint32_t n = 0xe6546b64; 1050a96aa3bSJed Brown uint32_t hash = 0; 1060a96aa3bSJed Brown int len = nblocks * 4; 1070a96aa3bSJed Brown uint32_t i; 1080a96aa3bSJed Brown 1090a96aa3bSJed Brown for (i = 0; i < nblocks; i++) { 1100a96aa3bSJed Brown uint32_t k; 1110a96aa3bSJed Brown 1120a96aa3bSJed Brown k = blocks[i]; 1130a96aa3bSJed Brown k *= c1; 1140a96aa3bSJed Brown k = DMPROT32(k, r1); 1150a96aa3bSJed Brown k *= c2; 1160a96aa3bSJed Brown 1170a96aa3bSJed Brown hash ^= k; 1180a96aa3bSJed Brown hash = DMPROT32(hash, r2) * m + n; 1190a96aa3bSJed Brown } 1200a96aa3bSJed Brown 1210a96aa3bSJed Brown hash ^= len; 1220a96aa3bSJed Brown hash ^= (hash >> 16); 1230a96aa3bSJed Brown hash *= 0x85ebca6b; 1240a96aa3bSJed Brown hash ^= (hash >> 13); 1250a96aa3bSJed Brown hash *= 0xc2b2ae35; 1260a96aa3bSJed Brown hash ^= (hash >> 16); 1270a96aa3bSJed Brown 1280a96aa3bSJed Brown return hash; 1290a96aa3bSJed Brown } 1300a96aa3bSJed Brown 1310a96aa3bSJed Brown #if defined(UINT32_MAX) 1320a96aa3bSJed Brown #define DMP4EST_HASH_MAX UINT32_MAX 1330a96aa3bSJed Brown #else 1340a96aa3bSJed Brown #define DMP4EST_HASH_MAX ((uint32_t) 0xffffffff) 1350a96aa3bSJed Brown #endif 1360a96aa3bSJed Brown 1370a96aa3bSJed Brown static int DMRefinePattern_Hash(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrant) 1380a96aa3bSJed Brown { 1390a96aa3bSJed Brown uint32_t data[5]; 1400a96aa3bSJed Brown uint32_t result; 1410a96aa3bSJed Brown DMRefinePatternCtx *ctx; 1420a96aa3bSJed Brown 1430a96aa3bSJed Brown ctx = (DMRefinePatternCtx*) p4est->user_pointer; 1440a96aa3bSJed Brown if (quadrant->level >= ctx->maxLevel) return 0; 1450a96aa3bSJed Brown data[0] = ((uint32_t) quadrant->level) << 24; 1460a96aa3bSJed Brown data[1] = (uint32_t) which_tree; 1470a96aa3bSJed Brown data[2] = (uint32_t) quadrant->x; 1480a96aa3bSJed Brown data[3] = (uint32_t) quadrant->y; 1490a96aa3bSJed Brown #if defined(P4_TO_P8) 1500a96aa3bSJed Brown data[4] = (uint32_t) quadrant->z; 1510a96aa3bSJed Brown #endif 1520a96aa3bSJed Brown 1530a96aa3bSJed Brown result = DMPforestHash(data,2+P4EST_DIM); 1540a96aa3bSJed Brown if (((double) result / (double) DMP4EST_HASH_MAX) < ctx->hashLikelihood) return 1; 1550a96aa3bSJed Brown return 0; 1560a96aa3bSJed Brown } 1570a96aa3bSJed Brown 1580a96aa3bSJed Brown #define DMConvert_pforest_plex _infix_pforest(DMConvert,_plex) 1590a96aa3bSJed Brown static PetscErrorCode DMConvert_pforest_plex(DM,DMType,DM*); 1600a96aa3bSJed Brown 1610a96aa3bSJed Brown #define DMFTopology_pforest _append_pforest(DMFTopology) 1620a96aa3bSJed Brown typedef struct { 1630a96aa3bSJed Brown PetscInt refct; 1640a96aa3bSJed Brown p4est_connectivity_t *conn; 1650a96aa3bSJed Brown p4est_geometry_t *geom; 1660a96aa3bSJed Brown PetscInt *tree_face_to_uniq; /* p4est does not explicitly enumerate facets, but we must to keep track of labels */ 1670a96aa3bSJed Brown } DMFTopology_pforest; 1680a96aa3bSJed Brown 1690a96aa3bSJed Brown #define DM_Forest_pforest _append_pforest(DM_Forest) 1700a96aa3bSJed Brown typedef struct { 1710a96aa3bSJed Brown DMFTopology_pforest *topo; 1720a96aa3bSJed Brown p4est_t *forest; 1730a96aa3bSJed Brown p4est_ghost_t *ghost; 1740a96aa3bSJed Brown p4est_lnodes_t *lnodes; 1750a96aa3bSJed Brown PetscBool partition_for_coarsening; 1760a96aa3bSJed Brown PetscBool coarsen_hierarchy; 1770a96aa3bSJed Brown PetscBool labelsFinalized; 1780a96aa3bSJed Brown PetscBool adaptivitySuccess; 1790a96aa3bSJed Brown PetscInt cLocalStart; 1800a96aa3bSJed Brown PetscInt cLocalEnd; 1810a96aa3bSJed Brown DM plex; 1820a96aa3bSJed Brown char *ghostName; 1830a96aa3bSJed Brown PetscSF pointAdaptToSelfSF; 1840a96aa3bSJed Brown PetscSF pointSelfToAdaptSF; 1850a96aa3bSJed Brown PetscInt *pointAdaptToSelfCids; 1860a96aa3bSJed Brown PetscInt *pointSelfToAdaptCids; 1870a96aa3bSJed Brown } DM_Forest_pforest; 1880a96aa3bSJed Brown 1890a96aa3bSJed Brown #define DM_Forest_geometry_pforest _append_pforest(DM_Forest_geometry) 1900a96aa3bSJed Brown typedef struct { 1910a96aa3bSJed Brown DM base; 1920a96aa3bSJed Brown PetscErrorCode (*map)(DM, PetscInt, PetscInt, const PetscReal[], PetscReal[], void*); 1930a96aa3bSJed Brown void *mapCtx; 1940a96aa3bSJed Brown PetscInt coordDim; 1950a96aa3bSJed Brown p4est_geometry_t *inner; 1960a96aa3bSJed Brown } 1970a96aa3bSJed Brown DM_Forest_geometry_pforest; 1980a96aa3bSJed Brown 1990a96aa3bSJed Brown #define GeometryMapping_pforest _append_pforest(GeometryMapping) 2000a96aa3bSJed Brown static void GeometryMapping_pforest(p4est_geometry_t *geom, p4est_topidx_t which_tree, const double abc[3], double xyz[3]) 2010a96aa3bSJed Brown { 2020a96aa3bSJed Brown DM_Forest_geometry_pforest *geom_pforest = (DM_Forest_geometry_pforest*)geom->user; 2030a96aa3bSJed Brown PetscReal PetscABC[3] = {0.}; 2040a96aa3bSJed Brown PetscReal PetscXYZ[3] = {0.}; 2050a96aa3bSJed Brown PetscInt i, d = PetscMin(3,geom_pforest->coordDim); 2060a96aa3bSJed Brown double ABC[3]; 2070a96aa3bSJed Brown PetscErrorCode ierr; 2080a96aa3bSJed Brown 2090a96aa3bSJed Brown (geom_pforest->inner->X)(geom_pforest->inner,which_tree,abc,ABC); 2100a96aa3bSJed Brown 2110a96aa3bSJed Brown for (i = 0; i < d; i++) PetscABC[i] = ABC[i]; 2120a96aa3bSJed Brown ierr = (geom_pforest->map)(geom_pforest->base,(PetscInt) which_tree,geom_pforest->coordDim,PetscABC,PetscXYZ,geom_pforest->mapCtx);PETSC_P4EST_ASSERT(!ierr); 2130a96aa3bSJed Brown for (i = 0; i < d; i++) xyz[i] = PetscXYZ[i]; 2140a96aa3bSJed Brown } 2150a96aa3bSJed Brown 2160a96aa3bSJed Brown #define GeometryDestroy_pforest _append_pforest(GeometryDestroy) 2170a96aa3bSJed Brown static void GeometryDestroy_pforest(p4est_geometry_t *geom) 2180a96aa3bSJed Brown { 2190a96aa3bSJed Brown DM_Forest_geometry_pforest *geom_pforest = (DM_Forest_geometry_pforest*)geom->user; 2200a96aa3bSJed Brown PetscErrorCode ierr; 2210a96aa3bSJed Brown 2220a96aa3bSJed Brown p4est_geometry_destroy(geom_pforest->inner); 2230a96aa3bSJed Brown ierr = PetscFree(geom->user);PETSC_P4EST_ASSERT(!ierr); 2240a96aa3bSJed Brown ierr = PetscFree(geom);PETSC_P4EST_ASSERT(!ierr); 2250a96aa3bSJed Brown } 2260a96aa3bSJed Brown 2270a96aa3bSJed Brown #define DMFTopologyDestroy_pforest _append_pforest(DMFTopologyDestroy) 2280a96aa3bSJed Brown static PetscErrorCode DMFTopologyDestroy_pforest(DMFTopology_pforest **topo) 2290a96aa3bSJed Brown { 2300a96aa3bSJed Brown PetscErrorCode ierr; 2310a96aa3bSJed Brown 2320a96aa3bSJed Brown PetscFunctionBegin; 2330a96aa3bSJed Brown if (!(*topo)) PetscFunctionReturn(0); 2340a96aa3bSJed Brown if (--((*topo)->refct) > 0) { 2350a96aa3bSJed Brown *topo = NULL; 2360a96aa3bSJed Brown PetscFunctionReturn(0); 2370a96aa3bSJed Brown } 2380a96aa3bSJed Brown if ((*topo)->geom) PetscStackCallP4est(p4est_geometry_destroy,((*topo)->geom)); 2390a96aa3bSJed Brown PetscStackCallP4est(p4est_connectivity_destroy,((*topo)->conn)); 2400a96aa3bSJed Brown ierr = PetscFree((*topo)->tree_face_to_uniq);CHKERRQ(ierr); 2410a96aa3bSJed Brown ierr = PetscFree(*topo);CHKERRQ(ierr); 2420a96aa3bSJed Brown *topo = NULL; 2430a96aa3bSJed Brown PetscFunctionReturn(0); 2440a96aa3bSJed Brown } 2450a96aa3bSJed Brown 2460a96aa3bSJed Brown static PetscErrorCode PforestConnectivityEnumerateFacets(p4est_connectivity_t*,PetscInt**); 2470a96aa3bSJed Brown 2480a96aa3bSJed Brown #define DMFTopologyCreateBrick_pforest _append_pforest(DMFTopologyCreateBrick) 2490a96aa3bSJed Brown static PetscErrorCode DMFTopologyCreateBrick_pforest(DM dm,PetscInt N[], PetscInt P[], PetscReal B[],DMFTopology_pforest **topo, PetscBool useMorton) 2500a96aa3bSJed Brown { 2510a96aa3bSJed Brown double *vertices; 2520a96aa3bSJed Brown PetscInt i, numVerts; 2530a96aa3bSJed Brown PetscErrorCode ierr; 2540a96aa3bSJed Brown 2550a96aa3bSJed Brown PetscFunctionBegin; 256*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!useMorton,PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Lexicographic ordering not implemented yet"); 2570a96aa3bSJed Brown ierr = PetscNewLog(dm,topo);CHKERRQ(ierr); 2580a96aa3bSJed Brown 2590a96aa3bSJed Brown (*topo)->refct = 1; 2600a96aa3bSJed Brown #if !defined(P4_TO_P8) 2610a96aa3bSJed Brown PetscStackCallP4estReturn((*topo)->conn,p4est_connectivity_new_brick,((int) N[0], (int) N[1], (P[0] == DM_BOUNDARY_NONE) ? 0 : 1, (P[1] == DM_BOUNDARY_NONE) ? 0 : 1)); 2620a96aa3bSJed Brown #else 2630a96aa3bSJed Brown PetscStackCallP4estReturn((*topo)->conn,p8est_connectivity_new_brick,((int) N[0], (int) N[1], (int) N[2], (P[0] == DM_BOUNDARY_NONE) ? 0 : 1, (P[1] == DM_BOUNDARY_NONE) ? 0 : 1, (P[2] == DM_BOUNDARY_NONE) ? 0 : 1)); 2640a96aa3bSJed Brown #endif 2650a96aa3bSJed Brown numVerts = (*topo)->conn->num_vertices; 2660a96aa3bSJed Brown vertices = (*topo)->conn->vertices; 2670a96aa3bSJed Brown for (i = 0; i < 3 * numVerts; i++) { 2680a96aa3bSJed Brown PetscInt j = i % 3; 2690a96aa3bSJed Brown 2700a96aa3bSJed Brown vertices[i] = B[2 * j] + (vertices[i]/N[j]) * (B[2 * j + 1] - B[2 * j]); 2710a96aa3bSJed Brown } 2720a96aa3bSJed Brown (*topo)->geom = NULL; 2730a96aa3bSJed Brown ierr = PforestConnectivityEnumerateFacets((*topo)->conn,&(*topo)->tree_face_to_uniq);CHKERRQ(ierr); 2740a96aa3bSJed Brown PetscFunctionReturn(0); 2750a96aa3bSJed Brown } 2760a96aa3bSJed Brown 2770a96aa3bSJed Brown #define DMFTopologyCreate_pforest _append_pforest(DMFTopologyCreate) 2780a96aa3bSJed Brown static PetscErrorCode DMFTopologyCreate_pforest(DM dm, DMForestTopology topologyName, DMFTopology_pforest **topo) 2790a96aa3bSJed Brown { 2800a96aa3bSJed Brown const char *name = (const char*) topologyName; 2810a96aa3bSJed Brown const char *prefix; 2820a96aa3bSJed Brown PetscBool isBrick, isShell, isSphere, isMoebius; 2830a96aa3bSJed Brown PetscErrorCode ierr; 2840a96aa3bSJed Brown 2850a96aa3bSJed Brown PetscFunctionBegin; 2860a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 2870a96aa3bSJed Brown PetscValidCharPointer(name,2); 2880a96aa3bSJed Brown PetscValidPointer(topo,3); 2890a96aa3bSJed Brown ierr = PetscStrcmp(name,"brick",&isBrick);CHKERRQ(ierr); 2900a96aa3bSJed Brown ierr = PetscStrcmp(name,"shell",&isShell);CHKERRQ(ierr); 2910a96aa3bSJed Brown ierr = PetscStrcmp(name,"sphere",&isSphere);CHKERRQ(ierr); 2920a96aa3bSJed Brown ierr = PetscStrcmp(name,"moebius",&isMoebius);CHKERRQ(ierr); 2930a96aa3bSJed Brown ierr = PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);CHKERRQ(ierr); 2940a96aa3bSJed Brown if (isBrick) { 2950a96aa3bSJed Brown PetscBool flgN, flgP, flgM, flgB, useMorton = PETSC_TRUE, periodic = PETSC_FALSE; 2960a96aa3bSJed Brown PetscInt N[3] = {2,2,2}, P[3] = {0,0,0}, nretN = P4EST_DIM, nretP = P4EST_DIM, nretB = 2 * P4EST_DIM, i; 2970a96aa3bSJed Brown PetscReal B[6] = {0.0,1.0,0.0,1.0,0.0,1.0}; 2980a96aa3bSJed Brown 2990a96aa3bSJed Brown if (dm->setfromoptionscalled) { 3000a96aa3bSJed Brown ierr = PetscOptionsGetIntArray(((PetscObject)dm)->options,prefix,"-dm_p4est_brick_size",N,&nretN,&flgN);CHKERRQ(ierr); 3010a96aa3bSJed Brown ierr = PetscOptionsGetIntArray(((PetscObject)dm)->options,prefix,"-dm_p4est_brick_periodicity",P,&nretP,&flgP);CHKERRQ(ierr); 3020a96aa3bSJed Brown ierr = PetscOptionsGetRealArray(((PetscObject)dm)->options,prefix,"-dm_p4est_brick_bounds",B,&nretB,&flgB);CHKERRQ(ierr); 3030a96aa3bSJed Brown ierr = PetscOptionsGetBool(((PetscObject)dm)->options,prefix,"-dm_p4est_brick_use_morton_curve",&useMorton,&flgM);CHKERRQ(ierr); 304*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(flgN && nretN != P4EST_DIM,PetscObjectComm((PetscObject)dm),PETSC_ERR_ARG_SIZ,"Need to give %d sizes in -dm_p4est_brick_size, gave %d",P4EST_DIM,nretN); 305*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(flgP && nretP != P4EST_DIM,PetscObjectComm((PetscObject)dm),PETSC_ERR_ARG_SIZ,"Need to give %d periodicities in -dm_p4est_brick_periodicity, gave %d",P4EST_DIM,nretP); 306*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(flgB && nretB != 2 * P4EST_DIM,PetscObjectComm((PetscObject)dm),PETSC_ERR_ARG_SIZ,"Need to give %d bounds in -dm_p4est_brick_bounds, gave %d",P4EST_DIM,nretP); 3070a96aa3bSJed Brown } 3080a96aa3bSJed Brown for (i = 0; i < P4EST_DIM; i++) { 3090a96aa3bSJed Brown P[i] = (P[i] ? DM_BOUNDARY_PERIODIC : DM_BOUNDARY_NONE); 3100a96aa3bSJed Brown periodic = (PetscBool)(P[i] || periodic); 3110a96aa3bSJed Brown if (!flgB) B[2 * i + 1] = N[i]; 3120a96aa3bSJed Brown } 3130a96aa3bSJed Brown ierr = DMFTopologyCreateBrick_pforest(dm,N,P,B,topo,useMorton);CHKERRQ(ierr); 3140a96aa3bSJed Brown /* the maxCell trick is not robust enough, localize on all cells if periodic */ 3150a96aa3bSJed Brown ierr = DMSetPeriodicity(dm,periodic,NULL,NULL,NULL);CHKERRQ(ierr); 3160a96aa3bSJed Brown } else { 3170a96aa3bSJed Brown ierr = PetscNewLog(dm,topo);CHKERRQ(ierr); 3180a96aa3bSJed Brown 3190a96aa3bSJed Brown (*topo)->refct = 1; 3200a96aa3bSJed Brown PetscStackCallP4estReturn((*topo)->conn,p4est_connectivity_new_byname,(name)); 3210a96aa3bSJed Brown (*topo)->geom = NULL; 3220a96aa3bSJed Brown if (isMoebius) { 3230a96aa3bSJed Brown ierr = DMSetCoordinateDim(dm,3);CHKERRQ(ierr); 3240a96aa3bSJed Brown } 3250a96aa3bSJed Brown #if defined(P4_TO_P8) 3260a96aa3bSJed Brown if (isShell) { 3270a96aa3bSJed Brown PetscReal R2 = 1., R1 = .55; 3280a96aa3bSJed Brown 3290a96aa3bSJed Brown if (dm->setfromoptionscalled) { 3300a96aa3bSJed Brown ierr = PetscOptionsGetReal(((PetscObject)dm)->options,prefix,"-dm_p4est_shell_outer_radius",&R2,NULL);CHKERRQ(ierr); 3310a96aa3bSJed Brown ierr = PetscOptionsGetReal(((PetscObject)dm)->options,prefix,"-dm_p4est_shell_inner_radius",&R1,NULL);CHKERRQ(ierr); 3320a96aa3bSJed Brown } 3330a96aa3bSJed Brown PetscStackCallP4estReturn((*topo)->geom,p8est_geometry_new_shell,((*topo)->conn,R2,R1)); 3340a96aa3bSJed Brown } else if (isSphere) { 3350a96aa3bSJed Brown PetscReal R2 = 1., R1 = 0.191728, R0 = 0.039856; 3360a96aa3bSJed Brown 3370a96aa3bSJed Brown if (dm->setfromoptionscalled) { 3380a96aa3bSJed Brown ierr = PetscOptionsGetReal(((PetscObject)dm)->options,prefix,"-dm_p4est_sphere_outer_radius",&R2,NULL);CHKERRQ(ierr); 3390a96aa3bSJed Brown ierr = PetscOptionsGetReal(((PetscObject)dm)->options,prefix,"-dm_p4est_sphere_inner_radius",&R1,NULL);CHKERRQ(ierr); 3400a96aa3bSJed Brown ierr = PetscOptionsGetReal(((PetscObject)dm)->options,prefix,"-dm_p4est_sphere_core_radius",&R0,NULL);CHKERRQ(ierr); 3410a96aa3bSJed Brown } 3420a96aa3bSJed Brown PetscStackCallP4estReturn((*topo)->geom,p8est_geometry_new_sphere,((*topo)->conn,R2,R1,R0)); 3430a96aa3bSJed Brown } 3440a96aa3bSJed Brown #endif 3450a96aa3bSJed Brown ierr = PforestConnectivityEnumerateFacets((*topo)->conn,&(*topo)->tree_face_to_uniq);CHKERRQ(ierr); 3460a96aa3bSJed Brown } 3470a96aa3bSJed Brown PetscFunctionReturn(0); 3480a96aa3bSJed Brown } 3490a96aa3bSJed Brown 3500a96aa3bSJed Brown #define DMConvert_plex_pforest _append_pforest(DMConvert_plex) 3510a96aa3bSJed Brown static PetscErrorCode DMConvert_plex_pforest(DM dm, DMType newtype, DM *pforest) 3520a96aa3bSJed Brown { 3530a96aa3bSJed Brown MPI_Comm comm; 3540a96aa3bSJed Brown PetscBool isPlex; 3550a96aa3bSJed Brown PetscInt dim; 3560a96aa3bSJed Brown void *ctx; 3570a96aa3bSJed Brown PetscErrorCode ierr; 3580a96aa3bSJed Brown 3590a96aa3bSJed Brown PetscFunctionBegin; 3600a96aa3bSJed Brown 3610a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 3620a96aa3bSJed Brown comm = PetscObjectComm((PetscObject)dm); 3630a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject)dm,DMPLEX,&isPlex);CHKERRQ(ierr); 364*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!isPlex,comm,PETSC_ERR_ARG_WRONG,"Expected DM type %s, got %s",DMPLEX,((PetscObject)dm)->type_name); 3650a96aa3bSJed Brown ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr); 366*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(dim != P4EST_DIM,comm,PETSC_ERR_ARG_WRONG,"Expected DM dimension %d, got %d",P4EST_DIM,dim); 3670a96aa3bSJed Brown ierr = DMCreate(comm,pforest);CHKERRQ(ierr); 3680a96aa3bSJed Brown ierr = DMSetType(*pforest,DMPFOREST);CHKERRQ(ierr); 3690a96aa3bSJed Brown ierr = DMForestSetBaseDM(*pforest,dm);CHKERRQ(ierr); 3700a96aa3bSJed Brown ierr = DMGetApplicationContext(dm,&ctx);CHKERRQ(ierr); 3710a96aa3bSJed Brown ierr = DMSetApplicationContext(*pforest,ctx);CHKERRQ(ierr); 3720a96aa3bSJed Brown ierr = DMCopyDisc(dm,*pforest);CHKERRQ(ierr); 3730a96aa3bSJed Brown PetscFunctionReturn(0); 3740a96aa3bSJed Brown } 3750a96aa3bSJed Brown 3760a96aa3bSJed Brown #define DMForestDestroy_pforest _append_pforest(DMForestDestroy) 3770a96aa3bSJed Brown static PetscErrorCode DMForestDestroy_pforest(DM dm) 3780a96aa3bSJed Brown { 3790a96aa3bSJed Brown DM_Forest *forest = (DM_Forest*) dm->data; 3800a96aa3bSJed Brown DM_Forest_pforest *pforest = (DM_Forest_pforest*) forest->data; 3810a96aa3bSJed Brown PetscErrorCode ierr; 3820a96aa3bSJed Brown 3830a96aa3bSJed Brown PetscFunctionBegin; 3840a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 3850a96aa3bSJed Brown if (pforest->lnodes) PetscStackCallP4est(p4est_lnodes_destroy,(pforest->lnodes)); 3860a96aa3bSJed Brown pforest->lnodes = NULL; 3870a96aa3bSJed Brown if (pforest->ghost) PetscStackCallP4est(p4est_ghost_destroy,(pforest->ghost)); 3880a96aa3bSJed Brown pforest->ghost = NULL; 3890a96aa3bSJed Brown if (pforest->forest) PetscStackCallP4est(p4est_destroy,(pforest->forest)); 3900a96aa3bSJed Brown pforest->forest = NULL; 3910a96aa3bSJed Brown ierr = DMFTopologyDestroy_pforest(&pforest->topo);CHKERRQ(ierr); 3920a96aa3bSJed Brown ierr = PetscObjectComposeFunction((PetscObject)dm,PetscStringize(DMConvert_plex_pforest) "_C",NULL);CHKERRQ(ierr); 3930a96aa3bSJed Brown ierr = PetscObjectComposeFunction((PetscObject)dm,PetscStringize(DMConvert_pforest_plex) "_C",NULL);CHKERRQ(ierr); 3940a96aa3bSJed Brown ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C",NULL);CHKERRQ(ierr); 3950a96aa3bSJed Brown ierr = PetscFree(pforest->ghostName);CHKERRQ(ierr); 3960a96aa3bSJed Brown ierr = DMDestroy(&pforest->plex);CHKERRQ(ierr); 3970a96aa3bSJed Brown ierr = PetscSFDestroy(&pforest->pointAdaptToSelfSF);CHKERRQ(ierr); 3980a96aa3bSJed Brown ierr = PetscSFDestroy(&pforest->pointSelfToAdaptSF);CHKERRQ(ierr); 3990a96aa3bSJed Brown ierr = PetscFree(pforest->pointAdaptToSelfCids);CHKERRQ(ierr); 4000a96aa3bSJed Brown ierr = PetscFree(pforest->pointSelfToAdaptCids);CHKERRQ(ierr); 4010a96aa3bSJed Brown ierr = PetscFree(forest->data);CHKERRQ(ierr); 4020a96aa3bSJed Brown PetscFunctionReturn(0); 4030a96aa3bSJed Brown } 4040a96aa3bSJed Brown 4050a96aa3bSJed Brown #define DMForestTemplate_pforest _append_pforest(DMForestTemplate) 4060a96aa3bSJed Brown static PetscErrorCode DMForestTemplate_pforest(DM dm, DM tdm) 4070a96aa3bSJed Brown { 4080a96aa3bSJed Brown DM_Forest_pforest *pforest = (DM_Forest_pforest*) ((DM_Forest*) dm->data)->data; 4090a96aa3bSJed Brown DM_Forest_pforest *tpforest = (DM_Forest_pforest*) ((DM_Forest*) tdm->data)->data; 4100a96aa3bSJed Brown PetscErrorCode ierr; 4110a96aa3bSJed Brown 4120a96aa3bSJed Brown PetscFunctionBegin; 4130a96aa3bSJed Brown if (pforest->topo) pforest->topo->refct++; 4140a96aa3bSJed Brown ierr = DMFTopologyDestroy_pforest(&(tpforest->topo));CHKERRQ(ierr); 4150a96aa3bSJed Brown tpforest->topo = pforest->topo; 4160a96aa3bSJed Brown PetscFunctionReturn(0); 4170a96aa3bSJed Brown } 4180a96aa3bSJed Brown 4190a96aa3bSJed Brown #define DMPlexCreateConnectivity_pforest _append_pforest(DMPlexCreateConnectivity) 4200a96aa3bSJed Brown static PetscErrorCode DMPlexCreateConnectivity_pforest(DM,p4est_connectivity_t**,PetscInt**); 4210a96aa3bSJed Brown 4220a96aa3bSJed Brown typedef struct _PforestAdaptCtx 4230a96aa3bSJed Brown { 4240a96aa3bSJed Brown PetscInt maxLevel; 4250a96aa3bSJed Brown PetscInt minLevel; 4260a96aa3bSJed Brown PetscInt currLevel; 4270a96aa3bSJed Brown PetscBool anyChange; 4280a96aa3bSJed Brown } 4290a96aa3bSJed Brown PforestAdaptCtx; 4300a96aa3bSJed Brown 4310a96aa3bSJed Brown static int pforest_coarsen_currlevel(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrants[]) 4320a96aa3bSJed Brown { 4330a96aa3bSJed Brown PforestAdaptCtx *ctx = (PforestAdaptCtx*) p4est->user_pointer; 4340a96aa3bSJed Brown PetscInt minLevel = ctx->minLevel; 4350a96aa3bSJed Brown PetscInt currLevel = ctx->currLevel; 4360a96aa3bSJed Brown 4370a96aa3bSJed Brown if (quadrants[0]->level <= minLevel) return 0; 4380a96aa3bSJed Brown return (int) ((PetscInt) quadrants[0]->level == currLevel); 4390a96aa3bSJed Brown } 4400a96aa3bSJed Brown 4410a96aa3bSJed Brown static int pforest_coarsen_uniform(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrants[]) 4420a96aa3bSJed Brown { 4430a96aa3bSJed Brown PforestAdaptCtx *ctx = (PforestAdaptCtx*) p4est->user_pointer; 4440a96aa3bSJed Brown PetscInt minLevel = ctx->minLevel; 4450a96aa3bSJed Brown 4460a96aa3bSJed Brown return (int) ((PetscInt) quadrants[0]->level > minLevel); 4470a96aa3bSJed Brown } 4480a96aa3bSJed Brown 4490a96aa3bSJed Brown static int pforest_coarsen_flag_any(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrants[]) 4500a96aa3bSJed Brown { 4510a96aa3bSJed Brown PetscInt i; 4520a96aa3bSJed Brown PetscBool any = PETSC_FALSE; 4530a96aa3bSJed Brown PforestAdaptCtx *ctx = (PforestAdaptCtx*) p4est->user_pointer; 4540a96aa3bSJed Brown PetscInt minLevel = ctx->minLevel; 4550a96aa3bSJed Brown 4560a96aa3bSJed Brown if (quadrants[0]->level <= minLevel) return 0; 4570a96aa3bSJed Brown for (i = 0; i < P4EST_CHILDREN; i++) { 4580a96aa3bSJed Brown if (quadrants[i]->p.user_int == DM_ADAPT_KEEP) { 4590a96aa3bSJed Brown any = PETSC_FALSE; 4600a96aa3bSJed Brown break; 4610a96aa3bSJed Brown } 4620a96aa3bSJed Brown if (quadrants[i]->p.user_int == DM_ADAPT_COARSEN) { 4630a96aa3bSJed Brown any = PETSC_TRUE; 4640a96aa3bSJed Brown break; 4650a96aa3bSJed Brown } 4660a96aa3bSJed Brown } 4670a96aa3bSJed Brown return any ? 1 : 0; 4680a96aa3bSJed Brown } 4690a96aa3bSJed Brown 4700a96aa3bSJed Brown static int pforest_coarsen_flag_all(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrants[]) 4710a96aa3bSJed Brown { 4720a96aa3bSJed Brown PetscInt i; 4730a96aa3bSJed Brown PetscBool all = PETSC_TRUE; 4740a96aa3bSJed Brown PforestAdaptCtx *ctx = (PforestAdaptCtx*) p4est->user_pointer; 4750a96aa3bSJed Brown PetscInt minLevel = ctx->minLevel; 4760a96aa3bSJed Brown 4770a96aa3bSJed Brown if (quadrants[0]->level <= minLevel) return 0; 4780a96aa3bSJed Brown for (i = 0; i < P4EST_CHILDREN; i++) { 4790a96aa3bSJed Brown if (quadrants[i]->p.user_int != DM_ADAPT_COARSEN) { 4800a96aa3bSJed Brown all = PETSC_FALSE; 4810a96aa3bSJed Brown break; 4820a96aa3bSJed Brown } 4830a96aa3bSJed Brown } 4840a96aa3bSJed Brown return all ? 1 : 0; 4850a96aa3bSJed Brown } 4860a96aa3bSJed Brown 4870a96aa3bSJed Brown static void pforest_init_determine(p4est_t *p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrant) 4880a96aa3bSJed Brown { 4890a96aa3bSJed Brown quadrant->p.user_int = DM_ADAPT_DETERMINE; 4900a96aa3bSJed Brown } 4910a96aa3bSJed Brown 4920a96aa3bSJed Brown static int pforest_refine_uniform(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrant) 4930a96aa3bSJed Brown { 4940a96aa3bSJed Brown PforestAdaptCtx *ctx = (PforestAdaptCtx*) p4est->user_pointer; 4950a96aa3bSJed Brown PetscInt maxLevel = ctx->maxLevel; 4960a96aa3bSJed Brown 4970a96aa3bSJed Brown return ((PetscInt) quadrant->level < maxLevel); 4980a96aa3bSJed Brown } 4990a96aa3bSJed Brown 5000a96aa3bSJed Brown static int pforest_refine_flag(p4est_t * p4est, p4est_topidx_t which_tree, p4est_quadrant_t *quadrant) 5010a96aa3bSJed Brown { 5020a96aa3bSJed Brown PforestAdaptCtx *ctx = (PforestAdaptCtx*) p4est->user_pointer; 5030a96aa3bSJed Brown PetscInt maxLevel = ctx->maxLevel; 5040a96aa3bSJed Brown 5050a96aa3bSJed Brown if ((PetscInt) quadrant->level >= maxLevel) return 0; 5060a96aa3bSJed Brown 5070a96aa3bSJed Brown return (quadrant->p.user_int == DM_ADAPT_REFINE); 5080a96aa3bSJed Brown } 5090a96aa3bSJed Brown 5100a96aa3bSJed Brown static PetscErrorCode DMPforestComputeLocalCellTransferSF_loop(p4est_t *p4estFrom, PetscInt FromOffset, p4est_t *p4estTo, PetscInt ToOffset, p4est_topidx_t flt, p4est_topidx_t llt, PetscInt *toFineLeavesCount, PetscInt *toLeaves, PetscSFNode *fromRoots, PetscInt *fromFineLeavesCount, PetscInt *fromLeaves, PetscSFNode *toRoots) 5110a96aa3bSJed Brown { 5120a96aa3bSJed Brown PetscMPIInt rank = p4estFrom->mpirank; 5130a96aa3bSJed Brown p4est_topidx_t t; 5140a96aa3bSJed Brown PetscInt toFineLeaves = 0, fromFineLeaves = 0; 5150a96aa3bSJed Brown 5160a96aa3bSJed Brown PetscFunctionBegin; 5170a96aa3bSJed Brown for (t = flt; t <= llt; t++) { /* count roots and leaves */ 5180a96aa3bSJed Brown p4est_tree_t *treeFrom = &(((p4est_tree_t*) p4estFrom->trees->array)[t]); 5190a96aa3bSJed Brown p4est_tree_t *treeTo = &(((p4est_tree_t*) p4estTo->trees->array)[t]); 5200a96aa3bSJed Brown p4est_quadrant_t *firstFrom = &treeFrom->first_desc; 5210a96aa3bSJed Brown p4est_quadrant_t *firstTo = &treeTo->first_desc; 5220a96aa3bSJed Brown PetscInt numFrom = (PetscInt) treeFrom->quadrants.elem_count; 5230a96aa3bSJed Brown PetscInt numTo = (PetscInt) treeTo->quadrants.elem_count; 5240a96aa3bSJed Brown p4est_quadrant_t *quadsFrom = (p4est_quadrant_t*) treeFrom->quadrants.array; 5250a96aa3bSJed Brown p4est_quadrant_t *quadsTo = (p4est_quadrant_t*) treeTo->quadrants.array; 5260a96aa3bSJed Brown PetscInt currentFrom, currentTo; 5270a96aa3bSJed Brown PetscInt treeOffsetFrom = (PetscInt) treeFrom->quadrants_offset; 5280a96aa3bSJed Brown PetscInt treeOffsetTo = (PetscInt) treeTo->quadrants_offset; 5290a96aa3bSJed Brown int comp; 5300a96aa3bSJed Brown 5310a96aa3bSJed Brown PetscStackCallP4estReturn(comp,p4est_quadrant_is_equal,(firstFrom,firstTo)); 532*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!comp,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"non-matching partitions"); 5330a96aa3bSJed Brown 5340a96aa3bSJed Brown for (currentFrom = 0, currentTo = 0; currentFrom < numFrom && currentTo < numTo;) { 5350a96aa3bSJed Brown p4est_quadrant_t *quadFrom = &quadsFrom[currentFrom]; 5360a96aa3bSJed Brown p4est_quadrant_t *quadTo = &quadsTo[currentTo]; 5370a96aa3bSJed Brown 5380a96aa3bSJed Brown if (quadFrom->level == quadTo->level) { 5390a96aa3bSJed Brown if (toLeaves) { 5400a96aa3bSJed Brown toLeaves[toFineLeaves] = currentTo + treeOffsetTo + ToOffset; 5410a96aa3bSJed Brown fromRoots[toFineLeaves].rank = rank; 5420a96aa3bSJed Brown fromRoots[toFineLeaves].index = currentFrom + treeOffsetFrom + FromOffset; 5430a96aa3bSJed Brown } 5440a96aa3bSJed Brown toFineLeaves++; 5450a96aa3bSJed Brown currentFrom++; 5460a96aa3bSJed Brown currentTo++; 5470a96aa3bSJed Brown } else { 5480a96aa3bSJed Brown int fromIsAncestor; 5490a96aa3bSJed Brown 5500a96aa3bSJed Brown PetscStackCallP4estReturn(fromIsAncestor,p4est_quadrant_is_ancestor,(quadFrom,quadTo)); 5510a96aa3bSJed Brown if (fromIsAncestor) { 5520a96aa3bSJed Brown p4est_quadrant_t lastDesc; 5530a96aa3bSJed Brown 5540a96aa3bSJed Brown if (toLeaves) { 5550a96aa3bSJed Brown toLeaves[toFineLeaves] = currentTo + treeOffsetTo + ToOffset; 5560a96aa3bSJed Brown fromRoots[toFineLeaves].rank = rank; 5570a96aa3bSJed Brown fromRoots[toFineLeaves].index = currentFrom + treeOffsetFrom + FromOffset; 5580a96aa3bSJed Brown } 5590a96aa3bSJed Brown toFineLeaves++; 5600a96aa3bSJed Brown currentTo++; 5610a96aa3bSJed Brown PetscStackCallP4est(p4est_quadrant_last_descendant,(quadFrom,&lastDesc,quadTo->level)); 5620a96aa3bSJed Brown PetscStackCallP4estReturn(comp,p4est_quadrant_is_equal,(quadTo,&lastDesc)); 5630a96aa3bSJed Brown if (comp) currentFrom++; 5640a96aa3bSJed Brown } else { 5650a96aa3bSJed Brown p4est_quadrant_t lastDesc; 5660a96aa3bSJed Brown 5670a96aa3bSJed Brown if (fromLeaves) { 5680a96aa3bSJed Brown fromLeaves[fromFineLeaves] = currentFrom + treeOffsetFrom + FromOffset; 5690a96aa3bSJed Brown toRoots[fromFineLeaves].rank = rank; 5700a96aa3bSJed Brown toRoots[fromFineLeaves].index = currentTo + treeOffsetTo + ToOffset; 5710a96aa3bSJed Brown } 5720a96aa3bSJed Brown fromFineLeaves++; 5730a96aa3bSJed Brown currentFrom++; 5740a96aa3bSJed Brown PetscStackCallP4est(p4est_quadrant_last_descendant,(quadTo,&lastDesc,quadFrom->level)); 5750a96aa3bSJed Brown PetscStackCallP4estReturn(comp,p4est_quadrant_is_equal,(quadFrom,&lastDesc)); 5760a96aa3bSJed Brown if (comp) currentTo++; 5770a96aa3bSJed Brown } 5780a96aa3bSJed Brown } 5790a96aa3bSJed Brown } 5800a96aa3bSJed Brown } 5810a96aa3bSJed Brown *toFineLeavesCount = toFineLeaves; 5820a96aa3bSJed Brown *fromFineLeavesCount = fromFineLeaves; 5830a96aa3bSJed Brown PetscFunctionReturn(0); 5840a96aa3bSJed Brown } 5850a96aa3bSJed Brown 5860a96aa3bSJed Brown /* Compute the maximum level across all the trees */ 5870a96aa3bSJed Brown static PetscErrorCode DMPforestGetRefinementLevel(DM dm, PetscInt *lev) 5880a96aa3bSJed Brown { 5890a96aa3bSJed Brown p4est_topidx_t t, flt, llt; 5900a96aa3bSJed Brown DM_Forest *forest = (DM_Forest*) dm->data; 5910a96aa3bSJed Brown DM_Forest_pforest *pforest = (DM_Forest_pforest*) forest->data; 5920a96aa3bSJed Brown PetscInt maxlevelloc = 0; 5930a96aa3bSJed Brown p4est_t *p4est; 5940a96aa3bSJed Brown PetscErrorCode ierr; 5950a96aa3bSJed Brown 5960a96aa3bSJed Brown PetscFunctionBegin; 597*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!pforest,PetscObjectComm((PetscObject)dm),PETSC_ERR_PLIB,"Missing DM_Forest_pforest"); 598*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!pforest->forest,PetscObjectComm((PetscObject)dm),PETSC_ERR_PLIB,"Missing p4est_t"); 5990a96aa3bSJed Brown p4est = pforest->forest; 6000a96aa3bSJed Brown flt = p4est->first_local_tree; 6010a96aa3bSJed Brown llt = p4est->last_local_tree; 6020a96aa3bSJed Brown for (t = flt; t <= llt; t++) { 6030a96aa3bSJed Brown p4est_tree_t *tree = &(((p4est_tree_t*) p4est->trees->array)[t]); 6040a96aa3bSJed Brown maxlevelloc = PetscMax((PetscInt)tree->maxlevel,maxlevelloc); 6050a96aa3bSJed Brown } 6060a96aa3bSJed Brown ierr = MPIU_Allreduce(&maxlevelloc,lev,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr); 6070a96aa3bSJed Brown PetscFunctionReturn(0); 6080a96aa3bSJed Brown } 6090a96aa3bSJed Brown 6100a96aa3bSJed Brown /* Puts identity in coarseToFine */ 6110a96aa3bSJed Brown /* assumes a matching partition */ 6120a96aa3bSJed Brown static PetscErrorCode DMPforestComputeLocalCellTransferSF(MPI_Comm comm, p4est_t *p4estFrom, PetscInt FromOffset, p4est_t *p4estTo, PetscInt ToOffset, PetscSF *fromCoarseToFine, PetscSF *toCoarseFromFine) 6130a96aa3bSJed Brown { 6140a96aa3bSJed Brown p4est_topidx_t flt, llt; 6150a96aa3bSJed Brown PetscSF fromCoarse, toCoarse; 6160a96aa3bSJed Brown PetscInt numRootsFrom, numRootsTo, numLeavesFrom, numLeavesTo; 6170a96aa3bSJed Brown PetscInt *fromLeaves = NULL, *toLeaves = NULL; 6180a96aa3bSJed Brown PetscSFNode *fromRoots = NULL, *toRoots = NULL; 6190a96aa3bSJed Brown PetscErrorCode ierr; 6200a96aa3bSJed Brown 6210a96aa3bSJed Brown PetscFunctionBegin; 6220a96aa3bSJed Brown flt = p4estFrom->first_local_tree; 6230a96aa3bSJed Brown llt = p4estFrom->last_local_tree; 6240a96aa3bSJed Brown ierr = PetscSFCreate(comm,&fromCoarse);CHKERRQ(ierr); 6250a96aa3bSJed Brown if (toCoarseFromFine) { 6260a96aa3bSJed Brown ierr = PetscSFCreate(comm,&toCoarse);CHKERRQ(ierr); 6270a96aa3bSJed Brown } 6280a96aa3bSJed Brown numRootsFrom = p4estFrom->local_num_quadrants + FromOffset; 6290a96aa3bSJed Brown numRootsTo = p4estTo->local_num_quadrants + ToOffset; 6300a96aa3bSJed Brown ierr = DMPforestComputeLocalCellTransferSF_loop(p4estFrom,FromOffset,p4estTo,ToOffset,flt,llt,&numLeavesTo,NULL,NULL,&numLeavesFrom,NULL,NULL);CHKERRQ(ierr); 6310a96aa3bSJed Brown ierr = PetscMalloc1(numLeavesTo,&toLeaves);CHKERRQ(ierr); 6320a96aa3bSJed Brown ierr = PetscMalloc1(numLeavesTo,&fromRoots);CHKERRQ(ierr); 6330a96aa3bSJed Brown if (toCoarseFromFine) { 6340a96aa3bSJed Brown ierr = PetscMalloc1(numLeavesFrom,&fromLeaves);CHKERRQ(ierr); 6350a96aa3bSJed Brown ierr = PetscMalloc1(numLeavesFrom,&fromRoots);CHKERRQ(ierr); 6360a96aa3bSJed Brown } 6370a96aa3bSJed Brown ierr = DMPforestComputeLocalCellTransferSF_loop(p4estFrom,FromOffset,p4estTo,ToOffset,flt,llt,&numLeavesTo,toLeaves,fromRoots,&numLeavesFrom,fromLeaves,toRoots);CHKERRQ(ierr); 6380a96aa3bSJed Brown if (!ToOffset && (numLeavesTo == numRootsTo)) { /* compress */ 6390a96aa3bSJed Brown ierr = PetscFree(toLeaves);CHKERRQ(ierr); 6400a96aa3bSJed Brown ierr = PetscSFSetGraph(fromCoarse,numRootsFrom,numLeavesTo,NULL,PETSC_OWN_POINTER,fromRoots,PETSC_OWN_POINTER);CHKERRQ(ierr); 6410a96aa3bSJed Brown } else { /* generic */ 6420a96aa3bSJed Brown ierr = PetscSFSetGraph(fromCoarse,numRootsFrom,numLeavesTo,toLeaves,PETSC_OWN_POINTER,fromRoots,PETSC_OWN_POINTER);CHKERRQ(ierr); 6430a96aa3bSJed Brown } 6440a96aa3bSJed Brown *fromCoarseToFine = fromCoarse; 6450a96aa3bSJed Brown if (toCoarseFromFine) { 6460a96aa3bSJed Brown ierr = PetscSFSetGraph(toCoarse,numRootsTo,numLeavesFrom,fromLeaves,PETSC_OWN_POINTER,toRoots,PETSC_OWN_POINTER);CHKERRQ(ierr); 6470a96aa3bSJed Brown *toCoarseFromFine = toCoarse; 6480a96aa3bSJed Brown } 6490a96aa3bSJed Brown PetscFunctionReturn(0); 6500a96aa3bSJed Brown } 6510a96aa3bSJed Brown 6520a96aa3bSJed Brown /* range of processes whose B sections overlap this ranks A section */ 6530a96aa3bSJed Brown static PetscErrorCode DMPforestComputeOverlappingRanks(PetscMPIInt size, PetscMPIInt rank, p4est_t *p4estA, p4est_t *p4estB, PetscInt *startB, PetscInt *endB) 6540a96aa3bSJed Brown { 6550a96aa3bSJed Brown p4est_quadrant_t * myCoarseStart = &(p4estA->global_first_position[rank]); 6560a96aa3bSJed Brown p4est_quadrant_t * myCoarseEnd = &(p4estA->global_first_position[rank+1]); 6570a96aa3bSJed Brown p4est_quadrant_t * globalFirstB = p4estB->global_first_position; 6580a96aa3bSJed Brown 6590a96aa3bSJed Brown PetscFunctionBegin; 6600a96aa3bSJed Brown *startB = -1; 6610a96aa3bSJed Brown *endB = -1; 6620a96aa3bSJed Brown if (p4estA->local_num_quadrants) { 6630a96aa3bSJed Brown PetscInt lo, hi, guess; 6640a96aa3bSJed Brown /* binary search to find interval containing myCoarseStart */ 6650a96aa3bSJed Brown lo = 0; 6660a96aa3bSJed Brown hi = size; 6670a96aa3bSJed Brown guess = rank; 6680a96aa3bSJed Brown while (1) { 6690a96aa3bSJed Brown int startCompMy, myCompEnd; 6700a96aa3bSJed Brown 6710a96aa3bSJed Brown PetscStackCallP4estReturn(startCompMy,p4est_quadrant_compare_piggy,(&globalFirstB[guess],myCoarseStart)); 6720a96aa3bSJed Brown PetscStackCallP4estReturn(myCompEnd,p4est_quadrant_compare_piggy,(myCoarseStart,&globalFirstB[guess+1])); 6730a96aa3bSJed Brown if (startCompMy <= 0 && myCompEnd < 0) { 6740a96aa3bSJed Brown *startB = guess; 6750a96aa3bSJed Brown break; 6760a96aa3bSJed Brown } else if (startCompMy > 0) { /* guess is to high */ 6770a96aa3bSJed Brown hi = guess; 6780a96aa3bSJed Brown } else { /* guess is to low */ 6790a96aa3bSJed Brown lo = guess + 1; 6800a96aa3bSJed Brown } 6810a96aa3bSJed Brown guess = lo + (hi - lo) / 2; 6820a96aa3bSJed Brown } 6830a96aa3bSJed Brown /* reset bounds, but not guess */ 6840a96aa3bSJed Brown lo = 0; 6850a96aa3bSJed Brown hi = size; 6860a96aa3bSJed Brown while (1) { 6870a96aa3bSJed Brown int startCompMy, myCompEnd; 6880a96aa3bSJed Brown 6890a96aa3bSJed Brown PetscStackCallP4estReturn(startCompMy,p4est_quadrant_compare_piggy,(&globalFirstB[guess],myCoarseEnd)); 6900a96aa3bSJed Brown PetscStackCallP4estReturn(myCompEnd,p4est_quadrant_compare_piggy,(myCoarseEnd,&globalFirstB[guess+1])); 6910a96aa3bSJed Brown if (startCompMy < 0 && myCompEnd <= 0) { /* notice that the comparison operators are different from above */ 6920a96aa3bSJed Brown *endB = guess + 1; 6930a96aa3bSJed Brown break; 6940a96aa3bSJed Brown } else if (startCompMy >= 0) { /* guess is to high */ 6950a96aa3bSJed Brown hi = guess; 6960a96aa3bSJed Brown } else { /* guess is to low */ 6970a96aa3bSJed Brown lo = guess + 1; 6980a96aa3bSJed Brown } 6990a96aa3bSJed Brown guess = lo + (hi - lo) / 2; 7000a96aa3bSJed Brown } 7010a96aa3bSJed Brown } 7020a96aa3bSJed Brown PetscFunctionReturn(0); 7030a96aa3bSJed Brown } 7040a96aa3bSJed Brown 7050a96aa3bSJed Brown static PetscErrorCode DMPforestGetPlex(DM,DM*); 7060a96aa3bSJed Brown 7070a96aa3bSJed Brown #define DMSetUp_pforest _append_pforest(DMSetUp) 7080a96aa3bSJed Brown static PetscErrorCode DMSetUp_pforest(DM dm) 7090a96aa3bSJed Brown { 7100a96aa3bSJed Brown DM_Forest *forest = (DM_Forest*) dm->data; 7110a96aa3bSJed Brown DM_Forest_pforest *pforest = (DM_Forest_pforest*) forest->data; 7120a96aa3bSJed Brown DM base, adaptFrom; 7130a96aa3bSJed Brown DMForestTopology topoName; 7140a96aa3bSJed Brown PetscSF preCoarseToFine = NULL, coarseToPreFine = NULL; 7150a96aa3bSJed Brown PforestAdaptCtx ctx; 7160a96aa3bSJed Brown PetscErrorCode ierr; 7170a96aa3bSJed Brown 7180a96aa3bSJed Brown PetscFunctionBegin; 7190a96aa3bSJed Brown ctx.minLevel = PETSC_MAX_INT; 7200a96aa3bSJed Brown ctx.maxLevel = 0; 7210a96aa3bSJed Brown ctx.currLevel = 0; 7220a96aa3bSJed Brown ctx.anyChange = PETSC_FALSE; 7230a96aa3bSJed Brown /* sanity check */ 7240a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(dm,&adaptFrom);CHKERRQ(ierr); 7250a96aa3bSJed Brown ierr = DMForestGetBaseDM(dm,&base);CHKERRQ(ierr); 7260a96aa3bSJed Brown ierr = DMForestGetTopology(dm,&topoName);CHKERRQ(ierr); 727*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!adaptFrom && !base && !topoName,PetscObjectComm((PetscObject)dm),PETSC_ERR_ARG_WRONGSTATE,"A forest needs a topology, a base DM, or a DM to adapt from"); 7280a96aa3bSJed Brown 7290a96aa3bSJed Brown /* === Step 1: DMFTopology === */ 7300a96aa3bSJed Brown if (adaptFrom) { /* reference already created topology */ 7310a96aa3bSJed Brown PetscBool ispforest; 7320a96aa3bSJed Brown DM_Forest *aforest = (DM_Forest*) adaptFrom->data; 7330a96aa3bSJed Brown DM_Forest_pforest *apforest = (DM_Forest_pforest*) aforest->data; 7340a96aa3bSJed Brown 7350a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject)adaptFrom,DMPFOREST,&ispforest);CHKERRQ(ierr); 736*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!ispforest,PetscObjectComm((PetscObject)dm),PETSC_ERR_ARG_NOTSAMETYPE,"Trying to adapt from %s, which is not %s",((PetscObject)adaptFrom)->type_name,DMPFOREST); 737*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!apforest->topo,PetscObjectComm((PetscObject)dm),PETSC_ERR_ARG_WRONGSTATE,"The pre-adaptation forest must have a topology"); 7380a96aa3bSJed Brown ierr = DMSetUp(adaptFrom);CHKERRQ(ierr); 7390a96aa3bSJed Brown ierr = DMForestGetBaseDM(dm,&base);CHKERRQ(ierr); 7400a96aa3bSJed Brown ierr = DMForestGetTopology(dm,&topoName);CHKERRQ(ierr); 7410a96aa3bSJed Brown } else if (base) { /* construct a connectivity from base */ 7420a96aa3bSJed Brown PetscBool isPlex, isDA; 7430a96aa3bSJed Brown 7440a96aa3bSJed Brown ierr = PetscObjectGetName((PetscObject)base,&topoName);CHKERRQ(ierr); 7450a96aa3bSJed Brown ierr = DMForestSetTopology(dm,topoName);CHKERRQ(ierr); 7460a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject)base,DMPLEX,&isPlex);CHKERRQ(ierr); 7470a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject)base,DMDA,&isDA);CHKERRQ(ierr); 7480a96aa3bSJed Brown if (isPlex) { 7490a96aa3bSJed Brown MPI_Comm comm = PetscObjectComm((PetscObject)dm); 7500a96aa3bSJed Brown PetscInt depth; 7510a96aa3bSJed Brown PetscMPIInt size; 7520a96aa3bSJed Brown p4est_connectivity_t *conn = NULL; 7530a96aa3bSJed Brown DMFTopology_pforest *topo; 7540a96aa3bSJed Brown PetscInt *tree_face_to_uniq = NULL; 7550a96aa3bSJed Brown PetscErrorCode ierr; 7560a96aa3bSJed Brown 7570a96aa3bSJed Brown ierr = DMPlexGetDepth(base,&depth);CHKERRQ(ierr); 7580a96aa3bSJed Brown if (depth == 1) { 7590a96aa3bSJed Brown DM connDM; 7600a96aa3bSJed Brown 7610a96aa3bSJed Brown ierr = DMPlexInterpolate(base,&connDM);CHKERRQ(ierr); 7620a96aa3bSJed Brown base = connDM; 7630a96aa3bSJed Brown ierr = DMForestSetBaseDM(dm,base);CHKERRQ(ierr); 7640a96aa3bSJed Brown ierr = DMDestroy(&connDM);CHKERRQ(ierr); 765*2c71b3e2SJacob Faibussowitsch } else PetscCheckFalse(depth != P4EST_DIM,comm,PETSC_ERR_ARG_WRONG,"Base plex is neither interpolated nor uninterpolated? depth %D, expected 2 or %d",depth,P4EST_DIM + 1); 7660a96aa3bSJed Brown ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 7670a96aa3bSJed Brown if (size > 1) { 7680a96aa3bSJed Brown DM dmRedundant; 7690a96aa3bSJed Brown PetscSF sf; 7700a96aa3bSJed Brown 7710a96aa3bSJed Brown ierr = DMPlexGetRedundantDM(base,&sf,&dmRedundant);CHKERRQ(ierr); 772*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!dmRedundant,comm,PETSC_ERR_PLIB,"Could not create redundant DM"); 7730a96aa3bSJed Brown ierr = PetscObjectCompose((PetscObject)dmRedundant,"_base_migration_sf",(PetscObject)sf);CHKERRQ(ierr); 7740a96aa3bSJed Brown ierr = PetscSFDestroy(&sf);CHKERRQ(ierr); 7750a96aa3bSJed Brown base = dmRedundant; 7760a96aa3bSJed Brown ierr = DMForestSetBaseDM(dm,base);CHKERRQ(ierr); 7770a96aa3bSJed Brown ierr = DMDestroy(&dmRedundant);CHKERRQ(ierr); 7780a96aa3bSJed Brown } 7790a96aa3bSJed Brown ierr = DMViewFromOptions(base,NULL,"-dm_p4est_base_view");CHKERRQ(ierr); 7800a96aa3bSJed Brown ierr = DMPlexCreateConnectivity_pforest(base,&conn,&tree_face_to_uniq);CHKERRQ(ierr); 7810a96aa3bSJed Brown ierr = PetscNewLog(dm,&topo);CHKERRQ(ierr); 7820a96aa3bSJed Brown topo->refct = 1; 7830a96aa3bSJed Brown topo->conn = conn; 7840a96aa3bSJed Brown topo->geom = NULL; 7850a96aa3bSJed Brown { 7860a96aa3bSJed Brown PetscErrorCode (*map)(DM,PetscInt,PetscInt,const PetscReal[],PetscReal[],void*); 7870a96aa3bSJed Brown void *mapCtx; 7880a96aa3bSJed Brown 7890a96aa3bSJed Brown ierr = DMForestGetBaseCoordinateMapping(dm,&map,&mapCtx);CHKERRQ(ierr); 7900a96aa3bSJed Brown if (map) { 7910a96aa3bSJed Brown DM_Forest_geometry_pforest *geom_pforest; 7920a96aa3bSJed Brown p4est_geometry_t *geom; 7930a96aa3bSJed Brown 7940a96aa3bSJed Brown ierr = PetscNew(&geom_pforest);CHKERRQ(ierr); 7950a96aa3bSJed Brown ierr = DMGetCoordinateDim(dm,&geom_pforest->coordDim);CHKERRQ(ierr); 7960a96aa3bSJed Brown geom_pforest->map = map; 7970a96aa3bSJed Brown geom_pforest->mapCtx = mapCtx; 7980a96aa3bSJed Brown PetscStackCallP4estReturn(geom_pforest->inner,p4est_geometry_new_connectivity,(conn)); 7990a96aa3bSJed Brown ierr = PetscNew(&geom);CHKERRQ(ierr); 8000a96aa3bSJed Brown geom->name = topoName; 8010a96aa3bSJed Brown geom->user = geom_pforest; 8020a96aa3bSJed Brown geom->X = GeometryMapping_pforest; 8030a96aa3bSJed Brown geom->destroy = GeometryDestroy_pforest; 8040a96aa3bSJed Brown topo->geom = geom; 8050a96aa3bSJed Brown } 8060a96aa3bSJed Brown } 8070a96aa3bSJed Brown topo->tree_face_to_uniq = tree_face_to_uniq; 8080a96aa3bSJed Brown pforest->topo = topo; 809*2c71b3e2SJacob Faibussowitsch } else PetscCheckFalse(isDA,PetscObjectComm((PetscObject)dm),PETSC_ERR_PLIB,"Not implemented yet"); 8100a96aa3bSJed Brown #if 0 8110a96aa3bSJed Brown PetscInt N[3], P[3]; 8120a96aa3bSJed Brown 8130a96aa3bSJed Brown /* get the sizes, periodicities */ 8140a96aa3bSJed Brown /* ... */ 8150a96aa3bSJed Brown /* don't use Morton order */ 8160a96aa3bSJed Brown ierr = DMFTopologyCreateBrick_pforest(dm,N,P,&pforest->topo,PETSC_FALSE);CHKERRQ(ierr); 8170a96aa3bSJed Brown #endif 8180a96aa3bSJed Brown { 8190a96aa3bSJed Brown PetscInt numLabels, l; 8200a96aa3bSJed Brown 8210a96aa3bSJed Brown ierr = DMGetNumLabels(base,&numLabels);CHKERRQ(ierr); 8220a96aa3bSJed Brown for (l = 0; l < numLabels; l++) { 8230a96aa3bSJed Brown PetscBool isDepth, isGhost, isVTK, isDim, isCellType; 8240a96aa3bSJed Brown DMLabel label, labelNew; 8250a96aa3bSJed Brown PetscInt defVal; 8260a96aa3bSJed Brown const char *name; 8270a96aa3bSJed Brown 8280a96aa3bSJed Brown ierr = DMGetLabelName(base, l, &name);CHKERRQ(ierr); 8290a96aa3bSJed Brown ierr = DMGetLabelByNum(base, l, &label);CHKERRQ(ierr); 8300a96aa3bSJed Brown ierr = PetscStrcmp(name,"depth",&isDepth);CHKERRQ(ierr); 8310a96aa3bSJed Brown if (isDepth) continue; 8320a96aa3bSJed Brown ierr = PetscStrcmp(name,"dim",&isDim);CHKERRQ(ierr); 8330a96aa3bSJed Brown if (isDim) continue; 8340a96aa3bSJed Brown ierr = PetscStrcmp(name,"celltype",&isCellType);CHKERRQ(ierr); 8350a96aa3bSJed Brown if (isCellType) continue; 8360a96aa3bSJed Brown ierr = PetscStrcmp(name,"ghost",&isGhost);CHKERRQ(ierr); 8370a96aa3bSJed Brown if (isGhost) continue; 8380a96aa3bSJed Brown ierr = PetscStrcmp(name,"vtk",&isVTK);CHKERRQ(ierr); 8390a96aa3bSJed Brown if (isVTK) continue; 8400a96aa3bSJed Brown ierr = DMCreateLabel(dm,name);CHKERRQ(ierr); 8410a96aa3bSJed Brown ierr = DMGetLabel(dm,name,&labelNew);CHKERRQ(ierr); 8420a96aa3bSJed Brown ierr = DMLabelGetDefaultValue(label,&defVal);CHKERRQ(ierr); 8430a96aa3bSJed Brown ierr = DMLabelSetDefaultValue(labelNew,defVal);CHKERRQ(ierr); 8440a96aa3bSJed Brown } 8450a96aa3bSJed Brown /* map dm points (internal plex) to base 8460a96aa3bSJed Brown we currently create the subpoint_map for the entire hierarchy, starting from the finest forest 8470a96aa3bSJed Brown and propagating back to the coarsest 8480a96aa3bSJed Brown This is not an optimal approach, since we need the map only on the coarsest level 8490a96aa3bSJed Brown during DMForestTransferVecFromBase */ 8500a96aa3bSJed Brown ierr = DMForestGetMinimumRefinement(dm,&l);CHKERRQ(ierr); 8510a96aa3bSJed Brown if (!l) { 8520a96aa3bSJed Brown ierr = DMCreateLabel(dm,"_forest_base_subpoint_map");CHKERRQ(ierr); 8530a96aa3bSJed Brown } 8540a96aa3bSJed Brown } 8550a96aa3bSJed Brown } else { /* construct from topology name */ 8560a96aa3bSJed Brown DMFTopology_pforest *topo; 8570a96aa3bSJed Brown 8580a96aa3bSJed Brown ierr = DMFTopologyCreate_pforest(dm,topoName,&topo);CHKERRQ(ierr); 8590a96aa3bSJed Brown pforest->topo = topo; 8600a96aa3bSJed Brown /* TODO: construct base? */ 8610a96aa3bSJed Brown } 8620a96aa3bSJed Brown 8630a96aa3bSJed Brown /* === Step 2: get the leaves of the forest === */ 8640a96aa3bSJed Brown if (adaptFrom) { /* start with the old forest */ 8650a96aa3bSJed Brown DMLabel adaptLabel; 8660a96aa3bSJed Brown PetscInt defaultValue; 8670a96aa3bSJed Brown PetscInt numValues, numValuesGlobal, cLocalStart, count; 8680a96aa3bSJed Brown DM_Forest *aforest = (DM_Forest*) adaptFrom->data; 8690a96aa3bSJed Brown DM_Forest_pforest *apforest = (DM_Forest_pforest*) aforest->data; 8700a96aa3bSJed Brown PetscBool computeAdaptSF; 8710a96aa3bSJed Brown p4est_topidx_t flt, llt, t; 8720a96aa3bSJed Brown 8730a96aa3bSJed Brown flt = apforest->forest->first_local_tree; 8740a96aa3bSJed Brown llt = apforest->forest->last_local_tree; 8750a96aa3bSJed Brown cLocalStart = apforest->cLocalStart; 8760a96aa3bSJed Brown ierr = DMForestGetComputeAdaptivitySF(dm,&computeAdaptSF);CHKERRQ(ierr); 8770a96aa3bSJed Brown PetscStackCallP4estReturn(pforest->forest,p4est_copy,(apforest->forest, 0)); /* 0 indicates no data copying */ 8780a96aa3bSJed Brown ierr = DMForestGetAdaptivityLabel(dm,&adaptLabel);CHKERRQ(ierr); 8790a96aa3bSJed Brown if (adaptLabel) { 8800a96aa3bSJed Brown /* apply the refinement/coarsening by flags, plus minimum/maximum refinement */ 8810a96aa3bSJed Brown ierr = DMLabelGetNumValues(adaptLabel,&numValues);CHKERRQ(ierr); 8820a96aa3bSJed Brown ierr = MPI_Allreduce(&numValues,&numValuesGlobal,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)adaptFrom));CHKERRMPI(ierr); 8830a96aa3bSJed Brown ierr = DMLabelGetDefaultValue(adaptLabel,&defaultValue);CHKERRQ(ierr); 8840a96aa3bSJed Brown if (!numValuesGlobal && defaultValue == DM_ADAPT_COARSEN_LAST) { /* uniform coarsen of the last level only (equivalent to DM_ADAPT_COARSEN for conforming grids) */ 8850a96aa3bSJed Brown ierr = DMForestGetMinimumRefinement(dm,&ctx.minLevel);CHKERRQ(ierr); 8860a96aa3bSJed Brown ierr = DMPforestGetRefinementLevel(dm,&ctx.currLevel);CHKERRQ(ierr); 8870a96aa3bSJed Brown pforest->forest->user_pointer = (void*) &ctx; 8880a96aa3bSJed Brown PetscStackCallP4est(p4est_coarsen,(pforest->forest,0,pforest_coarsen_currlevel,NULL)); 8890a96aa3bSJed Brown pforest->forest->user_pointer = (void*) dm; 8900a96aa3bSJed Brown PetscStackCallP4est(p4est_balance,(pforest->forest,P4EST_CONNECT_FULL,NULL)); 8910a96aa3bSJed Brown /* we will have to change the offset after we compute the overlap */ 8920a96aa3bSJed Brown if (computeAdaptSF) { 8930a96aa3bSJed Brown ierr = DMPforestComputeLocalCellTransferSF(PetscObjectComm((PetscObject)dm),pforest->forest,0,apforest->forest,apforest->cLocalStart,&coarseToPreFine,NULL);CHKERRQ(ierr); 8940a96aa3bSJed Brown } 8950a96aa3bSJed Brown } else if (!numValuesGlobal && defaultValue == DM_ADAPT_COARSEN) { /* uniform coarsen */ 8960a96aa3bSJed Brown ierr = DMForestGetMinimumRefinement(dm,&ctx.minLevel);CHKERRQ(ierr); 8970a96aa3bSJed Brown pforest->forest->user_pointer = (void*) &ctx; 8980a96aa3bSJed Brown PetscStackCallP4est(p4est_coarsen,(pforest->forest,0,pforest_coarsen_uniform,NULL)); 8990a96aa3bSJed Brown pforest->forest->user_pointer = (void*) dm; 9000a96aa3bSJed Brown PetscStackCallP4est(p4est_balance,(pforest->forest,P4EST_CONNECT_FULL,NULL)); 9010a96aa3bSJed Brown /* we will have to change the offset after we compute the overlap */ 9020a96aa3bSJed Brown if (computeAdaptSF) { 9030a96aa3bSJed Brown ierr = DMPforestComputeLocalCellTransferSF(PetscObjectComm((PetscObject)dm),pforest->forest,0,apforest->forest,apforest->cLocalStart,&coarseToPreFine,NULL);CHKERRQ(ierr); 9040a96aa3bSJed Brown } 9050a96aa3bSJed Brown } else if (!numValuesGlobal && defaultValue == DM_ADAPT_REFINE) { /* uniform refine */ 9060a96aa3bSJed Brown ierr = DMForestGetMaximumRefinement(dm,&ctx.maxLevel);CHKERRQ(ierr); 9070a96aa3bSJed Brown pforest->forest->user_pointer = (void*) &ctx; 9080a96aa3bSJed Brown PetscStackCallP4est(p4est_refine,(pforest->forest,0,pforest_refine_uniform,NULL)); 9090a96aa3bSJed Brown pforest->forest->user_pointer = (void*) dm; 9100a96aa3bSJed Brown PetscStackCallP4est(p4est_balance,(pforest->forest,P4EST_CONNECT_FULL,NULL)); 9110a96aa3bSJed Brown /* we will have to change the offset after we compute the overlap */ 9120a96aa3bSJed Brown if (computeAdaptSF) { 9130a96aa3bSJed Brown ierr = DMPforestComputeLocalCellTransferSF(PetscObjectComm((PetscObject)dm),apforest->forest,apforest->cLocalStart,pforest->forest,0,&preCoarseToFine,NULL);CHKERRQ(ierr); 9140a96aa3bSJed Brown } 9150a96aa3bSJed Brown } else if (numValuesGlobal) { 9160a96aa3bSJed Brown p4est_t *p4est = pforest->forest; 9170a96aa3bSJed Brown PetscInt *cellFlags; 9180a96aa3bSJed Brown DMForestAdaptivityStrategy strategy; 9190a96aa3bSJed Brown PetscSF cellSF; 9200a96aa3bSJed Brown PetscInt c, cStart, cEnd; 9210a96aa3bSJed Brown PetscBool adaptAny; 9220a96aa3bSJed Brown 9230a96aa3bSJed Brown ierr = DMForestGetMaximumRefinement(dm,&ctx.maxLevel);CHKERRQ(ierr); 9240a96aa3bSJed Brown ierr = DMForestGetMinimumRefinement(dm,&ctx.minLevel);CHKERRQ(ierr); 9250a96aa3bSJed Brown ierr = DMForestGetAdaptivityStrategy(dm,&strategy);CHKERRQ(ierr); 9260a96aa3bSJed Brown ierr = PetscStrncmp(strategy,"any",3,&adaptAny);CHKERRQ(ierr); 9270a96aa3bSJed Brown ierr = DMForestGetCellChart(adaptFrom,&cStart,&cEnd);CHKERRQ(ierr); 9280a96aa3bSJed Brown ierr = DMForestGetCellSF(adaptFrom,&cellSF);CHKERRQ(ierr); 9290a96aa3bSJed Brown ierr = PetscMalloc1(cEnd-cStart,&cellFlags);CHKERRQ(ierr); 9300a96aa3bSJed Brown for (c = cStart; c < cEnd; c++) {ierr = DMLabelGetValue(adaptLabel,c,&cellFlags[c-cStart]);CHKERRQ(ierr);} 9310a96aa3bSJed Brown if (cellSF) { 9320a96aa3bSJed Brown if (adaptAny) { 9330a96aa3bSJed Brown ierr = PetscSFReduceBegin(cellSF,MPIU_INT,cellFlags,cellFlags,MPI_MAX);CHKERRQ(ierr); 9340a96aa3bSJed Brown ierr = PetscSFReduceEnd(cellSF,MPIU_INT,cellFlags,cellFlags,MPI_MAX);CHKERRQ(ierr); 9350a96aa3bSJed Brown } else { 9360a96aa3bSJed Brown ierr = PetscSFReduceBegin(cellSF,MPIU_INT,cellFlags,cellFlags,MPI_MIN);CHKERRQ(ierr); 9370a96aa3bSJed Brown ierr = PetscSFReduceEnd(cellSF,MPIU_INT,cellFlags,cellFlags,MPI_MIN);CHKERRQ(ierr); 9380a96aa3bSJed Brown } 9390a96aa3bSJed Brown } 9400a96aa3bSJed Brown for (t = flt, count = cLocalStart; t <= llt; t++) { 9410a96aa3bSJed Brown p4est_tree_t *tree = &(((p4est_tree_t*) p4est->trees->array)[t]); 9420a96aa3bSJed Brown PetscInt numQuads = (PetscInt) tree->quadrants.elem_count, i; 9430a96aa3bSJed Brown p4est_quadrant_t *quads = (p4est_quadrant_t *) tree->quadrants.array; 9440a96aa3bSJed Brown 9450a96aa3bSJed Brown for (i = 0; i < numQuads; i++) { 9460a96aa3bSJed Brown p4est_quadrant_t *q = &quads[i]; 9470a96aa3bSJed Brown q->p.user_int = cellFlags[count++]; 9480a96aa3bSJed Brown } 9490a96aa3bSJed Brown } 9500a96aa3bSJed Brown ierr = PetscFree(cellFlags);CHKERRQ(ierr); 9510a96aa3bSJed Brown 9520a96aa3bSJed Brown pforest->forest->user_pointer = (void*) &ctx; 9530a96aa3bSJed Brown if (adaptAny) { 9540a96aa3bSJed Brown PetscStackCallP4est(p4est_coarsen,(pforest->forest,0,pforest_coarsen_flag_any,pforest_init_determine)); 9550a96aa3bSJed Brown } else { 9560a96aa3bSJed Brown PetscStackCallP4est(p4est_coarsen,(pforest->forest,0,pforest_coarsen_flag_all,pforest_init_determine)); 9570a96aa3bSJed Brown } 9580a96aa3bSJed Brown PetscStackCallP4est(p4est_refine,(pforest->forest,0,pforest_refine_flag,NULL)); 9590a96aa3bSJed Brown pforest->forest->user_pointer = (void*) dm; 9600a96aa3bSJed Brown PetscStackCallP4est(p4est_balance,(pforest->forest,P4EST_CONNECT_FULL,NULL)); 9610a96aa3bSJed Brown if (computeAdaptSF) { 9620a96aa3bSJed Brown ierr = DMPforestComputeLocalCellTransferSF(PetscObjectComm((PetscObject)dm),apforest->forest,apforest->cLocalStart,pforest->forest,0,&preCoarseToFine,&coarseToPreFine);CHKERRQ(ierr); 9630a96aa3bSJed Brown } 9640a96aa3bSJed Brown } 9650a96aa3bSJed Brown for (t = flt, count = cLocalStart; t <= llt; t++) { 9660a96aa3bSJed Brown p4est_tree_t *atree = &(((p4est_tree_t*) apforest->forest->trees->array)[t]); 9670a96aa3bSJed Brown p4est_tree_t *tree = &(((p4est_tree_t*) pforest->forest->trees->array)[t]); 9680a96aa3bSJed Brown PetscInt anumQuads = (PetscInt) atree->quadrants.elem_count, i; 9690a96aa3bSJed Brown PetscInt numQuads = (PetscInt) tree->quadrants.elem_count; 9700a96aa3bSJed Brown p4est_quadrant_t *aquads = (p4est_quadrant_t *) atree->quadrants.array; 9710a96aa3bSJed Brown p4est_quadrant_t *quads = (p4est_quadrant_t *) tree->quadrants.array; 9720a96aa3bSJed Brown 9730a96aa3bSJed Brown if (anumQuads != numQuads) { 9740a96aa3bSJed Brown ctx.anyChange = PETSC_TRUE; 9750a96aa3bSJed Brown } else { 9760a96aa3bSJed Brown for (i = 0; i < numQuads; i++) { 9770a96aa3bSJed Brown p4est_quadrant_t *aq = &aquads[i]; 9780a96aa3bSJed Brown p4est_quadrant_t *q = &quads[i]; 9790a96aa3bSJed Brown 9800a96aa3bSJed Brown if (aq->level != q->level) { 9810a96aa3bSJed Brown ctx.anyChange = PETSC_TRUE; 9820a96aa3bSJed Brown break; 9830a96aa3bSJed Brown } 9840a96aa3bSJed Brown } 9850a96aa3bSJed Brown } 9860a96aa3bSJed Brown if (ctx.anyChange) { 9870a96aa3bSJed Brown break; 9880a96aa3bSJed Brown } 9890a96aa3bSJed Brown } 9900a96aa3bSJed Brown } 9910a96aa3bSJed Brown { 9920a96aa3bSJed Brown PetscInt numLabels, l; 9930a96aa3bSJed Brown 9940a96aa3bSJed Brown ierr = DMGetNumLabels(adaptFrom,&numLabels);CHKERRQ(ierr); 9950a96aa3bSJed Brown for (l = 0; l < numLabels; l++) { 9960a96aa3bSJed Brown PetscBool isDepth, isCellType, isGhost, isVTK; 9970a96aa3bSJed Brown DMLabel label, labelNew; 9980a96aa3bSJed Brown PetscInt defVal; 9990a96aa3bSJed Brown const char *name; 10000a96aa3bSJed Brown 10010a96aa3bSJed Brown ierr = DMGetLabelName(adaptFrom, l, &name);CHKERRQ(ierr); 10020a96aa3bSJed Brown ierr = DMGetLabelByNum(adaptFrom, l, &label);CHKERRQ(ierr); 10030a96aa3bSJed Brown ierr = PetscStrcmp(name,"depth",&isDepth);CHKERRQ(ierr); 10040a96aa3bSJed Brown if (isDepth) continue; 10050a96aa3bSJed Brown ierr = PetscStrcmp(name,"celltype",&isCellType);CHKERRQ(ierr); 10060a96aa3bSJed Brown if (isCellType) continue; 10070a96aa3bSJed Brown ierr = PetscStrcmp(name,"ghost",&isGhost);CHKERRQ(ierr); 10080a96aa3bSJed Brown if (isGhost) continue; 10090a96aa3bSJed Brown ierr = PetscStrcmp(name,"vtk",&isVTK);CHKERRQ(ierr); 10100a96aa3bSJed Brown if (isVTK) continue; 10110a96aa3bSJed Brown ierr = DMCreateLabel(dm,name);CHKERRQ(ierr); 10120a96aa3bSJed Brown ierr = DMGetLabel(dm,name,&labelNew);CHKERRQ(ierr); 10130a96aa3bSJed Brown ierr = DMLabelGetDefaultValue(label,&defVal);CHKERRQ(ierr); 10140a96aa3bSJed Brown ierr = DMLabelSetDefaultValue(labelNew,defVal);CHKERRQ(ierr); 10150a96aa3bSJed Brown } 10160a96aa3bSJed Brown } 10170a96aa3bSJed Brown } else { /* initial */ 10180a96aa3bSJed Brown PetscInt initLevel, minLevel; 10190a96aa3bSJed Brown 10200a96aa3bSJed Brown ierr = DMForestGetInitialRefinement(dm,&initLevel);CHKERRQ(ierr); 10210a96aa3bSJed Brown ierr = DMForestGetMinimumRefinement(dm,&minLevel);CHKERRQ(ierr); 10220a96aa3bSJed Brown PetscStackCallP4estReturn(pforest->forest,p4est_new_ext,(PetscObjectComm((PetscObject)dm),pforest->topo->conn, 10230a96aa3bSJed Brown 0, /* minimum number of quadrants per processor */ 10240a96aa3bSJed Brown initLevel, /* level of refinement */ 10250a96aa3bSJed Brown 1, /* uniform refinement */ 10260a96aa3bSJed Brown 0, /* we don't allocate any per quadrant data */ 10270a96aa3bSJed Brown NULL, /* there is no special quadrant initialization */ 10280a96aa3bSJed Brown (void*)dm)); /* this dm is the user context */ 10290a96aa3bSJed Brown 10300a96aa3bSJed Brown if (initLevel > minLevel) pforest->coarsen_hierarchy = PETSC_TRUE; 10310a96aa3bSJed Brown if (dm->setfromoptionscalled) { 10320a96aa3bSJed Brown PetscBool flgPattern, flgFractal; 10330a96aa3bSJed Brown PetscInt corner = 0; 10340a96aa3bSJed Brown PetscInt corners[P4EST_CHILDREN], ncorner = P4EST_CHILDREN; 10350a96aa3bSJed Brown PetscReal likelihood = 1./ P4EST_DIM; 10360a96aa3bSJed Brown PetscInt pattern; 10370a96aa3bSJed Brown const char *prefix; 10380a96aa3bSJed Brown 10390a96aa3bSJed Brown ierr = PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);CHKERRQ(ierr); 10400a96aa3bSJed Brown ierr = PetscOptionsGetEList(((PetscObject)dm)->options,prefix,"-dm_p4est_refine_pattern",DMRefinePatternName,PATTERN_COUNT,&pattern,&flgPattern);CHKERRQ(ierr); 10410a96aa3bSJed Brown ierr = PetscOptionsGetInt(((PetscObject)dm)->options,prefix,"-dm_p4est_refine_corner",&corner,NULL);CHKERRQ(ierr); 10420a96aa3bSJed Brown ierr = PetscOptionsGetIntArray(((PetscObject)dm)->options,prefix,"-dm_p4est_refine_fractal_corners",corners,&ncorner,&flgFractal);CHKERRQ(ierr); 10430a96aa3bSJed Brown ierr = PetscOptionsGetReal(((PetscObject)dm)->options,prefix,"-dm_p4est_refine_hash_likelihood",&likelihood,NULL);CHKERRQ(ierr); 10440a96aa3bSJed Brown 10450a96aa3bSJed Brown if (flgPattern) { 10460a96aa3bSJed Brown DMRefinePatternCtx *ctx; 10470a96aa3bSJed Brown PetscInt maxLevel; 10480a96aa3bSJed Brown 10490a96aa3bSJed Brown ierr = DMForestGetMaximumRefinement(dm,&maxLevel);CHKERRQ(ierr); 10500a96aa3bSJed Brown ierr = PetscNewLog(dm,&ctx);CHKERRQ(ierr); 10510a96aa3bSJed Brown ctx->maxLevel = PetscMin(maxLevel,P4EST_QMAXLEVEL); 10520a96aa3bSJed Brown if (initLevel + ctx->maxLevel > minLevel) pforest->coarsen_hierarchy = PETSC_TRUE; 10530a96aa3bSJed Brown switch (pattern) { 10540a96aa3bSJed Brown case PATTERN_HASH: 10550a96aa3bSJed Brown ctx->refine_fn = DMRefinePattern_Hash; 10560a96aa3bSJed Brown ctx->hashLikelihood = likelihood; 10570a96aa3bSJed Brown break; 10580a96aa3bSJed Brown case PATTERN_CORNER: 10590a96aa3bSJed Brown ctx->corner = corner; 10600a96aa3bSJed Brown ctx->refine_fn = DMRefinePattern_Corner; 10610a96aa3bSJed Brown break; 10620a96aa3bSJed Brown case PATTERN_CENTER: 10630a96aa3bSJed Brown ctx->refine_fn = DMRefinePattern_Center; 10640a96aa3bSJed Brown break; 10650a96aa3bSJed Brown case PATTERN_FRACTAL: 10660a96aa3bSJed Brown if (flgFractal) { 10670a96aa3bSJed Brown PetscInt i; 10680a96aa3bSJed Brown 10690a96aa3bSJed Brown for (i = 0; i < ncorner; i++) ctx->fractal[corners[i]] = PETSC_TRUE; 10700a96aa3bSJed Brown } else { 10710a96aa3bSJed Brown #if !defined(P4_TO_P8) 10720a96aa3bSJed Brown ctx->fractal[0] = ctx->fractal[1] = ctx->fractal[2] = PETSC_TRUE; 10730a96aa3bSJed Brown #else 10740a96aa3bSJed Brown ctx->fractal[0] = ctx->fractal[3] = ctx->fractal[5] = ctx->fractal[6] = PETSC_TRUE; 10750a96aa3bSJed Brown #endif 10760a96aa3bSJed Brown } 10770a96aa3bSJed Brown ctx->refine_fn = DMRefinePattern_Fractal; 10780a96aa3bSJed Brown break; 10790a96aa3bSJed Brown default: 10800a96aa3bSJed Brown SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Not a valid refinement pattern"); 10810a96aa3bSJed Brown } 10820a96aa3bSJed Brown 10830a96aa3bSJed Brown pforest->forest->user_pointer = (void*) ctx; 10840a96aa3bSJed Brown PetscStackCallP4est(p4est_refine,(pforest->forest,1,ctx->refine_fn,NULL)); 10850a96aa3bSJed Brown PetscStackCallP4est(p4est_balance,(pforest->forest,P4EST_CONNECT_FULL,NULL)); 10860a96aa3bSJed Brown ierr = PetscFree(ctx);CHKERRQ(ierr); 10870a96aa3bSJed Brown pforest->forest->user_pointer = (void*) dm; 10880a96aa3bSJed Brown } 10890a96aa3bSJed Brown } 10900a96aa3bSJed Brown } 10910a96aa3bSJed Brown if (pforest->coarsen_hierarchy) { 10920a96aa3bSJed Brown PetscInt initLevel, currLevel, minLevel; 10930a96aa3bSJed Brown 10940a96aa3bSJed Brown ierr = DMPforestGetRefinementLevel(dm,&currLevel);CHKERRQ(ierr); 10950a96aa3bSJed Brown ierr = DMForestGetInitialRefinement(dm,&initLevel);CHKERRQ(ierr); 10960a96aa3bSJed Brown ierr = DMForestGetMinimumRefinement(dm,&minLevel);CHKERRQ(ierr); 10970a96aa3bSJed Brown if (currLevel > minLevel) { 10980a96aa3bSJed Brown DM_Forest_pforest *coarse_pforest; 10990a96aa3bSJed Brown DMLabel coarsen; 11000a96aa3bSJed Brown DM coarseDM; 11010a96aa3bSJed Brown 11020a96aa3bSJed Brown ierr = DMForestTemplate(dm,MPI_COMM_NULL,&coarseDM);CHKERRQ(ierr); 11030a96aa3bSJed Brown ierr = DMForestSetAdaptivityPurpose(coarseDM,DM_ADAPT_COARSEN);CHKERRQ(ierr); 11040a96aa3bSJed Brown ierr = DMLabelCreate(PETSC_COMM_SELF, "coarsen",&coarsen);CHKERRQ(ierr); 11050a96aa3bSJed Brown ierr = DMLabelSetDefaultValue(coarsen,DM_ADAPT_COARSEN);CHKERRQ(ierr); 11060a96aa3bSJed Brown ierr = DMForestSetAdaptivityLabel(coarseDM,coarsen);CHKERRQ(ierr); 11070a96aa3bSJed Brown ierr = DMLabelDestroy(&coarsen);CHKERRQ(ierr); 11080a96aa3bSJed Brown ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr); 11090a96aa3bSJed Brown ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr); 11100a96aa3bSJed Brown initLevel = currLevel == initLevel ? initLevel - 1 : initLevel; 11110a96aa3bSJed Brown ierr = DMForestSetInitialRefinement(coarseDM,initLevel);CHKERRQ(ierr); 11120a96aa3bSJed Brown ierr = DMForestSetMinimumRefinement(coarseDM,minLevel);CHKERRQ(ierr); 11130a96aa3bSJed Brown coarse_pforest = (DM_Forest_pforest*) ((DM_Forest*) coarseDM->data)->data; 11140a96aa3bSJed Brown coarse_pforest->coarsen_hierarchy = PETSC_TRUE; 11150a96aa3bSJed Brown } 11160a96aa3bSJed Brown } 11170a96aa3bSJed Brown 11180a96aa3bSJed Brown { /* repartitioning and overlap */ 11190a96aa3bSJed Brown PetscMPIInt size, rank; 11200a96aa3bSJed Brown 11210a96aa3bSJed Brown ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm),&size);CHKERRMPI(ierr); 11220a96aa3bSJed Brown ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm),&rank);CHKERRMPI(ierr); 11230a96aa3bSJed Brown if ((size > 1) && (pforest->partition_for_coarsening || forest->cellWeights || forest->weightCapacity != 1. || forest->weightsFactor != 1.)) { 11240a96aa3bSJed Brown PetscBool copyForest = PETSC_FALSE; 11250a96aa3bSJed Brown p4est_t *forest_copy = NULL; 11260a96aa3bSJed Brown p4est_gloidx_t shipped = 0; 11270a96aa3bSJed Brown 11280a96aa3bSJed Brown if (preCoarseToFine || coarseToPreFine) copyForest = PETSC_TRUE; 11290a96aa3bSJed Brown if (copyForest) PetscStackCallP4estReturn(forest_copy,p4est_copy,(pforest->forest,0)); 11300a96aa3bSJed Brown 11310a96aa3bSJed Brown if (!forest->cellWeights && forest->weightCapacity == 1. && forest->weightsFactor == 1.) { 11320a96aa3bSJed Brown PetscStackCallP4estReturn(shipped,p4est_partition_ext,(pforest->forest,(int)pforest->partition_for_coarsening,NULL)); 11330a96aa3bSJed Brown } else SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_PLIB,"Non-uniform partition cases not implemented yet"); 11340a96aa3bSJed Brown if (shipped) ctx.anyChange = PETSC_TRUE; 11350a96aa3bSJed Brown if (forest_copy) { 11360a96aa3bSJed Brown if (preCoarseToFine || coarseToPreFine) { 11370a96aa3bSJed Brown PetscSF repartSF; /* repartSF has roots in the old partition */ 11380a96aa3bSJed Brown PetscInt pStart = -1, pEnd = -1, p; 11390a96aa3bSJed Brown PetscInt numRoots, numLeaves; 11400a96aa3bSJed Brown PetscSFNode *repartRoots; 11410a96aa3bSJed Brown p4est_gloidx_t postStart = pforest->forest->global_first_quadrant[rank]; 11420a96aa3bSJed Brown p4est_gloidx_t postEnd = pforest->forest->global_first_quadrant[rank+1]; 11430a96aa3bSJed Brown p4est_gloidx_t partOffset = postStart; 11440a96aa3bSJed Brown 11450a96aa3bSJed Brown numRoots = (PetscInt) (forest_copy->global_first_quadrant[rank + 1] - forest_copy->global_first_quadrant[rank]); 11460a96aa3bSJed Brown numLeaves = (PetscInt) (postEnd - postStart); 11470a96aa3bSJed Brown ierr = DMPforestComputeOverlappingRanks(size,rank,pforest->forest,forest_copy,&pStart,&pEnd);CHKERRQ(ierr); 11480a96aa3bSJed Brown ierr = PetscMalloc1((PetscInt) pforest->forest->local_num_quadrants,&repartRoots);CHKERRQ(ierr); 11490a96aa3bSJed Brown for (p = pStart; p < pEnd; p++) { 11500a96aa3bSJed Brown p4est_gloidx_t preStart = forest_copy->global_first_quadrant[p]; 11510a96aa3bSJed Brown p4est_gloidx_t preEnd = forest_copy->global_first_quadrant[p+1]; 11520a96aa3bSJed Brown PetscInt q; 11530a96aa3bSJed Brown 11540a96aa3bSJed Brown if (preEnd == preStart) continue; 1155*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(preStart > postStart,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Bad partition overlap computation"); 11560a96aa3bSJed Brown preEnd = preEnd > postEnd ? postEnd : preEnd; 11570a96aa3bSJed Brown for (q = partOffset; q < preEnd; q++) { 11580a96aa3bSJed Brown repartRoots[q - postStart].rank = p; 11590a96aa3bSJed Brown repartRoots[q - postStart].index = partOffset - preStart; 11600a96aa3bSJed Brown } 11610a96aa3bSJed Brown partOffset = preEnd; 11620a96aa3bSJed Brown } 11630a96aa3bSJed Brown ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm),&repartSF);CHKERRQ(ierr); 11640a96aa3bSJed Brown ierr = PetscSFSetGraph(repartSF,numRoots,numLeaves,NULL,PETSC_OWN_POINTER,repartRoots,PETSC_OWN_POINTER);CHKERRQ(ierr); 11650a96aa3bSJed Brown ierr = PetscSFSetUp(repartSF);CHKERRQ(ierr); 11660a96aa3bSJed Brown if (preCoarseToFine) { 11670a96aa3bSJed Brown PetscSF repartSFembed, preCoarseToFineNew; 11680a96aa3bSJed Brown PetscInt nleaves; 11690a96aa3bSJed Brown const PetscInt *leaves; 11700a96aa3bSJed Brown 11710a96aa3bSJed Brown ierr = PetscSFSetUp(preCoarseToFine);CHKERRQ(ierr); 11720a96aa3bSJed Brown ierr = PetscSFGetGraph(preCoarseToFine,NULL,&nleaves,&leaves,NULL);CHKERRQ(ierr); 11730a96aa3bSJed Brown if (leaves) { 11740a96aa3bSJed Brown ierr = PetscSFCreateEmbeddedRootSF(repartSF,nleaves,leaves,&repartSFembed);CHKERRQ(ierr); 11750a96aa3bSJed Brown } else { 11760a96aa3bSJed Brown repartSFembed = repartSF; 11770a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)repartSFembed);CHKERRQ(ierr); 11780a96aa3bSJed Brown } 11790a96aa3bSJed Brown ierr = PetscSFCompose(preCoarseToFine,repartSFembed,&preCoarseToFineNew);CHKERRQ(ierr); 11800a96aa3bSJed Brown ierr = PetscSFDestroy(&preCoarseToFine);CHKERRQ(ierr); 11810a96aa3bSJed Brown ierr = PetscSFDestroy(&repartSFembed);CHKERRQ(ierr); 11820a96aa3bSJed Brown preCoarseToFine = preCoarseToFineNew; 11830a96aa3bSJed Brown } 11840a96aa3bSJed Brown if (coarseToPreFine) { 11850a96aa3bSJed Brown PetscSF repartSFinv, coarseToPreFineNew; 11860a96aa3bSJed Brown 11870a96aa3bSJed Brown ierr = PetscSFCreateInverseSF(repartSF,&repartSFinv);CHKERRQ(ierr); 11880a96aa3bSJed Brown ierr = PetscSFCompose(repartSFinv,coarseToPreFine,&coarseToPreFineNew);CHKERRQ(ierr); 11890a96aa3bSJed Brown ierr = PetscSFDestroy(&coarseToPreFine);CHKERRQ(ierr); 11900a96aa3bSJed Brown ierr = PetscSFDestroy(&repartSFinv);CHKERRQ(ierr); 11910a96aa3bSJed Brown coarseToPreFine = coarseToPreFineNew; 11920a96aa3bSJed Brown } 11930a96aa3bSJed Brown ierr = PetscSFDestroy(&repartSF);CHKERRQ(ierr); 11940a96aa3bSJed Brown } 11950a96aa3bSJed Brown PetscStackCallP4est(p4est_destroy,(forest_copy)); 11960a96aa3bSJed Brown } 11970a96aa3bSJed Brown } 11980a96aa3bSJed Brown if (size > 1) { 11990a96aa3bSJed Brown PetscInt overlap; 12000a96aa3bSJed Brown 12010a96aa3bSJed Brown ierr = DMForestGetPartitionOverlap(dm,&overlap);CHKERRQ(ierr); 12020a96aa3bSJed Brown 12030a96aa3bSJed Brown if (adaptFrom) { 12040a96aa3bSJed Brown PetscInt aoverlap; 12050a96aa3bSJed Brown 12060a96aa3bSJed Brown ierr = DMForestGetPartitionOverlap(adaptFrom,&aoverlap);CHKERRQ(ierr); 12070a96aa3bSJed Brown if (aoverlap != overlap) { 12080a96aa3bSJed Brown ctx.anyChange = PETSC_TRUE; 12090a96aa3bSJed Brown } 12100a96aa3bSJed Brown } 12110a96aa3bSJed Brown 12120a96aa3bSJed Brown if (overlap > 0) { 12130a96aa3bSJed Brown PetscInt i, cLocalStart; 12140a96aa3bSJed Brown PetscInt cEnd; 12150a96aa3bSJed Brown PetscSF preCellSF = NULL, cellSF = NULL; 12160a96aa3bSJed Brown 12170a96aa3bSJed Brown PetscStackCallP4estReturn(pforest->ghost,p4est_ghost_new,(pforest->forest,P4EST_CONNECT_FULL)); 12180a96aa3bSJed Brown PetscStackCallP4estReturn(pforest->lnodes,p4est_lnodes_new,(pforest->forest,pforest->ghost,-P4EST_DIM)); 12190a96aa3bSJed Brown PetscStackCallP4est(p4est_ghost_support_lnodes,(pforest->forest,pforest->lnodes,pforest->ghost)); 12200a96aa3bSJed Brown for (i = 1; i < overlap; i++) PetscStackCallP4est(p4est_ghost_expand_by_lnodes,(pforest->forest,pforest->lnodes,pforest->ghost)); 12210a96aa3bSJed Brown 12220a96aa3bSJed Brown cLocalStart = pforest->cLocalStart = pforest->ghost->proc_offsets[rank]; 12230a96aa3bSJed Brown cEnd = pforest->forest->local_num_quadrants + pforest->ghost->proc_offsets[size]; 12240a96aa3bSJed Brown 12250a96aa3bSJed Brown /* shift sfs by cLocalStart, expand by cell SFs */ 12260a96aa3bSJed Brown if (preCoarseToFine || coarseToPreFine) { 12270a96aa3bSJed Brown if (adaptFrom) {ierr = DMForestGetCellSF(adaptFrom,&preCellSF);CHKERRQ(ierr);} 12280a96aa3bSJed Brown dm->setupcalled = PETSC_TRUE; 12290a96aa3bSJed Brown ierr = DMForestGetCellSF(dm,&cellSF);CHKERRQ(ierr); 12300a96aa3bSJed Brown } 12310a96aa3bSJed Brown if (preCoarseToFine) { 12320a96aa3bSJed Brown PetscSF preCoarseToFineNew; 12330a96aa3bSJed Brown PetscInt nleaves, nroots, *leavesNew, i, nleavesNew; 12340a96aa3bSJed Brown const PetscInt *leaves; 12350a96aa3bSJed Brown const PetscSFNode *remotes; 12360a96aa3bSJed Brown PetscSFNode *remotesAll; 12370a96aa3bSJed Brown 12380a96aa3bSJed Brown ierr = PetscSFSetUp(preCoarseToFine);CHKERRQ(ierr); 12390a96aa3bSJed Brown ierr = PetscSFGetGraph(preCoarseToFine,&nroots,&nleaves,&leaves,&remotes);CHKERRQ(ierr); 12400a96aa3bSJed Brown ierr = PetscMalloc1(cEnd,&remotesAll);CHKERRQ(ierr); 12410a96aa3bSJed Brown for (i = 0; i < cEnd; i++) { 12420a96aa3bSJed Brown remotesAll[i].rank = -1; 12430a96aa3bSJed Brown remotesAll[i].index = -1; 12440a96aa3bSJed Brown } 12450a96aa3bSJed Brown for (i = 0; i < nleaves; i++) remotesAll[(leaves ? leaves[i] : i) + cLocalStart] = remotes[i]; 12460a96aa3bSJed Brown ierr = PetscSFSetUp(cellSF);CHKERRQ(ierr); 12470a96aa3bSJed Brown ierr = PetscSFBcastBegin(cellSF,MPIU_2INT,remotesAll,remotesAll,MPI_REPLACE);CHKERRQ(ierr); 12480a96aa3bSJed Brown ierr = PetscSFBcastEnd(cellSF,MPIU_2INT,remotesAll,remotesAll,MPI_REPLACE);CHKERRQ(ierr); 12490a96aa3bSJed Brown nleavesNew = 0; 12500a96aa3bSJed Brown for (i = 0; i < nleaves; i++) { 12510a96aa3bSJed Brown if (remotesAll[i].rank >= 0) nleavesNew++; 12520a96aa3bSJed Brown } 12530a96aa3bSJed Brown ierr = PetscMalloc1(nleavesNew,&leavesNew);CHKERRQ(ierr); 12540a96aa3bSJed Brown nleavesNew = 0; 12550a96aa3bSJed Brown for (i = 0; i < nleaves; i++) { 12560a96aa3bSJed Brown if (remotesAll[i].rank >= 0) { 12570a96aa3bSJed Brown leavesNew[nleavesNew] = i; 12580a96aa3bSJed Brown if (i > nleavesNew) remotesAll[nleavesNew] = remotesAll[i]; 12590a96aa3bSJed Brown nleavesNew++; 12600a96aa3bSJed Brown } 12610a96aa3bSJed Brown } 12620a96aa3bSJed Brown ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm),&preCoarseToFineNew);CHKERRQ(ierr); 12630a96aa3bSJed Brown if (nleavesNew < cEnd) { 12640a96aa3bSJed Brown ierr = PetscSFSetGraph(preCoarseToFineNew,nroots,nleavesNew,leavesNew,PETSC_OWN_POINTER,remotesAll,PETSC_COPY_VALUES);CHKERRQ(ierr); 12650a96aa3bSJed Brown } else { /* all cells are leaves */ 12660a96aa3bSJed Brown ierr = PetscFree(leavesNew);CHKERRQ(ierr); 12670a96aa3bSJed Brown ierr = PetscSFSetGraph(preCoarseToFineNew,nroots,nleavesNew,NULL,PETSC_OWN_POINTER,remotesAll,PETSC_COPY_VALUES);CHKERRQ(ierr); 12680a96aa3bSJed Brown } 12690a96aa3bSJed Brown ierr = PetscFree(remotesAll);CHKERRQ(ierr); 12700a96aa3bSJed Brown ierr = PetscSFDestroy(&preCoarseToFine);CHKERRQ(ierr); 12710a96aa3bSJed Brown preCoarseToFine = preCoarseToFineNew; 12720a96aa3bSJed Brown preCoarseToFine = preCoarseToFineNew; 12730a96aa3bSJed Brown } 12740a96aa3bSJed Brown if (coarseToPreFine) { 12750a96aa3bSJed Brown PetscSF coarseToPreFineNew; 12760a96aa3bSJed Brown PetscInt nleaves, nroots, i, nleavesCellSF, nleavesExpanded, *leavesNew; 12770a96aa3bSJed Brown const PetscInt *leaves; 12780a96aa3bSJed Brown const PetscSFNode *remotes; 12790a96aa3bSJed Brown PetscSFNode *remotesNew, *remotesNewRoot, *remotesExpanded; 12800a96aa3bSJed Brown 12810a96aa3bSJed Brown ierr = PetscSFSetUp(coarseToPreFine);CHKERRQ(ierr); 12820a96aa3bSJed Brown ierr = PetscSFGetGraph(coarseToPreFine,&nroots,&nleaves,&leaves,&remotes);CHKERRQ(ierr); 12830a96aa3bSJed Brown ierr = PetscSFGetGraph(preCellSF,NULL,&nleavesCellSF,NULL,NULL);CHKERRQ(ierr); 12840a96aa3bSJed Brown ierr = PetscMalloc1(nroots,&remotesNewRoot);CHKERRQ(ierr); 12850a96aa3bSJed Brown ierr = PetscMalloc1(nleaves,&remotesNew);CHKERRQ(ierr); 12860a96aa3bSJed Brown for (i = 0; i < nroots; i++) { 12870a96aa3bSJed Brown remotesNewRoot[i].rank = rank; 12880a96aa3bSJed Brown remotesNewRoot[i].index = i + cLocalStart; 12890a96aa3bSJed Brown } 12900a96aa3bSJed Brown ierr = PetscSFBcastBegin(coarseToPreFine,MPIU_2INT,remotesNewRoot,remotesNew,MPI_REPLACE);CHKERRQ(ierr); 12910a96aa3bSJed Brown ierr = PetscSFBcastEnd(coarseToPreFine,MPIU_2INT,remotesNewRoot,remotesNew,MPI_REPLACE);CHKERRQ(ierr); 12920a96aa3bSJed Brown ierr = PetscFree(remotesNewRoot);CHKERRQ(ierr); 12930a96aa3bSJed Brown ierr = PetscMalloc1(nleavesCellSF,&remotesExpanded);CHKERRQ(ierr); 12940a96aa3bSJed Brown for (i = 0; i < nleavesCellSF; i++) { 12950a96aa3bSJed Brown remotesExpanded[i].rank = -1; 12960a96aa3bSJed Brown remotesExpanded[i].index = -1; 12970a96aa3bSJed Brown } 12980a96aa3bSJed Brown for (i = 0; i < nleaves; i++) remotesExpanded[leaves ? leaves[i] : i] = remotesNew[i]; 12990a96aa3bSJed Brown ierr = PetscFree(remotesNew);CHKERRQ(ierr); 13000a96aa3bSJed Brown ierr = PetscSFBcastBegin(preCellSF,MPIU_2INT,remotesExpanded,remotesExpanded,MPI_REPLACE);CHKERRQ(ierr); 13010a96aa3bSJed Brown ierr = PetscSFBcastEnd(preCellSF,MPIU_2INT,remotesExpanded,remotesExpanded,MPI_REPLACE);CHKERRQ(ierr); 13020a96aa3bSJed Brown 13030a96aa3bSJed Brown nleavesExpanded = 0; 13040a96aa3bSJed Brown for (i = 0; i < nleavesCellSF; i++) { 13050a96aa3bSJed Brown if (remotesExpanded[i].rank >= 0) nleavesExpanded++; 13060a96aa3bSJed Brown } 13070a96aa3bSJed Brown ierr = PetscMalloc1(nleavesExpanded,&leavesNew);CHKERRQ(ierr); 13080a96aa3bSJed Brown nleavesExpanded = 0; 13090a96aa3bSJed Brown for (i = 0; i < nleavesCellSF; i++) { 13100a96aa3bSJed Brown if (remotesExpanded[i].rank >= 0) { 13110a96aa3bSJed Brown leavesNew[nleavesExpanded] = i; 13120a96aa3bSJed Brown if (i > nleavesExpanded) remotesExpanded[nleavesExpanded] = remotes[i]; 13130a96aa3bSJed Brown nleavesExpanded++; 13140a96aa3bSJed Brown } 13150a96aa3bSJed Brown } 13160a96aa3bSJed Brown ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm),&coarseToPreFineNew);CHKERRQ(ierr); 13170a96aa3bSJed Brown if (nleavesExpanded < nleavesCellSF) { 13180a96aa3bSJed Brown ierr = PetscSFSetGraph(coarseToPreFineNew,cEnd,nleavesExpanded,leavesNew,PETSC_OWN_POINTER,remotesExpanded,PETSC_COPY_VALUES);CHKERRQ(ierr); 13190a96aa3bSJed Brown } else { 13200a96aa3bSJed Brown ierr = PetscFree(leavesNew);CHKERRQ(ierr); 13210a96aa3bSJed Brown ierr = PetscSFSetGraph(coarseToPreFineNew,cEnd,nleavesExpanded,NULL,PETSC_OWN_POINTER,remotesExpanded,PETSC_COPY_VALUES);CHKERRQ(ierr); 13220a96aa3bSJed Brown } 13230a96aa3bSJed Brown ierr = PetscFree(remotesExpanded);CHKERRQ(ierr); 13240a96aa3bSJed Brown ierr = PetscSFDestroy(&coarseToPreFine);CHKERRQ(ierr); 13250a96aa3bSJed Brown coarseToPreFine = coarseToPreFineNew; 13260a96aa3bSJed Brown } 13270a96aa3bSJed Brown } 13280a96aa3bSJed Brown } 13290a96aa3bSJed Brown } 13300a96aa3bSJed Brown forest->preCoarseToFine = preCoarseToFine; 13310a96aa3bSJed Brown forest->coarseToPreFine = coarseToPreFine; 13320a96aa3bSJed Brown dm->setupcalled = PETSC_TRUE; 13330a96aa3bSJed Brown ierr = MPI_Allreduce(&ctx.anyChange,&(pforest->adaptivitySuccess),1,MPIU_BOOL,MPI_LOR,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr); 13340a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,NULL);CHKERRQ(ierr); 13350a96aa3bSJed Brown PetscFunctionReturn(0); 13360a96aa3bSJed Brown } 13370a96aa3bSJed Brown 13380a96aa3bSJed Brown #define DMForestGetAdaptivitySuccess_pforest _append_pforest(DMForestGetAdaptivitySuccess) 13390a96aa3bSJed Brown static PetscErrorCode DMForestGetAdaptivitySuccess_pforest(DM dm, PetscBool *success) 13400a96aa3bSJed Brown { 13410a96aa3bSJed Brown DM_Forest *forest; 13420a96aa3bSJed Brown DM_Forest_pforest *pforest; 13430a96aa3bSJed Brown 13440a96aa3bSJed Brown PetscFunctionBegin; 13450a96aa3bSJed Brown forest = (DM_Forest *) dm->data; 13460a96aa3bSJed Brown pforest = (DM_Forest_pforest *) forest->data; 13470a96aa3bSJed Brown *success = pforest->adaptivitySuccess; 13480a96aa3bSJed Brown PetscFunctionReturn(0); 13490a96aa3bSJed Brown } 13500a96aa3bSJed Brown 13510a96aa3bSJed Brown #define DMView_ASCII_pforest _append_pforest(DMView_ASCII) 13520a96aa3bSJed Brown static PetscErrorCode DMView_ASCII_pforest(PetscObject odm, PetscViewer viewer) 13530a96aa3bSJed Brown { 13540a96aa3bSJed Brown DM dm = (DM) odm; 13550a96aa3bSJed Brown PetscErrorCode ierr; 13560a96aa3bSJed Brown 13570a96aa3bSJed Brown PetscFunctionBegin; 13580a96aa3bSJed Brown PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 13590a96aa3bSJed Brown PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2); 13600a96aa3bSJed Brown ierr = DMSetUp(dm);CHKERRQ(ierr); 13610a96aa3bSJed Brown switch (viewer->format) { 13620a96aa3bSJed Brown case PETSC_VIEWER_DEFAULT: 13630a96aa3bSJed Brown case PETSC_VIEWER_ASCII_INFO: 13640a96aa3bSJed Brown { 13650a96aa3bSJed Brown PetscInt dim; 13660a96aa3bSJed Brown const char *name; 13670a96aa3bSJed Brown 13680a96aa3bSJed Brown ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr); 13690a96aa3bSJed Brown ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr); 13700a96aa3bSJed Brown if (name) {ierr = PetscViewerASCIIPrintf(viewer, "Forest %s in %D dimensions:\n", name, dim);CHKERRQ(ierr);} 13710a96aa3bSJed Brown else {ierr = PetscViewerASCIIPrintf(viewer, "Forest in %D dimensions:\n", dim);CHKERRQ(ierr);} 13720a96aa3bSJed Brown } 13730a96aa3bSJed Brown case PETSC_VIEWER_ASCII_INFO_DETAIL: 13740a96aa3bSJed Brown case PETSC_VIEWER_LOAD_BALANCE: 13750a96aa3bSJed Brown { 13760a96aa3bSJed Brown DM plex; 13770a96aa3bSJed Brown 13780a96aa3bSJed Brown ierr = DMPforestGetPlex(dm, &plex);CHKERRQ(ierr); 13790a96aa3bSJed Brown ierr = DMView(plex, viewer);CHKERRQ(ierr); 13800a96aa3bSJed Brown } 13810a96aa3bSJed Brown break; 138298921bdaSJacob Faibussowitsch default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "No support for format '%s'", PetscViewerFormats[viewer->format]); 13830a96aa3bSJed Brown } 13840a96aa3bSJed Brown PetscFunctionReturn(0); 13850a96aa3bSJed Brown } 13860a96aa3bSJed Brown 13870a96aa3bSJed Brown #define DMView_VTK_pforest _append_pforest(DMView_VTK) 13880a96aa3bSJed Brown static PetscErrorCode DMView_VTK_pforest(PetscObject odm, PetscViewer viewer) 13890a96aa3bSJed Brown { 13900a96aa3bSJed Brown DM dm = (DM) odm; 13910a96aa3bSJed Brown DM_Forest *forest = (DM_Forest*) dm->data; 13920a96aa3bSJed Brown DM_Forest_pforest *pforest = (DM_Forest_pforest*) forest->data; 13930a96aa3bSJed Brown PetscBool isvtk; 13940a96aa3bSJed Brown PetscReal vtkScale = 1. - PETSC_MACHINE_EPSILON; 13950a96aa3bSJed Brown PetscViewer_VTK *vtk = (PetscViewer_VTK*)viewer->data; 13960a96aa3bSJed Brown const char *name; 13970a96aa3bSJed Brown char *filenameStrip = NULL; 13980a96aa3bSJed Brown PetscBool hasExt; 13990a96aa3bSJed Brown size_t len; 14000a96aa3bSJed Brown p4est_geometry_t *geom; 14010a96aa3bSJed Brown PetscErrorCode ierr; 14020a96aa3bSJed Brown 14030a96aa3bSJed Brown PetscFunctionBegin; 14040a96aa3bSJed Brown PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 14050a96aa3bSJed Brown PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2); 14060a96aa3bSJed Brown ierr = DMSetUp(dm);CHKERRQ(ierr); 14070a96aa3bSJed Brown geom = pforest->topo->geom; 14080a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK, &isvtk);CHKERRQ(ierr); 1409*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!isvtk,PetscObjectComm((PetscObject)viewer), PETSC_ERR_ARG_INCOMP, "Cannot use viewer type %s", ((PetscObject)viewer)->type_name); 14100a96aa3bSJed Brown switch (viewer->format) { 14110a96aa3bSJed Brown case PETSC_VIEWER_VTK_VTU: 1412*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!pforest->forest,PetscObjectComm(odm),PETSC_ERR_ARG_WRONG,"DM has not been setup with a valid forest"); 14130a96aa3bSJed Brown name = vtk->filename; 14140a96aa3bSJed Brown ierr = PetscStrlen(name,&len);CHKERRQ(ierr); 14150a96aa3bSJed Brown ierr = PetscStrcasecmp(name+len-4,".vtu",&hasExt);CHKERRQ(ierr); 14160a96aa3bSJed Brown if (hasExt) { 14170a96aa3bSJed Brown ierr = PetscStrallocpy(name,&filenameStrip);CHKERRQ(ierr); 14180a96aa3bSJed Brown filenameStrip[len-4]='\0'; 14190a96aa3bSJed Brown name = filenameStrip; 14200a96aa3bSJed Brown } 14210a96aa3bSJed Brown if (!pforest->topo->geom) PetscStackCallP4estReturn(geom,p4est_geometry_new_connectivity,(pforest->topo->conn)); 14220a96aa3bSJed Brown { 14230a96aa3bSJed Brown p4est_vtk_context_t *pvtk; 14240a96aa3bSJed Brown int footerr; 14250a96aa3bSJed Brown 14260a96aa3bSJed Brown PetscStackCallP4estReturn(pvtk,p4est_vtk_context_new,(pforest->forest,name)); 14270a96aa3bSJed Brown PetscStackCallP4est(p4est_vtk_context_set_geom,(pvtk,geom)); 14280a96aa3bSJed Brown PetscStackCallP4est(p4est_vtk_context_set_scale,(pvtk,(double)vtkScale)); 14290a96aa3bSJed Brown PetscStackCallP4estReturn(pvtk,p4est_vtk_write_header,(pvtk)); 1430*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!pvtk,PetscObjectComm((PetscObject)odm),PETSC_ERR_LIB,P4EST_STRING "_vtk_write_header() failed"); 14310a96aa3bSJed Brown PetscStackCallP4estReturn(pvtk,p4est_vtk_write_cell_dataf,(pvtk, 14320a96aa3bSJed Brown 1, /* write tree */ 14330a96aa3bSJed Brown 1, /* write level */ 14340a96aa3bSJed Brown 1, /* write rank */ 14350a96aa3bSJed Brown 0, /* do not wrap rank */ 14360a96aa3bSJed Brown 0, /* no scalar fields */ 14370a96aa3bSJed Brown 0, /* no vector fields */ 14380a96aa3bSJed Brown pvtk)); 1439*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!pvtk,PetscObjectComm((PetscObject)odm),PETSC_ERR_LIB,P4EST_STRING "_vtk_write_cell_dataf() failed"); 14400a96aa3bSJed Brown PetscStackCallP4estReturn(footerr,p4est_vtk_write_footer,(pvtk)); 1441*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(footerr,PetscObjectComm((PetscObject)odm),PETSC_ERR_LIB,P4EST_STRING "_vtk_write_footer() failed"); 14420a96aa3bSJed Brown } 14430a96aa3bSJed Brown if (!pforest->topo->geom) PetscStackCallP4est(p4est_geometry_destroy,(geom)); 14440a96aa3bSJed Brown ierr = PetscFree(filenameStrip);CHKERRQ(ierr); 14450a96aa3bSJed Brown break; 144698921bdaSJacob Faibussowitsch default: SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "No support for format '%s'", PetscViewerFormats[viewer->format]); 14470a96aa3bSJed Brown } 14480a96aa3bSJed Brown PetscFunctionReturn(0); 14490a96aa3bSJed Brown } 14500a96aa3bSJed Brown 14510a96aa3bSJed Brown #define DMView_HDF5_pforest _append_pforest(DMView_HDF5) 14520a96aa3bSJed Brown static PetscErrorCode DMView_HDF5_pforest(DM dm, PetscViewer viewer) 14530a96aa3bSJed Brown { 14540a96aa3bSJed Brown DM plex; 14550a96aa3bSJed Brown PetscErrorCode ierr; 14560a96aa3bSJed Brown 14570a96aa3bSJed Brown PetscFunctionBegin; 14580a96aa3bSJed Brown ierr = DMSetUp(dm);CHKERRQ(ierr); 14590a96aa3bSJed Brown ierr = DMPforestGetPlex(dm, &plex);CHKERRQ(ierr); 14600a96aa3bSJed Brown ierr = DMView(plex, viewer);CHKERRQ(ierr); 14610a96aa3bSJed Brown PetscFunctionReturn(0); 14620a96aa3bSJed Brown } 14630a96aa3bSJed Brown 14640a96aa3bSJed Brown #define DMView_GLVis_pforest _append_pforest(DMView_GLVis) 14650a96aa3bSJed Brown static PetscErrorCode DMView_GLVis_pforest(DM dm, PetscViewer viewer) 14660a96aa3bSJed Brown { 14670a96aa3bSJed Brown DM plex; 14680a96aa3bSJed Brown PetscErrorCode ierr; 14690a96aa3bSJed Brown 14700a96aa3bSJed Brown PetscFunctionBegin; 14710a96aa3bSJed Brown ierr = DMSetUp(dm);CHKERRQ(ierr); 14720a96aa3bSJed Brown ierr = DMPforestGetPlex(dm, &plex);CHKERRQ(ierr); 14730a96aa3bSJed Brown ierr = DMView(plex, viewer);CHKERRQ(ierr); 14740a96aa3bSJed Brown PetscFunctionReturn(0); 14750a96aa3bSJed Brown } 14760a96aa3bSJed Brown 14770a96aa3bSJed Brown #define DMView_pforest _append_pforest(DMView) 14780a96aa3bSJed Brown static PetscErrorCode DMView_pforest(DM dm, PetscViewer viewer) 14790a96aa3bSJed Brown { 14800a96aa3bSJed Brown PetscBool isascii, isvtk, ishdf5, isglvis; 14810a96aa3bSJed Brown PetscErrorCode ierr; 14820a96aa3bSJed Brown 14830a96aa3bSJed Brown PetscFunctionBegin; 14840a96aa3bSJed Brown PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 14850a96aa3bSJed Brown PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2); 14860a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &isascii);CHKERRQ(ierr); 14870a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK, &isvtk);CHKERRQ(ierr); 14880a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr); 14890a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr); 14900a96aa3bSJed Brown if (isascii) { 14910a96aa3bSJed Brown ierr = DMView_ASCII_pforest((PetscObject) dm,viewer);CHKERRQ(ierr); 14920a96aa3bSJed Brown } else if (isvtk) { 14930a96aa3bSJed Brown ierr = DMView_VTK_pforest((PetscObject) dm,viewer);CHKERRQ(ierr); 14940a96aa3bSJed Brown } else if (ishdf5) { 14950a96aa3bSJed Brown ierr = DMView_HDF5_pforest(dm, viewer);CHKERRQ(ierr); 14960a96aa3bSJed Brown } else if (isglvis) { 14970a96aa3bSJed Brown ierr = DMView_GLVis_pforest(dm, viewer);CHKERRQ(ierr); 14980a96aa3bSJed Brown } else SETERRQ(PetscObjectComm((PetscObject) dm),PETSC_ERR_SUP,"Viewer not supported (not VTK, HDF5, or GLVis)"); 14990a96aa3bSJed Brown PetscFunctionReturn(0); 15000a96aa3bSJed Brown } 15010a96aa3bSJed Brown 15020a96aa3bSJed Brown static PetscErrorCode PforestConnectivityEnumerateFacets(p4est_connectivity_t *conn, PetscInt **tree_face_to_uniq) 15030a96aa3bSJed Brown { 15040a96aa3bSJed Brown PetscInt *ttf, f, t, g, count; 15050a96aa3bSJed Brown PetscInt numFacets; 15060a96aa3bSJed Brown PetscErrorCode ierr; 15070a96aa3bSJed Brown 15080a96aa3bSJed Brown PetscFunctionBegin; 15090a96aa3bSJed Brown numFacets = conn->num_trees * P4EST_FACES; 15100a96aa3bSJed Brown ierr = PetscMalloc1(numFacets,&ttf);CHKERRQ(ierr); 15110a96aa3bSJed Brown for (f = 0; f < numFacets; f++) ttf[f] = -1; 15120a96aa3bSJed Brown for (g = 0, count = 0, t = 0; t < conn->num_trees; t++) { 15130a96aa3bSJed Brown for (f = 0; f < P4EST_FACES; f++, g++) { 15140a96aa3bSJed Brown if (ttf[g] == -1) { 15150a96aa3bSJed Brown PetscInt ng; 15160a96aa3bSJed Brown 15170a96aa3bSJed Brown ttf[g] = count++; 15180a96aa3bSJed Brown ng = conn->tree_to_tree[g] * P4EST_FACES + (conn->tree_to_face[g] % P4EST_FACES); 15190a96aa3bSJed Brown ttf[ng] = ttf[g]; 15200a96aa3bSJed Brown } 15210a96aa3bSJed Brown } 15220a96aa3bSJed Brown } 15230a96aa3bSJed Brown *tree_face_to_uniq = ttf; 15240a96aa3bSJed Brown PetscFunctionReturn(0); 15250a96aa3bSJed Brown } 15260a96aa3bSJed Brown 15270a96aa3bSJed Brown static PetscErrorCode DMPlexCreateConnectivity_pforest(DM dm, p4est_connectivity_t **connOut, PetscInt **tree_face_to_uniq) 15280a96aa3bSJed Brown { 15290a96aa3bSJed Brown p4est_topidx_t numTrees, numVerts, numCorns, numCtt; 15300a96aa3bSJed Brown PetscSection ctt; 15310a96aa3bSJed Brown #if defined(P4_TO_P8) 15320a96aa3bSJed Brown p4est_topidx_t numEdges, numEtt; 15330a96aa3bSJed Brown PetscSection ett; 15340a96aa3bSJed Brown PetscInt eStart, eEnd, e, ettSize; 15350a96aa3bSJed Brown PetscInt vertOff = 1 + P4EST_FACES + P8EST_EDGES; 15360a96aa3bSJed Brown PetscInt edgeOff = 1 + P4EST_FACES; 15370a96aa3bSJed Brown #else 15380a96aa3bSJed Brown PetscInt vertOff = 1 + P4EST_FACES; 15390a96aa3bSJed Brown #endif 15400a96aa3bSJed Brown p4est_connectivity_t *conn; 15410a96aa3bSJed Brown PetscInt cStart, cEnd, c, vStart, vEnd, v, fStart, fEnd, f; 15420a96aa3bSJed Brown PetscInt *star = NULL, *closure = NULL, closureSize, starSize, cttSize; 15430a96aa3bSJed Brown PetscInt *ttf; 15440a96aa3bSJed Brown PetscErrorCode ierr; 15450a96aa3bSJed Brown 15460a96aa3bSJed Brown PetscFunctionBegin; 15470a96aa3bSJed Brown /* 1: count objects, allocate */ 15480a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr); 15490a96aa3bSJed Brown ierr = P4estTopidxCast(cEnd-cStart,&numTrees);CHKERRQ(ierr); 15500a96aa3bSJed Brown numVerts = P4EST_CHILDREN * numTrees; 15510a96aa3bSJed Brown ierr = DMPlexGetDepthStratum(dm,0,&vStart,&vEnd);CHKERRQ(ierr); 15520a96aa3bSJed Brown ierr = P4estTopidxCast(vEnd-vStart,&numCorns);CHKERRQ(ierr); 15530a96aa3bSJed Brown ierr = PetscSectionCreate(PETSC_COMM_SELF,&ctt);CHKERRQ(ierr); 15540a96aa3bSJed Brown ierr = PetscSectionSetChart(ctt,vStart,vEnd);CHKERRQ(ierr); 15550a96aa3bSJed Brown for (v = vStart; v < vEnd; v++) { 15560a96aa3bSJed Brown PetscInt s; 15570a96aa3bSJed Brown 15580a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(dm,v,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 15590a96aa3bSJed Brown for (s = 0; s < starSize; s++) { 15600a96aa3bSJed Brown PetscInt p = star[2*s]; 15610a96aa3bSJed Brown 15620a96aa3bSJed Brown if (p >= cStart && p < cEnd) { 15630a96aa3bSJed Brown /* we want to count every time cell p references v, so we see how many times it comes up in the closure. This 15640a96aa3bSJed Brown * only protects against periodicity problems */ 15650a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(dm,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 1566*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(closureSize != P4EST_INSUL,PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Cell %D with wrong closure size %D != %D", p, closureSize, P4EST_INSUL); 15670a96aa3bSJed Brown for (c = 0; c < P4EST_CHILDREN; c++) { 15680a96aa3bSJed Brown PetscInt cellVert = closure[2 * (c + vertOff)]; 15690a96aa3bSJed Brown 1570*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(cellVert < vStart || cellVert >= vEnd,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Non-standard closure: vertices"); 15710a96aa3bSJed Brown if (cellVert == v) { 15720a96aa3bSJed Brown ierr = PetscSectionAddDof(ctt,v,1);CHKERRQ(ierr); 15730a96aa3bSJed Brown } 15740a96aa3bSJed Brown } 15750a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(dm,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 15760a96aa3bSJed Brown } 15770a96aa3bSJed Brown } 15780a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(dm,v,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 15790a96aa3bSJed Brown } 15800a96aa3bSJed Brown ierr = PetscSectionSetUp(ctt);CHKERRQ(ierr); 15810a96aa3bSJed Brown ierr = PetscSectionGetStorageSize(ctt,&cttSize);CHKERRQ(ierr); 15820a96aa3bSJed Brown ierr = P4estTopidxCast(cttSize,&numCtt);CHKERRQ(ierr); 15830a96aa3bSJed Brown #if defined(P4_TO_P8) 15840a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(dm,P4EST_DIM-1,&eStart,&eEnd);CHKERRQ(ierr); 15850a96aa3bSJed Brown ierr = P4estTopidxCast(eEnd-eStart,&numEdges);CHKERRQ(ierr); 15860a96aa3bSJed Brown ierr = PetscSectionCreate(PETSC_COMM_SELF,&ett);CHKERRQ(ierr); 15870a96aa3bSJed Brown ierr = PetscSectionSetChart(ett,eStart,eEnd);CHKERRQ(ierr); 15880a96aa3bSJed Brown for (e = eStart; e < eEnd; e++) { 15890a96aa3bSJed Brown PetscInt s; 15900a96aa3bSJed Brown 15910a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(dm,e,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 15920a96aa3bSJed Brown for (s = 0; s < starSize; s++) { 15930a96aa3bSJed Brown PetscInt p = star[2*s]; 15940a96aa3bSJed Brown 15950a96aa3bSJed Brown if (p >= cStart && p < cEnd) { 15960a96aa3bSJed Brown /* we want to count every time cell p references e, so we see how many times it comes up in the closure. This 15970a96aa3bSJed Brown * only protects against periodicity problems */ 15980a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(dm,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 1599*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(closureSize != P4EST_INSUL,PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Cell with wrong closure size"); 16000a96aa3bSJed Brown for (c = 0; c < P8EST_EDGES; c++) { 16010a96aa3bSJed Brown PetscInt cellEdge = closure[2 * (c + edgeOff)]; 16020a96aa3bSJed Brown 1603*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(cellEdge < eStart || cellEdge >= eEnd,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Non-standard closure: edges"); 16040a96aa3bSJed Brown if (cellEdge == e) { 16050a96aa3bSJed Brown ierr = PetscSectionAddDof(ett,e,1);CHKERRQ(ierr); 16060a96aa3bSJed Brown } 16070a96aa3bSJed Brown } 16080a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(dm,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 16090a96aa3bSJed Brown } 16100a96aa3bSJed Brown } 16110a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(dm,e,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 16120a96aa3bSJed Brown } 16130a96aa3bSJed Brown ierr = PetscSectionSetUp(ett);CHKERRQ(ierr); 16140a96aa3bSJed Brown ierr = PetscSectionGetStorageSize(ett,&ettSize);CHKERRQ(ierr); 16150a96aa3bSJed Brown ierr = P4estTopidxCast(ettSize,&numEtt);CHKERRQ(ierr); 16160a96aa3bSJed Brown 16170a96aa3bSJed Brown /* This routine allocates space for the arrays, which we fill below */ 16180a96aa3bSJed Brown PetscStackCallP4estReturn(conn,p8est_connectivity_new,(numVerts,numTrees,numEdges,numEtt,numCorns,numCtt)); 16190a96aa3bSJed Brown #else 16200a96aa3bSJed Brown PetscStackCallP4estReturn(conn,p4est_connectivity_new,(numVerts,numTrees,numCorns,numCtt)); 16210a96aa3bSJed Brown #endif 16220a96aa3bSJed Brown 16230a96aa3bSJed Brown /* 2: visit every face, determine neighboring cells(trees) */ 16240a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(dm,1,&fStart,&fEnd);CHKERRQ(ierr); 16250a96aa3bSJed Brown ierr = PetscMalloc1((cEnd-cStart) * P4EST_FACES,&ttf);CHKERRQ(ierr); 16260a96aa3bSJed Brown for (f = fStart; f < fEnd; f++) { 16270a96aa3bSJed Brown PetscInt numSupp, s; 16280a96aa3bSJed Brown PetscInt myFace[2] = {-1, -1}; 16290a96aa3bSJed Brown PetscInt myOrnt[2] = {PETSC_MIN_INT, PETSC_MIN_INT}; 16300a96aa3bSJed Brown const PetscInt *supp; 16310a96aa3bSJed Brown 16320a96aa3bSJed Brown ierr = DMPlexGetSupportSize(dm, f, &numSupp);CHKERRQ(ierr); 1633*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(numSupp != 1 && numSupp != 2,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"point %D has facet with %D sides: must be 1 or 2 (boundary or conformal)",f,numSupp); 16340a96aa3bSJed Brown ierr = DMPlexGetSupport(dm, f, &supp);CHKERRQ(ierr); 16350a96aa3bSJed Brown 16360a96aa3bSJed Brown for (s = 0; s < numSupp; s++) { 16370a96aa3bSJed Brown PetscInt p = supp[s]; 16380a96aa3bSJed Brown 16390a96aa3bSJed Brown if (p >= cEnd) { 16400a96aa3bSJed Brown numSupp--; 16410a96aa3bSJed Brown if (s) supp = &supp[1 - s]; 16420a96aa3bSJed Brown break; 16430a96aa3bSJed Brown } 16440a96aa3bSJed Brown } 16450a96aa3bSJed Brown for (s = 0; s < numSupp; s++) { 16460a96aa3bSJed Brown PetscInt p = supp[s], i; 16470a96aa3bSJed Brown PetscInt numCone; 16480a96aa3bSJed Brown DMPolytopeType ct; 16490a96aa3bSJed Brown const PetscInt *cone; 16500a96aa3bSJed Brown const PetscInt *ornt; 16510a96aa3bSJed Brown PetscInt orient = PETSC_MIN_INT; 16520a96aa3bSJed Brown 16530a96aa3bSJed Brown ierr = DMPlexGetConeSize(dm, p, &numCone);CHKERRQ(ierr); 1654*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(numCone != P4EST_FACES,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"cell %D has %D facets, expect %d",p,numCone,P4EST_FACES); 16550a96aa3bSJed Brown ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr); 16560a96aa3bSJed Brown ierr = DMPlexGetCellType(dm, cone[0], &ct);CHKERRQ(ierr); 16570a96aa3bSJed Brown ierr = DMPlexGetConeOrientation(dm, p, &ornt);CHKERRQ(ierr); 16580a96aa3bSJed Brown for (i = 0; i < P4EST_FACES; i++) { 16590a96aa3bSJed Brown if (cone[i] == f) { 16600a96aa3bSJed Brown orient = DMPolytopeConvertNewOrientation_Internal(ct, ornt[i]); 16610a96aa3bSJed Brown break; 16620a96aa3bSJed Brown } 16630a96aa3bSJed Brown } 1664*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(i >= P4EST_FACES,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"cell %D faced %D mismatch",p,f); 16650a96aa3bSJed Brown if (p < cStart || p >= cEnd) { 16660a96aa3bSJed Brown DMPolytopeType ct; 16670a96aa3bSJed Brown ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr); 166898921bdaSJacob Faibussowitsch SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"cell %D (%s) should be in [%D, %D)",p,DMPolytopeTypes[ct],cStart,cEnd); 16690a96aa3bSJed Brown } 16700a96aa3bSJed Brown ttf[P4EST_FACES * (p - cStart) + PetscFaceToP4estFace[i]] = f - fStart; 16710a96aa3bSJed Brown if (numSupp == 1) { 16720a96aa3bSJed Brown /* boundary faces indicated by self reference */ 16730a96aa3bSJed Brown conn->tree_to_tree[P4EST_FACES * (p - cStart) + PetscFaceToP4estFace[i]] = p - cStart; 16740a96aa3bSJed Brown conn->tree_to_face[P4EST_FACES * (p - cStart) + PetscFaceToP4estFace[i]] = (int8_t) PetscFaceToP4estFace[i]; 16750a96aa3bSJed Brown } else { 16760a96aa3bSJed Brown const PetscInt N = P4EST_CHILDREN / 2; 16770a96aa3bSJed Brown 16780a96aa3bSJed Brown conn->tree_to_tree[P4EST_FACES * (p - cStart) + PetscFaceToP4estFace[i]] = supp[1 - s] - cStart; 16790a96aa3bSJed Brown myFace[s] = PetscFaceToP4estFace[i]; 16800a96aa3bSJed Brown /* get the orientation of cell p in p4est-type closure to facet f, by composing the p4est-closure to 16810a96aa3bSJed Brown * petsc-closure permutation and the petsc-closure to facet orientation */ 16820a96aa3bSJed Brown myOrnt[s] = DihedralCompose(N,orient,DMPolytopeConvertNewOrientation_Internal(ct, P4estFaceToPetscOrnt[myFace[s]])); 16830a96aa3bSJed Brown } 16840a96aa3bSJed Brown } 16850a96aa3bSJed Brown if (numSupp == 2) { 16860a96aa3bSJed Brown for (s = 0; s < numSupp; s++) { 16870a96aa3bSJed Brown PetscInt p = supp[s]; 16880a96aa3bSJed Brown PetscInt orntAtoB; 16890a96aa3bSJed Brown PetscInt p4estOrient; 16900a96aa3bSJed Brown const PetscInt N = P4EST_CHILDREN / 2; 16910a96aa3bSJed Brown 16920a96aa3bSJed Brown /* composing the forward permutation with the other cell's inverse permutation gives the self-to-neighbor 16930a96aa3bSJed Brown * permutation of this cell-facet's cone */ 16940a96aa3bSJed Brown orntAtoB = DihedralCompose(N,DihedralInvert(N,myOrnt[1-s]),myOrnt[s]); 16950a96aa3bSJed Brown 16960a96aa3bSJed Brown /* convert cone-description permutation (i.e., edges around facet) to cap-description permutation (i.e., 16970a96aa3bSJed Brown * vertices around facet) */ 16980a96aa3bSJed Brown #if !defined(P4_TO_P8) 16990a96aa3bSJed Brown p4estOrient = orntAtoB < 0 ? -(orntAtoB + 1) : orntAtoB; 17000a96aa3bSJed Brown #else 17010a96aa3bSJed Brown { 17020a96aa3bSJed Brown PetscInt firstVert = orntAtoB < 0 ? ((-orntAtoB) % N) : orntAtoB; 17030a96aa3bSJed Brown PetscInt p4estFirstVert = firstVert < 2 ? firstVert : (firstVert ^ 1); 17040a96aa3bSJed Brown 17050a96aa3bSJed Brown /* swap bits */ 17060a96aa3bSJed Brown p4estOrient = ((myFace[s] <= myFace[1 - s]) || (orntAtoB < 0)) ? p4estFirstVert : ((p4estFirstVert >> 1) | ((p4estFirstVert & 1) << 1)); 17070a96aa3bSJed Brown } 17080a96aa3bSJed Brown #endif 17090a96aa3bSJed Brown /* encode neighbor face and orientation in tree_to_face per p4est_connectivity standard (see 17100a96aa3bSJed Brown * p4est_connectivity.h, p8est_connectivity.h) */ 17110a96aa3bSJed Brown conn->tree_to_face[P4EST_FACES * (p - cStart) + myFace[s]] = (int8_t) myFace[1 - s] + p4estOrient * P4EST_FACES; 17120a96aa3bSJed Brown } 17130a96aa3bSJed Brown } 17140a96aa3bSJed Brown } 17150a96aa3bSJed Brown 17160a96aa3bSJed Brown #if defined(P4_TO_P8) 17170a96aa3bSJed Brown /* 3: visit every edge */ 17180a96aa3bSJed Brown conn->ett_offset[0] = 0; 17190a96aa3bSJed Brown for (e = eStart; e < eEnd; e++) { 17200a96aa3bSJed Brown PetscInt off, s; 17210a96aa3bSJed Brown 17220a96aa3bSJed Brown ierr = PetscSectionGetOffset(ett,e,&off);CHKERRQ(ierr); 17230a96aa3bSJed Brown conn->ett_offset[e - eStart] = (p4est_topidx_t) off; 17240a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(dm,e,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 17250a96aa3bSJed Brown for (s = 0; s < starSize; s++) { 17260a96aa3bSJed Brown PetscInt p = star[2 * s]; 17270a96aa3bSJed Brown 17280a96aa3bSJed Brown if (p >= cStart && p < cEnd) { 17290a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(dm,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 1730*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(closureSize != P4EST_INSUL,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Non-standard closure"); 17310a96aa3bSJed Brown for (c = 0; c < P8EST_EDGES; c++) { 17320a96aa3bSJed Brown PetscInt cellEdge = closure[2 * (c + edgeOff)]; 17330a96aa3bSJed Brown PetscInt cellOrnt = closure[2 * (c + edgeOff) + 1]; 17340a96aa3bSJed Brown DMPolytopeType ct; 17350a96aa3bSJed Brown 17360a96aa3bSJed Brown ierr = DMPlexGetCellType(dm, cellEdge, &ct);CHKERRQ(ierr); 17370a96aa3bSJed Brown cellOrnt = DMPolytopeConvertNewOrientation_Internal(ct, cellOrnt); 17380a96aa3bSJed Brown if (cellEdge == e) { 17390a96aa3bSJed Brown PetscInt p4estEdge = PetscEdgeToP4estEdge[c]; 17400a96aa3bSJed Brown PetscInt totalOrient; 17410a96aa3bSJed Brown 17420a96aa3bSJed Brown /* compose p4est-closure to petsc-closure permutation and petsc-closure to edge orientation */ 17430a96aa3bSJed Brown totalOrient = DihedralCompose(2,cellOrnt,DMPolytopeConvertNewOrientation_Internal(DM_POLYTOPE_SEGMENT, P4estEdgeToPetscOrnt[p4estEdge])); 17440a96aa3bSJed Brown /* p4est orientations are positive: -2 => 1, -1 => 0 */ 17450a96aa3bSJed Brown totalOrient = (totalOrient < 0) ? -(totalOrient + 1) : totalOrient; 17460a96aa3bSJed Brown conn->edge_to_tree[off] = (p4est_locidx_t) (p - cStart); 17470a96aa3bSJed Brown /* encode cell-edge and orientation in edge_to_edge per p8est_connectivity standart (see 17480a96aa3bSJed Brown * p8est_connectivity.h) */ 17490a96aa3bSJed Brown conn->edge_to_edge[off++] = (int8_t) p4estEdge + P8EST_EDGES * totalOrient; 17500a96aa3bSJed Brown conn->tree_to_edge[P8EST_EDGES * (p - cStart) + p4estEdge] = e - eStart; 17510a96aa3bSJed Brown } 17520a96aa3bSJed Brown } 17530a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(dm,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 17540a96aa3bSJed Brown } 17550a96aa3bSJed Brown } 17560a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(dm,e,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 17570a96aa3bSJed Brown } 17580a96aa3bSJed Brown ierr = PetscSectionDestroy(&ett);CHKERRQ(ierr); 17590a96aa3bSJed Brown #endif 17600a96aa3bSJed Brown 17610a96aa3bSJed Brown /* 4: visit every vertex */ 17620a96aa3bSJed Brown conn->ctt_offset[0] = 0; 17630a96aa3bSJed Brown for (v = vStart; v < vEnd; v++) { 17640a96aa3bSJed Brown PetscInt off, s; 17650a96aa3bSJed Brown 17660a96aa3bSJed Brown ierr = PetscSectionGetOffset(ctt,v,&off);CHKERRQ(ierr); 17670a96aa3bSJed Brown conn->ctt_offset[v - vStart] = (p4est_topidx_t) off; 17680a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(dm,v,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 17690a96aa3bSJed Brown for (s = 0; s < starSize; s++) { 17700a96aa3bSJed Brown PetscInt p = star[2 * s]; 17710a96aa3bSJed Brown 17720a96aa3bSJed Brown if (p >= cStart && p < cEnd) { 17730a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(dm,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 1774*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(closureSize != P4EST_INSUL,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Non-standard closure"); 17750a96aa3bSJed Brown for (c = 0; c < P4EST_CHILDREN; c++) { 17760a96aa3bSJed Brown PetscInt cellVert = closure[2 * (c + vertOff)]; 17770a96aa3bSJed Brown 17780a96aa3bSJed Brown if (cellVert == v) { 17790a96aa3bSJed Brown PetscInt p4estVert = PetscVertToP4estVert[c]; 17800a96aa3bSJed Brown 17810a96aa3bSJed Brown conn->corner_to_tree[off] = (p4est_locidx_t) (p - cStart); 17820a96aa3bSJed Brown conn->corner_to_corner[off++] = (int8_t) p4estVert; 17830a96aa3bSJed Brown conn->tree_to_corner[P4EST_CHILDREN * (p - cStart) + p4estVert] = v - vStart; 17840a96aa3bSJed Brown } 17850a96aa3bSJed Brown } 17860a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(dm,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 17870a96aa3bSJed Brown } 17880a96aa3bSJed Brown } 17890a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(dm,v,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 17900a96aa3bSJed Brown } 17910a96aa3bSJed Brown ierr = PetscSectionDestroy(&ctt);CHKERRQ(ierr); 17920a96aa3bSJed Brown 17930a96aa3bSJed Brown /* 5: Compute the coordinates */ 17940a96aa3bSJed Brown { 17950a96aa3bSJed Brown PetscInt coordDim; 17960a96aa3bSJed Brown Vec coordVec; 17970a96aa3bSJed Brown PetscSection coordSec; 17980a96aa3bSJed Brown PetscBool localized; 17990a96aa3bSJed Brown 18000a96aa3bSJed Brown ierr = DMGetCoordinateDim(dm, &coordDim);CHKERRQ(ierr); 18010a96aa3bSJed Brown ierr = DMGetCoordinatesLocal(dm, &coordVec);CHKERRQ(ierr); 18020a96aa3bSJed Brown ierr = DMGetCoordinatesLocalizedLocal(dm, &localized);CHKERRQ(ierr); 18030a96aa3bSJed Brown ierr = DMGetCoordinateSection(dm, &coordSec);CHKERRQ(ierr); 18040a96aa3bSJed Brown for (c = cStart; c < cEnd; c++) { 18050a96aa3bSJed Brown PetscInt dof; 18060a96aa3bSJed Brown PetscScalar *cellCoords = NULL; 18070a96aa3bSJed Brown 18080a96aa3bSJed Brown ierr = DMPlexVecGetClosure(dm, coordSec, coordVec, c, &dof, &cellCoords);CHKERRQ(ierr); 1809*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!localized && dof != P4EST_CHILDREN * coordDim,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Need coordinates at the corners: (dof) %D != %D * %D (sdim)", dof, P4EST_CHILDREN, coordDim); 18100a96aa3bSJed Brown for (v = 0; v < P4EST_CHILDREN; v++) { 18110a96aa3bSJed Brown PetscInt i, lim = PetscMin(3, coordDim); 18120a96aa3bSJed Brown PetscInt p4estVert = PetscVertToP4estVert[v]; 18130a96aa3bSJed Brown 18140a96aa3bSJed Brown conn->tree_to_vertex[P4EST_CHILDREN * (c - cStart) + v] = P4EST_CHILDREN * (c - cStart) + v; 18150a96aa3bSJed Brown /* p4est vertices are always embedded in R^3 */ 18160a96aa3bSJed Brown for (i = 0; i < 3; i++) conn->vertices[3 * (P4EST_CHILDREN * (c - cStart) + p4estVert) + i] = 0.; 18170a96aa3bSJed Brown for (i = 0; i < lim; i++) conn->vertices[3 * (P4EST_CHILDREN * (c - cStart) + p4estVert) + i] = PetscRealPart(cellCoords[v * coordDim + i]); 18180a96aa3bSJed Brown } 18190a96aa3bSJed Brown ierr = DMPlexVecRestoreClosure(dm, coordSec, coordVec, c, &dof, &cellCoords);CHKERRQ(ierr); 18200a96aa3bSJed Brown } 18210a96aa3bSJed Brown } 18220a96aa3bSJed Brown 18230a96aa3bSJed Brown #if defined(P4EST_ENABLE_DEBUG) 1824*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!p4est_connectivity_is_valid(conn),PETSC_COMM_SELF,PETSC_ERR_PLIB,"Plex to p4est conversion failed"); 18250a96aa3bSJed Brown #endif 18260a96aa3bSJed Brown 18270a96aa3bSJed Brown *connOut = conn; 18280a96aa3bSJed Brown 18290a96aa3bSJed Brown *tree_face_to_uniq = ttf; 18300a96aa3bSJed Brown 18310a96aa3bSJed Brown PetscFunctionReturn(0); 18320a96aa3bSJed Brown } 18330a96aa3bSJed Brown 18340a96aa3bSJed Brown static PetscErrorCode locidx_to_PetscInt(sc_array_t * array) 18350a96aa3bSJed Brown { 18360a96aa3bSJed Brown sc_array_t *newarray; 18370a96aa3bSJed Brown size_t zz, count = array->elem_count; 18380a96aa3bSJed Brown 18390a96aa3bSJed Brown PetscFunctionBegin; 1840*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(array->elem_size != sizeof(p4est_locidx_t),PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong locidx size"); 18410a96aa3bSJed Brown 18420a96aa3bSJed Brown if (sizeof(p4est_locidx_t) == sizeof(PetscInt)) PetscFunctionReturn(0); 18430a96aa3bSJed Brown 18440a96aa3bSJed Brown newarray = sc_array_new_size (sizeof(PetscInt), array->elem_count); 18450a96aa3bSJed Brown for (zz = 0; zz < count; zz++) { 18460a96aa3bSJed Brown p4est_locidx_t il = *((p4est_locidx_t*) sc_array_index (array, zz)); 18470a96aa3bSJed Brown PetscInt *ip = (PetscInt*) sc_array_index (newarray, zz); 18480a96aa3bSJed Brown 18490a96aa3bSJed Brown *ip = (PetscInt) il; 18500a96aa3bSJed Brown } 18510a96aa3bSJed Brown 18520a96aa3bSJed Brown sc_array_reset (array); 18530a96aa3bSJed Brown sc_array_init_size (array, sizeof(PetscInt), count); 18540a96aa3bSJed Brown sc_array_copy (array, newarray); 18550a96aa3bSJed Brown sc_array_destroy (newarray); 18560a96aa3bSJed Brown PetscFunctionReturn(0); 18570a96aa3bSJed Brown } 18580a96aa3bSJed Brown 18590a96aa3bSJed Brown static PetscErrorCode coords_double_to_PetscScalar(sc_array_t * array, PetscInt dim) 18600a96aa3bSJed Brown { 18610a96aa3bSJed Brown sc_array_t *newarray; 18620a96aa3bSJed Brown size_t zz, count = array->elem_count; 18630a96aa3bSJed Brown 18640a96aa3bSJed Brown PetscFunctionBegin; 1865*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(array->elem_size != 3 * sizeof(double),PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong coordinate size"); 18660a96aa3bSJed Brown #if !defined(PETSC_USE_COMPLEX) 18670a96aa3bSJed Brown if (sizeof(double) == sizeof(PetscScalar) && dim == 3) PetscFunctionReturn(0); 18680a96aa3bSJed Brown #endif 18690a96aa3bSJed Brown 18700a96aa3bSJed Brown newarray = sc_array_new_size (dim * sizeof(PetscScalar), array->elem_count); 18710a96aa3bSJed Brown for (zz = 0; zz < count; zz++) { 18720a96aa3bSJed Brown int i; 18730a96aa3bSJed Brown double *id = (double*) sc_array_index (array, zz); 18740a96aa3bSJed Brown PetscScalar *ip = (PetscScalar*) sc_array_index (newarray, zz); 18750a96aa3bSJed Brown 18760a96aa3bSJed Brown for (i = 0; i < dim; i++) ip[i] = 0.; 18770a96aa3bSJed Brown for (i = 0; i < PetscMin(dim,3); i++) ip[i] = (PetscScalar) id[i]; 18780a96aa3bSJed Brown } 18790a96aa3bSJed Brown 18800a96aa3bSJed Brown sc_array_reset (array); 18810a96aa3bSJed Brown sc_array_init_size (array, dim * sizeof(PetscScalar), count); 18820a96aa3bSJed Brown sc_array_copy (array, newarray); 18830a96aa3bSJed Brown sc_array_destroy (newarray); 18840a96aa3bSJed Brown PetscFunctionReturn(0); 18850a96aa3bSJed Brown } 18860a96aa3bSJed Brown 18870a96aa3bSJed Brown static PetscErrorCode locidx_pair_to_PetscSFNode(sc_array_t * array) 18880a96aa3bSJed Brown { 18890a96aa3bSJed Brown sc_array_t *newarray; 18900a96aa3bSJed Brown size_t zz, count = array->elem_count; 18910a96aa3bSJed Brown 18920a96aa3bSJed Brown PetscFunctionBegin; 1893*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(array->elem_size != 2 * sizeof(p4est_locidx_t),PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong locidx size"); 18940a96aa3bSJed Brown 18950a96aa3bSJed Brown newarray = sc_array_new_size (sizeof(PetscSFNode), array->elem_count); 18960a96aa3bSJed Brown for (zz = 0; zz < count; zz++) { 18970a96aa3bSJed Brown p4est_locidx_t *il = (p4est_locidx_t*) sc_array_index (array, zz); 18980a96aa3bSJed Brown PetscSFNode *ip = (PetscSFNode*) sc_array_index (newarray, zz); 18990a96aa3bSJed Brown 19000a96aa3bSJed Brown ip->rank = (PetscInt) il[0]; 19010a96aa3bSJed Brown ip->index = (PetscInt) il[1]; 19020a96aa3bSJed Brown } 19030a96aa3bSJed Brown 19040a96aa3bSJed Brown sc_array_reset (array); 19050a96aa3bSJed Brown sc_array_init_size (array, sizeof(PetscSFNode), count); 19060a96aa3bSJed Brown sc_array_copy (array, newarray); 19070a96aa3bSJed Brown sc_array_destroy (newarray); 19080a96aa3bSJed Brown PetscFunctionReturn(0); 19090a96aa3bSJed Brown } 19100a96aa3bSJed Brown 19110a96aa3bSJed Brown static PetscErrorCode P4estToPlex_Local(p4est_t *p4est, DM * plex) 19120a96aa3bSJed Brown { 19130a96aa3bSJed Brown PetscErrorCode ierr; 19140a96aa3bSJed Brown 19150a96aa3bSJed Brown PetscFunctionBegin; 19160a96aa3bSJed Brown { 19170a96aa3bSJed Brown sc_array_t *points_per_dim = sc_array_new(sizeof(p4est_locidx_t)); 19180a96aa3bSJed Brown sc_array_t *cone_sizes = sc_array_new(sizeof(p4est_locidx_t)); 19190a96aa3bSJed Brown sc_array_t *cones = sc_array_new(sizeof(p4est_locidx_t)); 19200a96aa3bSJed Brown sc_array_t *cone_orientations = sc_array_new(sizeof(p4est_locidx_t)); 19210a96aa3bSJed Brown sc_array_t *coords = sc_array_new(3 * sizeof(double)); 19220a96aa3bSJed Brown sc_array_t *children = sc_array_new(sizeof(p4est_locidx_t)); 19230a96aa3bSJed Brown sc_array_t *parents = sc_array_new(sizeof(p4est_locidx_t)); 19240a96aa3bSJed Brown sc_array_t *childids = sc_array_new(sizeof(p4est_locidx_t)); 19250a96aa3bSJed Brown sc_array_t *leaves = sc_array_new(sizeof(p4est_locidx_t)); 19260a96aa3bSJed Brown sc_array_t *remotes = sc_array_new(2 * sizeof(p4est_locidx_t)); 19270a96aa3bSJed Brown p4est_locidx_t first_local_quad; 19280a96aa3bSJed Brown 19290a96aa3bSJed Brown PetscStackCallP4est(p4est_get_plex_data,(p4est,P4EST_CONNECT_FULL,0,&first_local_quad,points_per_dim,cone_sizes,cones,cone_orientations,coords,children,parents,childids,leaves,remotes)); 19300a96aa3bSJed Brown 19310a96aa3bSJed Brown ierr = locidx_to_PetscInt(points_per_dim);CHKERRQ(ierr); 19320a96aa3bSJed Brown ierr = locidx_to_PetscInt(cone_sizes);CHKERRQ(ierr); 19330a96aa3bSJed Brown ierr = locidx_to_PetscInt(cones);CHKERRQ(ierr); 19340a96aa3bSJed Brown ierr = locidx_to_PetscInt(cone_orientations);CHKERRQ(ierr); 19350a96aa3bSJed Brown ierr = coords_double_to_PetscScalar(coords, P4EST_DIM);CHKERRQ(ierr); 19360a96aa3bSJed Brown 19370a96aa3bSJed Brown ierr = DMPlexCreate(PETSC_COMM_SELF,plex);CHKERRQ(ierr); 19380a96aa3bSJed Brown ierr = DMSetDimension(*plex,P4EST_DIM);CHKERRQ(ierr); 19390a96aa3bSJed Brown ierr = DMPlexCreateFromDAG(*plex,P4EST_DIM,(PetscInt*)points_per_dim->array,(PetscInt*)cone_sizes->array,(PetscInt*)cones->array,(PetscInt*)cone_orientations->array,(PetscScalar*)coords->array);CHKERRQ(ierr); 19400a96aa3bSJed Brown ierr = DMPlexConvertOldOrientations_Internal(*plex);CHKERRQ(ierr); 19410a96aa3bSJed Brown sc_array_destroy (points_per_dim); 19420a96aa3bSJed Brown sc_array_destroy (cone_sizes); 19430a96aa3bSJed Brown sc_array_destroy (cones); 19440a96aa3bSJed Brown sc_array_destroy (cone_orientations); 19450a96aa3bSJed Brown sc_array_destroy (coords); 19460a96aa3bSJed Brown sc_array_destroy (children); 19470a96aa3bSJed Brown sc_array_destroy (parents); 19480a96aa3bSJed Brown sc_array_destroy (childids); 19490a96aa3bSJed Brown sc_array_destroy (leaves); 19500a96aa3bSJed Brown sc_array_destroy (remotes); 19510a96aa3bSJed Brown } 19520a96aa3bSJed Brown PetscFunctionReturn(0); 19530a96aa3bSJed Brown } 19540a96aa3bSJed Brown 19550a96aa3bSJed Brown #define DMReferenceTreeGetChildSymmetry_pforest _append_pforest(DMReferenceTreeGetChildSymmetry) 19560a96aa3bSJed Brown static PetscErrorCode DMReferenceTreeGetChildSymmetry_pforest(DM dm, PetscInt parent, PetscInt parentOrientA, PetscInt childOrientA, PetscInt childA, PetscInt parentOrientB, PetscInt *childOrientB,PetscInt *childB) 19570a96aa3bSJed Brown { 19580a96aa3bSJed Brown PetscInt coneSize, dStart, dEnd, vStart, vEnd, dim, ABswap, oAvert, oBvert, ABswapVert; 19590a96aa3bSJed Brown PetscErrorCode ierr; 19600a96aa3bSJed Brown 19610a96aa3bSJed Brown PetscFunctionBegin; 19620a96aa3bSJed Brown if (parentOrientA == parentOrientB) { 19630a96aa3bSJed Brown if (childOrientB) *childOrientB = childOrientA; 19640a96aa3bSJed Brown if (childB) *childB = childA; 19650a96aa3bSJed Brown PetscFunctionReturn(0); 19660a96aa3bSJed Brown } 19670a96aa3bSJed Brown ierr = DMPlexGetDepthStratum(dm,0,&vStart,&vEnd);CHKERRQ(ierr); 19680a96aa3bSJed Brown if (childA >= vStart && childA < vEnd) { /* vertices (always in the middle) are invarient under rotation */ 19690a96aa3bSJed Brown if (childOrientB) *childOrientB = 0; 19700a96aa3bSJed Brown if (childB) *childB = childA; 19710a96aa3bSJed Brown PetscFunctionReturn(0); 19720a96aa3bSJed Brown } 19730a96aa3bSJed Brown for (dim = 0; dim < 3; dim++) { 19740a96aa3bSJed Brown ierr = DMPlexGetDepthStratum(dm,dim,&dStart,&dEnd);CHKERRQ(ierr); 19750a96aa3bSJed Brown if (parent >= dStart && parent <= dEnd) break; 19760a96aa3bSJed Brown } 1977*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(dim > 2,PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot perform child symmetry for %d-cells",dim); 1978*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!dim,PETSC_COMM_SELF,PETSC_ERR_PLIB,"A vertex has no children"); 19790a96aa3bSJed Brown if (childA < dStart || childA >= dEnd) { /* a 1-cell in a 2-cell */ 19800a96aa3bSJed Brown /* this is a lower-dimensional child: bootstrap */ 19810a96aa3bSJed Brown PetscInt size, i, sA = -1, sB, sOrientB, sConeSize; 19820a96aa3bSJed Brown const PetscInt *supp, *coneA, *coneB, *oA, *oB; 19830a96aa3bSJed Brown 19840a96aa3bSJed Brown ierr = DMPlexGetSupportSize(dm,childA,&size);CHKERRQ(ierr); 19850a96aa3bSJed Brown ierr = DMPlexGetSupport(dm,childA,&supp);CHKERRQ(ierr); 19860a96aa3bSJed Brown 19870a96aa3bSJed Brown /* find a point sA in supp(childA) that has the same parent */ 19880a96aa3bSJed Brown for (i = 0; i < size; i++) { 19890a96aa3bSJed Brown PetscInt sParent; 19900a96aa3bSJed Brown 19910a96aa3bSJed Brown sA = supp[i]; 19920a96aa3bSJed Brown if (sA == parent) continue; 19930a96aa3bSJed Brown ierr = DMPlexGetTreeParent(dm,sA,&sParent,NULL);CHKERRQ(ierr); 19940a96aa3bSJed Brown if (sParent == parent) break; 19950a96aa3bSJed Brown } 1996*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(i == size,PETSC_COMM_SELF,PETSC_ERR_PLIB,"could not find support in children"); 19970a96aa3bSJed Brown /* find out which point sB is in an equivalent position to sA under 19980a96aa3bSJed Brown * parentOrientB */ 19990a96aa3bSJed Brown ierr = DMReferenceTreeGetChildSymmetry_pforest(dm,parent,parentOrientA,0,sA,parentOrientB,&sOrientB,&sB);CHKERRQ(ierr); 20000a96aa3bSJed Brown ierr = DMPlexGetConeSize(dm,sA,&sConeSize);CHKERRQ(ierr); 20010a96aa3bSJed Brown ierr = DMPlexGetCone(dm,sA,&coneA);CHKERRQ(ierr); 20020a96aa3bSJed Brown ierr = DMPlexGetCone(dm,sB,&coneB);CHKERRQ(ierr); 20030a96aa3bSJed Brown ierr = DMPlexGetConeOrientation(dm,sA,&oA);CHKERRQ(ierr); 20040a96aa3bSJed Brown ierr = DMPlexGetConeOrientation(dm,sB,&oB);CHKERRQ(ierr); 20050a96aa3bSJed Brown /* step through the cone of sA in natural order */ 20060a96aa3bSJed Brown for (i = 0; i < sConeSize; i++) { 20070a96aa3bSJed Brown if (coneA[i] == childA) { 20080a96aa3bSJed Brown /* if childA is at position i in coneA, 20090a96aa3bSJed Brown * then we want the point that is at sOrientB*i in coneB */ 20100a96aa3bSJed Brown PetscInt j = (sOrientB >= 0) ? ((sOrientB + i) % sConeSize) : ((sConeSize -(sOrientB+1) - i) % sConeSize); 20110a96aa3bSJed Brown if (childB) *childB = coneB[j]; 20120a96aa3bSJed Brown if (childOrientB) { 20130a96aa3bSJed Brown DMPolytopeType ct; 20140a96aa3bSJed Brown PetscInt oBtrue; 20150a96aa3bSJed Brown 20160a96aa3bSJed Brown ierr = DMPlexGetConeSize(dm,childA,&coneSize);CHKERRQ(ierr); 20170a96aa3bSJed Brown /* compose sOrientB and oB[j] */ 2018*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(coneSize != 0 && coneSize != 2,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected a vertex or an edge"); 20190a96aa3bSJed Brown ct = coneSize ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT; 20200a96aa3bSJed Brown /* we may have to flip an edge */ 20210a96aa3bSJed Brown oBtrue = (sOrientB >= 0) ? oB[j] : DMPolytopeTypeComposeOrientationInv(ct, -1, oB[j]); 20220a96aa3bSJed Brown oBtrue = DMPolytopeConvertNewOrientation_Internal(ct, oBtrue); 20230a96aa3bSJed Brown ABswap = DihedralSwap(coneSize,DMPolytopeConvertNewOrientation_Internal(ct, oA[i]),oBtrue); 20240a96aa3bSJed Brown *childOrientB = DihedralCompose(coneSize,childOrientA,ABswap); 20250a96aa3bSJed Brown } 20260a96aa3bSJed Brown break; 20270a96aa3bSJed Brown } 20280a96aa3bSJed Brown } 2029*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(i == sConeSize,PETSC_COMM_SELF,PETSC_ERR_PLIB,"support cone mismatch"); 20300a96aa3bSJed Brown PetscFunctionReturn(0); 20310a96aa3bSJed Brown } 20320a96aa3bSJed Brown /* get the cone size and symmetry swap */ 20330a96aa3bSJed Brown ierr = DMPlexGetConeSize(dm,parent,&coneSize);CHKERRQ(ierr); 20340a96aa3bSJed Brown ABswap = DihedralSwap(coneSize, parentOrientA, parentOrientB); 20350a96aa3bSJed Brown if (dim == 2) { 20360a96aa3bSJed Brown /* orientations refer to cones: we want them to refer to vertices: 20370a96aa3bSJed Brown * if it's a rotation, they are the same, but if the order is reversed, a 20380a96aa3bSJed Brown * permutation that puts side i first does *not* put vertex i first */ 20390a96aa3bSJed Brown oAvert = (parentOrientA >= 0) ? parentOrientA : -((-parentOrientA % coneSize) + 1); 20400a96aa3bSJed Brown oBvert = (parentOrientB >= 0) ? parentOrientB : -((-parentOrientB % coneSize) + 1); 20410a96aa3bSJed Brown ABswapVert = DihedralSwap(coneSize, oAvert, oBvert); 20420a96aa3bSJed Brown } else { 20430a96aa3bSJed Brown oAvert = parentOrientA; 20440a96aa3bSJed Brown oBvert = parentOrientB; 20450a96aa3bSJed Brown ABswapVert = ABswap; 20460a96aa3bSJed Brown } 20470a96aa3bSJed Brown if (childB) { 20480a96aa3bSJed Brown /* assume that each child corresponds to a vertex, in the same order */ 20490a96aa3bSJed Brown PetscInt p, posA = -1, numChildren, i; 20500a96aa3bSJed Brown const PetscInt *children; 20510a96aa3bSJed Brown 20520a96aa3bSJed Brown /* count which position the child is in */ 20530a96aa3bSJed Brown ierr = DMPlexGetTreeChildren(dm,parent,&numChildren,&children);CHKERRQ(ierr); 20540a96aa3bSJed Brown for (i = 0; i < numChildren; i++) { 20550a96aa3bSJed Brown p = children[i]; 20560a96aa3bSJed Brown if (p == childA) { 20570a96aa3bSJed Brown if (dim == 1) { 20580a96aa3bSJed Brown posA = i; 20590a96aa3bSJed Brown } else { /* 2D Morton to rotation */ 20600a96aa3bSJed Brown posA = (i & 2) ? (i ^ 1) : i; 20610a96aa3bSJed Brown } 20620a96aa3bSJed Brown break; 20630a96aa3bSJed Brown } 20640a96aa3bSJed Brown } 20650a96aa3bSJed Brown if (posA >= coneSize) { 20660a96aa3bSJed Brown SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Could not find childA in children of parent"); 20670a96aa3bSJed Brown } else { 20680a96aa3bSJed Brown /* figure out position B by applying ABswapVert */ 20690a96aa3bSJed Brown PetscInt posB, childIdB; 20700a96aa3bSJed Brown 20710a96aa3bSJed Brown posB = (ABswapVert >= 0) ? ((ABswapVert + posA) % coneSize) : ((coneSize -(ABswapVert + 1) - posA) % coneSize); 20720a96aa3bSJed Brown if (dim == 1) { 20730a96aa3bSJed Brown childIdB = posB; 20740a96aa3bSJed Brown } else { /* 2D rotation to Morton */ 20750a96aa3bSJed Brown childIdB = (posB & 2) ? (posB ^ 1) : posB; 20760a96aa3bSJed Brown } 20770a96aa3bSJed Brown if (childB) *childB = children[childIdB]; 20780a96aa3bSJed Brown } 20790a96aa3bSJed Brown } 20800a96aa3bSJed Brown if (childOrientB) *childOrientB = DihedralCompose(coneSize,childOrientA,ABswap); 20810a96aa3bSJed Brown PetscFunctionReturn(0); 20820a96aa3bSJed Brown } 20830a96aa3bSJed Brown 20840a96aa3bSJed Brown #define DMCreateReferenceTree_pforest _append_pforest(DMCreateReferenceTree) 20850a96aa3bSJed Brown static PetscErrorCode DMCreateReferenceTree_pforest(MPI_Comm comm, DM *dm) 20860a96aa3bSJed Brown { 20870a96aa3bSJed Brown p4est_connectivity_t *refcube; 20880a96aa3bSJed Brown p4est_t *root, *refined; 20890a96aa3bSJed Brown DM dmRoot, dmRefined; 20900a96aa3bSJed Brown DM_Plex *mesh; 20910a96aa3bSJed Brown PetscMPIInt rank; 20920a96aa3bSJed Brown PetscErrorCode ierr; 20930a96aa3bSJed Brown 20940a96aa3bSJed Brown PetscFunctionBegin; 20950a96aa3bSJed Brown PetscStackCallP4estReturn(refcube,p4est_connectivity_new_byname,("unit")); 20960a96aa3bSJed Brown { /* [-1,1]^d geometry */ 20970a96aa3bSJed Brown PetscInt i, j; 20980a96aa3bSJed Brown 20990a96aa3bSJed Brown for (i = 0; i < P4EST_CHILDREN; i++) { 21000a96aa3bSJed Brown for (j = 0; j < 3; j++) { 21010a96aa3bSJed Brown refcube->vertices[3 * i + j] *= 2.; 21020a96aa3bSJed Brown refcube->vertices[3 * i + j] -= 1.; 21030a96aa3bSJed Brown } 21040a96aa3bSJed Brown } 21050a96aa3bSJed Brown } 21060a96aa3bSJed Brown PetscStackCallP4estReturn(root,p4est_new,(PETSC_COMM_SELF,refcube,0,NULL,NULL)); 21070a96aa3bSJed Brown PetscStackCallP4estReturn(refined,p4est_new_ext,(PETSC_COMM_SELF,refcube,0,1,1,0,NULL,NULL)); 21080a96aa3bSJed Brown ierr = P4estToPlex_Local(root,&dmRoot);CHKERRQ(ierr); 21090a96aa3bSJed Brown ierr = P4estToPlex_Local(refined,&dmRefined);CHKERRQ(ierr); 21100a96aa3bSJed Brown { 21110a96aa3bSJed Brown #if !defined(P4_TO_P8) 21120a96aa3bSJed Brown PetscInt nPoints = 25; 21130a96aa3bSJed Brown PetscInt perm[25] = {0, 1, 2, 3, 21140a96aa3bSJed Brown 4, 12, 8, 14, 21150a96aa3bSJed Brown 6, 9, 15, 21160a96aa3bSJed Brown 5, 13, 10, 21170a96aa3bSJed Brown 7, 11, 21180a96aa3bSJed Brown 16, 22, 20, 24, 21190a96aa3bSJed Brown 17, 21, 21200a96aa3bSJed Brown 18, 23, 21210a96aa3bSJed Brown 19}; 21220a96aa3bSJed Brown PetscInt ident[25] = {0, 0, 0, 0, 21230a96aa3bSJed Brown 1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0, 21240a96aa3bSJed Brown 5, 6, 7, 8, 1, 2, 3, 4, 0}; 21250a96aa3bSJed Brown #else 21260a96aa3bSJed Brown PetscInt nPoints = 125; 21270a96aa3bSJed Brown PetscInt perm[125] = {0, 1, 2, 3, 4, 5, 6, 7, 21280a96aa3bSJed Brown 8, 32, 16, 36, 24, 40, 21290a96aa3bSJed Brown 12, 17, 37, 25, 41, 21300a96aa3bSJed Brown 9, 33, 20, 26, 42, 21310a96aa3bSJed Brown 13, 21, 27, 43, 21320a96aa3bSJed Brown 10, 34, 18, 38, 28, 21330a96aa3bSJed Brown 14, 19, 39, 29, 21340a96aa3bSJed Brown 11, 35, 22, 30, 21350a96aa3bSJed Brown 15, 23, 31, 21360a96aa3bSJed Brown 44, 84, 76, 92, 52, 86, 68, 94, 60, 78, 70, 96, 21370a96aa3bSJed Brown 45, 85, 77, 93, 54, 72, 62, 74, 21380a96aa3bSJed Brown 46, 80, 53, 87, 69, 95, 64, 82, 21390a96aa3bSJed Brown 47, 81, 55, 73, 66, 21400a96aa3bSJed Brown 48, 88, 56, 90, 61, 79, 71, 97, 21410a96aa3bSJed Brown 49, 89, 58, 63, 75, 21420a96aa3bSJed Brown 50, 57, 91, 65, 83, 21430a96aa3bSJed Brown 51, 59, 67, 21440a96aa3bSJed Brown 98, 106, 110, 122, 114, 120, 118, 124, 21450a96aa3bSJed Brown 99, 111, 115, 119, 21460a96aa3bSJed Brown 100, 107, 116, 121, 21470a96aa3bSJed Brown 101, 117, 21480a96aa3bSJed Brown 102, 108, 112, 123, 21490a96aa3bSJed Brown 103, 113, 21500a96aa3bSJed Brown 104, 109, 21510a96aa3bSJed Brown 105}; 21520a96aa3bSJed Brown PetscInt ident[125] = {0, 0, 0, 0, 0, 0, 0, 0, 21530a96aa3bSJed Brown 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 21540a96aa3bSJed Brown 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21550a96aa3bSJed Brown 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 21560a96aa3bSJed Brown 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 21570a96aa3bSJed Brown 0, 0, 0, 0, 0, 0, 21580a96aa3bSJed Brown 19, 20, 21, 22, 23, 24, 25, 26, 21590a96aa3bSJed Brown 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21600a96aa3bSJed Brown 1, 2, 3, 4, 5, 6, 21610a96aa3bSJed Brown 0}; 21620a96aa3bSJed Brown 21630a96aa3bSJed Brown #endif 21640a96aa3bSJed Brown IS permIS; 21650a96aa3bSJed Brown DM dmPerm; 21660a96aa3bSJed Brown 21670a96aa3bSJed Brown ierr = ISCreateGeneral(PETSC_COMM_SELF,nPoints,perm,PETSC_USE_POINTER,&permIS);CHKERRQ(ierr); 21680a96aa3bSJed Brown ierr = DMPlexPermute(dmRefined,permIS,&dmPerm);CHKERRQ(ierr); 21690a96aa3bSJed Brown if (dmPerm) { 21700a96aa3bSJed Brown ierr = DMDestroy(&dmRefined);CHKERRQ(ierr); 21710a96aa3bSJed Brown dmRefined = dmPerm; 21720a96aa3bSJed Brown } 21730a96aa3bSJed Brown ierr = ISDestroy(&permIS);CHKERRQ(ierr); 21740a96aa3bSJed Brown { 21750a96aa3bSJed Brown PetscInt p; 21760a96aa3bSJed Brown ierr = DMCreateLabel(dmRoot,"identity");CHKERRQ(ierr); 21770a96aa3bSJed Brown ierr = DMCreateLabel(dmRefined,"identity");CHKERRQ(ierr); 21780a96aa3bSJed Brown for (p = 0; p < P4EST_INSUL; p++) { 21790a96aa3bSJed Brown ierr = DMSetLabelValue(dmRoot,"identity",p,p);CHKERRQ(ierr); 21800a96aa3bSJed Brown } 21810a96aa3bSJed Brown for (p = 0; p < nPoints; p++) { 21820a96aa3bSJed Brown ierr = DMSetLabelValue(dmRefined,"identity",p,ident[p]);CHKERRQ(ierr); 21830a96aa3bSJed Brown } 21840a96aa3bSJed Brown } 21850a96aa3bSJed Brown } 21860a96aa3bSJed Brown ierr = DMPlexCreateReferenceTree_Union(dmRoot,dmRefined,"identity",dm);CHKERRQ(ierr); 21870a96aa3bSJed Brown mesh = (DM_Plex*) (*dm)->data; 21880a96aa3bSJed Brown mesh->getchildsymmetry = DMReferenceTreeGetChildSymmetry_pforest; 21890a96aa3bSJed Brown ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 21900a96aa3bSJed Brown if (rank == 0) { 21910a96aa3bSJed Brown ierr = DMViewFromOptions(dmRoot, NULL,"-dm_p4est_ref_root_view");CHKERRQ(ierr); 21920a96aa3bSJed Brown ierr = DMViewFromOptions(dmRefined,NULL,"-dm_p4est_ref_refined_view");CHKERRQ(ierr); 21930a96aa3bSJed Brown ierr = DMViewFromOptions(dmRefined,NULL,"-dm_p4est_ref_tree_view");CHKERRQ(ierr); 21940a96aa3bSJed Brown } 21950a96aa3bSJed Brown ierr = DMDestroy(&dmRefined);CHKERRQ(ierr); 21960a96aa3bSJed Brown ierr = DMDestroy(&dmRoot);CHKERRQ(ierr); 21970a96aa3bSJed Brown PetscStackCallP4est(p4est_destroy,(refined)); 21980a96aa3bSJed Brown PetscStackCallP4est(p4est_destroy,(root)); 21990a96aa3bSJed Brown PetscStackCallP4est(p4est_connectivity_destroy,(refcube)); 22000a96aa3bSJed Brown PetscFunctionReturn(0); 22010a96aa3bSJed Brown } 22020a96aa3bSJed Brown 22030a96aa3bSJed Brown static PetscErrorCode DMShareDiscretization(DM dmA, DM dmB) 22040a96aa3bSJed Brown { 22050a96aa3bSJed Brown void *ctx; 22060a96aa3bSJed Brown PetscInt num; 22070a96aa3bSJed Brown PetscReal val; 22080a96aa3bSJed Brown PetscErrorCode ierr; 22090a96aa3bSJed Brown 22100a96aa3bSJed Brown PetscFunctionBegin; 22110a96aa3bSJed Brown ierr = DMGetApplicationContext(dmA,&ctx);CHKERRQ(ierr); 22120a96aa3bSJed Brown ierr = DMSetApplicationContext(dmB,ctx);CHKERRQ(ierr); 22130a96aa3bSJed Brown ierr = DMCopyDisc(dmA,dmB);CHKERRQ(ierr); 22140a96aa3bSJed Brown ierr = DMGetOutputSequenceNumber(dmA,&num,&val);CHKERRQ(ierr); 22150a96aa3bSJed Brown ierr = DMSetOutputSequenceNumber(dmB,num,val);CHKERRQ(ierr); 22160a96aa3bSJed Brown if (dmB->localSection != dmA->localSection || dmB->globalSection != dmA->globalSection) { 22170a96aa3bSJed Brown ierr = DMClearLocalVectors(dmB);CHKERRQ(ierr); 22180a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)dmA->localSection);CHKERRQ(ierr); 22190a96aa3bSJed Brown ierr = PetscSectionDestroy(&(dmB->localSection));CHKERRQ(ierr); 22200a96aa3bSJed Brown dmB->localSection = dmA->localSection; 22210a96aa3bSJed Brown ierr = DMClearGlobalVectors(dmB);CHKERRQ(ierr); 22220a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)dmA->globalSection);CHKERRQ(ierr); 22230a96aa3bSJed Brown ierr = PetscSectionDestroy(&(dmB->globalSection));CHKERRQ(ierr); 22240a96aa3bSJed Brown dmB->globalSection = dmA->globalSection; 22250a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)dmA->defaultConstraintSection);CHKERRQ(ierr); 22260a96aa3bSJed Brown ierr = PetscSectionDestroy(&(dmB->defaultConstraintSection));CHKERRQ(ierr); 22270a96aa3bSJed Brown dmB->defaultConstraintSection = dmA->defaultConstraintSection; 22280a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)dmA->defaultConstraintMat);CHKERRQ(ierr); 22290a96aa3bSJed Brown ierr = MatDestroy(&(dmB->defaultConstraintMat));CHKERRQ(ierr); 22300a96aa3bSJed Brown dmB->defaultConstraintMat = dmA->defaultConstraintMat; 22310a96aa3bSJed Brown if (dmA->map) {ierr = PetscLayoutReference(dmA->map, &dmB->map);CHKERRQ(ierr);} 22320a96aa3bSJed Brown } 22330a96aa3bSJed Brown if (dmB->sectionSF != dmA->sectionSF) { 22340a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)dmA->sectionSF);CHKERRQ(ierr); 22350a96aa3bSJed Brown ierr = PetscSFDestroy(&dmB->sectionSF);CHKERRQ(ierr); 22360a96aa3bSJed Brown dmB->sectionSF = dmA->sectionSF; 22370a96aa3bSJed Brown } 22380a96aa3bSJed Brown PetscFunctionReturn(0); 22390a96aa3bSJed Brown } 22400a96aa3bSJed Brown 22410a96aa3bSJed Brown /* Get an SF that broadcasts a coarse-cell covering of the local fine cells */ 22420a96aa3bSJed Brown static PetscErrorCode DMPforestGetCellCoveringSF(MPI_Comm comm,p4est_t *p4estC, p4est_t *p4estF, PetscInt cStart, PetscInt cEnd, PetscSF *coveringSF) 22430a96aa3bSJed Brown { 22440a96aa3bSJed Brown PetscInt startF, endF, startC, endC, p, nLeaves; 22450a96aa3bSJed Brown PetscSFNode *leaves; 22460a96aa3bSJed Brown PetscSF sf; 22470a96aa3bSJed Brown PetscInt *recv, *send; 22480a96aa3bSJed Brown PetscMPIInt tag; 22490a96aa3bSJed Brown MPI_Request *recvReqs, *sendReqs; 22500a96aa3bSJed Brown PetscSection section; 22510a96aa3bSJed Brown PetscErrorCode ierr; 22520a96aa3bSJed Brown 22530a96aa3bSJed Brown PetscFunctionBegin; 22540a96aa3bSJed Brown ierr = DMPforestComputeOverlappingRanks(p4estC->mpisize,p4estC->mpirank,p4estF,p4estC,&startC,&endC);CHKERRQ(ierr); 22550a96aa3bSJed Brown ierr = PetscMalloc2(2*(endC-startC),&recv,endC-startC,&recvReqs);CHKERRQ(ierr); 22560a96aa3bSJed Brown ierr = PetscCommGetNewTag(comm,&tag);CHKERRQ(ierr); 22570a96aa3bSJed Brown for (p = startC; p < endC; p++) { 22580a96aa3bSJed Brown recvReqs[p-startC] = MPI_REQUEST_NULL; /* just in case we don't initiate a receive */ 22590a96aa3bSJed Brown if (p4estC->global_first_quadrant[p] == p4estC->global_first_quadrant[p+1]) { /* empty coarse partition */ 22600a96aa3bSJed Brown recv[2*(p-startC)] = 0; 22610a96aa3bSJed Brown recv[2*(p-startC)+1] = 0; 22620a96aa3bSJed Brown continue; 22630a96aa3bSJed Brown } 22640a96aa3bSJed Brown 22650a96aa3bSJed Brown ierr = MPI_Irecv(&recv[2*(p-startC)],2,MPIU_INT,p,tag,comm,&recvReqs[p-startC]);CHKERRMPI(ierr); 22660a96aa3bSJed Brown } 22670a96aa3bSJed Brown ierr = DMPforestComputeOverlappingRanks(p4estC->mpisize,p4estC->mpirank,p4estC,p4estF,&startF,&endF);CHKERRQ(ierr); 22680a96aa3bSJed Brown ierr = PetscMalloc2(2*(endF-startF),&send,endF-startF,&sendReqs);CHKERRQ(ierr); 22690a96aa3bSJed Brown /* count the quadrants rank will send to each of [startF,endF) */ 22700a96aa3bSJed Brown for (p = startF; p < endF; p++) { 22710a96aa3bSJed Brown p4est_quadrant_t *myFineStart = &p4estF->global_first_position[p]; 22720a96aa3bSJed Brown p4est_quadrant_t *myFineEnd = &p4estF->global_first_position[p+1]; 22730a96aa3bSJed Brown PetscInt tStart = (PetscInt) myFineStart->p.which_tree; 22740a96aa3bSJed Brown PetscInt tEnd = (PetscInt) myFineEnd->p.which_tree; 22750a96aa3bSJed Brown PetscInt firstCell = -1, lastCell = -1; 22760a96aa3bSJed Brown p4est_tree_t *treeStart = &(((p4est_tree_t*) p4estC->trees->array)[tStart]); 22770a96aa3bSJed Brown p4est_tree_t *treeEnd = (size_t) tEnd < p4estC->trees->elem_count ? &(((p4est_tree_t*) p4estC->trees->array)[tEnd]) : NULL; 22780a96aa3bSJed Brown ssize_t overlapIndex; 22790a96aa3bSJed Brown 22800a96aa3bSJed Brown sendReqs[p-startF] = MPI_REQUEST_NULL; /* just in case we don't initiate a send */ 22810a96aa3bSJed Brown if (p4estF->global_first_quadrant[p] == p4estF->global_first_quadrant[p+1]) continue; 22820a96aa3bSJed Brown 22830a96aa3bSJed Brown /* locate myFineStart in (or before) a cell */ 22840a96aa3bSJed Brown if (treeStart->quadrants.elem_count) { 22850a96aa3bSJed Brown PetscStackCallP4estReturn(overlapIndex,sc_array_bsearch,(&(treeStart->quadrants),myFineStart,p4est_quadrant_disjoint)); 22860a96aa3bSJed Brown if (overlapIndex < 0) { 22870a96aa3bSJed Brown firstCell = 0; 22880a96aa3bSJed Brown } else { 22890a96aa3bSJed Brown firstCell = treeStart->quadrants_offset + overlapIndex; 22900a96aa3bSJed Brown } 22910a96aa3bSJed Brown } else { 22920a96aa3bSJed Brown firstCell = 0; 22930a96aa3bSJed Brown } 22940a96aa3bSJed Brown if (treeEnd && treeEnd->quadrants.elem_count) { 22950a96aa3bSJed Brown PetscStackCallP4estReturn(overlapIndex,sc_array_bsearch,(&(treeEnd->quadrants),myFineEnd,p4est_quadrant_disjoint)); 22960a96aa3bSJed Brown if (overlapIndex < 0) { /* all of this local section is overlapped */ 22970a96aa3bSJed Brown lastCell = p4estC->local_num_quadrants; 22980a96aa3bSJed Brown } else { 22990a96aa3bSJed Brown p4est_quadrant_t *container = &(((p4est_quadrant_t*) treeEnd->quadrants.array)[overlapIndex]); 23000a96aa3bSJed Brown p4est_quadrant_t first_desc; 23010a96aa3bSJed Brown int equal; 23020a96aa3bSJed Brown 23030a96aa3bSJed Brown PetscStackCallP4est(p4est_quadrant_first_descendant,(container,&first_desc,P4EST_QMAXLEVEL)); 23040a96aa3bSJed Brown PetscStackCallP4estReturn(equal,p4est_quadrant_is_equal,(myFineEnd,&first_desc)); 23050a96aa3bSJed Brown if (equal) { 23060a96aa3bSJed Brown lastCell = treeEnd->quadrants_offset + overlapIndex; 23070a96aa3bSJed Brown } else { 23080a96aa3bSJed Brown lastCell = treeEnd->quadrants_offset + overlapIndex + 1; 23090a96aa3bSJed Brown } 23100a96aa3bSJed Brown } 23110a96aa3bSJed Brown } else { 23120a96aa3bSJed Brown lastCell = p4estC->local_num_quadrants; 23130a96aa3bSJed Brown } 23140a96aa3bSJed Brown send[2*(p-startF)] = firstCell; 23150a96aa3bSJed Brown send[2*(p-startF)+1] = lastCell - firstCell; 23160a96aa3bSJed Brown ierr = MPI_Isend(&send[2*(p-startF)],2,MPIU_INT,p,tag,comm,&sendReqs[p-startF]);CHKERRMPI(ierr); 23170a96aa3bSJed Brown } 23180a96aa3bSJed Brown ierr = MPI_Waitall((PetscMPIInt)(endC-startC),recvReqs,MPI_STATUSES_IGNORE);CHKERRMPI(ierr); 23190a96aa3bSJed Brown ierr = PetscSectionCreate(PETSC_COMM_SELF,§ion);CHKERRQ(ierr); 23200a96aa3bSJed Brown ierr = PetscSectionSetChart(section,startC,endC);CHKERRQ(ierr); 23210a96aa3bSJed Brown for (p = startC; p < endC; p++) { 23220a96aa3bSJed Brown PetscInt numCells = recv[2*(p-startC)+1]; 23230a96aa3bSJed Brown ierr = PetscSectionSetDof(section,p,numCells);CHKERRQ(ierr); 23240a96aa3bSJed Brown } 23250a96aa3bSJed Brown ierr = PetscSectionSetUp(section);CHKERRQ(ierr); 23260a96aa3bSJed Brown ierr = PetscSectionGetStorageSize(section,&nLeaves);CHKERRQ(ierr); 23270a96aa3bSJed Brown ierr = PetscMalloc1(nLeaves,&leaves);CHKERRQ(ierr); 23280a96aa3bSJed Brown for (p = startC; p < endC; p++) { 23290a96aa3bSJed Brown PetscInt firstCell = recv[2*(p-startC)]; 23300a96aa3bSJed Brown PetscInt numCells = recv[2*(p-startC)+1]; 23310a96aa3bSJed Brown PetscInt off, i; 23320a96aa3bSJed Brown 23330a96aa3bSJed Brown ierr = PetscSectionGetOffset(section,p,&off);CHKERRQ(ierr); 23340a96aa3bSJed Brown for (i = 0; i < numCells; i++) { 23350a96aa3bSJed Brown leaves[off+i].rank = p; 23360a96aa3bSJed Brown leaves[off+i].index = firstCell + i; 23370a96aa3bSJed Brown } 23380a96aa3bSJed Brown } 23390a96aa3bSJed Brown ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr); 23400a96aa3bSJed Brown ierr = PetscSFSetGraph(sf,cEnd-cStart,nLeaves,NULL,PETSC_OWN_POINTER,leaves,PETSC_OWN_POINTER);CHKERRQ(ierr); 23410a96aa3bSJed Brown ierr = PetscSectionDestroy(§ion);CHKERRQ(ierr); 23420a96aa3bSJed Brown ierr = MPI_Waitall((PetscMPIInt)(endF-startF),sendReqs,MPI_STATUSES_IGNORE);CHKERRMPI(ierr); 23430a96aa3bSJed Brown ierr = PetscFree2(send,sendReqs);CHKERRQ(ierr); 23440a96aa3bSJed Brown ierr = PetscFree2(recv,recvReqs);CHKERRQ(ierr); 23450a96aa3bSJed Brown *coveringSF = sf; 23460a96aa3bSJed Brown PetscFunctionReturn(0); 23470a96aa3bSJed Brown } 23480a96aa3bSJed Brown 23490a96aa3bSJed Brown /* closure points for locally-owned cells */ 23500a96aa3bSJed Brown static PetscErrorCode DMPforestGetCellSFNodes(DM dm, PetscInt numClosureIndices, PetscInt *numClosurePoints, PetscSFNode **closurePoints,PetscBool redirect) 23510a96aa3bSJed Brown { 23520a96aa3bSJed Brown PetscInt cStart, cEnd; 23530a96aa3bSJed Brown PetscInt count, c; 23540a96aa3bSJed Brown PetscMPIInt rank; 23550a96aa3bSJed Brown PetscInt closureSize = -1; 23560a96aa3bSJed Brown PetscInt *closure = NULL; 23570a96aa3bSJed Brown PetscSF pointSF; 23580a96aa3bSJed Brown PetscInt nleaves, nroots; 23590a96aa3bSJed Brown const PetscInt *ilocal; 23600a96aa3bSJed Brown const PetscSFNode *iremote; 23610a96aa3bSJed Brown DM plex; 23620a96aa3bSJed Brown DM_Forest *forest; 23630a96aa3bSJed Brown DM_Forest_pforest *pforest; 23640a96aa3bSJed Brown PetscErrorCode ierr; 23650a96aa3bSJed Brown 23660a96aa3bSJed Brown PetscFunctionBegin; 23670a96aa3bSJed Brown forest = (DM_Forest *) dm->data; 23680a96aa3bSJed Brown pforest = (DM_Forest_pforest *) forest->data; 23690a96aa3bSJed Brown cStart = pforest->cLocalStart; 23700a96aa3bSJed Brown cEnd = pforest->cLocalEnd; 23710a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 23720a96aa3bSJed Brown ierr = DMGetPointSF(dm,&pointSF);CHKERRQ(ierr); 23730a96aa3bSJed Brown ierr = PetscSFGetGraph(pointSF,&nroots,&nleaves,&ilocal,&iremote);CHKERRQ(ierr); 23740a96aa3bSJed Brown nleaves = PetscMax(0,nleaves); 23750a96aa3bSJed Brown nroots = PetscMax(0,nroots); 23760a96aa3bSJed Brown *numClosurePoints = numClosureIndices * (cEnd - cStart); 23770a96aa3bSJed Brown ierr = PetscMalloc1(*numClosurePoints,closurePoints);CHKERRQ(ierr); 23780a96aa3bSJed Brown ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm),&rank);CHKERRMPI(ierr); 23790a96aa3bSJed Brown for (c = cStart, count = 0; c < cEnd; c++) { 23800a96aa3bSJed Brown PetscInt i; 23810a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(plex,c,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 23820a96aa3bSJed Brown 23830a96aa3bSJed Brown for (i = 0; i < numClosureIndices; i++, count++) { 23840a96aa3bSJed Brown PetscInt p = closure[2 * i]; 23850a96aa3bSJed Brown PetscInt loc = -1; 23860a96aa3bSJed Brown 23870a96aa3bSJed Brown ierr = PetscFindInt(p,nleaves,ilocal,&loc);CHKERRQ(ierr); 23880a96aa3bSJed Brown if (redirect && loc >= 0) { 23890a96aa3bSJed Brown (*closurePoints)[count].rank = iremote[loc].rank; 23900a96aa3bSJed Brown (*closurePoints)[count].index = iremote[loc].index; 23910a96aa3bSJed Brown } else { 23920a96aa3bSJed Brown (*closurePoints)[count].rank = rank; 23930a96aa3bSJed Brown (*closurePoints)[count].index = p; 23940a96aa3bSJed Brown } 23950a96aa3bSJed Brown } 23960a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(plex,c,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 23970a96aa3bSJed Brown } 23980a96aa3bSJed Brown PetscFunctionReturn(0); 23990a96aa3bSJed Brown } 24000a96aa3bSJed Brown 24010a96aa3bSJed Brown static void MPIAPI DMPforestMaxSFNode(void *a, void *b, PetscMPIInt *len, MPI_Datatype *type) 24020a96aa3bSJed Brown { 24030a96aa3bSJed Brown PetscMPIInt i; 24040a96aa3bSJed Brown 24050a96aa3bSJed Brown for (i = 0; i < *len; i++) { 24060a96aa3bSJed Brown PetscSFNode *A = (PetscSFNode*)a; 24070a96aa3bSJed Brown PetscSFNode *B = (PetscSFNode*)b; 24080a96aa3bSJed Brown 24090a96aa3bSJed Brown if (B->rank < 0) *B = *A; 24100a96aa3bSJed Brown } 24110a96aa3bSJed Brown } 24120a96aa3bSJed Brown 24130a96aa3bSJed Brown static PetscErrorCode DMPforestGetTransferSF_Point(DM coarse, DM fine, PetscSF *sf, PetscBool transferIdent, PetscInt *childIds[]) 24140a96aa3bSJed Brown { 24150a96aa3bSJed Brown MPI_Comm comm; 24160a96aa3bSJed Brown PetscMPIInt rank, size; 24170a96aa3bSJed Brown DM_Forest_pforest *pforestC, *pforestF; 24180a96aa3bSJed Brown p4est_t *p4estC, *p4estF; 24190a96aa3bSJed Brown PetscInt numClosureIndices; 24200a96aa3bSJed Brown PetscInt numClosurePointsC, numClosurePointsF; 24210a96aa3bSJed Brown PetscSFNode *closurePointsC, *closurePointsF; 24220a96aa3bSJed Brown p4est_quadrant_t *coverQuads = NULL; 24230a96aa3bSJed Brown p4est_quadrant_t **treeQuads; 24240a96aa3bSJed Brown PetscInt *treeQuadCounts; 24250a96aa3bSJed Brown MPI_Datatype nodeType; 24260a96aa3bSJed Brown MPI_Datatype nodeClosureType; 24270a96aa3bSJed Brown MPI_Op sfNodeReduce; 24280a96aa3bSJed Brown p4est_topidx_t fltF, lltF, t; 24290a96aa3bSJed Brown DM plexC, plexF; 24300a96aa3bSJed Brown PetscInt pStartF, pEndF, pStartC, pEndC; 24310a96aa3bSJed Brown PetscBool saveInCoarse = PETSC_FALSE; 24320a96aa3bSJed Brown PetscBool saveInFine = PETSC_FALSE; 24330a96aa3bSJed Brown PetscBool formCids = (childIds != NULL) ? PETSC_TRUE : PETSC_FALSE; 24340a96aa3bSJed Brown PetscInt *cids = NULL; 24350a96aa3bSJed Brown PetscErrorCode ierr; 24360a96aa3bSJed Brown 24370a96aa3bSJed Brown PetscFunctionBegin; 24380a96aa3bSJed Brown pforestC = (DM_Forest_pforest*) ((DM_Forest*) coarse->data)->data; 24390a96aa3bSJed Brown pforestF = (DM_Forest_pforest*) ((DM_Forest*) fine->data)->data; 24400a96aa3bSJed Brown p4estC = pforestC->forest; 24410a96aa3bSJed Brown p4estF = pforestF->forest; 2442*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(pforestC->topo != pforestF->topo,PetscObjectComm((PetscObject)coarse),PETSC_ERR_ARG_INCOMP,"DM's must have the same base DM"); 24430a96aa3bSJed Brown comm = PetscObjectComm((PetscObject)coarse); 24440a96aa3bSJed Brown ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 24450a96aa3bSJed Brown ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 24460a96aa3bSJed Brown ierr = DMPforestGetPlex(fine,&plexF);CHKERRQ(ierr); 24470a96aa3bSJed Brown ierr = DMPlexGetChart(plexF,&pStartF,&pEndF);CHKERRQ(ierr); 24480a96aa3bSJed Brown ierr = DMPforestGetPlex(coarse,&plexC);CHKERRQ(ierr); 24490a96aa3bSJed Brown ierr = DMPlexGetChart(plexC,&pStartC,&pEndC);CHKERRQ(ierr); 24500a96aa3bSJed Brown { /* check if the results have been cached */ 24510a96aa3bSJed Brown DM adaptCoarse, adaptFine; 24520a96aa3bSJed Brown 24530a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(coarse,&adaptCoarse);CHKERRQ(ierr); 24540a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(fine,&adaptFine);CHKERRQ(ierr); 24550a96aa3bSJed Brown if (adaptCoarse && adaptCoarse->data == fine->data) { /* coarse is adapted from fine */ 24560a96aa3bSJed Brown if (pforestC->pointSelfToAdaptSF) { 24570a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)(pforestC->pointSelfToAdaptSF));CHKERRQ(ierr); 24580a96aa3bSJed Brown *sf = pforestC->pointSelfToAdaptSF; 24590a96aa3bSJed Brown if (childIds) { 24600a96aa3bSJed Brown ierr = PetscMalloc1(pEndF-pStartF,&cids);CHKERRQ(ierr); 24610a96aa3bSJed Brown ierr = PetscArraycpy(cids,pforestC->pointSelfToAdaptCids,pEndF-pStartF);CHKERRQ(ierr); 24620a96aa3bSJed Brown *childIds = cids; 24630a96aa3bSJed Brown } 24640a96aa3bSJed Brown PetscFunctionReturn(0); 24650a96aa3bSJed Brown } else { 24660a96aa3bSJed Brown saveInCoarse = PETSC_TRUE; 24670a96aa3bSJed Brown formCids = PETSC_TRUE; 24680a96aa3bSJed Brown } 24690a96aa3bSJed Brown } else if (adaptFine && adaptFine->data == coarse->data) { /* fine is adapted from coarse */ 24700a96aa3bSJed Brown if (pforestF->pointAdaptToSelfSF) { 24710a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)(pforestF->pointAdaptToSelfSF));CHKERRQ(ierr); 24720a96aa3bSJed Brown *sf = pforestF->pointAdaptToSelfSF; 24730a96aa3bSJed Brown if (childIds) { 24740a96aa3bSJed Brown ierr = PetscMalloc1(pEndF-pStartF,&cids);CHKERRQ(ierr); 24750a96aa3bSJed Brown ierr = PetscArraycpy(cids,pforestF->pointAdaptToSelfCids,pEndF-pStartF);CHKERRQ(ierr); 24760a96aa3bSJed Brown *childIds = cids; 24770a96aa3bSJed Brown } 24780a96aa3bSJed Brown PetscFunctionReturn(0); 24790a96aa3bSJed Brown } else { 24800a96aa3bSJed Brown saveInFine = PETSC_TRUE; 24810a96aa3bSJed Brown formCids = PETSC_TRUE; 24820a96aa3bSJed Brown } 24830a96aa3bSJed Brown } 24840a96aa3bSJed Brown } 24850a96aa3bSJed Brown 24860a96aa3bSJed Brown /* count the number of closure points that have dofs and create a list */ 24870a96aa3bSJed Brown numClosureIndices = P4EST_INSUL; 24880a96aa3bSJed Brown /* create the datatype */ 24890a96aa3bSJed Brown ierr = MPI_Type_contiguous(2,MPIU_INT,&nodeType);CHKERRMPI(ierr); 24900a96aa3bSJed Brown ierr = MPI_Type_commit(&nodeType);CHKERRMPI(ierr); 24910a96aa3bSJed Brown ierr = MPI_Op_create(DMPforestMaxSFNode,PETSC_FALSE,&sfNodeReduce);CHKERRMPI(ierr); 24920a96aa3bSJed Brown ierr = MPI_Type_contiguous(numClosureIndices*2,MPIU_INT,&nodeClosureType);CHKERRMPI(ierr); 24930a96aa3bSJed Brown ierr = MPI_Type_commit(&nodeClosureType);CHKERRMPI(ierr); 24940a96aa3bSJed Brown /* everything has to go through cells: for each cell, create a list of the sfnodes in its closure */ 24950a96aa3bSJed Brown /* get lists of closure point SF nodes for every cell */ 24960a96aa3bSJed Brown ierr = DMPforestGetCellSFNodes(coarse,numClosureIndices,&numClosurePointsC,&closurePointsC,PETSC_TRUE);CHKERRQ(ierr); 24970a96aa3bSJed Brown ierr = DMPforestGetCellSFNodes(fine ,numClosureIndices,&numClosurePointsF,&closurePointsF,PETSC_FALSE);CHKERRQ(ierr); 24980a96aa3bSJed Brown /* create pointers for tree lists */ 24990a96aa3bSJed Brown fltF = p4estF->first_local_tree; 25000a96aa3bSJed Brown lltF = p4estF->last_local_tree; 25010a96aa3bSJed Brown ierr = PetscCalloc2(lltF + 1 - fltF, &treeQuads, lltF + 1 - fltF, &treeQuadCounts);CHKERRQ(ierr); 25020a96aa3bSJed Brown /* if the partitions don't match, ship the coarse to cover the fine */ 25030a96aa3bSJed Brown if (size > 1) { 25040a96aa3bSJed Brown PetscInt p; 25050a96aa3bSJed Brown 25060a96aa3bSJed Brown for (p = 0; p < size; p++) { 25070a96aa3bSJed Brown int equal; 25080a96aa3bSJed Brown 25090a96aa3bSJed Brown PetscStackCallP4estReturn(equal,p4est_quadrant_is_equal_piggy,(&p4estC->global_first_position[p],&p4estF->global_first_position[p])); 25100a96aa3bSJed Brown if (!equal) break; 25110a96aa3bSJed Brown } 25120a96aa3bSJed Brown if (p < size) { /* non-matching distribution: send the coarse to cover the fine */ 25130a96aa3bSJed Brown PetscInt cStartC, cEndC; 25140a96aa3bSJed Brown PetscSF coveringSF; 25150a96aa3bSJed Brown PetscInt nleaves; 25160a96aa3bSJed Brown PetscInt count; 25170a96aa3bSJed Brown PetscSFNode *newClosurePointsC; 25180a96aa3bSJed Brown p4est_quadrant_t *coverQuadsSend; 25190a96aa3bSJed Brown p4est_topidx_t fltC = p4estC->first_local_tree; 25200a96aa3bSJed Brown p4est_topidx_t lltC = p4estC->last_local_tree; 25210a96aa3bSJed Brown p4est_topidx_t t; 25220a96aa3bSJed Brown PetscMPIInt blockSizes[4] = {P4EST_DIM,2,1,1}; 25230a96aa3bSJed Brown MPI_Aint blockOffsets[4] = {offsetof(p4est_quadrant_t,x), 25240a96aa3bSJed Brown offsetof(p4est_quadrant_t,level), 25250a96aa3bSJed Brown offsetof(p4est_quadrant_t,pad16), 25260a96aa3bSJed Brown offsetof(p4est_quadrant_t,p)}; 25270a96aa3bSJed Brown MPI_Datatype blockTypes[4] = {MPI_INT32_T,MPI_INT8_T,MPI_INT16_T,MPI_INT32_T/* p.which_tree */}; 25280a96aa3bSJed Brown MPI_Datatype quadStruct,quadType; 25290a96aa3bSJed Brown 25300a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(plexC,0,&cStartC,&cEndC);CHKERRQ(ierr); 25310a96aa3bSJed Brown ierr = DMPforestGetCellCoveringSF(comm,p4estC,p4estF,pforestC->cLocalStart,pforestC->cLocalEnd,&coveringSF);CHKERRQ(ierr); 25320a96aa3bSJed Brown ierr = PetscSFGetGraph(coveringSF,NULL,&nleaves,NULL,NULL);CHKERRQ(ierr); 25330a96aa3bSJed Brown ierr = PetscMalloc1(numClosureIndices*nleaves,&newClosurePointsC);CHKERRQ(ierr); 25340a96aa3bSJed Brown ierr = PetscMalloc1(nleaves,&coverQuads);CHKERRQ(ierr); 25350a96aa3bSJed Brown ierr = PetscMalloc1(cEndC-cStartC,&coverQuadsSend);CHKERRQ(ierr); 25360a96aa3bSJed Brown count = 0; 25370a96aa3bSJed Brown for (t = fltC; t <= lltC; t++) { /* unfortunately, we need to pack a send array, since quads are not stored packed in p4est */ 25380a96aa3bSJed Brown p4est_tree_t *tree = &(((p4est_tree_t*) p4estC->trees->array)[t]); 25390a96aa3bSJed Brown PetscInt q; 25400a96aa3bSJed Brown 25410a96aa3bSJed Brown ierr = PetscMemcpy(&coverQuadsSend[count],tree->quadrants.array,tree->quadrants.elem_count * sizeof(p4est_quadrant_t));CHKERRQ(ierr); 25420a96aa3bSJed Brown for (q = 0; (size_t) q < tree->quadrants.elem_count; q++) coverQuadsSend[count+q].p.which_tree = t; 25430a96aa3bSJed Brown count += tree->quadrants.elem_count; 25440a96aa3bSJed Brown } 25450a96aa3bSJed Brown /* p is of a union type p4est_quadrant_data, but only the p.which_tree field is active at this time. So, we 25460a96aa3bSJed Brown have a simple blockTypes[] to use. Note that quadStruct does not count potential padding in array of 25470a96aa3bSJed Brown p4est_quadrant_t. We have to call MPI_Type_create_resized() to change upper-bound of quadStruct. 25480a96aa3bSJed Brown */ 25490a96aa3bSJed Brown ierr = MPI_Type_create_struct(4,blockSizes,blockOffsets,blockTypes,&quadStruct);CHKERRMPI(ierr); 25500a96aa3bSJed Brown ierr = MPI_Type_create_resized(quadStruct,0,sizeof(p4est_quadrant_t),&quadType);CHKERRMPI(ierr); 25510a96aa3bSJed Brown ierr = MPI_Type_commit(&quadType);CHKERRMPI(ierr); 25520a96aa3bSJed Brown ierr = PetscSFBcastBegin(coveringSF,nodeClosureType,closurePointsC,newClosurePointsC,MPI_REPLACE);CHKERRQ(ierr); 25530a96aa3bSJed Brown ierr = PetscSFBcastBegin(coveringSF,quadType,coverQuadsSend,coverQuads,MPI_REPLACE);CHKERRQ(ierr); 25540a96aa3bSJed Brown ierr = PetscSFBcastEnd(coveringSF,nodeClosureType,closurePointsC,newClosurePointsC,MPI_REPLACE);CHKERRQ(ierr); 25550a96aa3bSJed Brown ierr = PetscSFBcastEnd(coveringSF,quadType,coverQuadsSend,coverQuads,MPI_REPLACE);CHKERRQ(ierr); 25560a96aa3bSJed Brown ierr = MPI_Type_free(&quadStruct);CHKERRMPI(ierr); 25570a96aa3bSJed Brown ierr = MPI_Type_free(&quadType);CHKERRMPI(ierr); 25580a96aa3bSJed Brown ierr = PetscFree(coverQuadsSend);CHKERRQ(ierr); 25590a96aa3bSJed Brown ierr = PetscFree(closurePointsC);CHKERRQ(ierr); 25600a96aa3bSJed Brown ierr = PetscSFDestroy(&coveringSF);CHKERRQ(ierr); 25610a96aa3bSJed Brown closurePointsC = newClosurePointsC; 25620a96aa3bSJed Brown 25630a96aa3bSJed Brown /* assign tree quads based on locations in coverQuads */ 25640a96aa3bSJed Brown { 25650a96aa3bSJed Brown PetscInt q; 25660a96aa3bSJed Brown for (q = 0; q < nleaves; q++) { 25670a96aa3bSJed Brown p4est_locidx_t t = coverQuads[q].p.which_tree; 25680a96aa3bSJed Brown if (!treeQuadCounts[t-fltF]++) treeQuads[t-fltF] = &coverQuads[q]; 25690a96aa3bSJed Brown } 25700a96aa3bSJed Brown } 25710a96aa3bSJed Brown } 25720a96aa3bSJed Brown } 25730a96aa3bSJed Brown if (!coverQuads) { /* matching partitions: assign tree quads based on locations in p4est native arrays */ 25740a96aa3bSJed Brown for (t = fltF; t <= lltF; t++) { 25750a96aa3bSJed Brown p4est_tree_t *tree = &(((p4est_tree_t*) p4estC->trees->array)[t]); 25760a96aa3bSJed Brown 25770a96aa3bSJed Brown treeQuadCounts[t - fltF] = tree->quadrants.elem_count; 25780a96aa3bSJed Brown treeQuads[t - fltF] = (p4est_quadrant_t*) tree->quadrants.array; 25790a96aa3bSJed Brown } 25800a96aa3bSJed Brown } 25810a96aa3bSJed Brown 25820a96aa3bSJed Brown { 25830a96aa3bSJed Brown PetscInt p; 25840a96aa3bSJed Brown PetscInt cLocalStartF; 25850a96aa3bSJed Brown PetscSF pointSF; 25860a96aa3bSJed Brown PetscSFNode *roots; 25870a96aa3bSJed Brown PetscInt *rootType; 25880a96aa3bSJed Brown DM refTree = NULL; 25890a96aa3bSJed Brown DMLabel canonical; 25900a96aa3bSJed Brown PetscInt *childClosures[P4EST_CHILDREN] = {NULL}; 25910a96aa3bSJed Brown PetscInt *rootClosure = NULL; 25920a96aa3bSJed Brown PetscInt coarseOffset; 25930a96aa3bSJed Brown PetscInt numCoarseQuads; 25940a96aa3bSJed Brown 25950a96aa3bSJed Brown ierr = PetscMalloc1(pEndF-pStartF,&roots);CHKERRQ(ierr); 25960a96aa3bSJed Brown ierr = PetscMalloc1(pEndF-pStartF,&rootType);CHKERRQ(ierr); 25970a96aa3bSJed Brown ierr = DMGetPointSF(fine,&pointSF);CHKERRQ(ierr); 25980a96aa3bSJed Brown for (p = pStartF; p < pEndF; p++) { 25990a96aa3bSJed Brown roots[p-pStartF].rank = -1; 26000a96aa3bSJed Brown roots[p-pStartF].index = -1; 26010a96aa3bSJed Brown rootType[p-pStartF] = -1; 26020a96aa3bSJed Brown } 26030a96aa3bSJed Brown if (formCids) { 26040a96aa3bSJed Brown PetscInt child; 26050a96aa3bSJed Brown 26060a96aa3bSJed Brown ierr = PetscMalloc1(pEndF-pStartF,&cids);CHKERRQ(ierr); 26070a96aa3bSJed Brown for (p = pStartF; p < pEndF; p++) cids[p - pStartF] = -2; 26080a96aa3bSJed Brown ierr = DMPlexGetReferenceTree(plexF,&refTree);CHKERRQ(ierr); 26090a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(refTree,0,PETSC_TRUE,NULL,&rootClosure);CHKERRQ(ierr); 26100a96aa3bSJed Brown for (child = 0; child < P4EST_CHILDREN; child++) { /* get the closures of the child cells in the reference tree */ 26110a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(refTree,child+1,PETSC_TRUE,NULL,&childClosures[child]);CHKERRQ(ierr); 26120a96aa3bSJed Brown } 26130a96aa3bSJed Brown ierr = DMGetLabel(refTree,"canonical",&canonical);CHKERRQ(ierr); 26140a96aa3bSJed Brown } 26150a96aa3bSJed Brown cLocalStartF = pforestF->cLocalStart; 26160a96aa3bSJed Brown for (t = fltF, coarseOffset = 0, numCoarseQuads = 0; t <= lltF; t++, coarseOffset += numCoarseQuads) { 26170a96aa3bSJed Brown p4est_tree_t *tree = &(((p4est_tree_t*) p4estF->trees->array)[t]); 26180a96aa3bSJed Brown PetscInt numFineQuads = tree->quadrants.elem_count; 26190a96aa3bSJed Brown p4est_quadrant_t *coarseQuads = treeQuads[t - fltF]; 26200a96aa3bSJed Brown p4est_quadrant_t *fineQuads = (p4est_quadrant_t*) tree->quadrants.array; 26210a96aa3bSJed Brown PetscInt i, coarseCount = 0; 26220a96aa3bSJed Brown PetscInt offset = tree->quadrants_offset; 26230a96aa3bSJed Brown sc_array_t coarseQuadsArray; 26240a96aa3bSJed Brown 26250a96aa3bSJed Brown numCoarseQuads = treeQuadCounts[t - fltF]; 26260a96aa3bSJed Brown PetscStackCallP4est(sc_array_init_data,(&coarseQuadsArray,coarseQuads,sizeof(p4est_quadrant_t),(size_t) numCoarseQuads)); 26270a96aa3bSJed Brown for (i = 0; i < numFineQuads; i++) { 26280a96aa3bSJed Brown PetscInt c = i + offset; 26290a96aa3bSJed Brown p4est_quadrant_t *quad = &fineQuads[i]; 26300a96aa3bSJed Brown p4est_quadrant_t *quadCoarse = NULL; 26310a96aa3bSJed Brown ssize_t disjoint = -1; 26320a96aa3bSJed Brown 26330a96aa3bSJed Brown while (disjoint < 0 && coarseCount < numCoarseQuads) { 26340a96aa3bSJed Brown quadCoarse = &coarseQuads[coarseCount]; 26350a96aa3bSJed Brown PetscStackCallP4estReturn(disjoint,p4est_quadrant_disjoint,(quadCoarse,quad)); 26360a96aa3bSJed Brown if (disjoint < 0) coarseCount++; 26370a96aa3bSJed Brown } 2638*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(disjoint != 0,PETSC_COMM_SELF,PETSC_ERR_PLIB,"did not find overlapping coarse quad"); 26390a96aa3bSJed Brown if (quadCoarse->level > quad->level || (quadCoarse->level == quad->level && !transferIdent)) { /* the "coarse" mesh is finer than the fine mesh at the point: continue */ 26400a96aa3bSJed Brown if (transferIdent) { /* find corners */ 26410a96aa3bSJed Brown PetscInt j = 0; 26420a96aa3bSJed Brown 26430a96aa3bSJed Brown do { 26440a96aa3bSJed Brown if (j < P4EST_CHILDREN) { 26450a96aa3bSJed Brown p4est_quadrant_t cornerQuad; 26460a96aa3bSJed Brown int equal; 26470a96aa3bSJed Brown 26480a96aa3bSJed Brown PetscStackCallP4est(p4est_quadrant_corner_descendant,(quad,&cornerQuad,j,quadCoarse->level)); 26490a96aa3bSJed Brown PetscStackCallP4estReturn(equal,p4est_quadrant_is_equal,(&cornerQuad,quadCoarse)); 26500a96aa3bSJed Brown if (equal) { 26510a96aa3bSJed Brown PetscInt petscJ = P4estVertToPetscVert[j]; 26520a96aa3bSJed Brown PetscInt p = closurePointsF[numClosureIndices * c + (P4EST_INSUL - P4EST_CHILDREN) + petscJ].index; 26530a96aa3bSJed Brown PetscSFNode q = closurePointsC[numClosureIndices * (coarseCount + coarseOffset) + (P4EST_INSUL - P4EST_CHILDREN) + petscJ]; 26540a96aa3bSJed Brown 26550a96aa3bSJed Brown roots[p-pStartF] = q; 26560a96aa3bSJed Brown rootType[p-pStartF] = PETSC_MAX_INT; 26570a96aa3bSJed Brown cids[p-pStartF] = -1; 26580a96aa3bSJed Brown j++; 26590a96aa3bSJed Brown } 26600a96aa3bSJed Brown } 26610a96aa3bSJed Brown coarseCount++; 26620a96aa3bSJed Brown disjoint = 1; 26630a96aa3bSJed Brown if (coarseCount < numCoarseQuads) { 26640a96aa3bSJed Brown quadCoarse = &coarseQuads[coarseCount]; 26650a96aa3bSJed Brown PetscStackCallP4estReturn(disjoint,p4est_quadrant_disjoint,(quadCoarse,quad)); 26660a96aa3bSJed Brown } 26670a96aa3bSJed Brown } while (!disjoint); 26680a96aa3bSJed Brown } 26690a96aa3bSJed Brown continue; 26700a96aa3bSJed Brown } 26710a96aa3bSJed Brown if (quadCoarse->level == quad->level) { /* same quad present in coarse and fine mesh */ 26720a96aa3bSJed Brown PetscInt j; 26730a96aa3bSJed Brown for (j = 0; j < numClosureIndices; j++) { 26740a96aa3bSJed Brown PetscInt p = closurePointsF[numClosureIndices * c + j].index; 26750a96aa3bSJed Brown 26760a96aa3bSJed Brown roots[p-pStartF] = closurePointsC[numClosureIndices * (coarseCount + coarseOffset) + j]; 26770a96aa3bSJed Brown rootType[p-pStartF] = PETSC_MAX_INT; /* unconditionally accept */ 26780a96aa3bSJed Brown cids[p-pStartF] = -1; 26790a96aa3bSJed Brown } 26800a96aa3bSJed Brown } else { 26810a96aa3bSJed Brown PetscInt levelDiff = quad->level - quadCoarse->level; 26820a96aa3bSJed Brown PetscInt proposedCids[P4EST_INSUL] = {0}; 26830a96aa3bSJed Brown 26840a96aa3bSJed Brown if (formCids) { 26850a96aa3bSJed Brown PetscInt cl; 26860a96aa3bSJed Brown PetscInt *pointClosure = NULL; 26870a96aa3bSJed Brown int cid; 26880a96aa3bSJed Brown 2689*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(levelDiff > 1,PETSC_COMM_SELF,PETSC_ERR_USER,"Recursive child ids not implemented"); 26900a96aa3bSJed Brown PetscStackCallP4estReturn(cid,p4est_quadrant_child_id,(quad)); 26910a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(plexF,c + cLocalStartF,PETSC_TRUE,NULL,&pointClosure);CHKERRQ(ierr); 26920a96aa3bSJed Brown for (cl = 0; cl < P4EST_INSUL; cl++) { 26930a96aa3bSJed Brown PetscInt p = pointClosure[2 * cl]; 26940a96aa3bSJed Brown PetscInt point = childClosures[cid][2 * cl]; 26950a96aa3bSJed Brown PetscInt ornt = childClosures[cid][2 * cl + 1]; 26960a96aa3bSJed Brown PetscInt newcid = -1; 26970a96aa3bSJed Brown DMPolytopeType ct; 26980a96aa3bSJed Brown 26990a96aa3bSJed Brown if (rootType[p-pStartF] == PETSC_MAX_INT) continue; 27000a96aa3bSJed Brown ierr = DMPlexGetCellType(refTree, point, &ct);CHKERRQ(ierr); 27010a96aa3bSJed Brown ornt = DMPolytopeConvertNewOrientation_Internal(ct, ornt); 27020a96aa3bSJed Brown if (!cl) { 27030a96aa3bSJed Brown newcid = cid + 1; 27040a96aa3bSJed Brown } else { 27050a96aa3bSJed Brown PetscInt rcl, parent, parentOrnt = 0; 27060a96aa3bSJed Brown 27070a96aa3bSJed Brown ierr = DMPlexGetTreeParent(refTree,point,&parent,NULL);CHKERRQ(ierr); 27080a96aa3bSJed Brown if (parent == point) { 27090a96aa3bSJed Brown newcid = -1; 27100a96aa3bSJed Brown } else if (!parent) { /* in the root */ 27110a96aa3bSJed Brown newcid = point; 27120a96aa3bSJed Brown } else { 27130a96aa3bSJed Brown DMPolytopeType rct = DM_POLYTOPE_UNKNOWN; 27140a96aa3bSJed Brown 27150a96aa3bSJed Brown for (rcl = 1; rcl < P4EST_INSUL; rcl++) { 27160a96aa3bSJed Brown if (rootClosure[2 * rcl] == parent) { 27170a96aa3bSJed Brown ierr = DMPlexGetCellType(refTree, parent, &rct);CHKERRQ(ierr); 27180a96aa3bSJed Brown parentOrnt = DMPolytopeConvertNewOrientation_Internal(rct, rootClosure[2 * rcl + 1]); 27190a96aa3bSJed Brown break; 27200a96aa3bSJed Brown } 27210a96aa3bSJed Brown } 2722*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(rcl >= P4EST_INSUL,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Couldn't find parent in root closure"); 27230a96aa3bSJed Brown ierr = DMPlexReferenceTreeGetChildSymmetry(refTree,parent,parentOrnt,ornt,point,DMPolytopeConvertNewOrientation_Internal(rct, pointClosure[2 * rcl + 1]),NULL,&newcid);CHKERRQ(ierr); 27240a96aa3bSJed Brown } 27250a96aa3bSJed Brown } 27260a96aa3bSJed Brown if (newcid >= 0) { 27270a96aa3bSJed Brown 27280a96aa3bSJed Brown if (canonical) { 27290a96aa3bSJed Brown ierr = DMLabelGetValue(canonical,newcid,&newcid);CHKERRQ(ierr); 27300a96aa3bSJed Brown } 27310a96aa3bSJed Brown proposedCids[cl] = newcid; 27320a96aa3bSJed Brown } 27330a96aa3bSJed Brown } 27340a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(plexF,c + cLocalStartF,PETSC_TRUE,NULL,&pointClosure);CHKERRQ(ierr); 27350a96aa3bSJed Brown } 27360a96aa3bSJed Brown p4est_qcoord_t coarseBound[2][P4EST_DIM] = {{quadCoarse->x,quadCoarse->y, 27370a96aa3bSJed Brown #if defined(P4_TO_P8) 27380a96aa3bSJed Brown quadCoarse->z 27390a96aa3bSJed Brown #endif 27400a96aa3bSJed Brown },{0}}; 27410a96aa3bSJed Brown p4est_qcoord_t fineBound[2][P4EST_DIM] = {{quad->x,quad->y, 27420a96aa3bSJed Brown #if defined(P4_TO_P8) 27430a96aa3bSJed Brown quad->z 27440a96aa3bSJed Brown #endif 27450a96aa3bSJed Brown },{0}}; 27460a96aa3bSJed Brown PetscInt j; 27470a96aa3bSJed Brown for (j = 0; j < P4EST_DIM; j++) { /* get the coordinates of cell boundaries in each direction */ 27480a96aa3bSJed Brown coarseBound[1][j] = coarseBound[0][j] + P4EST_QUADRANT_LEN(quadCoarse->level); 27490a96aa3bSJed Brown fineBound[1][j] = fineBound[0][j] + P4EST_QUADRANT_LEN(quad->level); 27500a96aa3bSJed Brown } 27510a96aa3bSJed Brown for (j = 0; j < numClosureIndices; j++) { 27520a96aa3bSJed Brown PetscInt l, p; 27530a96aa3bSJed Brown PetscSFNode q; 27540a96aa3bSJed Brown 27550a96aa3bSJed Brown p = closurePointsF[numClosureIndices * c + j].index; 27560a96aa3bSJed Brown if (rootType[p-pStartF] == PETSC_MAX_INT) continue; 27570a96aa3bSJed Brown if (j == 0) { /* volume: ancestor is volume */ 27580a96aa3bSJed Brown l = 0; 27590a96aa3bSJed Brown } else if (j < 1 + P4EST_FACES) { /* facet */ 27600a96aa3bSJed Brown PetscInt face = PetscFaceToP4estFace[j - 1]; 27610a96aa3bSJed Brown PetscInt direction = face / 2; 27620a96aa3bSJed Brown PetscInt coarseFace = -1; 27630a96aa3bSJed Brown 27640a96aa3bSJed Brown if (coarseBound[face % 2][direction] == fineBound[face % 2][direction]) { 27650a96aa3bSJed Brown coarseFace = face; 27660a96aa3bSJed Brown l = 1 + P4estFaceToPetscFace[coarseFace]; 27670a96aa3bSJed Brown } else { 27680a96aa3bSJed Brown l = 0; 27690a96aa3bSJed Brown } 27700a96aa3bSJed Brown #if defined(P4_TO_P8) 27710a96aa3bSJed Brown } else if (j < 1 + P4EST_FACES + P8EST_EDGES) { 27720a96aa3bSJed Brown PetscInt edge = PetscEdgeToP4estEdge[j - (1 + P4EST_FACES)]; 27730a96aa3bSJed Brown PetscInt direction = edge / 4; 27740a96aa3bSJed Brown PetscInt mod = edge % 4; 27750a96aa3bSJed Brown PetscInt coarseEdge = -1, coarseFace = -1; 27760a96aa3bSJed Brown PetscInt minDir = PetscMin((direction + 1) % 3,(direction + 2) % 3); 27770a96aa3bSJed Brown PetscInt maxDir = PetscMax((direction + 1) % 3,(direction + 2) % 3); 27780a96aa3bSJed Brown PetscBool dirTest[2]; 27790a96aa3bSJed Brown 27800a96aa3bSJed Brown dirTest[0] = (PetscBool) (coarseBound[mod % 2][minDir] == fineBound[mod % 2][minDir]); 27810a96aa3bSJed Brown dirTest[1] = (PetscBool) (coarseBound[mod / 2][maxDir] == fineBound[mod / 2][maxDir]); 27820a96aa3bSJed Brown 27830a96aa3bSJed Brown if (dirTest[0] && dirTest[1]) { /* fine edge falls on coarse edge */ 27840a96aa3bSJed Brown coarseEdge = edge; 27850a96aa3bSJed Brown l = 1 + P4EST_FACES + P4estEdgeToPetscEdge[coarseEdge]; 27860a96aa3bSJed Brown } else if (dirTest[0]) { /* fine edge falls on a coarse face in the minDir direction */ 27870a96aa3bSJed Brown coarseFace = 2 * minDir + (mod % 2); 27880a96aa3bSJed Brown l = 1 + P4estFaceToPetscFace[coarseFace]; 27890a96aa3bSJed Brown } else if (dirTest[1]) { /* fine edge falls on a coarse face in the maxDir direction */ 27900a96aa3bSJed Brown coarseFace = 2 * maxDir + (mod / 2); 27910a96aa3bSJed Brown l = 1 + P4estFaceToPetscFace[coarseFace]; 27920a96aa3bSJed Brown } else { 27930a96aa3bSJed Brown l = 0; 27940a96aa3bSJed Brown } 27950a96aa3bSJed Brown #endif 27960a96aa3bSJed Brown } else { 27970a96aa3bSJed Brown PetscInt vertex = PetscVertToP4estVert[P4EST_CHILDREN - (P4EST_INSUL - j)]; 27980a96aa3bSJed Brown PetscBool dirTest[P4EST_DIM]; 27990a96aa3bSJed Brown PetscInt m; 28000a96aa3bSJed Brown PetscInt numMatch = 0; 28010a96aa3bSJed Brown PetscInt coarseVertex = -1, coarseFace = -1; 28020a96aa3bSJed Brown #if defined(P4_TO_P8) 28030a96aa3bSJed Brown PetscInt coarseEdge = -1; 28040a96aa3bSJed Brown #endif 28050a96aa3bSJed Brown 28060a96aa3bSJed Brown for (m = 0; m < P4EST_DIM; m++) { 28070a96aa3bSJed Brown dirTest[m] = (PetscBool) (coarseBound[(vertex >> m) & 1][m] == fineBound[(vertex >> m) & 1][m]); 28080a96aa3bSJed Brown if (dirTest[m]) numMatch++; 28090a96aa3bSJed Brown } 28100a96aa3bSJed Brown if (numMatch == P4EST_DIM) { /* vertex on vertex */ 28110a96aa3bSJed Brown coarseVertex = vertex; 28120a96aa3bSJed Brown l = P4EST_INSUL - (P4EST_CHILDREN - P4estVertToPetscVert[coarseVertex]); 28130a96aa3bSJed Brown } else if (numMatch == 1) { /* vertex on face */ 28140a96aa3bSJed Brown for (m = 0; m < P4EST_DIM; m++) { 28150a96aa3bSJed Brown if (dirTest[m]) { 28160a96aa3bSJed Brown coarseFace = 2 * m + ((vertex >> m) & 1); 28170a96aa3bSJed Brown break; 28180a96aa3bSJed Brown } 28190a96aa3bSJed Brown } 28200a96aa3bSJed Brown l = 1 + P4estFaceToPetscFace[coarseFace]; 28210a96aa3bSJed Brown #if defined(P4_TO_P8) 28220a96aa3bSJed Brown } else if (numMatch == 2) { /* vertex on edge */ 28230a96aa3bSJed Brown for (m = 0; m < P4EST_DIM; m++) { 28240a96aa3bSJed Brown if (!dirTest[m]) { 28250a96aa3bSJed Brown PetscInt otherDir1 = (m + 1) % 3; 28260a96aa3bSJed Brown PetscInt otherDir2 = (m + 2) % 3; 28270a96aa3bSJed Brown PetscInt minDir = PetscMin(otherDir1,otherDir2); 28280a96aa3bSJed Brown PetscInt maxDir = PetscMax(otherDir1,otherDir2); 28290a96aa3bSJed Brown 28300a96aa3bSJed Brown coarseEdge = m * 4 + 2 * ((vertex >> maxDir) & 1) + ((vertex >> minDir) & 1); 28310a96aa3bSJed Brown break; 28320a96aa3bSJed Brown } 28330a96aa3bSJed Brown } 28340a96aa3bSJed Brown l = 1 + P4EST_FACES + P4estEdgeToPetscEdge[coarseEdge]; 28350a96aa3bSJed Brown #endif 28360a96aa3bSJed Brown } else { /* volume */ 28370a96aa3bSJed Brown l = 0; 28380a96aa3bSJed Brown } 28390a96aa3bSJed Brown } 28400a96aa3bSJed Brown q = closurePointsC[numClosureIndices * (coarseCount + coarseOffset) + l]; 28410a96aa3bSJed Brown if (l > rootType[p-pStartF]) { 28420a96aa3bSJed Brown if (l >= P4EST_INSUL - P4EST_CHILDREN) { /* vertex on vertex: unconditional acceptance */ 28430a96aa3bSJed Brown if (transferIdent) { 28440a96aa3bSJed Brown roots[p-pStartF] = q; 28450a96aa3bSJed Brown rootType[p-pStartF] = PETSC_MAX_INT; 28460a96aa3bSJed Brown if (formCids) cids[p-pStartF] = -1; 28470a96aa3bSJed Brown } 28480a96aa3bSJed Brown } else { 28490a96aa3bSJed Brown PetscInt k, thisp = p, limit; 28500a96aa3bSJed Brown 28510a96aa3bSJed Brown roots[p-pStartF] = q; 28520a96aa3bSJed Brown rootType[p-pStartF] = l; 28530a96aa3bSJed Brown if (formCids) cids[p - pStartF] = proposedCids[j]; 28540a96aa3bSJed Brown limit = transferIdent ? levelDiff : (levelDiff - 1); 28550a96aa3bSJed Brown for (k = 0; k < limit; k++) { 28560a96aa3bSJed Brown PetscInt parent; 28570a96aa3bSJed Brown 28580a96aa3bSJed Brown ierr = DMPlexGetTreeParent(plexF,thisp,&parent,NULL);CHKERRQ(ierr); 28590a96aa3bSJed Brown if (parent == thisp) break; 28600a96aa3bSJed Brown 28610a96aa3bSJed Brown roots[parent-pStartF] = q; 28620a96aa3bSJed Brown rootType[parent-pStartF] = PETSC_MAX_INT; 28630a96aa3bSJed Brown if (formCids) cids[parent-pStartF] = -1; 28640a96aa3bSJed Brown thisp = parent; 28650a96aa3bSJed Brown } 28660a96aa3bSJed Brown } 28670a96aa3bSJed Brown } 28680a96aa3bSJed Brown } 28690a96aa3bSJed Brown } 28700a96aa3bSJed Brown } 28710a96aa3bSJed Brown } 28720a96aa3bSJed Brown 28730a96aa3bSJed Brown /* now every cell has labeled the points in its closure, so we first make sure everyone agrees by reducing to roots, and the broadcast the agreements */ 28740a96aa3bSJed Brown if (size > 1) { 28750a96aa3bSJed Brown PetscInt *rootTypeCopy, p; 28760a96aa3bSJed Brown 28770a96aa3bSJed Brown ierr = PetscMalloc1(pEndF-pStartF,&rootTypeCopy);CHKERRQ(ierr); 28780a96aa3bSJed Brown ierr = PetscArraycpy(rootTypeCopy,rootType,pEndF-pStartF);CHKERRQ(ierr); 28790a96aa3bSJed Brown ierr = PetscSFReduceBegin(pointSF,MPIU_INT,rootTypeCopy,rootTypeCopy,MPIU_MAX);CHKERRQ(ierr); 28800a96aa3bSJed Brown ierr = PetscSFReduceEnd(pointSF,MPIU_INT,rootTypeCopy,rootTypeCopy,MPIU_MAX);CHKERRQ(ierr); 28810a96aa3bSJed Brown ierr = PetscSFBcastBegin(pointSF,MPIU_INT,rootTypeCopy,rootTypeCopy,MPI_REPLACE);CHKERRQ(ierr); 28820a96aa3bSJed Brown ierr = PetscSFBcastEnd(pointSF,MPIU_INT,rootTypeCopy,rootTypeCopy,MPI_REPLACE);CHKERRQ(ierr); 28830a96aa3bSJed Brown for (p = pStartF; p < pEndF; p++) { 28840a96aa3bSJed Brown if (rootTypeCopy[p-pStartF] > rootType[p-pStartF]) { /* another process found a root of higher type (e.g. vertex instead of edge), which we want to accept, so nullify this */ 28850a96aa3bSJed Brown roots[p-pStartF].rank = -1; 28860a96aa3bSJed Brown roots[p-pStartF].index = -1; 28870a96aa3bSJed Brown } 28880a96aa3bSJed Brown if (formCids && rootTypeCopy[p-pStartF] == PETSC_MAX_INT) { 28890a96aa3bSJed Brown cids[p-pStartF] = -1; /* we have found an antecedent that is the same: no child id */ 28900a96aa3bSJed Brown } 28910a96aa3bSJed Brown } 28920a96aa3bSJed Brown ierr = PetscFree(rootTypeCopy);CHKERRQ(ierr); 28930a96aa3bSJed Brown ierr = PetscSFReduceBegin(pointSF,nodeType,roots,roots,sfNodeReduce);CHKERRQ(ierr); 28940a96aa3bSJed Brown ierr = PetscSFReduceEnd(pointSF,nodeType,roots,roots,sfNodeReduce);CHKERRQ(ierr); 28950a96aa3bSJed Brown ierr = PetscSFBcastBegin(pointSF,nodeType,roots,roots,MPI_REPLACE);CHKERRQ(ierr); 28960a96aa3bSJed Brown ierr = PetscSFBcastEnd(pointSF,nodeType,roots,roots,MPI_REPLACE);CHKERRQ(ierr); 28970a96aa3bSJed Brown } 28980a96aa3bSJed Brown ierr = PetscFree(rootType);CHKERRQ(ierr); 28990a96aa3bSJed Brown 29000a96aa3bSJed Brown { 29010a96aa3bSJed Brown PetscInt numRoots; 29020a96aa3bSJed Brown PetscInt numLeaves; 29030a96aa3bSJed Brown PetscInt *leaves; 29040a96aa3bSJed Brown PetscSFNode *iremote; 29050a96aa3bSJed Brown /* count leaves */ 29060a96aa3bSJed Brown 29070a96aa3bSJed Brown numRoots = pEndC - pStartC; 29080a96aa3bSJed Brown 29090a96aa3bSJed Brown numLeaves = 0; 29100a96aa3bSJed Brown for (p = pStartF; p < pEndF; p++) { 29110a96aa3bSJed Brown if (roots[p-pStartF].index >= 0) numLeaves++; 29120a96aa3bSJed Brown } 29130a96aa3bSJed Brown ierr = PetscMalloc1(numLeaves,&leaves);CHKERRQ(ierr); 29140a96aa3bSJed Brown ierr = PetscMalloc1(numLeaves,&iremote);CHKERRQ(ierr); 29150a96aa3bSJed Brown numLeaves = 0; 29160a96aa3bSJed Brown for (p = pStartF; p < pEndF; p++) { 29170a96aa3bSJed Brown if (roots[p-pStartF].index >= 0) { 29180a96aa3bSJed Brown leaves[numLeaves] = p-pStartF; 29190a96aa3bSJed Brown iremote[numLeaves] = roots[p-pStartF]; 29200a96aa3bSJed Brown numLeaves++; 29210a96aa3bSJed Brown } 29220a96aa3bSJed Brown } 29230a96aa3bSJed Brown ierr = PetscFree(roots);CHKERRQ(ierr); 29240a96aa3bSJed Brown ierr = PetscSFCreate(comm,sf);CHKERRQ(ierr); 29250a96aa3bSJed Brown if (numLeaves == (pEndF-pStartF)) { 29260a96aa3bSJed Brown ierr = PetscFree(leaves);CHKERRQ(ierr); 29270a96aa3bSJed Brown ierr = PetscSFSetGraph(*sf,numRoots,numLeaves,NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr); 29280a96aa3bSJed Brown } else { 29290a96aa3bSJed Brown ierr = PetscSFSetGraph(*sf,numRoots,numLeaves,leaves,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr); 29300a96aa3bSJed Brown } 29310a96aa3bSJed Brown } 29320a96aa3bSJed Brown if (formCids) { 29330a96aa3bSJed Brown PetscSF pointSF; 29340a96aa3bSJed Brown PetscInt child; 29350a96aa3bSJed Brown 29360a96aa3bSJed Brown ierr = DMPlexGetReferenceTree(plexF,&refTree);CHKERRQ(ierr); 29370a96aa3bSJed Brown ierr = DMGetPointSF(plexF,&pointSF);CHKERRQ(ierr); 29380a96aa3bSJed Brown ierr = PetscSFReduceBegin(pointSF,MPIU_INT,cids,cids,MPIU_MAX);CHKERRQ(ierr); 29390a96aa3bSJed Brown ierr = PetscSFReduceEnd(pointSF,MPIU_INT,cids,cids,MPIU_MAX);CHKERRQ(ierr); 29400a96aa3bSJed Brown if (childIds) *childIds = cids; 29410a96aa3bSJed Brown for (child = 0; child < P4EST_CHILDREN; child++) { 29420a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(refTree,child+1,PETSC_TRUE,NULL,&childClosures[child]);CHKERRQ(ierr); 29430a96aa3bSJed Brown } 29440a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(refTree,0,PETSC_TRUE,NULL,&rootClosure);CHKERRQ(ierr); 29450a96aa3bSJed Brown } 29460a96aa3bSJed Brown } 29470a96aa3bSJed Brown if (saveInCoarse) { /* cache results */ 29480a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)*sf);CHKERRQ(ierr); 29490a96aa3bSJed Brown pforestC->pointSelfToAdaptSF = *sf; 29500a96aa3bSJed Brown if (!childIds) { 29510a96aa3bSJed Brown pforestC->pointSelfToAdaptCids = cids; 29520a96aa3bSJed Brown } else { 29530a96aa3bSJed Brown ierr = PetscMalloc1(pEndF-pStartF,&pforestC->pointSelfToAdaptCids);CHKERRQ(ierr); 29540a96aa3bSJed Brown ierr = PetscArraycpy(pforestC->pointSelfToAdaptCids,cids,pEndF-pStartF);CHKERRQ(ierr); 29550a96aa3bSJed Brown } 29560a96aa3bSJed Brown } else if (saveInFine) { 29570a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)*sf);CHKERRQ(ierr); 29580a96aa3bSJed Brown pforestF->pointAdaptToSelfSF = *sf; 29590a96aa3bSJed Brown if (!childIds) { 29600a96aa3bSJed Brown pforestF->pointAdaptToSelfCids = cids; 29610a96aa3bSJed Brown } else { 29620a96aa3bSJed Brown ierr = PetscMalloc1(pEndF-pStartF,&pforestF->pointAdaptToSelfCids);CHKERRQ(ierr); 29630a96aa3bSJed Brown ierr = PetscArraycpy(pforestF->pointAdaptToSelfCids,cids,pEndF-pStartF);CHKERRQ(ierr); 29640a96aa3bSJed Brown } 29650a96aa3bSJed Brown } 29660a96aa3bSJed Brown ierr = PetscFree2(treeQuads,treeQuadCounts);CHKERRQ(ierr); 29670a96aa3bSJed Brown ierr = PetscFree(coverQuads);CHKERRQ(ierr); 29680a96aa3bSJed Brown ierr = PetscFree(closurePointsC);CHKERRQ(ierr); 29690a96aa3bSJed Brown ierr = PetscFree(closurePointsF);CHKERRQ(ierr); 29700a96aa3bSJed Brown ierr = MPI_Type_free(&nodeClosureType);CHKERRMPI(ierr); 29710a96aa3bSJed Brown ierr = MPI_Op_free(&sfNodeReduce);CHKERRMPI(ierr); 29720a96aa3bSJed Brown ierr = MPI_Type_free(&nodeType);CHKERRMPI(ierr); 29730a96aa3bSJed Brown PetscFunctionReturn(0); 29740a96aa3bSJed Brown } 29750a96aa3bSJed Brown 29760a96aa3bSJed Brown /* children are sf leaves of parents */ 29770a96aa3bSJed Brown static PetscErrorCode DMPforestGetTransferSF_Internal(DM coarse, DM fine, const PetscInt dofPerDim[], PetscSF *sf, PetscBool transferIdent, PetscInt *childIds[]) 29780a96aa3bSJed Brown { 29790a96aa3bSJed Brown MPI_Comm comm; 29800a96aa3bSJed Brown PetscMPIInt rank, size; 29810a96aa3bSJed Brown DM_Forest_pforest *pforestC, *pforestF; 29820a96aa3bSJed Brown PetscInt numClosureIndices; 29830a96aa3bSJed Brown DM plexC, plexF; 29840a96aa3bSJed Brown PetscInt pStartC, pEndC, pStartF, pEndF; 29850a96aa3bSJed Brown PetscSF pointTransferSF; 29860a96aa3bSJed Brown PetscBool allOnes = PETSC_TRUE; 29870a96aa3bSJed Brown PetscErrorCode ierr; 29880a96aa3bSJed Brown 29890a96aa3bSJed Brown PetscFunctionBegin; 29900a96aa3bSJed Brown pforestC = (DM_Forest_pforest*) ((DM_Forest*) coarse->data)->data; 29910a96aa3bSJed Brown pforestF = (DM_Forest_pforest*) ((DM_Forest*) fine->data)->data; 2992*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(pforestC->topo != pforestF->topo,PetscObjectComm((PetscObject)coarse),PETSC_ERR_ARG_INCOMP,"DM's must have the same base DM"); 29930a96aa3bSJed Brown comm = PetscObjectComm((PetscObject)coarse); 29940a96aa3bSJed Brown ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 29950a96aa3bSJed Brown ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 29960a96aa3bSJed Brown 29970a96aa3bSJed Brown /* count the number of closure points that have dofs and create a list */ 29980a96aa3bSJed Brown numClosureIndices = 0; 29990a96aa3bSJed Brown if (dofPerDim[P4EST_DIM] > 0) numClosureIndices += 1; 30000a96aa3bSJed Brown if (dofPerDim[P4EST_DIM - 1] > 0) numClosureIndices += P4EST_FACES; 30010a96aa3bSJed Brown #if defined(P4_TO_P8) 30020a96aa3bSJed Brown if (dofPerDim[P4EST_DIM - 2] > 0) numClosureIndices += P8EST_EDGES; 30030a96aa3bSJed Brown #endif 30040a96aa3bSJed Brown if (dofPerDim[0] > 0) numClosureIndices += P4EST_CHILDREN; 30050a96aa3bSJed Brown { 30060a96aa3bSJed Brown PetscInt i; 30070a96aa3bSJed Brown for (i = 0; i <= P4EST_DIM; i++) { 30080a96aa3bSJed Brown if (dofPerDim[i] != 1) { 30090a96aa3bSJed Brown allOnes = PETSC_FALSE; 30100a96aa3bSJed Brown break; 30110a96aa3bSJed Brown } 30120a96aa3bSJed Brown } 30130a96aa3bSJed Brown } 30140a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Point(coarse,fine,&pointTransferSF,transferIdent,childIds);CHKERRQ(ierr); 30150a96aa3bSJed Brown if (allOnes) { 30160a96aa3bSJed Brown *sf = pointTransferSF; 30170a96aa3bSJed Brown PetscFunctionReturn(0); 30180a96aa3bSJed Brown } 30190a96aa3bSJed Brown 30200a96aa3bSJed Brown ierr = DMPforestGetPlex(fine,&plexF);CHKERRQ(ierr); 30210a96aa3bSJed Brown ierr = DMPlexGetChart(plexF,&pStartF,&pEndF);CHKERRQ(ierr); 30220a96aa3bSJed Brown ierr = DMPforestGetPlex(coarse,&plexC);CHKERRQ(ierr); 30230a96aa3bSJed Brown ierr = DMPlexGetChart(plexC,&pStartC,&pEndC);CHKERRQ(ierr); 30240a96aa3bSJed Brown { 30250a96aa3bSJed Brown PetscInt numRoots; 30260a96aa3bSJed Brown PetscInt numLeaves; 30270a96aa3bSJed Brown const PetscInt *leaves; 30280a96aa3bSJed Brown const PetscSFNode *iremote; 30290a96aa3bSJed Brown PetscInt d; 30300a96aa3bSJed Brown PetscSection leafSection, rootSection; 30310a96aa3bSJed Brown /* count leaves */ 30320a96aa3bSJed Brown 30330a96aa3bSJed Brown ierr = PetscSFGetGraph(pointTransferSF,&numRoots,&numLeaves,&leaves,&iremote);CHKERRQ(ierr); 30340a96aa3bSJed Brown ierr = PetscSectionCreate(PETSC_COMM_SELF,&rootSection);CHKERRQ(ierr); 30350a96aa3bSJed Brown ierr = PetscSectionCreate(PETSC_COMM_SELF,&leafSection);CHKERRQ(ierr); 30360a96aa3bSJed Brown ierr = PetscSectionSetChart(rootSection,pStartC,pEndC);CHKERRQ(ierr); 30370a96aa3bSJed Brown ierr = PetscSectionSetChart(leafSection,pStartF,pEndF);CHKERRQ(ierr); 30380a96aa3bSJed Brown 30390a96aa3bSJed Brown for (d = 0; d <= P4EST_DIM; d++) { 30400a96aa3bSJed Brown PetscInt startC, endC, e; 30410a96aa3bSJed Brown 30420a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(plexC,P4EST_DIM-d,&startC,&endC);CHKERRQ(ierr); 30430a96aa3bSJed Brown for (e = startC; e < endC; e++) { 30440a96aa3bSJed Brown ierr = PetscSectionSetDof(rootSection,e,dofPerDim[d]);CHKERRQ(ierr); 30450a96aa3bSJed Brown } 30460a96aa3bSJed Brown } 30470a96aa3bSJed Brown 30480a96aa3bSJed Brown for (d = 0; d <= P4EST_DIM; d++) { 30490a96aa3bSJed Brown PetscInt startF, endF, e; 30500a96aa3bSJed Brown 30510a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(plexF,P4EST_DIM-d,&startF,&endF);CHKERRQ(ierr); 30520a96aa3bSJed Brown for (e = startF; e < endF; e++) { 30530a96aa3bSJed Brown ierr = PetscSectionSetDof(leafSection,e,dofPerDim[d]);CHKERRQ(ierr); 30540a96aa3bSJed Brown } 30550a96aa3bSJed Brown } 30560a96aa3bSJed Brown 30570a96aa3bSJed Brown ierr = PetscSectionSetUp(rootSection);CHKERRQ(ierr); 30580a96aa3bSJed Brown ierr = PetscSectionSetUp(leafSection);CHKERRQ(ierr); 30590a96aa3bSJed Brown { 30600a96aa3bSJed Brown PetscInt nroots, nleaves; 30610a96aa3bSJed Brown PetscInt *mine, i, p; 30620a96aa3bSJed Brown PetscInt *offsets, *offsetsRoot; 30630a96aa3bSJed Brown PetscSFNode *remote; 30640a96aa3bSJed Brown 30650a96aa3bSJed Brown ierr = PetscMalloc1(pEndF-pStartF,&offsets);CHKERRQ(ierr); 30660a96aa3bSJed Brown ierr = PetscMalloc1(pEndC-pStartC,&offsetsRoot);CHKERRQ(ierr); 30670a96aa3bSJed Brown for (p = pStartC; p < pEndC; p++) { 30680a96aa3bSJed Brown ierr = PetscSectionGetOffset(rootSection,p,&offsetsRoot[p-pStartC]);CHKERRQ(ierr); 30690a96aa3bSJed Brown } 30700a96aa3bSJed Brown ierr = PetscSFBcastBegin(pointTransferSF,MPIU_INT,offsetsRoot,offsets,MPI_REPLACE);CHKERRQ(ierr); 30710a96aa3bSJed Brown ierr = PetscSFBcastEnd(pointTransferSF,MPIU_INT,offsetsRoot,offsets,MPI_REPLACE);CHKERRQ(ierr); 30720a96aa3bSJed Brown ierr = PetscSectionGetStorageSize(rootSection,&nroots);CHKERRQ(ierr); 30730a96aa3bSJed Brown nleaves = 0; 30740a96aa3bSJed Brown for (i = 0; i < numLeaves; i++) { 30750a96aa3bSJed Brown PetscInt leaf = leaves ? leaves[i] : i; 30760a96aa3bSJed Brown PetscInt dof; 30770a96aa3bSJed Brown 30780a96aa3bSJed Brown ierr = PetscSectionGetDof(leafSection,leaf,&dof);CHKERRQ(ierr); 30790a96aa3bSJed Brown nleaves += dof; 30800a96aa3bSJed Brown } 30810a96aa3bSJed Brown ierr = PetscMalloc1(nleaves,&mine);CHKERRQ(ierr); 30820a96aa3bSJed Brown ierr = PetscMalloc1(nleaves,&remote);CHKERRQ(ierr); 30830a96aa3bSJed Brown nleaves = 0; 30840a96aa3bSJed Brown for (i = 0; i < numLeaves; i++) { 30850a96aa3bSJed Brown PetscInt leaf = leaves ? leaves[i] : i; 30860a96aa3bSJed Brown PetscInt dof; 30870a96aa3bSJed Brown PetscInt off, j; 30880a96aa3bSJed Brown 30890a96aa3bSJed Brown ierr = PetscSectionGetDof(leafSection,leaf,&dof);CHKERRQ(ierr); 30900a96aa3bSJed Brown ierr = PetscSectionGetOffset(leafSection,leaf,&off);CHKERRQ(ierr); 30910a96aa3bSJed Brown for (j = 0; j < dof; j++) { 30920a96aa3bSJed Brown remote[nleaves].rank = iremote[i].rank; 30930a96aa3bSJed Brown remote[nleaves].index = offsets[leaf] + j; 30940a96aa3bSJed Brown mine[nleaves++] = off + j; 30950a96aa3bSJed Brown } 30960a96aa3bSJed Brown } 30970a96aa3bSJed Brown ierr = PetscFree(offsetsRoot);CHKERRQ(ierr); 30980a96aa3bSJed Brown ierr = PetscFree(offsets);CHKERRQ(ierr); 30990a96aa3bSJed Brown ierr = PetscSFCreate(comm,sf);CHKERRQ(ierr); 31000a96aa3bSJed Brown ierr = PetscSFSetGraph(*sf,nroots,nleaves,mine,PETSC_OWN_POINTER,remote,PETSC_OWN_POINTER);CHKERRQ(ierr); 31010a96aa3bSJed Brown } 31020a96aa3bSJed Brown ierr = PetscSectionDestroy(&leafSection);CHKERRQ(ierr); 31030a96aa3bSJed Brown ierr = PetscSectionDestroy(&rootSection);CHKERRQ(ierr); 31040a96aa3bSJed Brown ierr = PetscSFDestroy(&pointTransferSF);CHKERRQ(ierr); 31050a96aa3bSJed Brown } 31060a96aa3bSJed Brown PetscFunctionReturn(0); 31070a96aa3bSJed Brown } 31080a96aa3bSJed Brown 31090a96aa3bSJed Brown static PetscErrorCode DMPforestGetTransferSF(DM dmA, DM dmB, const PetscInt dofPerDim[], PetscSF *sfAtoB, PetscSF *sfBtoA) 31100a96aa3bSJed Brown { 31110a96aa3bSJed Brown DM adaptA, adaptB; 31120a96aa3bSJed Brown DMAdaptFlag purpose; 31130a96aa3bSJed Brown PetscErrorCode ierr; 31140a96aa3bSJed Brown 31150a96aa3bSJed Brown PetscFunctionBegin; 31160a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(dmA,&adaptA);CHKERRQ(ierr); 31170a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(dmB,&adaptB);CHKERRQ(ierr); 31180a96aa3bSJed Brown /* it is more efficient when the coarser mesh is the first argument: reorder if we know one is coarser than the other */ 31190a96aa3bSJed Brown if (adaptA && adaptA->data == dmB->data) { /* dmA was adapted from dmB */ 31200a96aa3bSJed Brown ierr = DMForestGetAdaptivityPurpose(dmA,&purpose);CHKERRQ(ierr); 31210a96aa3bSJed Brown if (purpose == DM_ADAPT_REFINE) { 31220a96aa3bSJed Brown ierr = DMPforestGetTransferSF(dmB, dmA, dofPerDim, sfBtoA, sfAtoB);CHKERRQ(ierr); 31230a96aa3bSJed Brown PetscFunctionReturn(0); 31240a96aa3bSJed Brown } 31250a96aa3bSJed Brown } else if (adaptB && adaptB->data == dmA->data) { /* dmB was adapted from dmA */ 31260a96aa3bSJed Brown ierr = DMForestGetAdaptivityPurpose(dmB,&purpose);CHKERRQ(ierr); 31270a96aa3bSJed Brown if (purpose == DM_ADAPT_COARSEN) { 31280a96aa3bSJed Brown ierr = DMPforestGetTransferSF(dmB, dmA, dofPerDim, sfBtoA, sfAtoB);CHKERRQ(ierr); 31290a96aa3bSJed Brown PetscFunctionReturn(0); 31300a96aa3bSJed Brown } 31310a96aa3bSJed Brown } 31320a96aa3bSJed Brown if (sfAtoB) { 31330a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmA,dmB,dofPerDim,sfAtoB,PETSC_TRUE,NULL);CHKERRQ(ierr); 31340a96aa3bSJed Brown } 31350a96aa3bSJed Brown if (sfBtoA) { 31360a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmB,dmA,dofPerDim,sfBtoA,(PetscBool) (sfAtoB == NULL),NULL);CHKERRQ(ierr); 31370a96aa3bSJed Brown } 31380a96aa3bSJed Brown PetscFunctionReturn(0); 31390a96aa3bSJed Brown } 31400a96aa3bSJed Brown 31410a96aa3bSJed Brown static PetscErrorCode DMPforestLabelsInitialize(DM dm, DM plex) 31420a96aa3bSJed Brown { 31430a96aa3bSJed Brown DM_Forest *forest = (DM_Forest*) dm->data; 31440a96aa3bSJed Brown DM_Forest_pforest *pforest = (DM_Forest_pforest*) forest->data; 31450a96aa3bSJed Brown PetscInt cLocalStart, cLocalEnd, cStart, cEnd, fStart, fEnd, eStart, eEnd, vStart, vEnd; 31460a96aa3bSJed Brown PetscInt cStartBase, cEndBase, fStartBase, fEndBase, vStartBase, vEndBase, eStartBase, eEndBase; 31470a96aa3bSJed Brown PetscInt pStart, pEnd, pStartBase, pEndBase, p; 31480a96aa3bSJed Brown DM base; 31490a96aa3bSJed Brown PetscInt *star = NULL, starSize; 31500a96aa3bSJed Brown DMLabelLink next = dm->labels; 31510a96aa3bSJed Brown PetscInt guess = 0; 31520a96aa3bSJed Brown p4est_topidx_t num_trees = pforest->topo->conn->num_trees; 31530a96aa3bSJed Brown PetscErrorCode ierr; 31540a96aa3bSJed Brown 31550a96aa3bSJed Brown PetscFunctionBegin; 31560a96aa3bSJed Brown pforest->labelsFinalized = PETSC_TRUE; 31570a96aa3bSJed Brown cLocalStart = pforest->cLocalStart; 31580a96aa3bSJed Brown cLocalEnd = pforest->cLocalEnd; 31590a96aa3bSJed Brown ierr = DMForestGetBaseDM(dm,&base);CHKERRQ(ierr); 31600a96aa3bSJed Brown if (!base) { 31610a96aa3bSJed Brown if (pforest->ghostName) { /* insert a label to make the boundaries, with stratum values denoting which face of the element touches the boundary */ 31620a96aa3bSJed Brown p4est_connectivity_t *conn = pforest->topo->conn; 31630a96aa3bSJed Brown p4est_t *p4est = pforest->forest; 31640a96aa3bSJed Brown p4est_tree_t *trees = (p4est_tree_t*) p4est->trees->array; 31650a96aa3bSJed Brown p4est_topidx_t t, flt = p4est->first_local_tree; 31660a96aa3bSJed Brown p4est_topidx_t llt = pforest->forest->last_local_tree; 31670a96aa3bSJed Brown DMLabel ghostLabel; 31680a96aa3bSJed Brown PetscInt c; 31690a96aa3bSJed Brown 31700a96aa3bSJed Brown ierr = DMCreateLabel(plex,pforest->ghostName);CHKERRQ(ierr); 31710a96aa3bSJed Brown ierr = DMGetLabel(plex,pforest->ghostName,&ghostLabel);CHKERRQ(ierr); 31720a96aa3bSJed Brown for (c = cLocalStart, t = flt; t <= llt; t++) { 31730a96aa3bSJed Brown p4est_tree_t *tree = &trees[t]; 31740a96aa3bSJed Brown p4est_quadrant_t *quads = (p4est_quadrant_t*) tree->quadrants.array; 31750a96aa3bSJed Brown PetscInt numQuads = (PetscInt) tree->quadrants.elem_count; 31760a96aa3bSJed Brown PetscInt q; 31770a96aa3bSJed Brown 31780a96aa3bSJed Brown for (q = 0; q < numQuads; q++, c++) { 31790a96aa3bSJed Brown p4est_quadrant_t *quad = &quads[q]; 31800a96aa3bSJed Brown PetscInt f; 31810a96aa3bSJed Brown 31820a96aa3bSJed Brown for (f = 0; f < P4EST_FACES; f++) { 31830a96aa3bSJed Brown p4est_quadrant_t neigh; 31840a96aa3bSJed Brown int isOutside; 31850a96aa3bSJed Brown 31860a96aa3bSJed Brown PetscStackCallP4est(p4est_quadrant_face_neighbor,(quad,f,&neigh)); 31870a96aa3bSJed Brown PetscStackCallP4estReturn(isOutside,p4est_quadrant_is_outside_face,(&neigh)); 31880a96aa3bSJed Brown if (isOutside) { 31890a96aa3bSJed Brown p4est_topidx_t nt; 31900a96aa3bSJed Brown PetscInt nf; 31910a96aa3bSJed Brown 31920a96aa3bSJed Brown nt = conn->tree_to_tree[t * P4EST_FACES + f]; 31930a96aa3bSJed Brown nf = (PetscInt) conn->tree_to_face[t * P4EST_FACES + f]; 31940a96aa3bSJed Brown nf = nf % P4EST_FACES; 31950a96aa3bSJed Brown if (nt == t && nf == f) { 31960a96aa3bSJed Brown PetscInt plexF = P4estFaceToPetscFace[f]; 31970a96aa3bSJed Brown const PetscInt *cone; 31980a96aa3bSJed Brown 31990a96aa3bSJed Brown ierr = DMPlexGetCone(plex,c,&cone);CHKERRQ(ierr); 32000a96aa3bSJed Brown ierr = DMLabelSetValue(ghostLabel,cone[plexF],plexF+1);CHKERRQ(ierr); 32010a96aa3bSJed Brown } 32020a96aa3bSJed Brown } 32030a96aa3bSJed Brown } 32040a96aa3bSJed Brown } 32050a96aa3bSJed Brown } 32060a96aa3bSJed Brown } 32070a96aa3bSJed Brown PetscFunctionReturn(0); 32080a96aa3bSJed Brown } 32090a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(base,0,&cStartBase,&cEndBase);CHKERRQ(ierr); 32100a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(base,1,&fStartBase,&fEndBase);CHKERRQ(ierr); 32110a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(base,P4EST_DIM-1,&eStartBase,&eEndBase);CHKERRQ(ierr); 32120a96aa3bSJed Brown ierr = DMPlexGetDepthStratum(base,0,&vStartBase,&vEndBase);CHKERRQ(ierr); 32130a96aa3bSJed Brown 32140a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(plex,0,&cStart,&cEnd);CHKERRQ(ierr); 32150a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(plex,1,&fStart,&fEnd);CHKERRQ(ierr); 32160a96aa3bSJed Brown ierr = DMPlexGetSimplexOrBoxCells(plex,P4EST_DIM-1,&eStart,&eEnd);CHKERRQ(ierr); 32170a96aa3bSJed Brown ierr = DMPlexGetDepthStratum(plex,0,&vStart,&vEnd);CHKERRQ(ierr); 32180a96aa3bSJed Brown 32190a96aa3bSJed Brown ierr = DMPlexGetChart(plex,&pStart,&pEnd);CHKERRQ(ierr); 32200a96aa3bSJed Brown ierr = DMPlexGetChart(base,&pStartBase,&pEndBase);CHKERRQ(ierr); 32210a96aa3bSJed Brown /* go through the mesh: use star to find a quadrant that borders a point. Use the closure to determine the 32220a96aa3bSJed Brown * orientation of the quadrant relative to that point. Use that to relate the point to the numbering in the base 32230a96aa3bSJed Brown * mesh, and extract a label value (since the base mesh is redundantly distributed, can be found locally). */ 32240a96aa3bSJed Brown while (next) { 32250a96aa3bSJed Brown DMLabel baseLabel; 32260a96aa3bSJed Brown DMLabel label = next->label; 32270a96aa3bSJed Brown PetscBool isDepth, isCellType, isGhost, isVTK, isSpmap; 32280a96aa3bSJed Brown const char *name; 32290a96aa3bSJed Brown 32300a96aa3bSJed Brown ierr = PetscObjectGetName((PetscObject) label, &name);CHKERRQ(ierr); 32310a96aa3bSJed Brown ierr = PetscStrcmp(name,"depth",&isDepth);CHKERRQ(ierr); 32320a96aa3bSJed Brown if (isDepth) { 32330a96aa3bSJed Brown next = next->next; 32340a96aa3bSJed Brown continue; 32350a96aa3bSJed Brown } 32360a96aa3bSJed Brown ierr = PetscStrcmp(name,"celltype",&isCellType);CHKERRQ(ierr); 32370a96aa3bSJed Brown if (isCellType) { 32380a96aa3bSJed Brown next = next->next; 32390a96aa3bSJed Brown continue; 32400a96aa3bSJed Brown } 32410a96aa3bSJed Brown ierr = PetscStrcmp(name,"ghost",&isGhost);CHKERRQ(ierr); 32420a96aa3bSJed Brown if (isGhost) { 32430a96aa3bSJed Brown next = next->next; 32440a96aa3bSJed Brown continue; 32450a96aa3bSJed Brown } 32460a96aa3bSJed Brown ierr = PetscStrcmp(name,"vtk",&isVTK);CHKERRQ(ierr); 32470a96aa3bSJed Brown if (isVTK) { 32480a96aa3bSJed Brown next = next->next; 32490a96aa3bSJed Brown continue; 32500a96aa3bSJed Brown } 32510a96aa3bSJed Brown ierr = PetscStrcmp(name,"_forest_base_subpoint_map",&isSpmap);CHKERRQ(ierr); 32520a96aa3bSJed Brown if (!isSpmap) { 32530a96aa3bSJed Brown ierr = DMGetLabel(base,name,&baseLabel);CHKERRQ(ierr); 32540a96aa3bSJed Brown if (!baseLabel) { 32550a96aa3bSJed Brown next = next->next; 32560a96aa3bSJed Brown continue; 32570a96aa3bSJed Brown } 32580a96aa3bSJed Brown ierr = DMLabelCreateIndex(baseLabel,pStartBase,pEndBase);CHKERRQ(ierr); 32590a96aa3bSJed Brown } else baseLabel = NULL; 32600a96aa3bSJed Brown 32610a96aa3bSJed Brown for (p = pStart; p < pEnd; p++) { 32620a96aa3bSJed Brown PetscInt s, c = -1, l; 32630a96aa3bSJed Brown PetscInt *closure = NULL, closureSize; 32640a96aa3bSJed Brown p4est_quadrant_t * ghosts = (p4est_quadrant_t*) pforest->ghost->ghosts.array; 32650a96aa3bSJed Brown p4est_tree_t *trees = (p4est_tree_t*) pforest->forest->trees->array; 32660a96aa3bSJed Brown p4est_quadrant_t * q; 32670a96aa3bSJed Brown PetscInt t, val; 32680a96aa3bSJed Brown PetscBool zerosupportpoint = PETSC_FALSE; 32690a96aa3bSJed Brown 32700a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(plex,p,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 32710a96aa3bSJed Brown for (s = 0; s < starSize; s++) { 32720a96aa3bSJed Brown PetscInt point = star[2*s]; 32730a96aa3bSJed Brown 32740a96aa3bSJed Brown if (cStart <= point && point < cEnd) { 32750a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(plex,point,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 32760a96aa3bSJed Brown for (l = 0; l < closureSize; l++) { 32770a96aa3bSJed Brown PetscInt qParent = closure[2 * l], q, pp = p, pParent = p; 32780a96aa3bSJed Brown do { /* check parents of q */ 32790a96aa3bSJed Brown q = qParent; 32800a96aa3bSJed Brown if (q == p) { 32810a96aa3bSJed Brown c = point; 32820a96aa3bSJed Brown break; 32830a96aa3bSJed Brown } 32840a96aa3bSJed Brown ierr = DMPlexGetTreeParent(plex,q,&qParent,NULL);CHKERRQ(ierr); 32850a96aa3bSJed Brown } while (qParent != q); 32860a96aa3bSJed Brown if (c != -1) break; 32870a96aa3bSJed Brown ierr = DMPlexGetTreeParent(plex,pp,&pParent,NULL);CHKERRQ(ierr); 32880a96aa3bSJed Brown q = closure[2 * l]; 32890a96aa3bSJed Brown while (pParent != pp) { /* check parents of p */ 32900a96aa3bSJed Brown pp = pParent; 32910a96aa3bSJed Brown if (pp == q) { 32920a96aa3bSJed Brown c = point; 32930a96aa3bSJed Brown break; 32940a96aa3bSJed Brown } 32950a96aa3bSJed Brown ierr = DMPlexGetTreeParent(plex,pp,&pParent,NULL);CHKERRQ(ierr); 32960a96aa3bSJed Brown } 32970a96aa3bSJed Brown if (c != -1) break; 32980a96aa3bSJed Brown } 32990a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(plex,point,PETSC_TRUE,NULL,&closure);CHKERRQ(ierr); 33000a96aa3bSJed Brown if (l < closureSize) break; 33010a96aa3bSJed Brown } else { 33020a96aa3bSJed Brown PetscInt supportSize; 33030a96aa3bSJed Brown 33040a96aa3bSJed Brown ierr = DMPlexGetSupportSize(plex,point,&supportSize);CHKERRQ(ierr); 33050a96aa3bSJed Brown zerosupportpoint = (PetscBool) (zerosupportpoint || !supportSize); 33060a96aa3bSJed Brown } 33070a96aa3bSJed Brown } 33080a96aa3bSJed Brown if (c < 0) { 33090a96aa3bSJed Brown const char* prefix; 33100a96aa3bSJed Brown PetscBool print = PETSC_FALSE; 33110a96aa3bSJed Brown 33120a96aa3bSJed Brown ierr = PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);CHKERRQ(ierr); 33130a96aa3bSJed Brown ierr = PetscOptionsGetBool(((PetscObject)dm)->options,prefix,"-dm_forest_print_label_error",&print,NULL);CHKERRQ(ierr); 33140a96aa3bSJed Brown if (print) { 33150a96aa3bSJed Brown PetscInt i; 33160a96aa3bSJed Brown 33170a96aa3bSJed Brown ierr = PetscPrintf(PETSC_COMM_SELF,"[%d] Failed to find cell with point %D in its closure for label %s (starSize %D)\n",PetscGlobalRank,p,baseLabel ? ((PetscObject)baseLabel)->name : "_forest_base_subpoint_map",starSize);CHKERRQ(ierr); 33180a96aa3bSJed Brown for (i = 0; i < starSize; i++) { ierr = PetscPrintf(PETSC_COMM_SELF," star[%D] = %D,%D\n",i,star[2*i],star[2*i+1]);CHKERRQ(ierr); } 33190a96aa3bSJed Brown } 33200a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(plex,p,PETSC_FALSE,NULL,&star);CHKERRQ(ierr); 33210a96aa3bSJed Brown if (zerosupportpoint) continue; 332298921bdaSJacob Faibussowitsch else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Failed to find cell with point %D in its closure for label %s. Rerun with -dm_forest_print_label_error for more information",p,baseLabel ? ((PetscObject) baseLabel)->name : "_forest_base_subpoint_map"); 33230a96aa3bSJed Brown } 33240a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(plex,p,PETSC_FALSE,NULL,&star);CHKERRQ(ierr); 33250a96aa3bSJed Brown 33260a96aa3bSJed Brown if (c < cLocalStart) { 33270a96aa3bSJed Brown /* get from the beginning of the ghost layer */ 33280a96aa3bSJed Brown q = &(ghosts[c]); 33290a96aa3bSJed Brown t = (PetscInt) q->p.which_tree; 33300a96aa3bSJed Brown } else if (c < cLocalEnd) { 33310a96aa3bSJed Brown PetscInt lo = 0, hi = num_trees; 33320a96aa3bSJed Brown /* get from local quadrants: have to find the right tree */ 33330a96aa3bSJed Brown 33340a96aa3bSJed Brown c -= cLocalStart; 33350a96aa3bSJed Brown 33360a96aa3bSJed Brown do { 33370a96aa3bSJed Brown p4est_tree_t *tree; 33380a96aa3bSJed Brown 3339*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(guess < lo || guess >= num_trees || lo >= hi,PETSC_COMM_SELF,PETSC_ERR_PLIB,"failed binary search"); 33400a96aa3bSJed Brown tree = &trees[guess]; 33410a96aa3bSJed Brown if (c < tree->quadrants_offset) { 33420a96aa3bSJed Brown hi = guess; 33430a96aa3bSJed Brown } else if (c < tree->quadrants_offset + (PetscInt) tree->quadrants.elem_count) { 33440a96aa3bSJed Brown q = &((p4est_quadrant_t *)tree->quadrants.array)[c - (PetscInt) tree->quadrants_offset]; 33450a96aa3bSJed Brown t = guess; 33460a96aa3bSJed Brown break; 33470a96aa3bSJed Brown } else { 33480a96aa3bSJed Brown lo = guess + 1; 33490a96aa3bSJed Brown } 33500a96aa3bSJed Brown guess = lo + (hi - lo) / 2; 33510a96aa3bSJed Brown } while (1); 33520a96aa3bSJed Brown } else { 33530a96aa3bSJed Brown /* get from the end of the ghost layer */ 33540a96aa3bSJed Brown c -= (cLocalEnd - cLocalStart); 33550a96aa3bSJed Brown 33560a96aa3bSJed Brown q = &(ghosts[c]); 33570a96aa3bSJed Brown t = (PetscInt) q->p.which_tree; 33580a96aa3bSJed Brown } 33590a96aa3bSJed Brown 33600a96aa3bSJed Brown if (l == 0) { /* cell */ 33610a96aa3bSJed Brown if (baseLabel) { 33620a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,t+cStartBase,&val);CHKERRQ(ierr); 33630a96aa3bSJed Brown } else { 33640a96aa3bSJed Brown val = t+cStartBase; 33650a96aa3bSJed Brown } 33660a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 33670a96aa3bSJed Brown } else if (l >= 1 && l < 1 + P4EST_FACES) { /* facet */ 33680a96aa3bSJed Brown p4est_quadrant_t nq; 33690a96aa3bSJed Brown int isInside; 33700a96aa3bSJed Brown 33710a96aa3bSJed Brown l = PetscFaceToP4estFace[l - 1]; 33720a96aa3bSJed Brown PetscStackCallP4est(p4est_quadrant_face_neighbor,(q,l,&nq)); 33730a96aa3bSJed Brown PetscStackCallP4estReturn(isInside,p4est_quadrant_is_inside_root,(&nq)); 33740a96aa3bSJed Brown if (isInside) { 33750a96aa3bSJed Brown /* this facet is in the interior of a tree, so it inherits the label of the tree */ 33760a96aa3bSJed Brown if (baseLabel) { 33770a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,t+cStartBase,&val);CHKERRQ(ierr); 33780a96aa3bSJed Brown } else { 33790a96aa3bSJed Brown val = t+cStartBase; 33800a96aa3bSJed Brown } 33810a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 33820a96aa3bSJed Brown } else { 33830a96aa3bSJed Brown PetscInt f = pforest->topo->tree_face_to_uniq[P4EST_FACES * t + l]; 33840a96aa3bSJed Brown 33850a96aa3bSJed Brown if (baseLabel) { 33860a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,f+fStartBase,&val);CHKERRQ(ierr); 33870a96aa3bSJed Brown } else { 33880a96aa3bSJed Brown val = f+fStartBase; 33890a96aa3bSJed Brown } 33900a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 33910a96aa3bSJed Brown } 33920a96aa3bSJed Brown #if defined(P4_TO_P8) 33930a96aa3bSJed Brown } else if (l >= 1 + P4EST_FACES && l < 1 + P4EST_FACES + P8EST_EDGES) { /* edge */ 33940a96aa3bSJed Brown p4est_quadrant_t nq; 33950a96aa3bSJed Brown int isInside; 33960a96aa3bSJed Brown 33970a96aa3bSJed Brown l = PetscEdgeToP4estEdge[l - (1 + P4EST_FACES)]; 33980a96aa3bSJed Brown PetscStackCallP4est(p8est_quadrant_edge_neighbor,(q,l,&nq)); 33990a96aa3bSJed Brown PetscStackCallP4estReturn(isInside,p4est_quadrant_is_inside_root,(&nq)); 34000a96aa3bSJed Brown if (isInside) { 34010a96aa3bSJed Brown /* this edge is in the interior of a tree, so it inherits the label of the tree */ 34020a96aa3bSJed Brown if (baseLabel) { 34030a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,t+cStartBase,&val);CHKERRQ(ierr); 34040a96aa3bSJed Brown } else { 34050a96aa3bSJed Brown val = t+cStartBase; 34060a96aa3bSJed Brown } 34070a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 34080a96aa3bSJed Brown } else { 34090a96aa3bSJed Brown int isOutsideFace; 34100a96aa3bSJed Brown 34110a96aa3bSJed Brown PetscStackCallP4estReturn(isOutsideFace,p4est_quadrant_is_outside_face,(&nq)); 34120a96aa3bSJed Brown if (isOutsideFace) { 34130a96aa3bSJed Brown PetscInt f; 34140a96aa3bSJed Brown 34150a96aa3bSJed Brown if (nq.x < 0) { 34160a96aa3bSJed Brown f = 0; 34170a96aa3bSJed Brown } else if (nq.x >= P4EST_ROOT_LEN) { 34180a96aa3bSJed Brown f = 1; 34190a96aa3bSJed Brown } else if (nq.y < 0) { 34200a96aa3bSJed Brown f = 2; 34210a96aa3bSJed Brown } else if (nq.y >= P4EST_ROOT_LEN) { 34220a96aa3bSJed Brown f = 3; 34230a96aa3bSJed Brown } else if (nq.z < 0) { 34240a96aa3bSJed Brown f = 4; 34250a96aa3bSJed Brown } else { 34260a96aa3bSJed Brown f = 5; 34270a96aa3bSJed Brown } 34280a96aa3bSJed Brown f = pforest->topo->tree_face_to_uniq[P4EST_FACES * t + f]; 34290a96aa3bSJed Brown if (baseLabel) { 34300a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,f+fStartBase,&val);CHKERRQ(ierr); 34310a96aa3bSJed Brown } else { 34320a96aa3bSJed Brown val = f+fStartBase; 34330a96aa3bSJed Brown } 34340a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 34350a96aa3bSJed Brown } else { /* the quadrant edge corresponds to the tree edge */ 34360a96aa3bSJed Brown PetscInt e = pforest->topo->conn->tree_to_edge[P8EST_EDGES * t + l]; 34370a96aa3bSJed Brown 34380a96aa3bSJed Brown if (baseLabel) { 34390a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,e+eStartBase,&val);CHKERRQ(ierr); 34400a96aa3bSJed Brown } else { 34410a96aa3bSJed Brown val = e+eStartBase; 34420a96aa3bSJed Brown } 34430a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 34440a96aa3bSJed Brown } 34450a96aa3bSJed Brown } 34460a96aa3bSJed Brown #endif 34470a96aa3bSJed Brown } else { /* vertex */ 34480a96aa3bSJed Brown p4est_quadrant_t nq; 34490a96aa3bSJed Brown int isInside; 34500a96aa3bSJed Brown 34510a96aa3bSJed Brown #if defined(P4_TO_P8) 34520a96aa3bSJed Brown l = PetscVertToP4estVert[l - (1 + P4EST_FACES + P8EST_EDGES)]; 34530a96aa3bSJed Brown #else 34540a96aa3bSJed Brown l = PetscVertToP4estVert[l - (1 + P4EST_FACES)]; 34550a96aa3bSJed Brown #endif 34560a96aa3bSJed Brown PetscStackCallP4est(p4est_quadrant_corner_neighbor,(q,l,&nq)); 34570a96aa3bSJed Brown PetscStackCallP4estReturn(isInside,p4est_quadrant_is_inside_root,(&nq)); 34580a96aa3bSJed Brown if (isInside) { 34590a96aa3bSJed Brown if (baseLabel) { 34600a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,t+cStartBase,&val);CHKERRQ(ierr); 34610a96aa3bSJed Brown } else { 34620a96aa3bSJed Brown val = t+cStartBase; 34630a96aa3bSJed Brown } 34640a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 34650a96aa3bSJed Brown } else { 34660a96aa3bSJed Brown int isOutside; 34670a96aa3bSJed Brown 34680a96aa3bSJed Brown PetscStackCallP4estReturn(isOutside,p4est_quadrant_is_outside_face,(&nq)); 34690a96aa3bSJed Brown if (isOutside) { 34700a96aa3bSJed Brown PetscInt f = -1; 34710a96aa3bSJed Brown 34720a96aa3bSJed Brown if (nq.x < 0) { 34730a96aa3bSJed Brown f = 0; 34740a96aa3bSJed Brown } else if (nq.x >= P4EST_ROOT_LEN) { 34750a96aa3bSJed Brown f = 1; 34760a96aa3bSJed Brown } else if (nq.y < 0) { 34770a96aa3bSJed Brown f = 2; 34780a96aa3bSJed Brown } else if (nq.y >= P4EST_ROOT_LEN) { 34790a96aa3bSJed Brown f = 3; 34800a96aa3bSJed Brown #if defined(P4_TO_P8) 34810a96aa3bSJed Brown } else if (nq.z < 0) { 34820a96aa3bSJed Brown f = 4; 34830a96aa3bSJed Brown } else { 34840a96aa3bSJed Brown f = 5; 34850a96aa3bSJed Brown #endif 34860a96aa3bSJed Brown } 34870a96aa3bSJed Brown f = pforest->topo->tree_face_to_uniq[P4EST_FACES * t + f]; 34880a96aa3bSJed Brown if (baseLabel) { 34890a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,f+fStartBase,&val);CHKERRQ(ierr); 34900a96aa3bSJed Brown } else { 34910a96aa3bSJed Brown val = f+fStartBase; 34920a96aa3bSJed Brown } 34930a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 34940a96aa3bSJed Brown continue; 34950a96aa3bSJed Brown } 34960a96aa3bSJed Brown #if defined(P4_TO_P8) 34970a96aa3bSJed Brown PetscStackCallP4estReturn(isOutside,p8est_quadrant_is_outside_edge,(&nq)); 34980a96aa3bSJed Brown if (isOutside) { 34990a96aa3bSJed Brown /* outside edge */ 35000a96aa3bSJed Brown PetscInt e = -1; 35010a96aa3bSJed Brown 35020a96aa3bSJed Brown if (nq.x >= 0 && nq.x < P4EST_ROOT_LEN) { 35030a96aa3bSJed Brown if (nq.z < 0) { 35040a96aa3bSJed Brown if (nq.y < 0) { 35050a96aa3bSJed Brown e = 0; 35060a96aa3bSJed Brown } else { 35070a96aa3bSJed Brown e = 1; 35080a96aa3bSJed Brown } 35090a96aa3bSJed Brown } else { 35100a96aa3bSJed Brown if (nq.y < 0) { 35110a96aa3bSJed Brown e = 2; 35120a96aa3bSJed Brown } else { 35130a96aa3bSJed Brown e = 3; 35140a96aa3bSJed Brown } 35150a96aa3bSJed Brown } 35160a96aa3bSJed Brown } else if (nq.y >= 0 && nq.y < P4EST_ROOT_LEN) { 35170a96aa3bSJed Brown if (nq.z < 0) { 35180a96aa3bSJed Brown if (nq.x < 0) { 35190a96aa3bSJed Brown e = 4; 35200a96aa3bSJed Brown } else { 35210a96aa3bSJed Brown e = 5; 35220a96aa3bSJed Brown } 35230a96aa3bSJed Brown } else { 35240a96aa3bSJed Brown if (nq.x < 0) { 35250a96aa3bSJed Brown e = 6; 35260a96aa3bSJed Brown } else { 35270a96aa3bSJed Brown e = 7; 35280a96aa3bSJed Brown } 35290a96aa3bSJed Brown } 35300a96aa3bSJed Brown } else { 35310a96aa3bSJed Brown if (nq.y < 0) { 35320a96aa3bSJed Brown if (nq.x < 0) { 35330a96aa3bSJed Brown e = 8; 35340a96aa3bSJed Brown } else { 35350a96aa3bSJed Brown e = 9; 35360a96aa3bSJed Brown } 35370a96aa3bSJed Brown } else { 35380a96aa3bSJed Brown if (nq.x < 0) { 35390a96aa3bSJed Brown e = 10; 35400a96aa3bSJed Brown } else { 35410a96aa3bSJed Brown e = 11; 35420a96aa3bSJed Brown } 35430a96aa3bSJed Brown } 35440a96aa3bSJed Brown } 35450a96aa3bSJed Brown 35460a96aa3bSJed Brown e = pforest->topo->conn->tree_to_edge[P8EST_EDGES * t + e]; 35470a96aa3bSJed Brown if (baseLabel) { 35480a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,e+eStartBase,&val);CHKERRQ(ierr); 35490a96aa3bSJed Brown } else { 35500a96aa3bSJed Brown val = e+eStartBase; 35510a96aa3bSJed Brown } 35520a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 35530a96aa3bSJed Brown continue; 35540a96aa3bSJed Brown } 35550a96aa3bSJed Brown #endif 35560a96aa3bSJed Brown { 35570a96aa3bSJed Brown /* outside vertex: same corner as quadrant corner */ 35580a96aa3bSJed Brown PetscInt v = pforest->topo->conn->tree_to_corner[P4EST_CHILDREN * t + l]; 35590a96aa3bSJed Brown 35600a96aa3bSJed Brown if (baseLabel) { 35610a96aa3bSJed Brown ierr = DMLabelGetValue(baseLabel,v+vStartBase,&val);CHKERRQ(ierr); 35620a96aa3bSJed Brown } else { 35630a96aa3bSJed Brown val = v+vStartBase; 35640a96aa3bSJed Brown } 35650a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,val);CHKERRQ(ierr); 35660a96aa3bSJed Brown } 35670a96aa3bSJed Brown } 35680a96aa3bSJed Brown } 35690a96aa3bSJed Brown } 35700a96aa3bSJed Brown next = next->next; 35710a96aa3bSJed Brown } 35720a96aa3bSJed Brown PetscFunctionReturn(0); 35730a96aa3bSJed Brown } 35740a96aa3bSJed Brown 35750a96aa3bSJed Brown static PetscErrorCode DMPforestLabelsFinalize(DM dm, DM plex) 35760a96aa3bSJed Brown { 35770a96aa3bSJed Brown DM_Forest_pforest *pforest = (DM_Forest_pforest*) ((DM_Forest*) dm->data)->data; 35780a96aa3bSJed Brown DM adapt; 35790a96aa3bSJed Brown PetscErrorCode ierr; 35800a96aa3bSJed Brown 35810a96aa3bSJed Brown PetscFunctionBegin; 35820a96aa3bSJed Brown if (pforest->labelsFinalized) PetscFunctionReturn(0); 35830a96aa3bSJed Brown pforest->labelsFinalized = PETSC_TRUE; 35840a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(dm,&adapt);CHKERRQ(ierr); 35850a96aa3bSJed Brown if (!adapt) { 35860a96aa3bSJed Brown /* Initialize labels from the base dm */ 35870a96aa3bSJed Brown ierr = DMPforestLabelsInitialize(dm,plex);CHKERRQ(ierr); 35880a96aa3bSJed Brown } else { 35890a96aa3bSJed Brown PetscInt dofPerDim[4]={1, 1, 1, 1}; 35900a96aa3bSJed Brown PetscSF transferForward, transferBackward, pointSF; 35910a96aa3bSJed Brown PetscInt pStart, pEnd, pStartA, pEndA; 35920a96aa3bSJed Brown PetscInt *values, *adaptValues; 35930a96aa3bSJed Brown DMLabelLink next = adapt->labels; 35940a96aa3bSJed Brown DMLabel adaptLabel; 35950a96aa3bSJed Brown DM adaptPlex; 35960a96aa3bSJed Brown 35970a96aa3bSJed Brown ierr = DMForestGetAdaptivityLabel(dm,&adaptLabel);CHKERRQ(ierr); 35980a96aa3bSJed Brown ierr = DMPforestGetPlex(adapt,&adaptPlex);CHKERRQ(ierr); 35990a96aa3bSJed Brown ierr = DMPforestGetTransferSF(adapt,dm,dofPerDim,&transferForward,&transferBackward);CHKERRQ(ierr); 36000a96aa3bSJed Brown ierr = DMPlexGetChart(plex,&pStart,&pEnd);CHKERRQ(ierr); 36010a96aa3bSJed Brown ierr = DMPlexGetChart(adaptPlex,&pStartA,&pEndA);CHKERRQ(ierr); 36020a96aa3bSJed Brown ierr = PetscMalloc2(pEnd-pStart,&values,pEndA-pStartA,&adaptValues);CHKERRQ(ierr); 36030a96aa3bSJed Brown ierr = DMGetPointSF(plex,&pointSF);CHKERRQ(ierr); 36040a96aa3bSJed Brown if (PetscDefined(USE_DEBUG)) { 36050a96aa3bSJed Brown PetscInt p; 36060a96aa3bSJed Brown for (p = pStartA; p < pEndA; p++) adaptValues[p-pStartA] = -1; 36070a96aa3bSJed Brown for (p = pStart; p < pEnd; p++) values[p-pStart] = -2; 36080a96aa3bSJed Brown if (transferForward) { 36090a96aa3bSJed Brown ierr = PetscSFBcastBegin(transferForward,MPIU_INT,adaptValues,values,MPI_REPLACE);CHKERRQ(ierr); 36100a96aa3bSJed Brown ierr = PetscSFBcastEnd(transferForward,MPIU_INT,adaptValues,values,MPI_REPLACE);CHKERRQ(ierr); 36110a96aa3bSJed Brown } 36120a96aa3bSJed Brown if (transferBackward) { 36130a96aa3bSJed Brown ierr = PetscSFReduceBegin(transferBackward,MPIU_INT,adaptValues,values,MPIU_MAX);CHKERRQ(ierr); 36140a96aa3bSJed Brown ierr = PetscSFReduceEnd(transferBackward,MPIU_INT,adaptValues,values,MPIU_MAX);CHKERRQ(ierr); 36150a96aa3bSJed Brown } 36160a96aa3bSJed Brown for (p = pStart; p < pEnd; p++) { 36170a96aa3bSJed Brown PetscInt q = p, parent; 36180a96aa3bSJed Brown 36190a96aa3bSJed Brown ierr = DMPlexGetTreeParent(plex,q,&parent,NULL);CHKERRQ(ierr); 36200a96aa3bSJed Brown while (parent != q) { 36210a96aa3bSJed Brown if (values[parent] == -2) values[parent] = values[q]; 36220a96aa3bSJed Brown q = parent; 36230a96aa3bSJed Brown ierr = DMPlexGetTreeParent(plex,q,&parent,NULL);CHKERRQ(ierr); 36240a96aa3bSJed Brown } 36250a96aa3bSJed Brown } 36260a96aa3bSJed Brown ierr = PetscSFReduceBegin(pointSF,MPIU_INT,values,values,MPIU_MAX);CHKERRQ(ierr); 36270a96aa3bSJed Brown ierr = PetscSFReduceEnd(pointSF,MPIU_INT,values,values,MPIU_MAX);CHKERRQ(ierr); 36280a96aa3bSJed Brown ierr = PetscSFBcastBegin(pointSF,MPIU_INT,values,values,MPI_REPLACE);CHKERRQ(ierr); 36290a96aa3bSJed Brown ierr = PetscSFBcastEnd(pointSF,MPIU_INT,values,values,MPI_REPLACE);CHKERRQ(ierr); 36300a96aa3bSJed Brown for (p = pStart; p < pEnd; p++) { 3631*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(values[p-pStart] == -2,PETSC_COMM_SELF,PETSC_ERR_PLIB,"uncovered point %D",p); 36320a96aa3bSJed Brown } 36330a96aa3bSJed Brown } 36340a96aa3bSJed Brown while (next) { 36350a96aa3bSJed Brown DMLabel nextLabel = next->label; 36360a96aa3bSJed Brown const char *name; 36370a96aa3bSJed Brown PetscBool isDepth, isCellType, isGhost, isVTK; 36380a96aa3bSJed Brown DMLabel label; 36390a96aa3bSJed Brown PetscInt p; 36400a96aa3bSJed Brown 36410a96aa3bSJed Brown ierr = PetscObjectGetName((PetscObject) nextLabel, &name);CHKERRQ(ierr); 36420a96aa3bSJed Brown ierr = PetscStrcmp(name,"depth",&isDepth);CHKERRQ(ierr); 36430a96aa3bSJed Brown if (isDepth) { 36440a96aa3bSJed Brown next = next->next; 36450a96aa3bSJed Brown continue; 36460a96aa3bSJed Brown } 36470a96aa3bSJed Brown ierr = PetscStrcmp(name,"celltype",&isCellType);CHKERRQ(ierr); 36480a96aa3bSJed Brown if (isCellType) { 36490a96aa3bSJed Brown next = next->next; 36500a96aa3bSJed Brown continue; 36510a96aa3bSJed Brown } 36520a96aa3bSJed Brown ierr = PetscStrcmp(name,"ghost",&isGhost);CHKERRQ(ierr); 36530a96aa3bSJed Brown if (isGhost) { 36540a96aa3bSJed Brown next = next->next; 36550a96aa3bSJed Brown continue; 36560a96aa3bSJed Brown } 36570a96aa3bSJed Brown ierr = PetscStrcmp(name,"vtk",&isVTK);CHKERRQ(ierr); 36580a96aa3bSJed Brown if (isVTK) { 36590a96aa3bSJed Brown next = next->next; 36600a96aa3bSJed Brown continue; 36610a96aa3bSJed Brown } 36620a96aa3bSJed Brown if (nextLabel == adaptLabel) { 36630a96aa3bSJed Brown next = next->next; 36640a96aa3bSJed Brown continue; 36650a96aa3bSJed Brown } 36660a96aa3bSJed Brown /* label was created earlier */ 36670a96aa3bSJed Brown ierr = DMGetLabel(dm,name,&label);CHKERRQ(ierr); 36680a96aa3bSJed Brown for (p = pStartA; p < pEndA; p++) { 36690a96aa3bSJed Brown ierr = DMLabelGetValue(nextLabel,p,&adaptValues[p]);CHKERRQ(ierr); 36700a96aa3bSJed Brown } 36710a96aa3bSJed Brown for (p = pStart; p < pEnd; p++) values[p] = PETSC_MIN_INT; 36720a96aa3bSJed Brown 36730a96aa3bSJed Brown if (transferForward) { 36740a96aa3bSJed Brown ierr = PetscSFBcastBegin(transferForward,MPIU_INT,adaptValues,values,MPI_REPLACE);CHKERRQ(ierr); 36750a96aa3bSJed Brown } 36760a96aa3bSJed Brown if (transferBackward) { 36770a96aa3bSJed Brown ierr = PetscSFReduceBegin(transferBackward,MPIU_INT,adaptValues,values,MPIU_MAX);CHKERRQ(ierr); 36780a96aa3bSJed Brown } 36790a96aa3bSJed Brown if (transferForward) { 36800a96aa3bSJed Brown ierr = PetscSFBcastEnd(transferForward,MPIU_INT,adaptValues,values,MPI_REPLACE);CHKERRQ(ierr); 36810a96aa3bSJed Brown } 36820a96aa3bSJed Brown if (transferBackward) { 36830a96aa3bSJed Brown ierr = PetscSFReduceEnd(transferBackward,MPIU_INT,adaptValues,values,MPIU_MAX);CHKERRQ(ierr); 36840a96aa3bSJed Brown } 36850a96aa3bSJed Brown for (p = pStart; p < pEnd; p++) { 36860a96aa3bSJed Brown PetscInt q = p, parent; 36870a96aa3bSJed Brown 36880a96aa3bSJed Brown ierr = DMPlexGetTreeParent(plex,q,&parent,NULL);CHKERRQ(ierr); 36890a96aa3bSJed Brown while (parent != q) { 36900a96aa3bSJed Brown if (values[parent] == PETSC_MIN_INT) values[parent] = values[q]; 36910a96aa3bSJed Brown q = parent; 36920a96aa3bSJed Brown ierr = DMPlexGetTreeParent(plex,q,&parent,NULL);CHKERRQ(ierr); 36930a96aa3bSJed Brown } 36940a96aa3bSJed Brown } 36950a96aa3bSJed Brown ierr = PetscSFReduceBegin(pointSF,MPIU_INT,values,values,MPIU_MAX);CHKERRQ(ierr); 36960a96aa3bSJed Brown ierr = PetscSFReduceEnd(pointSF,MPIU_INT,values,values,MPIU_MAX);CHKERRQ(ierr); 36970a96aa3bSJed Brown ierr = PetscSFBcastBegin(pointSF,MPIU_INT,values,values,MPI_REPLACE);CHKERRQ(ierr); 36980a96aa3bSJed Brown ierr = PetscSFBcastEnd(pointSF,MPIU_INT,values,values,MPI_REPLACE);CHKERRQ(ierr); 36990a96aa3bSJed Brown 37000a96aa3bSJed Brown for (p = pStart; p < pEnd; p++) { 37010a96aa3bSJed Brown ierr = DMLabelSetValue(label,p,values[p]);CHKERRQ(ierr); 37020a96aa3bSJed Brown } 37030a96aa3bSJed Brown next = next->next; 37040a96aa3bSJed Brown } 37050a96aa3bSJed Brown ierr = PetscFree2(values,adaptValues);CHKERRQ(ierr); 37060a96aa3bSJed Brown ierr = PetscSFDestroy(&transferForward);CHKERRQ(ierr); 37070a96aa3bSJed Brown ierr = PetscSFDestroy(&transferBackward);CHKERRQ(ierr); 37080a96aa3bSJed Brown pforest->labelsFinalized = PETSC_TRUE; 37090a96aa3bSJed Brown } 37100a96aa3bSJed Brown PetscFunctionReturn(0); 37110a96aa3bSJed Brown } 37120a96aa3bSJed Brown 37130a96aa3bSJed Brown static PetscErrorCode DMPforestMapCoordinates_Cell(DM plex, p4est_geometry_t *geom, PetscInt cell, p4est_quadrant_t *q, p4est_topidx_t t, p4est_connectivity_t * conn, PetscScalar *coords) 37140a96aa3bSJed Brown { 37150a96aa3bSJed Brown PetscInt closureSize, c, coordStart, coordEnd, coordDim; 37160a96aa3bSJed Brown PetscInt *closure = NULL; 37170a96aa3bSJed Brown PetscSection coordSec; 37180a96aa3bSJed Brown PetscErrorCode ierr; 37190a96aa3bSJed Brown 37200a96aa3bSJed Brown PetscFunctionBegin; 37210a96aa3bSJed Brown ierr = DMGetCoordinateSection(plex,&coordSec);CHKERRQ(ierr); 37220a96aa3bSJed Brown ierr = PetscSectionGetChart(coordSec,&coordStart,&coordEnd);CHKERRQ(ierr); 37230a96aa3bSJed Brown ierr = DMGetCoordinateDim(plex,&coordDim);CHKERRQ(ierr); 37240a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(plex,cell,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 37250a96aa3bSJed Brown for (c = 0; c < closureSize; c++) { 37260a96aa3bSJed Brown PetscInt point = closure[2 * c]; 37270a96aa3bSJed Brown 37280a96aa3bSJed Brown if (point >= coordStart && point < coordEnd) { 37290a96aa3bSJed Brown PetscInt dof, off; 37300a96aa3bSJed Brown PetscInt nCoords, i; 37310a96aa3bSJed Brown ierr = PetscSectionGetDof(coordSec,point,&dof);CHKERRQ(ierr); 3732*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(dof % coordDim,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Did not understand coordinate layout"); 37330a96aa3bSJed Brown nCoords = dof / coordDim; 37340a96aa3bSJed Brown ierr = PetscSectionGetOffset(coordSec,point,&off);CHKERRQ(ierr); 37350a96aa3bSJed Brown for (i = 0; i < nCoords; i++) { 37360a96aa3bSJed Brown PetscScalar *coord = &coords[off + i * coordDim]; 37370a96aa3bSJed Brown double coordP4est[3] = {0.}; 37380a96aa3bSJed Brown double coordP4estMapped[3] = {0.}; 37390a96aa3bSJed Brown PetscInt j; 37400a96aa3bSJed Brown PetscReal treeCoords[P4EST_CHILDREN][3] = {{0.}}; 37410a96aa3bSJed Brown PetscReal eta[3] = {0.}; 37420a96aa3bSJed Brown PetscInt numRounds = 10; 37430a96aa3bSJed Brown PetscReal coordGuess[3] = {0.}; 37440a96aa3bSJed Brown 37450a96aa3bSJed Brown eta[0] = (PetscReal) q->x / (PetscReal) P4EST_ROOT_LEN; 37460a96aa3bSJed Brown eta[1] = (PetscReal) q->y / (PetscReal) P4EST_ROOT_LEN; 37470a96aa3bSJed Brown #if defined(P4_TO_P8) 37480a96aa3bSJed Brown eta[2] = (PetscReal) q->z / (PetscReal) P4EST_ROOT_LEN; 37490a96aa3bSJed Brown #endif 37500a96aa3bSJed Brown 37510a96aa3bSJed Brown for (j = 0; j < P4EST_CHILDREN; j++) { 37520a96aa3bSJed Brown PetscInt k; 37530a96aa3bSJed Brown 37540a96aa3bSJed Brown for (k = 0; k < 3; k++) treeCoords[j][k] = conn->vertices[3 * conn->tree_to_vertex[P4EST_CHILDREN * t + j] + k]; 37550a96aa3bSJed Brown } 37560a96aa3bSJed Brown 37570a96aa3bSJed Brown for (j = 0; j < P4EST_CHILDREN; j++) { 37580a96aa3bSJed Brown PetscInt k; 37590a96aa3bSJed Brown PetscReal prod = 1.; 37600a96aa3bSJed Brown 37610a96aa3bSJed Brown for (k = 0; k < P4EST_DIM; k++) prod *= (j & (1 << k)) ? eta[k] : (1. - eta[k]); 37620a96aa3bSJed Brown for (k = 0; k < 3; k++) coordGuess[k] += prod * treeCoords[j][k]; 37630a96aa3bSJed Brown } 37640a96aa3bSJed Brown 37650a96aa3bSJed Brown for (j = 0; j < numRounds; j++) { 37660a96aa3bSJed Brown PetscInt dir; 37670a96aa3bSJed Brown 37680a96aa3bSJed Brown for (dir = 0; dir < P4EST_DIM; dir++) { 37690a96aa3bSJed Brown PetscInt k; 37700a96aa3bSJed Brown PetscReal diff[3]; 37710a96aa3bSJed Brown PetscReal dXdeta[3] = {0.}; 37720a96aa3bSJed Brown PetscReal rhs, scale, update; 37730a96aa3bSJed Brown 37740a96aa3bSJed Brown for (k = 0; k < 3; k++) diff[k] = coordP4est[k] - coordGuess[k]; 37750a96aa3bSJed Brown for (k = 0; k < P4EST_CHILDREN; k++) { 37760a96aa3bSJed Brown PetscInt l; 37770a96aa3bSJed Brown PetscReal prod = 1.; 37780a96aa3bSJed Brown 37790a96aa3bSJed Brown for (l = 0; l < P4EST_DIM; l++) { 37800a96aa3bSJed Brown if (l == dir) { 37810a96aa3bSJed Brown prod *= (k & (1 << l)) ? 1. : -1.; 37820a96aa3bSJed Brown } else { 37830a96aa3bSJed Brown prod *= (k & (1 << l)) ? eta[l] : (1. - eta[l]); 37840a96aa3bSJed Brown } 37850a96aa3bSJed Brown } 37860a96aa3bSJed Brown for (l = 0; l < 3; l++) dXdeta[l] += prod * treeCoords[k][l]; 37870a96aa3bSJed Brown } 37880a96aa3bSJed Brown rhs = 0.; 37890a96aa3bSJed Brown scale = 0; 37900a96aa3bSJed Brown for (k = 0; k < 3; k++) { 37910a96aa3bSJed Brown rhs += diff[k] * dXdeta[k]; 37920a96aa3bSJed Brown scale += dXdeta[k] * dXdeta[k]; 37930a96aa3bSJed Brown } 37940a96aa3bSJed Brown update = rhs / scale; 37950a96aa3bSJed Brown eta[dir] += update; 37960a96aa3bSJed Brown eta[dir] = PetscMin(eta[dir],1.); 37970a96aa3bSJed Brown eta[dir] = PetscMax(eta[dir],0.); 37980a96aa3bSJed Brown 37990a96aa3bSJed Brown coordGuess[0] = coordGuess[1] = coordGuess[2] = 0.; 38000a96aa3bSJed Brown for (k = 0; k < P4EST_CHILDREN; k++) { 38010a96aa3bSJed Brown PetscInt l; 38020a96aa3bSJed Brown PetscReal prod = 1.; 38030a96aa3bSJed Brown 38040a96aa3bSJed Brown for (l = 0; l < P4EST_DIM; l++) prod *= (k & (1 << l)) ? eta[l] : (1. - eta[l]); 38050a96aa3bSJed Brown for (l = 0; l < 3; l++) coordGuess[l] += prod * treeCoords[k][l]; 38060a96aa3bSJed Brown } 38070a96aa3bSJed Brown } 38080a96aa3bSJed Brown } 38090a96aa3bSJed Brown for (j = 0; j < 3; j++) coordP4est[j] = (double) eta[j]; 38100a96aa3bSJed Brown 38110a96aa3bSJed Brown if (geom) { 38120a96aa3bSJed Brown (geom->X)(geom,t,coordP4est,coordP4estMapped); 38130a96aa3bSJed Brown for (j = 0; j < coordDim; j++) coord[j] = (PetscScalar) coordP4estMapped[j]; 38140a96aa3bSJed Brown } else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Not coded"); 38150a96aa3bSJed Brown } 38160a96aa3bSJed Brown } 38170a96aa3bSJed Brown } 38180a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(plex,cell,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 38190a96aa3bSJed Brown PetscFunctionReturn(0); 38200a96aa3bSJed Brown } 38210a96aa3bSJed Brown 38220a96aa3bSJed Brown static PetscErrorCode DMPforestMapCoordinates(DM dm, DM plex) 38230a96aa3bSJed Brown { 38240a96aa3bSJed Brown DM_Forest *forest; 38250a96aa3bSJed Brown DM_Forest_pforest *pforest; 38260a96aa3bSJed Brown p4est_geometry_t *geom; 38270a96aa3bSJed Brown PetscInt cLocalStart, cLocalEnd; 38280a96aa3bSJed Brown Vec coordLocalVec; 38290a96aa3bSJed Brown PetscScalar *coords; 38300a96aa3bSJed Brown p4est_topidx_t flt, llt, t; 38310a96aa3bSJed Brown p4est_tree_t *trees; 38320a96aa3bSJed Brown PetscErrorCode (*map)(DM,PetscInt, PetscInt, const PetscReal [], PetscReal [], void*); 38330a96aa3bSJed Brown void *mapCtx; 38340a96aa3bSJed Brown PetscErrorCode ierr; 38350a96aa3bSJed Brown 38360a96aa3bSJed Brown PetscFunctionBegin; 38370a96aa3bSJed Brown forest = (DM_Forest*) dm->data; 38380a96aa3bSJed Brown pforest = (DM_Forest_pforest*) forest->data; 38390a96aa3bSJed Brown geom = pforest->topo->geom; 38400a96aa3bSJed Brown ierr = DMForestGetBaseCoordinateMapping(dm,&map,&mapCtx);CHKERRQ(ierr); 38410a96aa3bSJed Brown if (!geom && !map) PetscFunctionReturn(0); 38420a96aa3bSJed Brown ierr = DMGetCoordinatesLocal(plex,&coordLocalVec);CHKERRQ(ierr); 38430a96aa3bSJed Brown ierr = VecGetArray(coordLocalVec,&coords);CHKERRQ(ierr); 38440a96aa3bSJed Brown cLocalStart = pforest->cLocalStart; 38450a96aa3bSJed Brown cLocalEnd = pforest->cLocalEnd; 38460a96aa3bSJed Brown flt = pforest->forest->first_local_tree; 38470a96aa3bSJed Brown llt = pforest->forest->last_local_tree; 38480a96aa3bSJed Brown trees = (p4est_tree_t*) pforest->forest->trees->array; 38490a96aa3bSJed Brown if (map) { /* apply the map directly to the existing coordinates */ 38500a96aa3bSJed Brown PetscSection coordSec; 38510a96aa3bSJed Brown PetscInt coordStart, coordEnd, p, coordDim, p4estCoordDim, cStart, cEnd, cEndInterior; 38520a96aa3bSJed Brown DM base; 38530a96aa3bSJed Brown 38540a96aa3bSJed Brown ierr = DMPlexGetHeightStratum(plex,0,&cStart,&cEnd);CHKERRQ(ierr); 38550a96aa3bSJed Brown ierr = DMPlexGetGhostCellStratum(plex,&cEndInterior,NULL);CHKERRQ(ierr); 38560a96aa3bSJed Brown cEnd = cEndInterior < 0 ? cEnd : cEndInterior; 38570a96aa3bSJed Brown ierr = DMForestGetBaseDM(dm,&base);CHKERRQ(ierr); 38580a96aa3bSJed Brown ierr = DMGetCoordinateSection(plex,&coordSec);CHKERRQ(ierr); 38590a96aa3bSJed Brown ierr = PetscSectionGetChart(coordSec,&coordStart,&coordEnd);CHKERRQ(ierr); 38600a96aa3bSJed Brown ierr = DMGetCoordinateDim(plex,&coordDim);CHKERRQ(ierr); 38610a96aa3bSJed Brown p4estCoordDim = PetscMin(coordDim,3); 38620a96aa3bSJed Brown for (p = coordStart; p < coordEnd; p++) { 38630a96aa3bSJed Brown PetscInt *star = NULL, starSize; 38640a96aa3bSJed Brown PetscInt dof, off, cell = -1, coarsePoint = -1; 38650a96aa3bSJed Brown PetscInt nCoords, i; 38660a96aa3bSJed Brown ierr = PetscSectionGetDof(coordSec,p,&dof);CHKERRQ(ierr); 3867*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(dof % coordDim,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Did not understand coordinate layout"); 38680a96aa3bSJed Brown nCoords = dof / coordDim; 38690a96aa3bSJed Brown ierr = PetscSectionGetOffset(coordSec,p,&off);CHKERRQ(ierr); 38700a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(plex,p,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 38710a96aa3bSJed Brown for (i = 0; i < starSize; i++) { 38720a96aa3bSJed Brown PetscInt point = star[2 * i]; 38730a96aa3bSJed Brown 38740a96aa3bSJed Brown if (cStart <= point && point < cEnd) { 38750a96aa3bSJed Brown cell = point; 38760a96aa3bSJed Brown break; 38770a96aa3bSJed Brown } 38780a96aa3bSJed Brown } 38790a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(plex,p,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 38800a96aa3bSJed Brown if (cell >= 0) { 38810a96aa3bSJed Brown if (cell < cLocalStart) { 38820a96aa3bSJed Brown p4est_quadrant_t *ghosts = (p4est_quadrant_t*) pforest->ghost->ghosts.array; 38830a96aa3bSJed Brown 38840a96aa3bSJed Brown coarsePoint = ghosts[cell].p.which_tree; 38850a96aa3bSJed Brown } else if (cell < cLocalEnd) { 38860a96aa3bSJed Brown cell -= cLocalStart; 38870a96aa3bSJed Brown for (t = flt; t <= llt; t++) { 38880a96aa3bSJed Brown p4est_tree_t *tree = &(trees[t]); 38890a96aa3bSJed Brown 38900a96aa3bSJed Brown if (cell >= tree->quadrants_offset && (size_t) cell < tree->quadrants_offset + tree->quadrants.elem_count) { 38910a96aa3bSJed Brown coarsePoint = t; 38920a96aa3bSJed Brown break; 38930a96aa3bSJed Brown } 38940a96aa3bSJed Brown } 38950a96aa3bSJed Brown } else { 38960a96aa3bSJed Brown p4est_quadrant_t *ghosts = (p4est_quadrant_t*) pforest->ghost->ghosts.array; 38970a96aa3bSJed Brown 38980a96aa3bSJed Brown coarsePoint = ghosts[cell - cLocalEnd].p.which_tree; 38990a96aa3bSJed Brown } 39000a96aa3bSJed Brown } 39010a96aa3bSJed Brown for (i = 0; i < nCoords; i++) { 39020a96aa3bSJed Brown PetscScalar *coord = &coords[off + i * coordDim]; 39030a96aa3bSJed Brown PetscReal coordP4est[3] = {0.}; 39040a96aa3bSJed Brown PetscReal coordP4estMapped[3] = {0.}; 39050a96aa3bSJed Brown PetscInt j; 39060a96aa3bSJed Brown 39070a96aa3bSJed Brown for (j = 0; j < p4estCoordDim; j++) coordP4est[j] = PetscRealPart(coord[j]); 39080a96aa3bSJed Brown ierr = (map)(base,coarsePoint,p4estCoordDim,coordP4est,coordP4estMapped,mapCtx);CHKERRQ(ierr); 39090a96aa3bSJed Brown for (j = 0; j < p4estCoordDim; j++) coord[j] = (PetscScalar) coordP4estMapped[j]; 39100a96aa3bSJed Brown } 39110a96aa3bSJed Brown } 39120a96aa3bSJed Brown } else { /* we have to transform coordinates back to the unit cube (where geom is defined), and then apply geom */ 39130a96aa3bSJed Brown PetscInt cStart, cEnd, cEndInterior; 39140a96aa3bSJed Brown 39150a96aa3bSJed Brown ierr = DMPlexGetHeightStratum(plex,0,&cStart,&cEnd);CHKERRQ(ierr); 39160a96aa3bSJed Brown ierr = DMPlexGetGhostCellStratum(plex,&cEndInterior,NULL);CHKERRQ(ierr); 39170a96aa3bSJed Brown cEnd = cEndInterior < 0 ? cEnd : cEndInterior; 39180a96aa3bSJed Brown if (cLocalStart > 0) { 39190a96aa3bSJed Brown p4est_quadrant_t *ghosts = (p4est_quadrant_t*) pforest->ghost->ghosts.array; 39200a96aa3bSJed Brown PetscInt count; 39210a96aa3bSJed Brown 39220a96aa3bSJed Brown for (count = 0; count < cLocalStart; count++) { 39230a96aa3bSJed Brown p4est_quadrant_t *quad = &ghosts[count]; 39240a96aa3bSJed Brown p4est_topidx_t t = quad->p.which_tree; 39250a96aa3bSJed Brown 39260a96aa3bSJed Brown ierr = DMPforestMapCoordinates_Cell(plex,geom,count,quad,t,pforest->topo->conn,coords);CHKERRQ(ierr); 39270a96aa3bSJed Brown } 39280a96aa3bSJed Brown } 39290a96aa3bSJed Brown for (t = flt; t <= llt; t++) { 39300a96aa3bSJed Brown p4est_tree_t *tree = &(trees[t]); 39310a96aa3bSJed Brown PetscInt offset = cLocalStart + tree->quadrants_offset, i; 39320a96aa3bSJed Brown PetscInt numQuads = (PetscInt) tree->quadrants.elem_count; 39330a96aa3bSJed Brown p4est_quadrant_t *quads = (p4est_quadrant_t*) tree->quadrants.array; 39340a96aa3bSJed Brown 39350a96aa3bSJed Brown for (i = 0; i < numQuads; i++) { 39360a96aa3bSJed Brown PetscInt count = i + offset; 39370a96aa3bSJed Brown 39380a96aa3bSJed Brown ierr = DMPforestMapCoordinates_Cell(plex,geom,count,&quads[i],t,pforest->topo->conn,coords);CHKERRQ(ierr); 39390a96aa3bSJed Brown } 39400a96aa3bSJed Brown } 39410a96aa3bSJed Brown if (cLocalEnd - cLocalStart < cEnd - cStart) { 39420a96aa3bSJed Brown p4est_quadrant_t *ghosts = (p4est_quadrant_t*) pforest->ghost->ghosts.array; 39430a96aa3bSJed Brown PetscInt numGhosts = (PetscInt) pforest->ghost->ghosts.elem_count; 39440a96aa3bSJed Brown PetscInt count; 39450a96aa3bSJed Brown 39460a96aa3bSJed Brown for (count = 0; count < numGhosts - cLocalStart; count++) { 39470a96aa3bSJed Brown p4est_quadrant_t *quad = &ghosts[count + cLocalStart]; 39480a96aa3bSJed Brown p4est_topidx_t t = quad->p.which_tree; 39490a96aa3bSJed Brown 39500a96aa3bSJed Brown ierr = DMPforestMapCoordinates_Cell(plex,geom,count + cLocalEnd,quad,t,pforest->topo->conn,coords);CHKERRQ(ierr); 39510a96aa3bSJed Brown } 39520a96aa3bSJed Brown } 39530a96aa3bSJed Brown } 39540a96aa3bSJed Brown ierr = VecRestoreArray(coordLocalVec,&coords);CHKERRQ(ierr); 39550a96aa3bSJed Brown PetscFunctionReturn(0); 39560a96aa3bSJed Brown } 39570a96aa3bSJed Brown 39580a96aa3bSJed Brown static PetscErrorCode DMPforestLocalizeCoordinates(DM dm, DM plex) 39590a96aa3bSJed Brown { 39600a96aa3bSJed Brown DM_Forest *forest; 39610a96aa3bSJed Brown DM_Forest_pforest *pforest; 39620a96aa3bSJed Brown DM base; 39630a96aa3bSJed Brown Vec coordinates, cVec; 39640a96aa3bSJed Brown PetscSection oldSection, baseSection = NULL, newSection; 39650a96aa3bSJed Brown const PetscScalar *coords; 39660a96aa3bSJed Brown PetscScalar *coords2; 39670a96aa3bSJed Brown PetscInt cLocalStart, cLocalEnd, coarsePoint; 39680a96aa3bSJed Brown PetscInt cDim, newStart, newEnd, dof, cdof = -1; 39690a96aa3bSJed Brown PetscInt v, vStart, vEnd, cp, cStart, cEnd, cEndInterior, *coarsePoints; 39700a96aa3bSJed Brown PetscInt *localize, overlap; 39710a96aa3bSJed Brown p4est_topidx_t flt, llt, t; 39720a96aa3bSJed Brown p4est_tree_t *trees; 39730a96aa3bSJed Brown PetscBool isper, baseLocalized = PETSC_FALSE; 39740a96aa3bSJed Brown PetscErrorCode ierr; 39750a96aa3bSJed Brown 39760a96aa3bSJed Brown PetscFunctionBegin; 39770a96aa3bSJed Brown ierr = DMGetPeriodicity(dm,&isper,NULL,NULL,NULL);CHKERRQ(ierr); 39780a96aa3bSJed Brown if (!isper) PetscFunctionReturn(0); 39790a96aa3bSJed Brown /* we localize on all cells if we don't have a base DM or the base DM coordinates have not been localized */ 39800a96aa3bSJed Brown ierr = DMGetCoordinateDim(dm, &cDim);CHKERRQ(ierr); 39810a96aa3bSJed Brown cdof = P4EST_CHILDREN*cDim; 39820a96aa3bSJed Brown ierr = DMForestGetBaseDM(dm,&base);CHKERRQ(ierr); 39830a96aa3bSJed Brown if (base) { 39840a96aa3bSJed Brown ierr = DMGetCoordinatesLocalized(base,&baseLocalized);CHKERRQ(ierr); 39850a96aa3bSJed Brown } 39860a96aa3bSJed Brown if (!baseLocalized) base = NULL; 39870a96aa3bSJed Brown ierr = DMPlexGetChart(plex, &newStart, &newEnd);CHKERRQ(ierr); 39880a96aa3bSJed Brown 39890a96aa3bSJed Brown ierr = DMForestGetPartitionOverlap(dm,&overlap);CHKERRQ(ierr); 39900a96aa3bSJed Brown ierr = PetscCalloc1(overlap ? newEnd - newStart : 0,&localize);CHKERRQ(ierr); 39910a96aa3bSJed Brown 39920a96aa3bSJed Brown ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &newSection);CHKERRQ(ierr); 39930a96aa3bSJed Brown ierr = PetscSectionSetNumFields(newSection, 1);CHKERRQ(ierr); 39940a96aa3bSJed Brown ierr = PetscSectionSetFieldComponents(newSection, 0, cDim);CHKERRQ(ierr); 39950a96aa3bSJed Brown ierr = PetscSectionSetChart(newSection, newStart, newEnd);CHKERRQ(ierr); 39960a96aa3bSJed Brown 39970a96aa3bSJed Brown ierr = DMGetCoordinateSection(plex, &oldSection);CHKERRQ(ierr); 39980a96aa3bSJed Brown if (base) { ierr = DMGetCoordinateSection(base, &baseSection);CHKERRQ(ierr); } 39990a96aa3bSJed Brown ierr = DMPlexGetDepthStratum(plex,0,&vStart,&vEnd);CHKERRQ(ierr); 40000a96aa3bSJed Brown for (v = vStart; v < vEnd; ++v) { 40010a96aa3bSJed Brown ierr = PetscSectionGetDof(oldSection, v, &dof);CHKERRQ(ierr); 40020a96aa3bSJed Brown ierr = PetscSectionSetDof(newSection, v, dof);CHKERRQ(ierr); 40030a96aa3bSJed Brown ierr = PetscSectionSetFieldDof(newSection, v, 0, dof);CHKERRQ(ierr); 40040a96aa3bSJed Brown if (overlap) localize[v] = dof; 40050a96aa3bSJed Brown } 40060a96aa3bSJed Brown 40070a96aa3bSJed Brown forest = (DM_Forest*) dm->data; 40080a96aa3bSJed Brown pforest = (DM_Forest_pforest*) forest->data; 40090a96aa3bSJed Brown cLocalStart = pforest->cLocalStart; 40100a96aa3bSJed Brown cLocalEnd = pforest->cLocalEnd; 40110a96aa3bSJed Brown flt = pforest->forest->first_local_tree; 40120a96aa3bSJed Brown llt = pforest->forest->last_local_tree; 40130a96aa3bSJed Brown trees = (p4est_tree_t*) pforest->forest->trees->array; 40140a96aa3bSJed Brown 40150a96aa3bSJed Brown cp = 0; 40160a96aa3bSJed Brown ierr = DMPlexGetHeightStratum(plex,0,&cStart,&cEnd);CHKERRQ(ierr); 40170a96aa3bSJed Brown ierr = DMPlexGetGhostCellStratum(plex,&cEndInterior,NULL);CHKERRQ(ierr); 40180a96aa3bSJed Brown cEnd = cEndInterior < 0 ? cEnd : cEndInterior; 40190a96aa3bSJed Brown ierr = PetscMalloc1(cEnd-cStart,&coarsePoints);CHKERRQ(ierr); 40200a96aa3bSJed Brown if (cLocalStart > 0) { 40210a96aa3bSJed Brown p4est_quadrant_t *ghosts = (p4est_quadrant_t*) pforest->ghost->ghosts.array; 40220a96aa3bSJed Brown PetscInt count; 40230a96aa3bSJed Brown 40240a96aa3bSJed Brown for (count = 0; count < cLocalStart; count++) { 40250a96aa3bSJed Brown p4est_quadrant_t *quad = &ghosts[count]; 40260a96aa3bSJed Brown coarsePoint = quad->p.which_tree; 40270a96aa3bSJed Brown 40280a96aa3bSJed Brown if (baseSection) { ierr = PetscSectionGetFieldDof(baseSection, coarsePoint, 0, &cdof);CHKERRQ(ierr); } 40290a96aa3bSJed Brown ierr = PetscSectionSetDof(newSection, count, cdof);CHKERRQ(ierr); 40300a96aa3bSJed Brown ierr = PetscSectionSetFieldDof(newSection, count, 0, cdof);CHKERRQ(ierr); 40310a96aa3bSJed Brown coarsePoints[cp++] = cdof ? coarsePoint : -1; 40320a96aa3bSJed Brown if (overlap) localize[count] = cdof; 40330a96aa3bSJed Brown } 40340a96aa3bSJed Brown } 40350a96aa3bSJed Brown for (t = flt; t <= llt; t++) { 40360a96aa3bSJed Brown p4est_tree_t *tree = &(trees[t]); 40370a96aa3bSJed Brown PetscInt offset = cLocalStart + tree->quadrants_offset; 40380a96aa3bSJed Brown PetscInt numQuads = (PetscInt) tree->quadrants.elem_count; 40390a96aa3bSJed Brown PetscInt i; 40400a96aa3bSJed Brown 40410a96aa3bSJed Brown if (!numQuads) continue; 40420a96aa3bSJed Brown coarsePoint = t; 40430a96aa3bSJed Brown if (baseSection) { ierr = PetscSectionGetFieldDof(baseSection, coarsePoint, 0, &cdof);CHKERRQ(ierr); } 40440a96aa3bSJed Brown for (i = 0; i < numQuads; i++) { 40450a96aa3bSJed Brown PetscInt newCell = i + offset; 40460a96aa3bSJed Brown 40470a96aa3bSJed Brown ierr = PetscSectionSetDof(newSection, newCell, cdof);CHKERRQ(ierr); 40480a96aa3bSJed Brown ierr = PetscSectionSetFieldDof(newSection, newCell, 0, cdof);CHKERRQ(ierr); 40490a96aa3bSJed Brown coarsePoints[cp++] = cdof ? coarsePoint : -1; 40500a96aa3bSJed Brown if (overlap) localize[newCell] = cdof; 40510a96aa3bSJed Brown } 40520a96aa3bSJed Brown } 40530a96aa3bSJed Brown if (cLocalEnd - cLocalStart < cEnd - cStart) { 40540a96aa3bSJed Brown p4est_quadrant_t *ghosts = (p4est_quadrant_t*) pforest->ghost->ghosts.array; 40550a96aa3bSJed Brown PetscInt numGhosts = (PetscInt) pforest->ghost->ghosts.elem_count; 40560a96aa3bSJed Brown PetscInt count; 40570a96aa3bSJed Brown 40580a96aa3bSJed Brown for (count = 0; count < numGhosts - cLocalStart; count++) { 40590a96aa3bSJed Brown p4est_quadrant_t *quad = &ghosts[count + cLocalStart]; 40600a96aa3bSJed Brown coarsePoint = quad->p.which_tree; 40610a96aa3bSJed Brown PetscInt newCell = count + cLocalEnd; 40620a96aa3bSJed Brown 40630a96aa3bSJed Brown if (baseSection) { ierr = PetscSectionGetFieldDof(baseSection, coarsePoint, 0, &cdof);CHKERRQ(ierr); } 40640a96aa3bSJed Brown ierr = PetscSectionSetDof(newSection, newCell, cdof);CHKERRQ(ierr); 40650a96aa3bSJed Brown ierr = PetscSectionSetFieldDof(newSection, newCell, 0, cdof);CHKERRQ(ierr); 40660a96aa3bSJed Brown coarsePoints[cp++] = cdof ? coarsePoint : -1; 40670a96aa3bSJed Brown if (overlap) localize[newCell] = cdof; 40680a96aa3bSJed Brown } 40690a96aa3bSJed Brown } 4070*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(cp != cEnd - cStart,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected number of fine cells %D != %D",cp,cEnd-cStart); 40710a96aa3bSJed Brown 40720a96aa3bSJed Brown if (base) { /* we need to localize on all the cells in the star of the coarse cell vertices */ 40730a96aa3bSJed Brown PetscInt *closure = NULL, closureSize; 40740a96aa3bSJed Brown PetscInt p, i, c, vStartBase, vEndBase, cStartBase, cEndBase; 40750a96aa3bSJed Brown 40760a96aa3bSJed Brown ierr = DMPlexGetHeightStratum(base,0,&cStartBase,&cEndBase);CHKERRQ(ierr); 40770a96aa3bSJed Brown ierr = DMPlexGetDepthStratum(base,0,&vStartBase,&vEndBase);CHKERRQ(ierr); 40780a96aa3bSJed Brown for (p = cStart; p < cEnd; p++) { 40790a96aa3bSJed Brown coarsePoint = coarsePoints[p-cStart]; 40800a96aa3bSJed Brown if (coarsePoint < 0) continue; 40810a96aa3bSJed Brown if (baseSection) { ierr = PetscSectionGetFieldDof(baseSection, coarsePoint, 0, &cdof);CHKERRQ(ierr); } 40820a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(base,coarsePoint,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 40830a96aa3bSJed Brown for (c = 0; c < closureSize; c++) { 40840a96aa3bSJed Brown PetscInt *star = NULL, starSize; 40850a96aa3bSJed Brown PetscInt j, v = closure[2 * c]; 40860a96aa3bSJed Brown 40870a96aa3bSJed Brown if (v < vStartBase || v > vEndBase) continue; 40880a96aa3bSJed Brown ierr = DMPlexGetTransitiveClosure(base,v,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 40890a96aa3bSJed Brown for (j = 0; j < starSize; j++) { 40900a96aa3bSJed Brown PetscInt cell = star[2 * j]; 40910a96aa3bSJed Brown 40920a96aa3bSJed Brown if (cStartBase <= cell && cell < cEndBase) { 40930a96aa3bSJed Brown p4est_tree_t *tree; 40940a96aa3bSJed Brown PetscInt offset,numQuads; 40950a96aa3bSJed Brown 40960a96aa3bSJed Brown if (cell < flt || cell > llt) continue; 40970a96aa3bSJed Brown tree = &(trees[cell]); 40980a96aa3bSJed Brown offset = cLocalStart + tree->quadrants_offset; 40990a96aa3bSJed Brown numQuads = (PetscInt) tree->quadrants.elem_count; 41000a96aa3bSJed Brown for (i = 0; i < numQuads; i++) { 41010a96aa3bSJed Brown PetscInt newCell = i + offset; 41020a96aa3bSJed Brown 41030a96aa3bSJed Brown ierr = PetscSectionSetDof(newSection, newCell, cdof);CHKERRQ(ierr); 41040a96aa3bSJed Brown ierr = PetscSectionSetFieldDof(newSection, newCell, 0, cdof);CHKERRQ(ierr); 41050a96aa3bSJed Brown if (overlap) localize[newCell] = cdof; 41060a96aa3bSJed Brown } 41070a96aa3bSJed Brown } 41080a96aa3bSJed Brown } 41090a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(base,v,PETSC_FALSE,&starSize,&star);CHKERRQ(ierr); 41100a96aa3bSJed Brown } 41110a96aa3bSJed Brown ierr = DMPlexRestoreTransitiveClosure(base,coarsePoint,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr); 41120a96aa3bSJed Brown } 41130a96aa3bSJed Brown } 41140a96aa3bSJed Brown ierr = PetscFree(coarsePoints);CHKERRQ(ierr); 41150a96aa3bSJed Brown 41160a96aa3bSJed Brown /* final consensus with overlap */ 41170a96aa3bSJed Brown if (overlap) { 41180a96aa3bSJed Brown PetscSF sf; 41190a96aa3bSJed Brown PetscInt *localizeGlobal; 41200a96aa3bSJed Brown 41210a96aa3bSJed Brown ierr = DMGetPointSF(plex,&sf);CHKERRQ(ierr); 41220a96aa3bSJed Brown ierr = PetscMalloc1(newEnd-newStart,&localizeGlobal);CHKERRQ(ierr); 41230a96aa3bSJed Brown for (v = newStart; v < newEnd; v++) localizeGlobal[v - newStart] = localize[v - newStart]; 41240a96aa3bSJed Brown ierr = PetscSFBcastBegin(sf,MPIU_INT,localize,localizeGlobal,MPI_REPLACE);CHKERRQ(ierr); 41250a96aa3bSJed Brown ierr = PetscSFBcastEnd(sf,MPIU_INT,localize,localizeGlobal,MPI_REPLACE);CHKERRQ(ierr); 41260a96aa3bSJed Brown for (v = newStart; v < newEnd; v++) { 41270a96aa3bSJed Brown ierr = PetscSectionSetDof(newSection, v, localizeGlobal[v-newStart]);CHKERRQ(ierr); 41280a96aa3bSJed Brown ierr = PetscSectionSetFieldDof(newSection, v, 0, localizeGlobal[v-newStart]);CHKERRQ(ierr); 41290a96aa3bSJed Brown } 41300a96aa3bSJed Brown ierr = PetscFree(localizeGlobal);CHKERRQ(ierr); 41310a96aa3bSJed Brown } 41320a96aa3bSJed Brown ierr = PetscFree(localize);CHKERRQ(ierr); 41330a96aa3bSJed Brown ierr = PetscSectionSetUp(newSection);CHKERRQ(ierr); 41340a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)oldSection);CHKERRQ(ierr); 41350a96aa3bSJed Brown ierr = DMSetCoordinateSection(plex, cDim, newSection);CHKERRQ(ierr); 41360a96aa3bSJed Brown ierr = PetscSectionGetStorageSize(newSection, &v);CHKERRQ(ierr); 41370a96aa3bSJed Brown ierr = VecCreate(PETSC_COMM_SELF, &cVec);CHKERRQ(ierr); 41380a96aa3bSJed Brown ierr = PetscObjectSetName((PetscObject)cVec,"coordinates");CHKERRQ(ierr); 41390a96aa3bSJed Brown ierr = VecSetBlockSize(cVec, cDim);CHKERRQ(ierr); 41400a96aa3bSJed Brown ierr = VecSetSizes(cVec, v, PETSC_DETERMINE);CHKERRQ(ierr); 41410a96aa3bSJed Brown ierr = VecSetType(cVec, VECSTANDARD);CHKERRQ(ierr); 41420a96aa3bSJed Brown ierr = VecSet(cVec, PETSC_MIN_REAL);CHKERRQ(ierr); 41430a96aa3bSJed Brown 41440a96aa3bSJed Brown /* Copy over vertex coordinates */ 41450a96aa3bSJed Brown ierr = DMGetCoordinatesLocal(plex, &coordinates);CHKERRQ(ierr); 4146*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!coordinates,PetscObjectComm((PetscObject)plex),PETSC_ERR_SUP,"Missing local coordinates vector"); 41470a96aa3bSJed Brown ierr = VecGetArray(cVec, &coords2);CHKERRQ(ierr); 41480a96aa3bSJed Brown ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr); 41490a96aa3bSJed Brown for (v = vStart; v < vEnd; ++v) { 41500a96aa3bSJed Brown PetscInt d, off,off2; 41510a96aa3bSJed Brown 41520a96aa3bSJed Brown ierr = PetscSectionGetDof(oldSection, v, &dof);CHKERRQ(ierr); 41530a96aa3bSJed Brown ierr = PetscSectionGetOffset(oldSection, v, &off);CHKERRQ(ierr); 41540a96aa3bSJed Brown ierr = PetscSectionGetOffset(newSection, v, &off2);CHKERRQ(ierr); 41550a96aa3bSJed Brown for (d = 0; d < dof; ++d) coords2[off2+d] = coords[off+d]; 41560a96aa3bSJed Brown } 41570a96aa3bSJed Brown ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr); 41580a96aa3bSJed Brown 41590a96aa3bSJed Brown /* Localize coordinates on cells if needed */ 41600a96aa3bSJed Brown for (t = flt; t <= llt; t++) { 41610a96aa3bSJed Brown p4est_tree_t *tree = &(trees[t]); 41620a96aa3bSJed Brown const double *v = pforest->topo->conn->vertices; 41630a96aa3bSJed Brown p4est_quadrant_t *quads = (p4est_quadrant_t*) tree->quadrants.array; 41640a96aa3bSJed Brown PetscInt offset = cLocalStart + tree->quadrants_offset; 41650a96aa3bSJed Brown PetscInt numQuads = (PetscInt) tree->quadrants.elem_count; 41660a96aa3bSJed Brown p4est_topidx_t vt[8] = {0,0,0,0,0,0,0,0}; 41670a96aa3bSJed Brown PetscInt i,k; 41680a96aa3bSJed Brown 41690a96aa3bSJed Brown if (!numQuads) continue; 41700a96aa3bSJed Brown for (k = 0; k < P4EST_CHILDREN; ++k) { 41710a96aa3bSJed Brown vt[k] = pforest->topo->conn->tree_to_vertex[t * P4EST_CHILDREN + k]; 41720a96aa3bSJed Brown } 41730a96aa3bSJed Brown 41740a96aa3bSJed Brown for (i = 0; i < numQuads; i++) { 41750a96aa3bSJed Brown p4est_quadrant_t *quad = &quads[i]; 41760a96aa3bSJed Brown const PetscReal intsize = 1.0 / P4EST_ROOT_LEN; 41770a96aa3bSJed Brown PetscReal h2; 41780a96aa3bSJed Brown PetscScalar xyz[3]; 41790a96aa3bSJed Brown #ifdef P4_TO_P8 41800a96aa3bSJed Brown PetscInt zi; 41810a96aa3bSJed Brown #endif 41820a96aa3bSJed Brown PetscInt yi,xi; 41830a96aa3bSJed Brown PetscInt off2; 41840a96aa3bSJed Brown PetscInt newCell = i + offset; 41850a96aa3bSJed Brown 41860a96aa3bSJed Brown ierr = PetscSectionGetFieldDof(newSection, newCell, 0, &cdof);CHKERRQ(ierr); 41870a96aa3bSJed Brown if (!cdof) continue; 41880a96aa3bSJed Brown 41890a96aa3bSJed Brown h2 = .5 * intsize * P4EST_QUADRANT_LEN (quad->level); 41900a96aa3bSJed Brown k = 0; 41910a96aa3bSJed Brown ierr = PetscSectionGetOffset(newSection, newCell, &off2);CHKERRQ(ierr); 41920a96aa3bSJed Brown #ifdef P4_TO_P8 41930a96aa3bSJed Brown for (zi = 0; zi < 2; ++zi) { 41940a96aa3bSJed Brown const PetscReal eta_z = intsize * quad->z + h2 * (1. + (zi * 2 - 1)); 41950a96aa3bSJed Brown #else 41960a96aa3bSJed Brown { 41970a96aa3bSJed Brown const PetscReal eta_z = 0.0; 41980a96aa3bSJed Brown #endif 41990a96aa3bSJed Brown for (yi = 0; yi < 2; ++yi) { 42000a96aa3bSJed Brown const PetscReal eta_y = intsize * quad->y + h2 * (1. + (yi * 2 - 1)); 42010a96aa3bSJed Brown for (xi = 0; xi < 2; ++xi) { 42020a96aa3bSJed Brown const PetscReal eta_x = intsize * quad->x + h2 * (1. + (xi * 2 - 1)); 42030a96aa3bSJed Brown PetscInt j; 42040a96aa3bSJed Brown 42050a96aa3bSJed Brown for (j = 0; j < 3; ++j) { 42060a96aa3bSJed Brown xyz[j] = ((1. - eta_z) * ((1. - eta_y) * ((1. - eta_x) * v[3 * vt[0] + j] + 42070a96aa3bSJed Brown eta_x * v[3 * vt[1] + j]) + 42080a96aa3bSJed Brown eta_y * ((1. - eta_x) * v[3 * vt[2] + j] + 42090a96aa3bSJed Brown eta_x * v[3 * vt[3] + j])) 42100a96aa3bSJed Brown + eta_z * ((1. - eta_y) * ((1. - eta_x) * v[3 * vt[4] + j] + 42110a96aa3bSJed Brown eta_x * v[3 * vt[5] + j]) + 42120a96aa3bSJed Brown eta_y * ((1. - eta_x) * v[3 * vt[6] + j] + 42130a96aa3bSJed Brown eta_x * v[3 * vt[7] + j]))); 42140a96aa3bSJed Brown } 42150a96aa3bSJed Brown for (j = 0; j < cDim; ++j) coords2[off2 + cDim*P4estVertToPetscVert[k] + j] = xyz[j]; 42160a96aa3bSJed Brown ++k; 42170a96aa3bSJed Brown } 42180a96aa3bSJed Brown } 42190a96aa3bSJed Brown } 42200a96aa3bSJed Brown } 42210a96aa3bSJed Brown } 42220a96aa3bSJed Brown ierr = VecRestoreArray(cVec, &coords2);CHKERRQ(ierr); 42230a96aa3bSJed Brown ierr = DMSetCoordinatesLocal(plex, cVec);CHKERRQ(ierr); 42240a96aa3bSJed Brown ierr = VecDestroy(&cVec);CHKERRQ(ierr); 42250a96aa3bSJed Brown ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr); 42260a96aa3bSJed Brown ierr = PetscSectionDestroy(&oldSection);CHKERRQ(ierr); 42270a96aa3bSJed Brown PetscFunctionReturn(0); 42280a96aa3bSJed Brown } 42290a96aa3bSJed Brown 42300a96aa3bSJed Brown #define DMForestClearAdaptivityForest_pforest _append_pforest(DMForestClearAdaptivityForest) 42310a96aa3bSJed Brown static PetscErrorCode DMForestClearAdaptivityForest_pforest(DM dm) 42320a96aa3bSJed Brown { 42330a96aa3bSJed Brown DM_Forest *forest; 42340a96aa3bSJed Brown DM_Forest_pforest *pforest; 42350a96aa3bSJed Brown PetscErrorCode ierr; 42360a96aa3bSJed Brown 42370a96aa3bSJed Brown PetscFunctionBegin; 42380a96aa3bSJed Brown forest = (DM_Forest*) dm->data; 42390a96aa3bSJed Brown pforest = (DM_Forest_pforest *) forest->data; 42400a96aa3bSJed Brown ierr = PetscSFDestroy(&(pforest->pointAdaptToSelfSF));CHKERRQ(ierr); 42410a96aa3bSJed Brown ierr = PetscSFDestroy(&(pforest->pointSelfToAdaptSF));CHKERRQ(ierr); 42420a96aa3bSJed Brown ierr = PetscFree(pforest->pointAdaptToSelfCids);CHKERRQ(ierr); 42430a96aa3bSJed Brown ierr = PetscFree(pforest->pointSelfToAdaptCids);CHKERRQ(ierr); 42440a96aa3bSJed Brown PetscFunctionReturn(0); 42450a96aa3bSJed Brown } 42460a96aa3bSJed Brown 42470a96aa3bSJed Brown static PetscErrorCode DMConvert_pforest_plex(DM dm, DMType newtype, DM *plex) 42480a96aa3bSJed Brown { 42490a96aa3bSJed Brown DM_Forest *forest; 42500a96aa3bSJed Brown DM_Forest_pforest *pforest; 42510a96aa3bSJed Brown DM refTree, newPlex, base; 42520a96aa3bSJed Brown PetscInt adjDim, adjCodim, coordDim; 42530a96aa3bSJed Brown MPI_Comm comm; 42540a96aa3bSJed Brown PetscBool isPforest; 42550a96aa3bSJed Brown PetscInt dim; 42560a96aa3bSJed Brown PetscInt overlap; 42570a96aa3bSJed Brown p4est_connect_type_t ctype; 42580a96aa3bSJed Brown p4est_locidx_t first_local_quad = -1; 42590a96aa3bSJed Brown sc_array_t *points_per_dim, *cone_sizes, *cones, *cone_orientations, *coords, *children, *parents, *childids, *leaves, *remotes; 42600a96aa3bSJed Brown PetscSection parentSection; 42610a96aa3bSJed Brown PetscSF pointSF; 42620a96aa3bSJed Brown size_t zz, count; 42630a96aa3bSJed Brown PetscInt pStart, pEnd; 42640a96aa3bSJed Brown DMLabel ghostLabelBase = NULL; 42650a96aa3bSJed Brown PetscErrorCode ierr; 42660a96aa3bSJed Brown 42670a96aa3bSJed Brown PetscFunctionBegin; 42680a96aa3bSJed Brown 42690a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 42700a96aa3bSJed Brown comm = PetscObjectComm((PetscObject)dm); 42710a96aa3bSJed Brown ierr = PetscObjectTypeCompare((PetscObject)dm,DMPFOREST,&isPforest);CHKERRQ(ierr); 4272*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!isPforest,comm,PETSC_ERR_ARG_WRONG,"Expected DM type %s, got %s",DMPFOREST,((PetscObject)dm)->type_name); 42730a96aa3bSJed Brown ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr); 4274*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(dim != P4EST_DIM,comm,PETSC_ERR_ARG_WRONG,"Expected DM dimension %d, got %d",P4EST_DIM,dim); 42750a96aa3bSJed Brown forest = (DM_Forest*) dm->data; 42760a96aa3bSJed Brown pforest = (DM_Forest_pforest*) forest->data; 42770a96aa3bSJed Brown ierr = DMForestGetBaseDM(dm,&base);CHKERRQ(ierr); 42780a96aa3bSJed Brown if (base) { 42790a96aa3bSJed Brown ierr = DMGetLabel(base,"ghost",&ghostLabelBase);CHKERRQ(ierr); 42800a96aa3bSJed Brown } 42810a96aa3bSJed Brown if (!pforest->plex) { 42820a96aa3bSJed Brown PetscMPIInt size; 42830a96aa3bSJed Brown 42840a96aa3bSJed Brown ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 42850a96aa3bSJed Brown ierr = DMCreate(comm,&newPlex);CHKERRQ(ierr); 42860a96aa3bSJed Brown ierr = DMSetType(newPlex,DMPLEX);CHKERRQ(ierr); 42870a96aa3bSJed Brown ierr = DMSetMatType(newPlex,dm->mattype);CHKERRQ(ierr); 42880a96aa3bSJed Brown /* share labels */ 42890a96aa3bSJed Brown ierr = DMCopyLabels(dm, newPlex, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL);CHKERRQ(ierr); 42900a96aa3bSJed Brown ierr = DMForestGetAdjacencyDimension(dm,&adjDim);CHKERRQ(ierr); 42910a96aa3bSJed Brown ierr = DMForestGetAdjacencyCodimension(dm,&adjCodim);CHKERRQ(ierr); 42920a96aa3bSJed Brown ierr = DMGetCoordinateDim(dm,&coordDim);CHKERRQ(ierr); 42930a96aa3bSJed Brown if (adjDim == 0) { 42940a96aa3bSJed Brown ctype = P4EST_CONNECT_FULL; 42950a96aa3bSJed Brown } else if (adjCodim == 1) { 42960a96aa3bSJed Brown ctype = P4EST_CONNECT_FACE; 42970a96aa3bSJed Brown #if defined(P4_TO_P8) 42980a96aa3bSJed Brown } else if (adjDim == 1) { 42990a96aa3bSJed Brown ctype = P8EST_CONNECT_EDGE; 43000a96aa3bSJed Brown #endif 43010a96aa3bSJed Brown } else { 430298921bdaSJacob Faibussowitsch SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_ARG_WRONG,"Invalid adjacency dimension %d",adjDim); 43030a96aa3bSJed Brown } 4304*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(ctype != P4EST_CONNECT_FULL,PetscObjectComm((PetscObject)dm),PETSC_ERR_ARG_WRONG,"Adjacency dimension %D / codimension %D not supported yet",adjDim,adjCodim); 43050a96aa3bSJed Brown ierr = DMForestGetPartitionOverlap(dm,&overlap);CHKERRQ(ierr); 43060a96aa3bSJed Brown ((DM_Plex *) newPlex->data)->overlap = overlap; 43070a96aa3bSJed Brown 43080a96aa3bSJed Brown points_per_dim = sc_array_new(sizeof(p4est_locidx_t)); 43090a96aa3bSJed Brown cone_sizes = sc_array_new(sizeof(p4est_locidx_t)); 43100a96aa3bSJed Brown cones = sc_array_new(sizeof(p4est_locidx_t)); 43110a96aa3bSJed Brown cone_orientations = sc_array_new(sizeof(p4est_locidx_t)); 43120a96aa3bSJed Brown coords = sc_array_new(3 * sizeof(double)); 43130a96aa3bSJed Brown children = sc_array_new(sizeof(p4est_locidx_t)); 43140a96aa3bSJed Brown parents = sc_array_new(sizeof(p4est_locidx_t)); 43150a96aa3bSJed Brown childids = sc_array_new(sizeof(p4est_locidx_t)); 43160a96aa3bSJed Brown leaves = sc_array_new(sizeof(p4est_locidx_t)); 43170a96aa3bSJed Brown remotes = sc_array_new(2 * sizeof(p4est_locidx_t)); 43180a96aa3bSJed Brown 43190a96aa3bSJed Brown PetscStackCallP4est(p4est_get_plex_data_ext,(pforest->forest,&pforest->ghost,&pforest->lnodes,ctype,(int)((size > 1) ? overlap : 0),&first_local_quad,points_per_dim,cone_sizes,cones,cone_orientations,coords,children,parents,childids,leaves,remotes,1)); 43200a96aa3bSJed Brown 43210a96aa3bSJed Brown pforest->cLocalStart = (PetscInt) first_local_quad; 43220a96aa3bSJed Brown pforest->cLocalEnd = pforest->cLocalStart + (PetscInt) pforest->forest->local_num_quadrants; 43230a96aa3bSJed Brown ierr = locidx_to_PetscInt(points_per_dim);CHKERRQ(ierr); 43240a96aa3bSJed Brown ierr = locidx_to_PetscInt(cone_sizes);CHKERRQ(ierr); 43250a96aa3bSJed Brown ierr = locidx_to_PetscInt(cones);CHKERRQ(ierr); 43260a96aa3bSJed Brown ierr = locidx_to_PetscInt(cone_orientations);CHKERRQ(ierr); 43270a96aa3bSJed Brown ierr = coords_double_to_PetscScalar(coords, coordDim);CHKERRQ(ierr); 43280a96aa3bSJed Brown ierr = locidx_to_PetscInt(children);CHKERRQ(ierr); 43290a96aa3bSJed Brown ierr = locidx_to_PetscInt(parents);CHKERRQ(ierr); 43300a96aa3bSJed Brown ierr = locidx_to_PetscInt(childids);CHKERRQ(ierr); 43310a96aa3bSJed Brown ierr = locidx_to_PetscInt(leaves);CHKERRQ(ierr); 43320a96aa3bSJed Brown ierr = locidx_pair_to_PetscSFNode(remotes);CHKERRQ(ierr); 43330a96aa3bSJed Brown 43340a96aa3bSJed Brown ierr = DMSetDimension(newPlex,P4EST_DIM);CHKERRQ(ierr); 43350a96aa3bSJed Brown ierr = DMSetCoordinateDim(newPlex,coordDim);CHKERRQ(ierr); 43360a96aa3bSJed Brown ierr = DMPlexSetMaxProjectionHeight(newPlex,P4EST_DIM - 1);CHKERRQ(ierr); 43370a96aa3bSJed Brown ierr = DMPlexCreateFromDAG(newPlex,P4EST_DIM,(PetscInt*)points_per_dim->array,(PetscInt*)cone_sizes->array,(PetscInt*)cones->array,(PetscInt*)cone_orientations->array,(PetscScalar*)coords->array);CHKERRQ(ierr); 43380a96aa3bSJed Brown ierr = DMPlexConvertOldOrientations_Internal(newPlex);CHKERRQ(ierr); 43390a96aa3bSJed Brown ierr = DMCreateReferenceTree_pforest(comm,&refTree);CHKERRQ(ierr); 43400a96aa3bSJed Brown ierr = DMPlexSetReferenceTree(newPlex,refTree);CHKERRQ(ierr); 43410a96aa3bSJed Brown ierr = PetscSectionCreate(comm,&parentSection);CHKERRQ(ierr); 43420a96aa3bSJed Brown ierr = DMPlexGetChart(newPlex,&pStart,&pEnd);CHKERRQ(ierr); 43430a96aa3bSJed Brown ierr = PetscSectionSetChart(parentSection,pStart,pEnd);CHKERRQ(ierr); 43440a96aa3bSJed Brown count = children->elem_count; 43450a96aa3bSJed Brown for (zz = 0; zz < count; zz++) { 43460a96aa3bSJed Brown PetscInt child = *((PetscInt*) sc_array_index(children,zz)); 43470a96aa3bSJed Brown 43480a96aa3bSJed Brown ierr = PetscSectionSetDof(parentSection,child,1);CHKERRQ(ierr); 43490a96aa3bSJed Brown } 43500a96aa3bSJed Brown ierr = PetscSectionSetUp(parentSection);CHKERRQ(ierr); 43510a96aa3bSJed Brown ierr = DMPlexSetTree(newPlex,parentSection,(PetscInt*)parents->array,(PetscInt*)childids->array);CHKERRQ(ierr); 43520a96aa3bSJed Brown ierr = PetscSectionDestroy(&parentSection);CHKERRQ(ierr); 43530a96aa3bSJed Brown ierr = PetscSFCreate(comm,&pointSF);CHKERRQ(ierr); 43540a96aa3bSJed Brown /* 43550a96aa3bSJed Brown These arrays defining the sf are from the p4est library, but the code there shows the leaves being populated in increasing order. 43560a96aa3bSJed Brown https://gitlab.com/petsc/petsc/merge_requests/2248#note_240186391 43570a96aa3bSJed Brown */ 43580a96aa3bSJed Brown ierr = PetscSFSetGraph(pointSF,pEnd - pStart,(PetscInt)leaves->elem_count,(PetscInt*)leaves->array,PETSC_COPY_VALUES,(PetscSFNode*)remotes->array,PETSC_COPY_VALUES);CHKERRQ(ierr); 43590a96aa3bSJed Brown ierr = DMSetPointSF(newPlex,pointSF);CHKERRQ(ierr); 43600a96aa3bSJed Brown ierr = DMSetPointSF(dm,pointSF);CHKERRQ(ierr); 43610a96aa3bSJed Brown { 43620a96aa3bSJed Brown DM coordDM; 43630a96aa3bSJed Brown 43640a96aa3bSJed Brown ierr = DMGetCoordinateDM(newPlex,&coordDM);CHKERRQ(ierr); 43650a96aa3bSJed Brown ierr = DMSetPointSF(coordDM,pointSF);CHKERRQ(ierr); 43660a96aa3bSJed Brown } 43670a96aa3bSJed Brown ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr); 43680a96aa3bSJed Brown sc_array_destroy (points_per_dim); 43690a96aa3bSJed Brown sc_array_destroy (cone_sizes); 43700a96aa3bSJed Brown sc_array_destroy (cones); 43710a96aa3bSJed Brown sc_array_destroy (cone_orientations); 43720a96aa3bSJed Brown sc_array_destroy (coords); 43730a96aa3bSJed Brown sc_array_destroy (children); 43740a96aa3bSJed Brown sc_array_destroy (parents); 43750a96aa3bSJed Brown sc_array_destroy (childids); 43760a96aa3bSJed Brown sc_array_destroy (leaves); 43770a96aa3bSJed Brown sc_array_destroy (remotes); 43780a96aa3bSJed Brown 43790a96aa3bSJed Brown { 43800a96aa3bSJed Brown PetscBool isper; 43810a96aa3bSJed Brown const PetscReal *maxCell, *L; 43820a96aa3bSJed Brown const DMBoundaryType *bd; 43830a96aa3bSJed Brown 43840a96aa3bSJed Brown ierr = DMGetPeriodicity(dm,&isper,&maxCell,&L,&bd);CHKERRQ(ierr); 43850a96aa3bSJed Brown ierr = DMSetPeriodicity(newPlex,isper,maxCell,L,bd);CHKERRQ(ierr); 43860a96aa3bSJed Brown ierr = DMPforestLocalizeCoordinates(dm,newPlex);CHKERRQ(ierr); 43870a96aa3bSJed Brown } 43880a96aa3bSJed Brown 43890a96aa3bSJed Brown if (overlap > 0) { /* the p4est routine can't set all of the coordinates in its routine if there is overlap */ 43900a96aa3bSJed Brown Vec coordsGlobal, coordsLocal; 43910a96aa3bSJed Brown const PetscScalar *globalArray; 43920a96aa3bSJed Brown PetscScalar *localArray; 43930a96aa3bSJed Brown PetscSF coordSF; 43940a96aa3bSJed Brown DM coordDM; 43950a96aa3bSJed Brown 43960a96aa3bSJed Brown ierr = DMGetCoordinateDM(newPlex,&coordDM);CHKERRQ(ierr); 43970a96aa3bSJed Brown ierr = DMGetSectionSF(coordDM,&coordSF);CHKERRQ(ierr); 43980a96aa3bSJed Brown ierr = DMGetCoordinates(newPlex, &coordsGlobal);CHKERRQ(ierr); 43990a96aa3bSJed Brown ierr = DMGetCoordinatesLocal(newPlex, &coordsLocal);CHKERRQ(ierr); 44000a96aa3bSJed Brown ierr = VecGetArrayRead(coordsGlobal, &globalArray);CHKERRQ(ierr); 44010a96aa3bSJed Brown ierr = VecGetArray(coordsLocal, &localArray);CHKERRQ(ierr); 44020a96aa3bSJed Brown ierr = PetscSFBcastBegin(coordSF,MPIU_SCALAR,globalArray,localArray,MPI_REPLACE);CHKERRQ(ierr); 44030a96aa3bSJed Brown ierr = PetscSFBcastEnd(coordSF,MPIU_SCALAR,globalArray,localArray,MPI_REPLACE);CHKERRQ(ierr); 44040a96aa3bSJed Brown ierr = VecRestoreArray(coordsLocal, &localArray);CHKERRQ(ierr); 44050a96aa3bSJed Brown ierr = VecRestoreArrayRead(coordsGlobal, &globalArray);CHKERRQ(ierr); 44060a96aa3bSJed Brown ierr = DMSetCoordinatesLocal(newPlex, coordsLocal);CHKERRQ(ierr); 44070a96aa3bSJed Brown } 44080a96aa3bSJed Brown ierr = DMPforestMapCoordinates(dm,newPlex);CHKERRQ(ierr); 44090a96aa3bSJed Brown 44100a96aa3bSJed Brown pforest->plex = newPlex; 44110a96aa3bSJed Brown 44120a96aa3bSJed Brown /* copy labels */ 44130a96aa3bSJed Brown ierr = DMPforestLabelsFinalize(dm,newPlex);CHKERRQ(ierr); 44140a96aa3bSJed Brown 44150a96aa3bSJed Brown if (ghostLabelBase || pforest->ghostName) { /* we have to do this after copying labels because the labels drive the construction of ghost cells */ 44160a96aa3bSJed Brown PetscInt numAdded; 44170a96aa3bSJed Brown DM newPlexGhosted; 44180a96aa3bSJed Brown void *ctx; 44190a96aa3bSJed Brown 44200a96aa3bSJed Brown ierr = DMPlexConstructGhostCells(newPlex,pforest->ghostName,&numAdded,&newPlexGhosted);CHKERRQ(ierr); 44210a96aa3bSJed Brown ierr = DMGetApplicationContext(newPlex,&ctx);CHKERRQ(ierr); 44220a96aa3bSJed Brown ierr = DMSetApplicationContext(newPlexGhosted,ctx);CHKERRQ(ierr); 44230a96aa3bSJed Brown /* we want the sf for the ghost dm to be the one for the p4est dm as well */ 44240a96aa3bSJed Brown ierr = DMGetPointSF(newPlexGhosted,&pointSF);CHKERRQ(ierr); 44250a96aa3bSJed Brown ierr = DMSetPointSF(dm,pointSF);CHKERRQ(ierr); 44260a96aa3bSJed Brown ierr = DMDestroy(&newPlex);CHKERRQ(ierr); 44270a96aa3bSJed Brown ierr = DMPlexSetReferenceTree(newPlexGhosted,refTree);CHKERRQ(ierr); 44280a96aa3bSJed Brown ierr = DMForestClearAdaptivityForest_pforest(dm);CHKERRQ(ierr); 44290a96aa3bSJed Brown newPlex = newPlexGhosted; 44300a96aa3bSJed Brown 44310a96aa3bSJed Brown /* share the labels back */ 44320a96aa3bSJed Brown ierr = DMDestroyLabelLinkList_Internal(dm);CHKERRQ(ierr); 44330a96aa3bSJed Brown ierr = DMCopyLabels(newPlex, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL);CHKERRQ(ierr); 44340a96aa3bSJed Brown pforest->plex = newPlex; 44350a96aa3bSJed Brown } 44360a96aa3bSJed Brown ierr = DMDestroy(&refTree);CHKERRQ(ierr); 44370a96aa3bSJed Brown if (dm->setfromoptionscalled) { 44380a96aa3bSJed Brown ierr = PetscObjectOptionsBegin((PetscObject)newPlex);CHKERRQ(ierr); 44390a96aa3bSJed Brown ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject,newPlex);CHKERRQ(ierr); 44400a96aa3bSJed Brown ierr = PetscObjectProcessOptionsHandlers(PetscOptionsObject,(PetscObject) newPlex);CHKERRQ(ierr); 44410a96aa3bSJed Brown ierr = PetscOptionsEnd();CHKERRQ(ierr); 44420a96aa3bSJed Brown } 44430a96aa3bSJed Brown ierr = DMViewFromOptions(newPlex,NULL,"-dm_p4est_plex_view");CHKERRQ(ierr); 44440a96aa3bSJed Brown { 44450a96aa3bSJed Brown PetscSection coordsSec; 44460a96aa3bSJed Brown Vec coords; 44470a96aa3bSJed Brown PetscInt cDim; 44480a96aa3bSJed Brown 44490a96aa3bSJed Brown ierr = DMGetCoordinateDim(newPlex,&cDim);CHKERRQ(ierr); 44500a96aa3bSJed Brown ierr = DMGetCoordinateSection(newPlex,&coordsSec);CHKERRQ(ierr); 44510a96aa3bSJed Brown ierr = DMSetCoordinateSection(dm,cDim,coordsSec);CHKERRQ(ierr); 44520a96aa3bSJed Brown ierr = DMGetCoordinatesLocal(newPlex,&coords);CHKERRQ(ierr); 44530a96aa3bSJed Brown ierr = DMSetCoordinatesLocal(dm,coords);CHKERRQ(ierr); 44540a96aa3bSJed Brown } 44550a96aa3bSJed Brown } 44560a96aa3bSJed Brown newPlex = pforest->plex; 44570a96aa3bSJed Brown if (plex) { 44580a96aa3bSJed Brown DM coordDM; 44590a96aa3bSJed Brown 44600a96aa3bSJed Brown ierr = DMClone(newPlex,plex);CHKERRQ(ierr); 44610a96aa3bSJed Brown ierr = DMGetCoordinateDM(newPlex,&coordDM);CHKERRQ(ierr); 44620a96aa3bSJed Brown ierr = DMSetCoordinateDM(*plex,coordDM);CHKERRQ(ierr); 44630a96aa3bSJed Brown ierr = DMShareDiscretization(dm,*plex);CHKERRQ(ierr); 44640a96aa3bSJed Brown } 44650a96aa3bSJed Brown PetscFunctionReturn(0); 44660a96aa3bSJed Brown } 44670a96aa3bSJed Brown 44680a96aa3bSJed Brown static PetscErrorCode DMSetFromOptions_pforest(PetscOptionItems *PetscOptionsObject,DM dm) 44690a96aa3bSJed Brown { 44700a96aa3bSJed Brown DM_Forest_pforest *pforest = (DM_Forest_pforest*) ((DM_Forest*) dm->data)->data; 44710a96aa3bSJed Brown char stringBuffer[256]; 44720a96aa3bSJed Brown PetscBool flg; 44730a96aa3bSJed Brown PetscErrorCode ierr; 44740a96aa3bSJed Brown 44750a96aa3bSJed Brown PetscFunctionBegin; 44760a96aa3bSJed Brown ierr = DMSetFromOptions_Forest(PetscOptionsObject,dm);CHKERRQ(ierr); 44770a96aa3bSJed Brown ierr = PetscOptionsHead(PetscOptionsObject,"DM" P4EST_STRING " options");CHKERRQ(ierr); 44780a96aa3bSJed Brown ierr = PetscOptionsBool("-dm_p4est_partition_for_coarsening","partition forest to allow for coarsening","DMP4estSetPartitionForCoarsening",pforest->partition_for_coarsening,&(pforest->partition_for_coarsening),NULL);CHKERRQ(ierr); 44790a96aa3bSJed Brown ierr = PetscOptionsString("-dm_p4est_ghost_label_name","the name of the ghost label when converting from a DMPlex",NULL,NULL,stringBuffer,sizeof(stringBuffer),&flg);CHKERRQ(ierr); 44800a96aa3bSJed Brown ierr = PetscOptionsTail();CHKERRQ(ierr); 44810a96aa3bSJed Brown if (flg) { 44820a96aa3bSJed Brown ierr = PetscFree(pforest->ghostName);CHKERRQ(ierr); 44830a96aa3bSJed Brown ierr = PetscStrallocpy(stringBuffer,&pforest->ghostName);CHKERRQ(ierr); 44840a96aa3bSJed Brown } 44850a96aa3bSJed Brown PetscFunctionReturn(0); 44860a96aa3bSJed Brown } 44870a96aa3bSJed Brown 44880a96aa3bSJed Brown #if !defined(P4_TO_P8) 44890a96aa3bSJed Brown #define DMPforestGetPartitionForCoarsening DMP4estGetPartitionForCoarsening 44900a96aa3bSJed Brown #define DMPforestSetPartitionForCoarsening DMP4estSetPartitionForCoarsening 44910a96aa3bSJed Brown #else 44920a96aa3bSJed Brown #define DMPforestGetPartitionForCoarsening DMP8estGetPartitionForCoarsening 44930a96aa3bSJed Brown #define DMPforestSetPartitionForCoarsening DMP8estSetPartitionForCoarsening 44940a96aa3bSJed Brown #endif 44950a96aa3bSJed Brown 44960a96aa3bSJed Brown PETSC_EXTERN PetscErrorCode DMPforestGetPartitionForCoarsening(DM dm, PetscBool *flg) 44970a96aa3bSJed Brown { 44980a96aa3bSJed Brown DM_Forest_pforest *pforest; 44990a96aa3bSJed Brown 45000a96aa3bSJed Brown PetscFunctionBegin; 45010a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 45020a96aa3bSJed Brown pforest = (DM_Forest_pforest*) ((DM_Forest*) dm->data)->data; 45030a96aa3bSJed Brown *flg = pforest->partition_for_coarsening; 45040a96aa3bSJed Brown PetscFunctionReturn(0); 45050a96aa3bSJed Brown } 45060a96aa3bSJed Brown 45070a96aa3bSJed Brown PETSC_EXTERN PetscErrorCode DMPforestSetPartitionForCoarsening(DM dm, PetscBool flg) 45080a96aa3bSJed Brown { 45090a96aa3bSJed Brown DM_Forest_pforest *pforest; 45100a96aa3bSJed Brown 45110a96aa3bSJed Brown PetscFunctionBegin; 45120a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 45130a96aa3bSJed Brown pforest = (DM_Forest_pforest*) ((DM_Forest*) dm->data)->data; 45140a96aa3bSJed Brown pforest->partition_for_coarsening = flg; 45150a96aa3bSJed Brown PetscFunctionReturn(0); 45160a96aa3bSJed Brown } 45170a96aa3bSJed Brown 45180a96aa3bSJed Brown static PetscErrorCode DMPforestGetPlex(DM dm,DM *plex) 45190a96aa3bSJed Brown { 45200a96aa3bSJed Brown DM_Forest_pforest *pforest; 45210a96aa3bSJed Brown PetscErrorCode ierr; 45220a96aa3bSJed Brown 45230a96aa3bSJed Brown PetscFunctionBegin; 45240a96aa3bSJed Brown if (plex) *plex = NULL; 45250a96aa3bSJed Brown ierr = DMSetUp(dm);CHKERRQ(ierr); 45260a96aa3bSJed Brown pforest = (DM_Forest_pforest*) ((DM_Forest*) dm->data)->data; 45270a96aa3bSJed Brown if (!pforest->plex) { 45280a96aa3bSJed Brown ierr = DMConvert_pforest_plex(dm,DMPLEX,NULL);CHKERRQ(ierr); 45290a96aa3bSJed Brown } 45300a96aa3bSJed Brown ierr = DMShareDiscretization(dm,pforest->plex);CHKERRQ(ierr); 45310a96aa3bSJed Brown if (plex) *plex = pforest->plex; 45320a96aa3bSJed Brown PetscFunctionReturn(0); 45330a96aa3bSJed Brown } 45340a96aa3bSJed Brown 45350a96aa3bSJed Brown #define DMCreateInterpolation_pforest _append_pforest(DMCreateInterpolation) 45360a96aa3bSJed Brown static PetscErrorCode DMCreateInterpolation_pforest(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling) 45370a96aa3bSJed Brown { 45380a96aa3bSJed Brown PetscSection gsc, gsf; 45390a96aa3bSJed Brown PetscInt m, n; 45400a96aa3bSJed Brown DM cdm; 45410a96aa3bSJed Brown PetscErrorCode ierr; 45420a96aa3bSJed Brown 45430a96aa3bSJed Brown PetscFunctionBegin; 45440a96aa3bSJed Brown ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr); 45450a96aa3bSJed Brown ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr); 45460a96aa3bSJed Brown ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr); 45470a96aa3bSJed Brown ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr); 45480a96aa3bSJed Brown 45490a96aa3bSJed Brown ierr = MatCreate(PetscObjectComm((PetscObject) dmFine), interpolation);CHKERRQ(ierr); 45500a96aa3bSJed Brown ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr); 45510a96aa3bSJed Brown ierr = MatSetType(*interpolation, MATAIJ);CHKERRQ(ierr); 45520a96aa3bSJed Brown 45530a96aa3bSJed Brown ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr); 4554*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(cdm != dmCoarse,PetscObjectComm((PetscObject)dmFine),PETSC_ERR_SUP,"Only interpolation from coarse DM for now"); 45550a96aa3bSJed Brown 45560a96aa3bSJed Brown { 45570a96aa3bSJed Brown DM plexF, plexC; 45580a96aa3bSJed Brown PetscSF sf; 45590a96aa3bSJed Brown PetscInt *cids; 45600a96aa3bSJed Brown PetscInt dofPerDim[4] = {1,1,1,1}; 45610a96aa3bSJed Brown 45620a96aa3bSJed Brown ierr = DMPforestGetPlex(dmCoarse,&plexC);CHKERRQ(ierr); 45630a96aa3bSJed Brown ierr = DMPforestGetPlex(dmFine,&plexF);CHKERRQ(ierr); 45640a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmCoarse, dmFine, dofPerDim, &sf, PETSC_TRUE, &cids);CHKERRQ(ierr); 45650a96aa3bSJed Brown ierr = PetscSFSetUp(sf);CHKERRQ(ierr); 45660a96aa3bSJed Brown ierr = DMPlexComputeInterpolatorTree(plexC, plexF, sf, cids, *interpolation);CHKERRQ(ierr); 45670a96aa3bSJed Brown ierr = PetscSFDestroy(&sf);CHKERRQ(ierr); 45680a96aa3bSJed Brown ierr = PetscFree(cids);CHKERRQ(ierr); 45690a96aa3bSJed Brown } 45700a96aa3bSJed Brown ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr); 45710a96aa3bSJed Brown /* Use naive scaling */ 45720a96aa3bSJed Brown ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr); 45730a96aa3bSJed Brown PetscFunctionReturn(0); 45740a96aa3bSJed Brown } 45750a96aa3bSJed Brown 45760a96aa3bSJed Brown #define DMCreateInjection_pforest _append_pforest(DMCreateInjection) 45770a96aa3bSJed Brown static PetscErrorCode DMCreateInjection_pforest(DM dmCoarse, DM dmFine, Mat *injection) 45780a96aa3bSJed Brown { 45790a96aa3bSJed Brown PetscSection gsc, gsf; 45800a96aa3bSJed Brown PetscInt m, n; 45810a96aa3bSJed Brown DM cdm; 45820a96aa3bSJed Brown PetscErrorCode ierr; 45830a96aa3bSJed Brown 45840a96aa3bSJed Brown PetscFunctionBegin; 45850a96aa3bSJed Brown ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr); 45860a96aa3bSJed Brown ierr = PetscSectionGetConstrainedStorageSize(gsf, &n);CHKERRQ(ierr); 45870a96aa3bSJed Brown ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr); 45880a96aa3bSJed Brown ierr = PetscSectionGetConstrainedStorageSize(gsc, &m);CHKERRQ(ierr); 45890a96aa3bSJed Brown 45900a96aa3bSJed Brown ierr = MatCreate(PetscObjectComm((PetscObject) dmFine), injection);CHKERRQ(ierr); 45910a96aa3bSJed Brown ierr = MatSetSizes(*injection, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr); 45920a96aa3bSJed Brown ierr = MatSetType(*injection, MATAIJ);CHKERRQ(ierr); 45930a96aa3bSJed Brown 45940a96aa3bSJed Brown ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr); 4595*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(cdm != dmCoarse,PetscObjectComm((PetscObject)dmFine),PETSC_ERR_SUP,"Only injection to coarse DM for now"); 45960a96aa3bSJed Brown 45970a96aa3bSJed Brown { 45980a96aa3bSJed Brown DM plexF, plexC; 45990a96aa3bSJed Brown PetscSF sf; 46000a96aa3bSJed Brown PetscInt *cids; 46010a96aa3bSJed Brown PetscInt dofPerDim[4] = {1,1,1,1}; 46020a96aa3bSJed Brown 46030a96aa3bSJed Brown ierr = DMPforestGetPlex(dmCoarse,&plexC);CHKERRQ(ierr); 46040a96aa3bSJed Brown ierr = DMPforestGetPlex(dmFine,&plexF);CHKERRQ(ierr); 46050a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmCoarse, dmFine, dofPerDim, &sf, PETSC_TRUE, &cids);CHKERRQ(ierr); 46060a96aa3bSJed Brown ierr = PetscSFSetUp(sf);CHKERRQ(ierr); 46070a96aa3bSJed Brown ierr = DMPlexComputeInjectorTree(plexC, plexF, sf, cids, *injection);CHKERRQ(ierr); 46080a96aa3bSJed Brown ierr = PetscSFDestroy(&sf);CHKERRQ(ierr); 46090a96aa3bSJed Brown ierr = PetscFree(cids);CHKERRQ(ierr); 46100a96aa3bSJed Brown } 46110a96aa3bSJed Brown ierr = MatViewFromOptions(*injection, NULL, "-inject_mat_view");CHKERRQ(ierr); 46120a96aa3bSJed Brown /* Use naive scaling */ 46130a96aa3bSJed Brown PetscFunctionReturn(0); 46140a96aa3bSJed Brown } 46150a96aa3bSJed Brown 46160a96aa3bSJed Brown #define DMForestTransferVecFromBase_pforest _append_pforest(DMForestTransferVecFromBase) 46170a96aa3bSJed Brown static PetscErrorCode DMForestTransferVecFromBase_pforest(DM dm, Vec vecIn, Vec vecOut) 46180a96aa3bSJed Brown { 46190a96aa3bSJed Brown DM dmIn, dmVecIn, base, basec, plex, coarseDM; 46200a96aa3bSJed Brown DM *hierarchy; 46210a96aa3bSJed Brown PetscSF sfRed = NULL; 46220a96aa3bSJed Brown PetscDS ds; 46230a96aa3bSJed Brown Vec vecInLocal, vecOutLocal; 46240a96aa3bSJed Brown DMLabel subpointMap; 46250a96aa3bSJed Brown PetscInt minLevel, mh, n_hi, i; 46260a96aa3bSJed Brown PetscBool hiforest, *hierarchy_forest; 46270a96aa3bSJed Brown PetscErrorCode ierr; 46280a96aa3bSJed Brown 46290a96aa3bSJed Brown PetscFunctionBegin; 46300a96aa3bSJed Brown ierr = VecGetDM(vecIn,&dmVecIn);CHKERRQ(ierr); 46310a96aa3bSJed Brown ierr = DMGetDS(dmVecIn,&ds);CHKERRQ(ierr); 4632*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!ds,PetscObjectComm((PetscObject)dmVecIn),PETSC_ERR_SUP,"Cannot transfer without a PetscDS object"); 46330a96aa3bSJed Brown { /* we cannot stick user contexts into function callbacks for DMProjectFieldLocal! */ 46340a96aa3bSJed Brown PetscSection section; 46350a96aa3bSJed Brown PetscInt Nf; 46360a96aa3bSJed Brown 46370a96aa3bSJed Brown ierr = DMGetLocalSection(dmVecIn,§ion);CHKERRQ(ierr); 46380a96aa3bSJed Brown ierr = PetscSectionGetNumFields(section,&Nf);CHKERRQ(ierr); 4639*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(Nf > 3,PetscObjectComm((PetscObject)dmVecIn),PETSC_ERR_SUP,"Number of fields %D are currently not supported! Send an email at petsc-dev@mcs.anl.gov",Nf); 46400a96aa3bSJed Brown } 46410a96aa3bSJed Brown ierr = DMForestGetMinimumRefinement(dm,&minLevel);CHKERRQ(ierr); 4642*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(minLevel,PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Cannot transfer with minimum refinement set to %D. Rerun with DMForestSetMinimumRefinement(dm,0)",minLevel); 46430a96aa3bSJed Brown ierr = DMForestGetBaseDM(dm,&base);CHKERRQ(ierr); 4644*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!base,PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Missing base DM"); 46450a96aa3bSJed Brown 46460a96aa3bSJed Brown ierr = VecSet(vecOut,0.0);CHKERRQ(ierr); 46470a96aa3bSJed Brown if (dmVecIn == base) { /* sequential runs */ 46480a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)vecIn);CHKERRQ(ierr); 46490a96aa3bSJed Brown } else { 46500a96aa3bSJed Brown PetscSection secIn, secInRed; 46510a96aa3bSJed Brown Vec vecInRed, vecInLocal; 46520a96aa3bSJed Brown 46530a96aa3bSJed Brown ierr = PetscObjectQuery((PetscObject)base,"_base_migration_sf",(PetscObject*)&sfRed);CHKERRQ(ierr); 4654*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!sfRed,PETSC_COMM_SELF,PETSC_ERR_SUP,"Not the DM set with DMForestSetBaseDM()"); 46550a96aa3bSJed Brown ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dmVecIn),&secInRed);CHKERRQ(ierr); 46560a96aa3bSJed Brown ierr = VecCreate(PETSC_COMM_SELF,&vecInRed);CHKERRQ(ierr); 46570a96aa3bSJed Brown ierr = DMGetLocalSection(dmVecIn,&secIn);CHKERRQ(ierr); 46580a96aa3bSJed Brown ierr = DMGetLocalVector(dmVecIn,&vecInLocal);CHKERRQ(ierr); 46590a96aa3bSJed Brown ierr = DMGlobalToLocalBegin(dmVecIn,vecIn,INSERT_VALUES,vecInLocal);CHKERRQ(ierr); 46600a96aa3bSJed Brown ierr = DMGlobalToLocalEnd(dmVecIn,vecIn,INSERT_VALUES,vecInLocal);CHKERRQ(ierr); 46610a96aa3bSJed Brown ierr = DMPlexDistributeField(dmVecIn,sfRed,secIn,vecInLocal,secInRed,vecInRed);CHKERRQ(ierr); 46620a96aa3bSJed Brown ierr = DMRestoreLocalVector(dmVecIn,&vecInLocal);CHKERRQ(ierr); 46630a96aa3bSJed Brown ierr = PetscSectionDestroy(&secInRed);CHKERRQ(ierr); 46640a96aa3bSJed Brown vecIn = vecInRed; 46650a96aa3bSJed Brown } 46660a96aa3bSJed Brown 46670a96aa3bSJed Brown /* we first search through the AdaptivityForest hierarchy 46680a96aa3bSJed Brown once we found the first disconnected forest, we upsweep the DM hierarchy */ 46690a96aa3bSJed Brown hiforest = PETSC_TRUE; 46700a96aa3bSJed Brown 46710a96aa3bSJed Brown /* upsweep to the coarsest DM */ 46720a96aa3bSJed Brown n_hi = 0; 46730a96aa3bSJed Brown coarseDM = dm; 46740a96aa3bSJed Brown do { 46750a96aa3bSJed Brown PetscBool isforest; 46760a96aa3bSJed Brown 46770a96aa3bSJed Brown dmIn = coarseDM; 46780a96aa3bSJed Brown /* need to call DMSetUp to have the hierarchy recursively setup */ 46790a96aa3bSJed Brown ierr = DMSetUp(dmIn);CHKERRQ(ierr); 46800a96aa3bSJed Brown ierr = DMIsForest(dmIn,&isforest);CHKERRQ(ierr); 4681*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!isforest,PetscObjectComm((PetscObject)dmIn),PETSC_ERR_SUP,"Cannot currently transfer through a mixed hierarchy! Found DM type %s",((PetscObject)dmIn)->type_name); 46820a96aa3bSJed Brown coarseDM = NULL; 46830a96aa3bSJed Brown if (hiforest) { 46840a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(dmIn,&coarseDM);CHKERRQ(ierr); 46850a96aa3bSJed Brown } 46860a96aa3bSJed Brown if (!coarseDM) { /* DMForest hierarchy ended, we keep upsweeping through the DM hierarchy */ 46870a96aa3bSJed Brown hiforest = PETSC_FALSE; 46880a96aa3bSJed Brown ierr = DMGetCoarseDM(dmIn,&coarseDM);CHKERRQ(ierr); 46890a96aa3bSJed Brown } 46900a96aa3bSJed Brown n_hi++; 46910a96aa3bSJed Brown } while (coarseDM); 46920a96aa3bSJed Brown 46930a96aa3bSJed Brown ierr = PetscMalloc2(n_hi,&hierarchy,n_hi,&hierarchy_forest);CHKERRQ(ierr); 46940a96aa3bSJed Brown 46950a96aa3bSJed Brown i = 0; 46960a96aa3bSJed Brown hiforest = PETSC_TRUE; 46970a96aa3bSJed Brown coarseDM = dm; 46980a96aa3bSJed Brown do { 46990a96aa3bSJed Brown dmIn = coarseDM; 47000a96aa3bSJed Brown coarseDM = NULL; 47010a96aa3bSJed Brown if (hiforest) { 47020a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(dmIn,&coarseDM);CHKERRQ(ierr); 47030a96aa3bSJed Brown } 47040a96aa3bSJed Brown if (!coarseDM) { /* DMForest hierarchy ended, we keep upsweeping through the DM hierarchy */ 47050a96aa3bSJed Brown hiforest = PETSC_FALSE; 47060a96aa3bSJed Brown ierr = DMGetCoarseDM(dmIn,&coarseDM);CHKERRQ(ierr); 47070a96aa3bSJed Brown } 47080a96aa3bSJed Brown i++; 47090a96aa3bSJed Brown hierarchy[n_hi - i] = dmIn; 47100a96aa3bSJed Brown } while (coarseDM); 47110a96aa3bSJed Brown 47120a96aa3bSJed Brown /* project base vector on the coarsest forest (minimum refinement = 0) */ 47130a96aa3bSJed Brown ierr = DMPforestGetPlex(dmIn,&plex);CHKERRQ(ierr); 47140a96aa3bSJed Brown 47150a96aa3bSJed Brown /* Check this plex is compatible with the base */ 47160a96aa3bSJed Brown { 47170a96aa3bSJed Brown IS gnum[2]; 47180a96aa3bSJed Brown PetscInt ncells[2],gncells[2]; 47190a96aa3bSJed Brown 47200a96aa3bSJed Brown ierr = DMPlexGetCellNumbering(base,&gnum[0]);CHKERRQ(ierr); 47210a96aa3bSJed Brown ierr = DMPlexGetCellNumbering(plex,&gnum[1]);CHKERRQ(ierr); 47220a96aa3bSJed Brown ierr = ISGetMinMax(gnum[0],NULL,&ncells[0]);CHKERRQ(ierr); 47230a96aa3bSJed Brown ierr = ISGetMinMax(gnum[1],NULL,&ncells[1]);CHKERRQ(ierr); 47240a96aa3bSJed Brown ierr = MPIU_Allreduce(ncells,gncells,2,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr); 4725*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(gncells[0] != gncells[1],PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Invalid number of base cells! Expected %D, found %D",gncells[0]+1,gncells[1]+1); 47260a96aa3bSJed Brown } 47270a96aa3bSJed Brown 47280a96aa3bSJed Brown ierr = DMGetLabel(dmIn,"_forest_base_subpoint_map",&subpointMap);CHKERRQ(ierr); 4729*2c71b3e2SJacob Faibussowitsch PetscCheckFalse(!subpointMap,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Missing _forest_base_subpoint_map label"); 47300a96aa3bSJed Brown 47310a96aa3bSJed Brown ierr = DMPlexGetMaxProjectionHeight(base,&mh);CHKERRQ(ierr); 47320a96aa3bSJed Brown ierr = DMPlexSetMaxProjectionHeight(plex,mh);CHKERRQ(ierr); 47330a96aa3bSJed Brown 47340a96aa3bSJed Brown ierr = DMClone(base,&basec);CHKERRQ(ierr); 47350a96aa3bSJed Brown ierr = DMCopyDisc(dmVecIn,basec);CHKERRQ(ierr); 47360a96aa3bSJed Brown if (sfRed) { 47370a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)vecIn);CHKERRQ(ierr); 47380a96aa3bSJed Brown vecInLocal = vecIn; 47390a96aa3bSJed Brown } else { 47400a96aa3bSJed Brown ierr = DMCreateLocalVector(basec,&vecInLocal);CHKERRQ(ierr); 47410a96aa3bSJed Brown ierr = DMGlobalToLocalBegin(basec,vecIn,INSERT_VALUES,vecInLocal);CHKERRQ(ierr); 47420a96aa3bSJed Brown ierr = DMGlobalToLocalEnd(basec,vecIn,INSERT_VALUES,vecInLocal);CHKERRQ(ierr); 47430a96aa3bSJed Brown } 47440a96aa3bSJed Brown 47450a96aa3bSJed Brown ierr = DMGetLocalVector(dmIn,&vecOutLocal);CHKERRQ(ierr); 47460a96aa3bSJed Brown { /* get degrees of freedom ordered onto dmIn */ 47470a96aa3bSJed Brown PetscSF basetocoarse; 47480a96aa3bSJed Brown PetscInt bStart, bEnd, nroots; 47490a96aa3bSJed Brown PetscInt iStart, iEnd, nleaves, leaf; 47500a96aa3bSJed Brown PetscMPIInt rank; 47510a96aa3bSJed Brown PetscSFNode *remotes; 47520a96aa3bSJed Brown PetscSection secIn, secOut; 47530a96aa3bSJed Brown PetscInt *remoteOffsets; 47540a96aa3bSJed Brown PetscSF transferSF; 47550a96aa3bSJed Brown const PetscScalar *inArray; 47560a96aa3bSJed Brown PetscScalar *outArray; 47570a96aa3bSJed Brown 47580a96aa3bSJed Brown ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)basec), &rank);CHKERRMPI(ierr); 47590a96aa3bSJed Brown ierr = DMPlexGetChart(basec, &bStart, &bEnd);CHKERRQ(ierr); 47600a96aa3bSJed Brown nroots = PetscMax(bEnd - bStart, 0); 47610a96aa3bSJed Brown ierr = DMPlexGetChart(plex, &iStart, &iEnd);CHKERRQ(ierr); 47620a96aa3bSJed Brown nleaves = PetscMax(iEnd - iStart, 0); 47630a96aa3bSJed Brown 47640a96aa3bSJed Brown ierr = PetscMalloc1(nleaves, &remotes);CHKERRQ(ierr); 47650a96aa3bSJed Brown for (leaf = iStart; leaf < iEnd; leaf++) { 47660a96aa3bSJed Brown PetscInt index; 47670a96aa3bSJed Brown 47680a96aa3bSJed Brown remotes[leaf - iStart].rank = rank; 47690a96aa3bSJed Brown ierr = DMLabelGetValue(subpointMap, leaf, &index);CHKERRQ(ierr); 47700a96aa3bSJed Brown remotes[leaf - iStart].index = index; 47710a96aa3bSJed Brown } 47720a96aa3bSJed Brown 47730a96aa3bSJed Brown ierr = PetscSFCreate(PetscObjectComm((PetscObject)basec), &basetocoarse);CHKERRQ(ierr); 47740a96aa3bSJed Brown ierr = PetscSFSetGraph(basetocoarse, nroots, nleaves, NULL, PETSC_OWN_POINTER, remotes, PETSC_OWN_POINTER);CHKERRQ(ierr); 47750a96aa3bSJed Brown ierr = PetscSFSetUp(basetocoarse);CHKERRQ(ierr); 47760a96aa3bSJed Brown ierr = DMGetLocalSection(basec,&secIn);CHKERRQ(ierr); 47770a96aa3bSJed Brown ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dmIn),&secOut);CHKERRQ(ierr); 47780a96aa3bSJed Brown ierr = PetscSFDistributeSection(basetocoarse, secIn, &remoteOffsets, secOut);CHKERRQ(ierr); 47790a96aa3bSJed Brown ierr = PetscSFCreateSectionSF(basetocoarse, secIn, remoteOffsets, secOut, &transferSF);CHKERRQ(ierr); 47800a96aa3bSJed Brown ierr = PetscFree(remoteOffsets);CHKERRQ(ierr); 47810a96aa3bSJed Brown ierr = VecGetArrayWrite(vecOutLocal, &outArray);CHKERRQ(ierr); 47820a96aa3bSJed Brown ierr = VecGetArrayRead(vecInLocal, &inArray);CHKERRQ(ierr); 47830a96aa3bSJed Brown ierr = PetscSFBcastBegin(transferSF, MPIU_SCALAR, inArray, outArray,MPI_REPLACE);CHKERRQ(ierr); 47840a96aa3bSJed Brown ierr = PetscSFBcastEnd(transferSF, MPIU_SCALAR, inArray, outArray,MPI_REPLACE);CHKERRQ(ierr); 47850a96aa3bSJed Brown ierr = VecRestoreArrayRead(vecInLocal, &inArray);CHKERRQ(ierr); 47860a96aa3bSJed Brown ierr = VecRestoreArrayWrite(vecOutLocal, &outArray);CHKERRQ(ierr); 47870a96aa3bSJed Brown ierr = PetscSFDestroy(&transferSF);CHKERRQ(ierr); 47880a96aa3bSJed Brown ierr = PetscSectionDestroy(&secOut);CHKERRQ(ierr); 47890a96aa3bSJed Brown ierr = PetscSFDestroy(&basetocoarse);CHKERRQ(ierr); 47900a96aa3bSJed Brown } 47910a96aa3bSJed Brown ierr = VecDestroy(&vecInLocal);CHKERRQ(ierr); 47920a96aa3bSJed Brown ierr = DMDestroy(&basec);CHKERRQ(ierr); 47930a96aa3bSJed Brown ierr = VecDestroy(&vecIn);CHKERRQ(ierr); 47940a96aa3bSJed Brown 47950a96aa3bSJed Brown /* output */ 47960a96aa3bSJed Brown if (n_hi > 1) { /* downsweep the stored hierarchy */ 47970a96aa3bSJed Brown Vec vecOut1, vecOut2; 47980a96aa3bSJed Brown DM fineDM; 47990a96aa3bSJed Brown 48000a96aa3bSJed Brown ierr = DMGetGlobalVector(dmIn,&vecOut1);CHKERRQ(ierr); 48010a96aa3bSJed Brown ierr = DMLocalToGlobal(dmIn,vecOutLocal,INSERT_VALUES,vecOut1);CHKERRQ(ierr); 48020a96aa3bSJed Brown ierr = DMRestoreLocalVector(dmIn,&vecOutLocal);CHKERRQ(ierr); 48030a96aa3bSJed Brown for (i = 1; i < n_hi-1; i++) { 48040a96aa3bSJed Brown fineDM = hierarchy[i]; 48050a96aa3bSJed Brown ierr = DMGetGlobalVector(fineDM,&vecOut2);CHKERRQ(ierr); 48060a96aa3bSJed Brown ierr = DMForestTransferVec(dmIn,vecOut1,fineDM,vecOut2,PETSC_TRUE,0.0);CHKERRQ(ierr); 48070a96aa3bSJed Brown ierr = DMRestoreGlobalVector(dmIn,&vecOut1);CHKERRQ(ierr); 48080a96aa3bSJed Brown vecOut1 = vecOut2; 48090a96aa3bSJed Brown dmIn = fineDM; 48100a96aa3bSJed Brown } 48110a96aa3bSJed Brown ierr = DMForestTransferVec(dmIn,vecOut1,dm,vecOut,PETSC_TRUE,0.0);CHKERRQ(ierr); 48120a96aa3bSJed Brown ierr = DMRestoreGlobalVector(dmIn,&vecOut1);CHKERRQ(ierr); 48130a96aa3bSJed Brown } else { 48140a96aa3bSJed Brown ierr = DMLocalToGlobal(dmIn,vecOutLocal,INSERT_VALUES,vecOut);CHKERRQ(ierr); 48150a96aa3bSJed Brown ierr = DMRestoreLocalVector(dmIn,&vecOutLocal);CHKERRQ(ierr); 48160a96aa3bSJed Brown } 48170a96aa3bSJed Brown ierr = PetscFree2(hierarchy,hierarchy_forest);CHKERRQ(ierr); 48180a96aa3bSJed Brown PetscFunctionReturn(0); 48190a96aa3bSJed Brown } 48200a96aa3bSJed Brown 48210a96aa3bSJed Brown #define DMForestTransferVec_pforest _append_pforest(DMForestTransferVec) 48220a96aa3bSJed Brown static PetscErrorCode DMForestTransferVec_pforest(DM dmIn, Vec vecIn, DM dmOut, Vec vecOut, PetscBool useBCs, PetscReal time) 48230a96aa3bSJed Brown { 48240a96aa3bSJed Brown DM adaptIn, adaptOut, plexIn, plexOut; 48250a96aa3bSJed Brown DM_Forest *forestIn, *forestOut, *forestAdaptIn, *forestAdaptOut; 48260a96aa3bSJed Brown PetscInt dofPerDim[] = {1, 1, 1, 1}; 48270a96aa3bSJed Brown PetscSF inSF = NULL, outSF = NULL; 48280a96aa3bSJed Brown PetscInt *inCids = NULL, *outCids = NULL; 48290a96aa3bSJed Brown DMAdaptFlag purposeIn, purposeOut; 48300a96aa3bSJed Brown PetscErrorCode ierr; 48310a96aa3bSJed Brown 48320a96aa3bSJed Brown PetscFunctionBegin; 48330a96aa3bSJed Brown forestOut = (DM_Forest *) dmOut->data; 48340a96aa3bSJed Brown forestIn = (DM_Forest *) dmIn->data; 48350a96aa3bSJed Brown 48360a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(dmOut,&adaptOut);CHKERRQ(ierr); 48370a96aa3bSJed Brown ierr = DMForestGetAdaptivityPurpose(dmOut,&purposeOut);CHKERRQ(ierr); 48380a96aa3bSJed Brown forestAdaptOut = adaptOut ? (DM_Forest *) adaptOut->data : NULL; 48390a96aa3bSJed Brown 48400a96aa3bSJed Brown ierr = DMForestGetAdaptivityForest(dmIn,&adaptIn);CHKERRQ(ierr); 48410a96aa3bSJed Brown ierr = DMForestGetAdaptivityPurpose(dmIn,&purposeIn);CHKERRQ(ierr); 48420a96aa3bSJed Brown forestAdaptIn = adaptIn ? (DM_Forest *) adaptIn->data : NULL; 48430a96aa3bSJed Brown 48440a96aa3bSJed Brown if (forestAdaptOut == forestIn) { 48450a96aa3bSJed Brown switch (purposeOut) { 48460a96aa3bSJed Brown case DM_ADAPT_REFINE: 48470a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmIn,dmOut,dofPerDim,&inSF,PETSC_TRUE,&inCids);CHKERRQ(ierr); 48480a96aa3bSJed Brown ierr = PetscSFSetUp(inSF);CHKERRQ(ierr); 48490a96aa3bSJed Brown break; 48500a96aa3bSJed Brown case DM_ADAPT_COARSEN: 48510a96aa3bSJed Brown case DM_ADAPT_COARSEN_LAST: 48520a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmOut,dmIn,dofPerDim,&outSF,PETSC_TRUE,&outCids);CHKERRQ(ierr); 48530a96aa3bSJed Brown ierr = PetscSFSetUp(outSF);CHKERRQ(ierr); 48540a96aa3bSJed Brown break; 48550a96aa3bSJed Brown default: 48560a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmIn,dmOut,dofPerDim,&inSF,PETSC_TRUE,&inCids);CHKERRQ(ierr); 48570a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmOut,dmIn,dofPerDim,&outSF,PETSC_FALSE,&outCids);CHKERRQ(ierr); 48580a96aa3bSJed Brown ierr = PetscSFSetUp(inSF);CHKERRQ(ierr); 48590a96aa3bSJed Brown ierr = PetscSFSetUp(outSF);CHKERRQ(ierr); 48600a96aa3bSJed Brown } 48610a96aa3bSJed Brown } else if (forestAdaptIn == forestOut) { 48620a96aa3bSJed Brown switch (purposeIn) { 48630a96aa3bSJed Brown case DM_ADAPT_REFINE: 48640a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmOut,dmIn,dofPerDim,&outSF,PETSC_TRUE,&inCids);CHKERRQ(ierr); 48650a96aa3bSJed Brown ierr = PetscSFSetUp(outSF);CHKERRQ(ierr); 48660a96aa3bSJed Brown break; 48670a96aa3bSJed Brown case DM_ADAPT_COARSEN: 48680a96aa3bSJed Brown case DM_ADAPT_COARSEN_LAST: 48690a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmIn,dmOut,dofPerDim,&inSF,PETSC_TRUE,&inCids);CHKERRQ(ierr); 48700a96aa3bSJed Brown ierr = PetscSFSetUp(inSF);CHKERRQ(ierr); 48710a96aa3bSJed Brown break; 48720a96aa3bSJed Brown default: 48730a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmIn,dmOut,dofPerDim,&inSF,PETSC_TRUE,&inCids);CHKERRQ(ierr); 48740a96aa3bSJed Brown ierr = DMPforestGetTransferSF_Internal(dmOut,dmIn,dofPerDim,&outSF,PETSC_FALSE,&outCids);CHKERRQ(ierr); 48750a96aa3bSJed Brown ierr = PetscSFSetUp(inSF);CHKERRQ(ierr); 48760a96aa3bSJed Brown ierr = PetscSFSetUp(outSF);CHKERRQ(ierr); 48770a96aa3bSJed Brown } 48780a96aa3bSJed Brown } else SETERRQ(PetscObjectComm((PetscObject)dmIn),PETSC_ERR_SUP,"Only support transfer from pre-adaptivity to post-adaptivity right now"); 48790a96aa3bSJed Brown ierr = DMPforestGetPlex(dmIn,&plexIn);CHKERRQ(ierr); 48800a96aa3bSJed Brown ierr = DMPforestGetPlex(dmOut,&plexOut);CHKERRQ(ierr); 48810a96aa3bSJed Brown 48820a96aa3bSJed Brown ierr = DMPlexTransferVecTree(plexIn,vecIn,plexOut,vecOut,inSF,outSF,inCids,outCids,useBCs,time);CHKERRQ(ierr); 48830a96aa3bSJed Brown ierr = PetscFree(inCids);CHKERRQ(ierr); 48840a96aa3bSJed Brown ierr = PetscFree(outCids);CHKERRQ(ierr); 48850a96aa3bSJed Brown ierr = PetscSFDestroy(&inSF);CHKERRQ(ierr); 48860a96aa3bSJed Brown ierr = PetscSFDestroy(&outSF);CHKERRQ(ierr); 48870a96aa3bSJed Brown ierr = PetscFree(inCids);CHKERRQ(ierr); 48880a96aa3bSJed Brown ierr = PetscFree(outCids);CHKERRQ(ierr); 48890a96aa3bSJed Brown PetscFunctionReturn(0); 48900a96aa3bSJed Brown } 48910a96aa3bSJed Brown 48920a96aa3bSJed Brown #define DMCreateCoordinateDM_pforest _append_pforest(DMCreateCoordinateDM) 48930a96aa3bSJed Brown static PetscErrorCode DMCreateCoordinateDM_pforest(DM dm,DM *cdm) 48940a96aa3bSJed Brown { 48950a96aa3bSJed Brown DM plex; 48960a96aa3bSJed Brown PetscErrorCode ierr; 48970a96aa3bSJed Brown 48980a96aa3bSJed Brown PetscFunctionBegin; 48990a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 49000a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 49010a96aa3bSJed Brown ierr = DMGetCoordinateDM(plex,cdm);CHKERRQ(ierr); 49020a96aa3bSJed Brown ierr = PetscObjectReference((PetscObject)*cdm);CHKERRQ(ierr); 49030a96aa3bSJed Brown PetscFunctionReturn(0); 49040a96aa3bSJed Brown } 49050a96aa3bSJed Brown 49060a96aa3bSJed Brown #define VecViewLocal_pforest _append_pforest(VecViewLocal) 49070a96aa3bSJed Brown static PetscErrorCode VecViewLocal_pforest(Vec vec,PetscViewer viewer) 49080a96aa3bSJed Brown { 49090a96aa3bSJed Brown DM dm, plex; 49100a96aa3bSJed Brown PetscErrorCode ierr; 49110a96aa3bSJed Brown 49120a96aa3bSJed Brown PetscFunctionBegin; 49130a96aa3bSJed Brown ierr = VecGetDM(vec,&dm);CHKERRQ(ierr); 49140a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 49150a96aa3bSJed Brown ierr = VecSetDM(vec,plex);CHKERRQ(ierr); 49160a96aa3bSJed Brown ierr = VecView_Plex_Local(vec,viewer);CHKERRQ(ierr); 49170a96aa3bSJed Brown ierr = VecSetDM(vec,dm);CHKERRQ(ierr); 49180a96aa3bSJed Brown PetscFunctionReturn(0); 49190a96aa3bSJed Brown } 49200a96aa3bSJed Brown 49210a96aa3bSJed Brown #define VecView_pforest _append_pforest(VecView) 49220a96aa3bSJed Brown static PetscErrorCode VecView_pforest(Vec vec,PetscViewer viewer) 49230a96aa3bSJed Brown { 49240a96aa3bSJed Brown DM dm, plex; 49250a96aa3bSJed Brown PetscErrorCode ierr; 49260a96aa3bSJed Brown 49270a96aa3bSJed Brown PetscFunctionBegin; 49280a96aa3bSJed Brown ierr = VecGetDM(vec,&dm);CHKERRQ(ierr); 49290a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 49300a96aa3bSJed Brown ierr = VecSetDM(vec,plex);CHKERRQ(ierr); 49310a96aa3bSJed Brown ierr = VecView_Plex(vec,viewer);CHKERRQ(ierr); 49320a96aa3bSJed Brown ierr = VecSetDM(vec,dm);CHKERRQ(ierr); 49330a96aa3bSJed Brown PetscFunctionReturn(0); 49340a96aa3bSJed Brown } 49350a96aa3bSJed Brown 49360a96aa3bSJed Brown #define VecView_pforest_Native _infix_pforest(VecView,_Native) 49370a96aa3bSJed Brown static PetscErrorCode VecView_pforest_Native(Vec vec,PetscViewer viewer) 49380a96aa3bSJed Brown { 49390a96aa3bSJed Brown DM dm, plex; 49400a96aa3bSJed Brown PetscErrorCode ierr; 49410a96aa3bSJed Brown 49420a96aa3bSJed Brown PetscFunctionBegin; 49430a96aa3bSJed Brown ierr = VecGetDM(vec,&dm);CHKERRQ(ierr); 49440a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 49450a96aa3bSJed Brown ierr = VecSetDM(vec,plex);CHKERRQ(ierr); 49460a96aa3bSJed Brown ierr = VecView_Plex_Native(vec,viewer);CHKERRQ(ierr); 49470a96aa3bSJed Brown ierr = VecSetDM(vec,dm);CHKERRQ(ierr); 49480a96aa3bSJed Brown PetscFunctionReturn(0); 49490a96aa3bSJed Brown } 49500a96aa3bSJed Brown 49510a96aa3bSJed Brown #define VecLoad_pforest _append_pforest(VecLoad) 49520a96aa3bSJed Brown static PetscErrorCode VecLoad_pforest(Vec vec,PetscViewer viewer) 49530a96aa3bSJed Brown { 49540a96aa3bSJed Brown DM dm, plex; 49550a96aa3bSJed Brown PetscErrorCode ierr; 49560a96aa3bSJed Brown 49570a96aa3bSJed Brown PetscFunctionBegin; 49580a96aa3bSJed Brown ierr = VecGetDM(vec,&dm);CHKERRQ(ierr); 49590a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 49600a96aa3bSJed Brown ierr = VecSetDM(vec,plex);CHKERRQ(ierr); 49610a96aa3bSJed Brown ierr = VecLoad_Plex(vec,viewer);CHKERRQ(ierr); 49620a96aa3bSJed Brown ierr = VecSetDM(vec,dm);CHKERRQ(ierr); 49630a96aa3bSJed Brown PetscFunctionReturn(0); 49640a96aa3bSJed Brown } 49650a96aa3bSJed Brown 49660a96aa3bSJed Brown #define VecLoad_pforest_Native _infix_pforest(VecLoad,_Native) 49670a96aa3bSJed Brown static PetscErrorCode VecLoad_pforest_Native(Vec vec,PetscViewer viewer) 49680a96aa3bSJed Brown { 49690a96aa3bSJed Brown DM dm, plex; 49700a96aa3bSJed Brown PetscErrorCode ierr; 49710a96aa3bSJed Brown 49720a96aa3bSJed Brown PetscFunctionBegin; 49730a96aa3bSJed Brown ierr = VecGetDM(vec,&dm);CHKERRQ(ierr); 49740a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 49750a96aa3bSJed Brown ierr = VecSetDM(vec,plex);CHKERRQ(ierr); 49760a96aa3bSJed Brown ierr = VecLoad_Plex_Native(vec,viewer);CHKERRQ(ierr); 49770a96aa3bSJed Brown ierr = VecSetDM(vec,dm);CHKERRQ(ierr); 49780a96aa3bSJed Brown PetscFunctionReturn(0); 49790a96aa3bSJed Brown } 49800a96aa3bSJed Brown 49810a96aa3bSJed Brown #define DMCreateGlobalVector_pforest _append_pforest(DMCreateGlobalVector) 49820a96aa3bSJed Brown static PetscErrorCode DMCreateGlobalVector_pforest(DM dm,Vec *vec) 49830a96aa3bSJed Brown { 49840a96aa3bSJed Brown PetscErrorCode ierr; 49850a96aa3bSJed Brown 49860a96aa3bSJed Brown PetscFunctionBegin; 49870a96aa3bSJed Brown ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr); 49880a96aa3bSJed Brown /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */ 49890a96aa3bSJed Brown ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_pforest);CHKERRQ(ierr); 49900a96aa3bSJed Brown ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_pforest_Native);CHKERRQ(ierr); 49910a96aa3bSJed Brown ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_pforest);CHKERRQ(ierr); 49920a96aa3bSJed Brown ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_pforest_Native);CHKERRQ(ierr); 49930a96aa3bSJed Brown PetscFunctionReturn(0); 49940a96aa3bSJed Brown } 49950a96aa3bSJed Brown 49960a96aa3bSJed Brown #define DMCreateLocalVector_pforest _append_pforest(DMCreateLocalVector) 49970a96aa3bSJed Brown static PetscErrorCode DMCreateLocalVector_pforest(DM dm,Vec *vec) 49980a96aa3bSJed Brown { 49990a96aa3bSJed Brown PetscErrorCode ierr; 50000a96aa3bSJed Brown 50010a96aa3bSJed Brown PetscFunctionBegin; 50020a96aa3bSJed Brown ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr); 50030a96aa3bSJed Brown ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecViewLocal_pforest);CHKERRQ(ierr); 50040a96aa3bSJed Brown PetscFunctionReturn(0); 50050a96aa3bSJed Brown } 50060a96aa3bSJed Brown 50070a96aa3bSJed Brown #define DMCreateMatrix_pforest _append_pforest(DMCreateMatrix) 50080a96aa3bSJed Brown static PetscErrorCode DMCreateMatrix_pforest(DM dm,Mat *mat) 50090a96aa3bSJed Brown { 50100a96aa3bSJed Brown DM plex; 50110a96aa3bSJed Brown PetscErrorCode ierr; 50120a96aa3bSJed Brown 50130a96aa3bSJed Brown PetscFunctionBegin; 50140a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 50150a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 50160a96aa3bSJed Brown if (plex->prealloc_only != dm->prealloc_only) plex->prealloc_only = dm->prealloc_only; /* maybe this should go into forest->plex */ 50170a96aa3bSJed Brown ierr = DMCreateMatrix(plex,mat);CHKERRQ(ierr); 50180a96aa3bSJed Brown ierr = MatSetDM(*mat,dm);CHKERRQ(ierr); 50190a96aa3bSJed Brown PetscFunctionReturn(0); 50200a96aa3bSJed Brown } 50210a96aa3bSJed Brown 50220a96aa3bSJed Brown #define DMProjectFunctionLocal_pforest _append_pforest(DMProjectFunctionLocal) 50230a96aa3bSJed Brown static PetscErrorCode DMProjectFunctionLocal_pforest(DM dm, PetscReal time, PetscErrorCode (**funcs) (PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void*), void **ctxs, InsertMode mode, Vec localX) 50240a96aa3bSJed Brown { 50250a96aa3bSJed Brown DM plex; 50260a96aa3bSJed Brown PetscErrorCode ierr; 50270a96aa3bSJed Brown 50280a96aa3bSJed Brown PetscFunctionBegin; 50290a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 50300a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 50310a96aa3bSJed Brown ierr = DMProjectFunctionLocal(plex,time,funcs,ctxs,mode,localX);CHKERRQ(ierr); 50320a96aa3bSJed Brown PetscFunctionReturn(0); 50330a96aa3bSJed Brown } 50340a96aa3bSJed Brown 50350a96aa3bSJed Brown #define DMProjectFunctionLabelLocal_pforest _append_pforest(DMProjectFunctionLabelLocal) 50360a96aa3bSJed Brown static PetscErrorCode DMProjectFunctionLabelLocal_pforest(DM dm, PetscReal time, DMLabel label, PetscInt numIds, const PetscInt ids[], PetscInt Ncc, const PetscInt comps[], PetscErrorCode (**funcs) (PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void*), void **ctxs, InsertMode mode, Vec localX) 50370a96aa3bSJed Brown { 50380a96aa3bSJed Brown DM plex; 50390a96aa3bSJed Brown PetscErrorCode ierr; 50400a96aa3bSJed Brown 50410a96aa3bSJed Brown PetscFunctionBegin; 50420a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 50430a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 50440a96aa3bSJed Brown ierr = DMProjectFunctionLabelLocal(plex,time,label,numIds,ids,Ncc,comps,funcs,ctxs,mode,localX);CHKERRQ(ierr); 50450a96aa3bSJed Brown PetscFunctionReturn(0); 50460a96aa3bSJed Brown } 50470a96aa3bSJed Brown 50480a96aa3bSJed Brown #define DMProjectFieldLocal_pforest _append_pforest(DMProjectFieldLocal) 50490a96aa3bSJed Brown PetscErrorCode DMProjectFieldLocal_pforest(DM dm, PetscReal time, Vec localU,void (**funcs) (PetscInt, PetscInt, PetscInt, 50500a96aa3bSJed Brown const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], 50510a96aa3bSJed Brown const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], 50520a96aa3bSJed Brown PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]),InsertMode mode, Vec localX) 50530a96aa3bSJed Brown { 50540a96aa3bSJed Brown DM plex; 50550a96aa3bSJed Brown PetscErrorCode ierr; 50560a96aa3bSJed Brown 50570a96aa3bSJed Brown PetscFunctionBegin; 50580a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 50590a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 50600a96aa3bSJed Brown ierr = DMProjectFieldLocal(plex,time,localU,funcs,mode,localX);CHKERRQ(ierr); 50610a96aa3bSJed Brown PetscFunctionReturn(0); 50620a96aa3bSJed Brown } 50630a96aa3bSJed Brown 50640a96aa3bSJed Brown #define DMComputeL2Diff_pforest _append_pforest(DMComputeL2Diff) 50650a96aa3bSJed Brown PetscErrorCode DMComputeL2Diff_pforest(DM dm, PetscReal time, PetscErrorCode (**funcs) (PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void*), void **ctxs, Vec X, PetscReal *diff) 50660a96aa3bSJed Brown { 50670a96aa3bSJed Brown DM plex; 50680a96aa3bSJed Brown PetscErrorCode ierr; 50690a96aa3bSJed Brown 50700a96aa3bSJed Brown PetscFunctionBegin; 50710a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 50720a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 50730a96aa3bSJed Brown ierr = DMComputeL2Diff(plex,time,funcs,ctxs,X,diff);CHKERRQ(ierr); 50740a96aa3bSJed Brown PetscFunctionReturn(0); 50750a96aa3bSJed Brown } 50760a96aa3bSJed Brown 50770a96aa3bSJed Brown #define DMComputeL2FieldDiff_pforest _append_pforest(DMComputeL2FieldDiff) 50780a96aa3bSJed Brown PetscErrorCode DMComputeL2FieldDiff_pforest(DM dm, PetscReal time, PetscErrorCode (**funcs) (PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void*), void **ctxs, Vec X, PetscReal diff[]) 50790a96aa3bSJed Brown { 50800a96aa3bSJed Brown DM plex; 50810a96aa3bSJed Brown PetscErrorCode ierr; 50820a96aa3bSJed Brown 50830a96aa3bSJed Brown PetscFunctionBegin; 50840a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 50850a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 50860a96aa3bSJed Brown ierr = DMComputeL2FieldDiff(plex,time,funcs,ctxs,X,diff);CHKERRQ(ierr); 50870a96aa3bSJed Brown PetscFunctionReturn(0); 50880a96aa3bSJed Brown } 50890a96aa3bSJed Brown 50900a96aa3bSJed Brown #define DMCreatelocalsection_pforest _append_pforest(DMCreatelocalsection) 50910a96aa3bSJed Brown static PetscErrorCode DMCreatelocalsection_pforest(DM dm) 50920a96aa3bSJed Brown { 50930a96aa3bSJed Brown DM plex; 50940a96aa3bSJed Brown PetscSection section; 50950a96aa3bSJed Brown PetscErrorCode ierr; 50960a96aa3bSJed Brown 50970a96aa3bSJed Brown PetscFunctionBegin; 50980a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 50990a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 51000a96aa3bSJed Brown ierr = DMGetLocalSection(plex,§ion);CHKERRQ(ierr); 51010a96aa3bSJed Brown ierr = DMSetLocalSection(dm,section);CHKERRQ(ierr); 51020a96aa3bSJed Brown PetscFunctionReturn(0); 51030a96aa3bSJed Brown } 51040a96aa3bSJed Brown 51050a96aa3bSJed Brown #define DMCreateDefaultConstraints_pforest _append_pforest(DMCreateDefaultConstraints) 51060a96aa3bSJed Brown static PetscErrorCode DMCreateDefaultConstraints_pforest(DM dm) 51070a96aa3bSJed Brown { 51080a96aa3bSJed Brown DM plex; 51090a96aa3bSJed Brown Mat mat; 51100a96aa3bSJed Brown PetscSection section; 51110a96aa3bSJed Brown PetscErrorCode ierr; 51120a96aa3bSJed Brown 51130a96aa3bSJed Brown PetscFunctionBegin; 51140a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 51150a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 51160a96aa3bSJed Brown ierr = DMGetDefaultConstraints(plex,§ion,&mat);CHKERRQ(ierr); 51170a96aa3bSJed Brown ierr = DMSetDefaultConstraints(dm,section,mat);CHKERRQ(ierr); 51180a96aa3bSJed Brown PetscFunctionReturn(0); 51190a96aa3bSJed Brown } 51200a96aa3bSJed Brown 51210a96aa3bSJed Brown #define DMGetDimPoints_pforest _append_pforest(DMGetDimPoints) 51220a96aa3bSJed Brown static PetscErrorCode DMGetDimPoints_pforest(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd) 51230a96aa3bSJed Brown { 51240a96aa3bSJed Brown DM plex; 51250a96aa3bSJed Brown PetscErrorCode ierr; 51260a96aa3bSJed Brown 51270a96aa3bSJed Brown PetscFunctionBegin; 51280a96aa3bSJed Brown PetscValidHeaderSpecific(dm,DM_CLASSID,1); 51290a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 51300a96aa3bSJed Brown ierr = DMGetDimPoints(plex,dim,cStart,cEnd);CHKERRQ(ierr); 51310a96aa3bSJed Brown PetscFunctionReturn(0); 51320a96aa3bSJed Brown } 51330a96aa3bSJed Brown 51340a96aa3bSJed Brown /* Need to forward declare */ 51350a96aa3bSJed Brown #define DMInitialize_pforest _append_pforest(DMInitialize) 51360a96aa3bSJed Brown static PetscErrorCode DMInitialize_pforest(DM dm); 51370a96aa3bSJed Brown 51380a96aa3bSJed Brown #define DMClone_pforest _append_pforest(DMClone) 51390a96aa3bSJed Brown static PetscErrorCode DMClone_pforest(DM dm, DM *newdm) 51400a96aa3bSJed Brown { 51410a96aa3bSJed Brown PetscErrorCode ierr; 51420a96aa3bSJed Brown 51430a96aa3bSJed Brown PetscFunctionBegin; 51440a96aa3bSJed Brown ierr = DMClone_Forest(dm,newdm);CHKERRQ(ierr); 51450a96aa3bSJed Brown ierr = DMInitialize_pforest(*newdm);CHKERRQ(ierr); 51460a96aa3bSJed Brown PetscFunctionReturn(0); 51470a96aa3bSJed Brown } 51480a96aa3bSJed Brown 51490a96aa3bSJed Brown #define DMForestCreateCellChart_pforest _append_pforest(DMForestCreateCellChart) 51500a96aa3bSJed Brown static PetscErrorCode DMForestCreateCellChart_pforest(DM dm, PetscInt *cStart, PetscInt *cEnd) 51510a96aa3bSJed Brown { 51520a96aa3bSJed Brown DM_Forest *forest; 51530a96aa3bSJed Brown DM_Forest_pforest *pforest; 51540a96aa3bSJed Brown PetscInt overlap; 51550a96aa3bSJed Brown PetscErrorCode ierr; 51560a96aa3bSJed Brown 51570a96aa3bSJed Brown PetscFunctionBegin; 51580a96aa3bSJed Brown ierr = DMSetUp(dm);CHKERRQ(ierr); 51590a96aa3bSJed Brown forest = (DM_Forest*) dm->data; 51600a96aa3bSJed Brown pforest = (DM_Forest_pforest*) forest->data; 51610a96aa3bSJed Brown *cStart = 0; 51620a96aa3bSJed Brown ierr = DMForestGetPartitionOverlap(dm,&overlap);CHKERRQ(ierr); 51630a96aa3bSJed Brown if (overlap && pforest->ghost) { 51640a96aa3bSJed Brown *cEnd = pforest->forest->local_num_quadrants + pforest->ghost->proc_offsets[pforest->forest->mpisize]; 51650a96aa3bSJed Brown } else { 51660a96aa3bSJed Brown *cEnd = pforest->forest->local_num_quadrants; 51670a96aa3bSJed Brown } 51680a96aa3bSJed Brown PetscFunctionReturn(0); 51690a96aa3bSJed Brown } 51700a96aa3bSJed Brown 51710a96aa3bSJed Brown #define DMForestCreateCellSF_pforest _append_pforest(DMForestCreateCellSF) 51720a96aa3bSJed Brown static PetscErrorCode DMForestCreateCellSF_pforest(DM dm, PetscSF *cellSF) 51730a96aa3bSJed Brown { 51740a96aa3bSJed Brown DM_Forest *forest; 51750a96aa3bSJed Brown DM_Forest_pforest *pforest; 51760a96aa3bSJed Brown PetscMPIInt rank; 51770a96aa3bSJed Brown PetscInt overlap; 51780a96aa3bSJed Brown PetscInt cStart, cEnd, cLocalStart, cLocalEnd; 51790a96aa3bSJed Brown PetscInt nRoots, nLeaves, *mine = NULL; 51800a96aa3bSJed Brown PetscSFNode *remote = NULL; 51810a96aa3bSJed Brown PetscSF sf; 51820a96aa3bSJed Brown PetscErrorCode ierr; 51830a96aa3bSJed Brown 51840a96aa3bSJed Brown PetscFunctionBegin; 51850a96aa3bSJed Brown ierr = DMForestGetCellChart(dm,&cStart,&cEnd);CHKERRQ(ierr); 51860a96aa3bSJed Brown forest = (DM_Forest*) dm->data; 51870a96aa3bSJed Brown pforest = (DM_Forest_pforest*) forest->data; 51880a96aa3bSJed Brown nRoots = cEnd - cStart; 51890a96aa3bSJed Brown cLocalStart = pforest->cLocalStart; 51900a96aa3bSJed Brown cLocalEnd = pforest->cLocalEnd; 51910a96aa3bSJed Brown nLeaves = 0; 51920a96aa3bSJed Brown ierr = DMForestGetPartitionOverlap(dm,&overlap);CHKERRQ(ierr); 51930a96aa3bSJed Brown ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm),&rank);CHKERRMPI(ierr); 51940a96aa3bSJed Brown if (overlap && pforest->ghost) { 51950a96aa3bSJed Brown PetscSFNode *mirror; 51960a96aa3bSJed Brown p4est_quadrant_t *mirror_array; 51970a96aa3bSJed Brown PetscInt nMirror, nGhostPre, nSelf, q; 51980a96aa3bSJed Brown void **mirrorPtrs; 51990a96aa3bSJed Brown 52000a96aa3bSJed Brown nMirror = (PetscInt) pforest->ghost->mirrors.elem_count; 52010a96aa3bSJed Brown nSelf = cLocalEnd - cLocalStart; 52020a96aa3bSJed Brown nLeaves = nRoots - nSelf; 52030a96aa3bSJed Brown nGhostPre = (PetscInt) pforest->ghost->proc_offsets[rank]; 52040a96aa3bSJed Brown ierr = PetscMalloc1(nLeaves,&mine);CHKERRQ(ierr); 52050a96aa3bSJed Brown ierr = PetscMalloc1(nLeaves,&remote);CHKERRQ(ierr); 52060a96aa3bSJed Brown ierr = PetscMalloc2(nMirror,&mirror,nMirror,&mirrorPtrs);CHKERRQ(ierr); 52070a96aa3bSJed Brown mirror_array = (p4est_quadrant_t*) pforest->ghost->mirrors.array; 52080a96aa3bSJed Brown for (q = 0; q < nMirror; q++) { 52090a96aa3bSJed Brown p4est_quadrant_t *mir = &(mirror_array[q]); 52100a96aa3bSJed Brown 52110a96aa3bSJed Brown mirror[q].rank = rank; 52120a96aa3bSJed Brown mirror[q].index = (PetscInt) mir->p.piggy3.local_num + cLocalStart; 52130a96aa3bSJed Brown mirrorPtrs[q] = (void*) &(mirror[q]); 52140a96aa3bSJed Brown } 52150a96aa3bSJed Brown PetscStackCallP4est(p4est_ghost_exchange_custom,(pforest->forest,pforest->ghost,sizeof(PetscSFNode),mirrorPtrs,remote)); 52160a96aa3bSJed Brown ierr = PetscFree2(mirror,mirrorPtrs);CHKERRQ(ierr); 52170a96aa3bSJed Brown for (q = 0; q < nGhostPre; q++) mine[q] = q; 52180a96aa3bSJed Brown for (; q < nLeaves; q++) mine[q] = (q - nGhostPre) + cLocalEnd; 52190a96aa3bSJed Brown } 52200a96aa3bSJed Brown ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm),&sf);CHKERRQ(ierr); 52210a96aa3bSJed Brown ierr = PetscSFSetGraph(sf,nRoots,nLeaves,mine,PETSC_OWN_POINTER,remote,PETSC_OWN_POINTER);CHKERRQ(ierr); 52220a96aa3bSJed Brown *cellSF = sf; 52230a96aa3bSJed Brown PetscFunctionReturn(0); 52240a96aa3bSJed Brown } 52250a96aa3bSJed Brown 52260a96aa3bSJed Brown static PetscErrorCode DMCreateNeumannOverlap_pforest(DM dm, IS* ovl, Mat *J, PetscErrorCode (**setup)(Mat, PetscReal, Vec, Vec, PetscReal, IS, void*), void **setup_ctx) 52270a96aa3bSJed Brown { 52280a96aa3bSJed Brown DM plex; 52290a96aa3bSJed Brown PetscErrorCode ierr; 52300a96aa3bSJed Brown 52310a96aa3bSJed Brown PetscFunctionBegin; 52320a96aa3bSJed Brown ierr = DMPforestGetPlex(dm,&plex);CHKERRQ(ierr); 52330a96aa3bSJed Brown ierr = DMCreateNeumannOverlap_Plex(plex,ovl,J,setup,setup_ctx);CHKERRQ(ierr); 52340a96aa3bSJed Brown if (!*setup) { 52350a96aa3bSJed Brown ierr = PetscObjectQueryFunction((PetscObject)dm, "MatComputeNeumannOverlap_C", setup);CHKERRQ(ierr); 52360a96aa3bSJed Brown if (*setup) { 52370a96aa3bSJed Brown ierr = PetscObjectCompose((PetscObject)*ovl, "_DM_Original_HPDDM", (PetscObject)dm);CHKERRQ(ierr); 52380a96aa3bSJed Brown } 52390a96aa3bSJed Brown } 52400a96aa3bSJed Brown PetscFunctionReturn(0); 52410a96aa3bSJed Brown } 52420a96aa3bSJed Brown 52430a96aa3bSJed Brown static PetscErrorCode DMInitialize_pforest(DM dm) 52440a96aa3bSJed Brown { 52450a96aa3bSJed Brown PetscErrorCode ierr; 52460a96aa3bSJed Brown 52470a96aa3bSJed Brown PetscFunctionBegin; 52480a96aa3bSJed Brown dm->ops->setup = DMSetUp_pforest; 52490a96aa3bSJed Brown dm->ops->view = DMView_pforest; 52500a96aa3bSJed Brown dm->ops->clone = DMClone_pforest; 52510a96aa3bSJed Brown dm->ops->createinterpolation = DMCreateInterpolation_pforest; 52520a96aa3bSJed Brown dm->ops->createinjection = DMCreateInjection_pforest; 52530a96aa3bSJed Brown dm->ops->setfromoptions = DMSetFromOptions_pforest; 52540a96aa3bSJed Brown dm->ops->createcoordinatedm = DMCreateCoordinateDM_pforest; 52550a96aa3bSJed Brown dm->ops->createglobalvector = DMCreateGlobalVector_pforest; 52560a96aa3bSJed Brown dm->ops->createlocalvector = DMCreateLocalVector_pforest; 52570a96aa3bSJed Brown dm->ops->creatematrix = DMCreateMatrix_pforest; 52580a96aa3bSJed Brown dm->ops->projectfunctionlocal = DMProjectFunctionLocal_pforest; 52590a96aa3bSJed Brown dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_pforest; 52600a96aa3bSJed Brown dm->ops->projectfieldlocal = DMProjectFieldLocal_pforest; 52610a96aa3bSJed Brown dm->ops->createlocalsection = DMCreatelocalsection_pforest; 52620a96aa3bSJed Brown dm->ops->createdefaultconstraints = DMCreateDefaultConstraints_pforest; 52630a96aa3bSJed Brown dm->ops->computel2diff = DMComputeL2Diff_pforest; 52640a96aa3bSJed Brown dm->ops->computel2fielddiff = DMComputeL2FieldDiff_pforest; 52650a96aa3bSJed Brown dm->ops->getdimpoints = DMGetDimPoints_pforest; 52660a96aa3bSJed Brown 52670a96aa3bSJed Brown ierr = PetscObjectComposeFunction((PetscObject)dm,PetscStringize(DMConvert_plex_pforest) "_C",DMConvert_plex_pforest);CHKERRQ(ierr); 52680a96aa3bSJed Brown ierr = PetscObjectComposeFunction((PetscObject)dm,PetscStringize(DMConvert_pforest_plex) "_C",DMConvert_pforest_plex);CHKERRQ(ierr); 52690a96aa3bSJed Brown ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C",DMCreateNeumannOverlap_pforest);CHKERRQ(ierr); 52700a96aa3bSJed Brown ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexGetOverlap_C",DMForestGetPartitionOverlap);CHKERRQ(ierr); 52710a96aa3bSJed Brown PetscFunctionReturn(0); 52720a96aa3bSJed Brown } 52730a96aa3bSJed Brown 52740a96aa3bSJed Brown #define DMCreate_pforest _append_pforest(DMCreate) 52750a96aa3bSJed Brown PETSC_EXTERN PetscErrorCode DMCreate_pforest(DM dm) 52760a96aa3bSJed Brown { 52770a96aa3bSJed Brown DM_Forest *forest; 52780a96aa3bSJed Brown DM_Forest_pforest *pforest; 52790a96aa3bSJed Brown PetscErrorCode ierr; 52800a96aa3bSJed Brown 52810a96aa3bSJed Brown PetscFunctionBegin; 52820a96aa3bSJed Brown ierr = PetscP4estInitialize();CHKERRQ(ierr); 52830a96aa3bSJed Brown ierr = DMCreate_Forest(dm);CHKERRQ(ierr); 52840a96aa3bSJed Brown ierr = DMInitialize_pforest(dm);CHKERRQ(ierr); 52850a96aa3bSJed Brown ierr = DMSetDimension(dm,P4EST_DIM);CHKERRQ(ierr); 52860a96aa3bSJed Brown 52870a96aa3bSJed Brown /* set forest defaults */ 52880a96aa3bSJed Brown ierr = DMForestSetTopology(dm,"unit");CHKERRQ(ierr); 52890a96aa3bSJed Brown ierr = DMForestSetMinimumRefinement(dm,0);CHKERRQ(ierr); 52900a96aa3bSJed Brown ierr = DMForestSetInitialRefinement(dm,0);CHKERRQ(ierr); 52910a96aa3bSJed Brown ierr = DMForestSetMaximumRefinement(dm,P4EST_QMAXLEVEL);CHKERRQ(ierr); 52920a96aa3bSJed Brown ierr = DMForestSetGradeFactor(dm,2);CHKERRQ(ierr); 52930a96aa3bSJed Brown ierr = DMForestSetAdjacencyDimension(dm,0);CHKERRQ(ierr); 52940a96aa3bSJed Brown ierr = DMForestSetPartitionOverlap(dm,0);CHKERRQ(ierr); 52950a96aa3bSJed Brown 52960a96aa3bSJed Brown /* create p4est data */ 52970a96aa3bSJed Brown ierr = PetscNewLog(dm,&pforest);CHKERRQ(ierr); 52980a96aa3bSJed Brown 52990a96aa3bSJed Brown forest = (DM_Forest*) dm->data; 53000a96aa3bSJed Brown forest->data = pforest; 53010a96aa3bSJed Brown forest->destroy = DMForestDestroy_pforest; 53020a96aa3bSJed Brown forest->ftemplate = DMForestTemplate_pforest; 53030a96aa3bSJed Brown forest->transfervec = DMForestTransferVec_pforest; 53040a96aa3bSJed Brown forest->transfervecfrombase = DMForestTransferVecFromBase_pforest; 53050a96aa3bSJed Brown forest->createcellchart = DMForestCreateCellChart_pforest; 53060a96aa3bSJed Brown forest->createcellsf = DMForestCreateCellSF_pforest; 53070a96aa3bSJed Brown forest->clearadaptivityforest = DMForestClearAdaptivityForest_pforest; 53080a96aa3bSJed Brown forest->getadaptivitysuccess = DMForestGetAdaptivitySuccess_pforest; 53090a96aa3bSJed Brown pforest->topo = NULL; 53100a96aa3bSJed Brown pforest->forest = NULL; 53110a96aa3bSJed Brown pforest->ghost = NULL; 53120a96aa3bSJed Brown pforest->lnodes = NULL; 53130a96aa3bSJed Brown pforest->partition_for_coarsening = PETSC_TRUE; 53140a96aa3bSJed Brown pforest->coarsen_hierarchy = PETSC_FALSE; 53150a96aa3bSJed Brown pforest->cLocalStart = -1; 53160a96aa3bSJed Brown pforest->cLocalEnd = -1; 53170a96aa3bSJed Brown pforest->labelsFinalized = PETSC_FALSE; 53180a96aa3bSJed Brown pforest->ghostName = NULL; 53190a96aa3bSJed Brown PetscFunctionReturn(0); 53200a96aa3bSJed Brown } 53210a96aa3bSJed Brown 53220a96aa3bSJed Brown #endif /* defined(PETSC_HAVE_P4EST) */ 5323