1af0996ceSBarry Smith #include <petsc/private/dmpleximpl.h> /*I "petscdmplex.h" I*/ 2af0996ceSBarry Smith #include <petsc/private/isimpl.h> 3af0996ceSBarry Smith #include <petsc/private/petscfeimpl.h> 4d6a7ad0dSToby Isaac #include <petscsf.h> 50c37af3bSToby Isaac #include <petscds.h> 6d6a7ad0dSToby Isaac 70e3d61c9SBarry Smith /* hierarchy routines */ 8d6a7ad0dSToby Isaac 9d6a7ad0dSToby Isaac /*@ 10d6a7ad0dSToby Isaac DMPlexSetReferenceTree - set the reference tree for hierarchically non-conforming meshes. 11d6a7ad0dSToby Isaac 12d6a7ad0dSToby Isaac Not collective 13d6a7ad0dSToby Isaac 14d6a7ad0dSToby Isaac Input Parameters: 15d6a7ad0dSToby Isaac + dm - The DMPlex object 16d6a7ad0dSToby Isaac - ref - The reference tree DMPlex object 17d6a7ad0dSToby Isaac 180b7167a0SToby Isaac Level: intermediate 19d6a7ad0dSToby Isaac 20db781477SPatrick Sanan .seealso: `DMPlexGetReferenceTree()`, `DMPlexCreateDefaultReferenceTree()` 21d6a7ad0dSToby Isaac @*/ 22*9371c9d4SSatish Balay PetscErrorCode DMPlexSetReferenceTree(DM dm, DM ref) { 23d6a7ad0dSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 24d6a7ad0dSToby Isaac 25d6a7ad0dSToby Isaac PetscFunctionBegin; 26d6a7ad0dSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 2747a1df27SMatthew G. Knepley if (ref) { PetscValidHeaderSpecific(ref, DM_CLASSID, 2); } 289566063dSJacob Faibussowitsch PetscCall(PetscObjectReference((PetscObject)ref)); 299566063dSJacob Faibussowitsch PetscCall(DMDestroy(&mesh->referenceTree)); 30d6a7ad0dSToby Isaac mesh->referenceTree = ref; 31d6a7ad0dSToby Isaac PetscFunctionReturn(0); 32d6a7ad0dSToby Isaac } 33d6a7ad0dSToby Isaac 34d6a7ad0dSToby Isaac /*@ 35d6a7ad0dSToby Isaac DMPlexGetReferenceTree - get the reference tree for hierarchically non-conforming meshes. 36d6a7ad0dSToby Isaac 37d6a7ad0dSToby Isaac Not collective 38d6a7ad0dSToby Isaac 39d6a7ad0dSToby Isaac Input Parameters: 40d6a7ad0dSToby Isaac . dm - The DMPlex object 41d6a7ad0dSToby Isaac 427a7aea1fSJed Brown Output Parameters: 43d6a7ad0dSToby Isaac . ref - The reference tree DMPlex object 44d6a7ad0dSToby Isaac 450b7167a0SToby Isaac Level: intermediate 46d6a7ad0dSToby Isaac 47db781477SPatrick Sanan .seealso: `DMPlexSetReferenceTree()`, `DMPlexCreateDefaultReferenceTree()` 48d6a7ad0dSToby Isaac @*/ 49*9371c9d4SSatish Balay PetscErrorCode DMPlexGetReferenceTree(DM dm, DM *ref) { 50d6a7ad0dSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 51d6a7ad0dSToby Isaac 52d6a7ad0dSToby Isaac PetscFunctionBegin; 53d6a7ad0dSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 54d6a7ad0dSToby Isaac PetscValidPointer(ref, 2); 55d6a7ad0dSToby Isaac *ref = mesh->referenceTree; 56d6a7ad0dSToby Isaac PetscFunctionReturn(0); 57d6a7ad0dSToby Isaac } 58d6a7ad0dSToby Isaac 59*9371c9d4SSatish Balay static PetscErrorCode DMPlexReferenceTreeGetChildSymmetry_Default(DM dm, PetscInt parent, PetscInt parentOrientA, PetscInt childOrientA, PetscInt childA, PetscInt parentOrientB, PetscInt *childOrientB, PetscInt *childB) { 60dcbd3bf7SToby Isaac PetscInt coneSize, dStart, dEnd, dim, ABswap, oAvert, oBvert, ABswapVert; 61dcbd3bf7SToby Isaac 62dcbd3bf7SToby Isaac PetscFunctionBegin; 63dcbd3bf7SToby Isaac if (parentOrientA == parentOrientB) { 64dcbd3bf7SToby Isaac if (childOrientB) *childOrientB = childOrientA; 65dcbd3bf7SToby Isaac if (childB) *childB = childA; 66dcbd3bf7SToby Isaac PetscFunctionReturn(0); 67dcbd3bf7SToby Isaac } 68dcbd3bf7SToby Isaac for (dim = 0; dim < 3; dim++) { 699566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(dm, dim, &dStart, &dEnd)); 70*9371c9d4SSatish Balay if (parent >= dStart && parent <= dEnd) { break; } 71dcbd3bf7SToby Isaac } 7263a3b9bcSJacob Faibussowitsch PetscCheck(dim <= 2, PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot perform child symmetry for %" PetscInt_FMT "-cells", dim); 7328b400f6SJacob Faibussowitsch PetscCheck(dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "A vertex has no children"); 74dcbd3bf7SToby Isaac if (childA < dStart || childA >= dEnd) { 75dcbd3bf7SToby Isaac /* this is a lower-dimensional child: bootstrap */ 76dcbd3bf7SToby Isaac PetscInt size, i, sA = -1, sB, sOrientB, sConeSize; 77dcbd3bf7SToby Isaac const PetscInt *supp, *coneA, *coneB, *oA, *oB; 78dcbd3bf7SToby Isaac 799566063dSJacob Faibussowitsch PetscCall(DMPlexGetSupportSize(dm, childA, &size)); 809566063dSJacob Faibussowitsch PetscCall(DMPlexGetSupport(dm, childA, &supp)); 81dcbd3bf7SToby Isaac 82dcbd3bf7SToby Isaac /* find a point sA in supp(childA) that has the same parent */ 83dcbd3bf7SToby Isaac for (i = 0; i < size; i++) { 84dcbd3bf7SToby Isaac PetscInt sParent; 85dcbd3bf7SToby Isaac 86dcbd3bf7SToby Isaac sA = supp[i]; 87dcbd3bf7SToby Isaac if (sA == parent) continue; 889566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, sA, &sParent, NULL)); 89*9371c9d4SSatish Balay if (sParent == parent) { break; } 90dcbd3bf7SToby Isaac } 9108401ef6SPierre Jolivet PetscCheck(i != size, PETSC_COMM_SELF, PETSC_ERR_PLIB, "could not find support in children"); 92dcbd3bf7SToby Isaac /* find out which point sB is in an equivalent position to sA under 93dcbd3bf7SToby Isaac * parentOrientB */ 949566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildSymmetry_Default(dm, parent, parentOrientA, 0, sA, parentOrientB, &sOrientB, &sB)); 959566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, sA, &sConeSize)); 969566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, sA, &coneA)); 979566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, sB, &coneB)); 989566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(dm, sA, &oA)); 999566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(dm, sB, &oB)); 100dcbd3bf7SToby Isaac /* step through the cone of sA in natural order */ 101dcbd3bf7SToby Isaac for (i = 0; i < sConeSize; i++) { 102dcbd3bf7SToby Isaac if (coneA[i] == childA) { 103dcbd3bf7SToby Isaac /* if childA is at position i in coneA, 104dcbd3bf7SToby Isaac * then we want the point that is at sOrientB*i in coneB */ 105dcbd3bf7SToby Isaac PetscInt j = (sOrientB >= 0) ? ((sOrientB + i) % sConeSize) : ((sConeSize - (sOrientB + 1) - i) % sConeSize); 106dcbd3bf7SToby Isaac if (childB) *childB = coneB[j]; 107dcbd3bf7SToby Isaac if (childOrientB) { 108b5a892a1SMatthew G. Knepley DMPolytopeType ct; 109dcbd3bf7SToby Isaac PetscInt oBtrue; 110dcbd3bf7SToby Isaac 1119566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, childA, &coneSize)); 112dcbd3bf7SToby Isaac /* compose sOrientB and oB[j] */ 1131dca8a05SBarry Smith PetscCheck(coneSize == 0 || coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected a vertex or an edge"); 114b5a892a1SMatthew G. Knepley ct = coneSize ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT; 115dcbd3bf7SToby Isaac /* we may have to flip an edge */ 116b5a892a1SMatthew G. Knepley oBtrue = (sOrientB >= 0) ? oB[j] : DMPolytopeTypeComposeOrientation(ct, -1, oB[j]); 117b5a892a1SMatthew G. Knepley oBtrue = DMPolytopeConvertNewOrientation_Internal(ct, oBtrue); 118b5a892a1SMatthew G. Knepley ABswap = DihedralSwap(coneSize, DMPolytopeConvertNewOrientation_Internal(ct, oA[i]), oBtrue); 119dcbd3bf7SToby Isaac *childOrientB = DihedralCompose(coneSize, childOrientA, ABswap); 120dcbd3bf7SToby Isaac } 121dcbd3bf7SToby Isaac break; 122dcbd3bf7SToby Isaac } 123dcbd3bf7SToby Isaac } 12408401ef6SPierre Jolivet PetscCheck(i != sConeSize, PETSC_COMM_SELF, PETSC_ERR_PLIB, "support cone mismatch"); 125dcbd3bf7SToby Isaac PetscFunctionReturn(0); 126dcbd3bf7SToby Isaac } 127dcbd3bf7SToby Isaac /* get the cone size and symmetry swap */ 1289566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, parent, &coneSize)); 129dcbd3bf7SToby Isaac ABswap = DihedralSwap(coneSize, parentOrientA, parentOrientB); 130dcbd3bf7SToby Isaac if (dim == 2) { 131dcbd3bf7SToby Isaac /* orientations refer to cones: we want them to refer to vertices: 132dcbd3bf7SToby Isaac * if it's a rotation, they are the same, but if the order is reversed, a 133dcbd3bf7SToby Isaac * permutation that puts side i first does *not* put vertex i first */ 134dcbd3bf7SToby Isaac oAvert = (parentOrientA >= 0) ? parentOrientA : -((-parentOrientA % coneSize) + 1); 135dcbd3bf7SToby Isaac oBvert = (parentOrientB >= 0) ? parentOrientB : -((-parentOrientB % coneSize) + 1); 136dcbd3bf7SToby Isaac ABswapVert = DihedralSwap(coneSize, oAvert, oBvert); 137947b95d8SBarry Smith } else { 138dcbd3bf7SToby Isaac ABswapVert = ABswap; 139dcbd3bf7SToby Isaac } 140dcbd3bf7SToby Isaac if (childB) { 141dcbd3bf7SToby Isaac /* assume that each child corresponds to a vertex, in the same order */ 142dcbd3bf7SToby Isaac PetscInt p, posA = -1, numChildren, i; 143dcbd3bf7SToby Isaac const PetscInt *children; 144dcbd3bf7SToby Isaac 145dcbd3bf7SToby Isaac /* count which position the child is in */ 1469566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(dm, parent, &numChildren, &children)); 147dcbd3bf7SToby Isaac for (i = 0; i < numChildren; i++) { 148dcbd3bf7SToby Isaac p = children[i]; 149dcbd3bf7SToby Isaac if (p == childA) { 150dcbd3bf7SToby Isaac posA = i; 151dcbd3bf7SToby Isaac break; 152dcbd3bf7SToby Isaac } 153dcbd3bf7SToby Isaac } 154dcbd3bf7SToby Isaac if (posA >= coneSize) { 155dcbd3bf7SToby Isaac /* this is the triangle in the middle of a uniformly refined triangle: it 156dcbd3bf7SToby Isaac * is invariant */ 1571dca8a05SBarry Smith PetscCheck(dim == 2 && posA == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Expected a middle triangle, got something else"); 158dcbd3bf7SToby Isaac *childB = childA; 159*9371c9d4SSatish Balay } else { 160dcbd3bf7SToby Isaac /* figure out position B by applying ABswapVert */ 161dcbd3bf7SToby Isaac PetscInt posB; 162dcbd3bf7SToby Isaac 163dcbd3bf7SToby Isaac posB = (ABswapVert >= 0) ? ((ABswapVert + posA) % coneSize) : ((coneSize - (ABswapVert + 1) - posA) % coneSize); 164dcbd3bf7SToby Isaac if (childB) *childB = children[posB]; 165dcbd3bf7SToby Isaac } 166dcbd3bf7SToby Isaac } 167dcbd3bf7SToby Isaac if (childOrientB) *childOrientB = DihedralCompose(coneSize, childOrientA, ABswap); 168dcbd3bf7SToby Isaac PetscFunctionReturn(0); 169dcbd3bf7SToby Isaac } 170dcbd3bf7SToby Isaac 171dcbd3bf7SToby Isaac /*@ 172dcbd3bf7SToby Isaac DMPlexReferenceTreeGetChildSymmetry - Given a reference tree, transform a childid and orientation from one parent frame to another 173dcbd3bf7SToby Isaac 174dcbd3bf7SToby Isaac Input Parameters: 175dcbd3bf7SToby Isaac + dm - the reference tree DMPlex object 176dcbd3bf7SToby Isaac . parent - the parent point 177dcbd3bf7SToby Isaac . parentOrientA - the reference orientation for describing the parent 178dcbd3bf7SToby Isaac . childOrientA - the reference orientation for describing the child 179dcbd3bf7SToby Isaac . childA - the reference childID for describing the child 180dcbd3bf7SToby Isaac - parentOrientB - the new orientation for describing the parent 181dcbd3bf7SToby Isaac 182dcbd3bf7SToby Isaac Output Parameters: 183dcbd3bf7SToby Isaac + childOrientB - if not NULL, set to the new oreintation for describing the child 184ff1f73f7SToby Isaac - childB - if not NULL, the new childID for describing the child 185dcbd3bf7SToby Isaac 186dcbd3bf7SToby Isaac Level: developer 187dcbd3bf7SToby Isaac 188db781477SPatrick Sanan .seealso: `DMPlexGetReferenceTree()`, `DMPlexSetReferenceTree()`, `DMPlexSetTree()` 189dcbd3bf7SToby Isaac @*/ 190*9371c9d4SSatish Balay PetscErrorCode DMPlexReferenceTreeGetChildSymmetry(DM dm, PetscInt parent, PetscInt parentOrientA, PetscInt childOrientA, PetscInt childA, PetscInt parentOrientB, PetscInt *childOrientB, PetscInt *childB) { 191dcbd3bf7SToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 192dcbd3bf7SToby Isaac 193dcbd3bf7SToby Isaac PetscFunctionBegin; 194dcbd3bf7SToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 19528b400f6SJacob Faibussowitsch PetscCheck(mesh->getchildsymmetry, PETSC_COMM_SELF, PETSC_ERR_SUP, "DMPlexReferenceTreeGetChildSymmetry not implemented"); 1969566063dSJacob Faibussowitsch PetscCall(mesh->getchildsymmetry(dm, parent, parentOrientA, childOrientA, childA, parentOrientB, childOrientB, childB)); 197dcbd3bf7SToby Isaac PetscFunctionReturn(0); 198dcbd3bf7SToby Isaac } 199dcbd3bf7SToby Isaac 200776742edSToby Isaac static PetscErrorCode DMPlexSetTree_Internal(DM, PetscSection, PetscInt *, PetscInt *, PetscBool, PetscBool); 201f9f063d4SToby Isaac 202*9371c9d4SSatish Balay PetscErrorCode DMPlexCreateReferenceTree_SetTree(DM dm, PetscSection parentSection, PetscInt parents[], PetscInt childIDs[]) { 203f2c1aa1dSLisandro Dalcin PetscFunctionBegin; 2049566063dSJacob Faibussowitsch PetscCall(DMPlexSetTree_Internal(dm, parentSection, parents, childIDs, PETSC_TRUE, PETSC_FALSE)); 205f2c1aa1dSLisandro Dalcin PetscFunctionReturn(0); 206f2c1aa1dSLisandro Dalcin } 207f2c1aa1dSLisandro Dalcin 208*9371c9d4SSatish Balay PetscErrorCode DMPlexCreateReferenceTree_Union(DM K, DM Kref, const char *labelName, DM *ref) { 2090e2cc29aSToby Isaac MPI_Comm comm; 2100e2cc29aSToby Isaac PetscInt dim, p, pStart, pEnd, pRefStart, pRefEnd, d, offset, parentSize, *parents, *childIDs; 211da43764aSToby Isaac PetscInt *permvals, *unionCones, *coneSizes, *unionOrientations, numUnionPoints, *numDimPoints, numCones, numVerts; 212da43764aSToby Isaac DMLabel identity, identityRef; 21310f7e118SToby Isaac PetscSection unionSection, unionConeSection, parentSection; 214da43764aSToby Isaac PetscScalar *unionCoords; 215da43764aSToby Isaac IS perm; 216da43764aSToby Isaac 217da43764aSToby Isaac PetscFunctionBegin; 2180e2cc29aSToby Isaac comm = PetscObjectComm((PetscObject)K); 2199566063dSJacob Faibussowitsch PetscCall(DMGetDimension(K, &dim)); 2209566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(K, &pStart, &pEnd)); 2219566063dSJacob Faibussowitsch PetscCall(DMGetLabel(K, labelName, &identity)); 2229566063dSJacob Faibussowitsch PetscCall(DMGetLabel(Kref, labelName, &identityRef)); 2239566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(Kref, &pRefStart, &pRefEnd)); 2249566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(comm, &unionSection)); 2259566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(unionSection, 0, (pEnd - pStart) + (pRefEnd - pRefStart))); 226da43764aSToby Isaac /* count points that will go in the union */ 227*9371c9d4SSatish Balay for (p = pStart; p < pEnd; p++) { PetscCall(PetscSectionSetDof(unionSection, p - pStart, 1)); } 228da43764aSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 229da43764aSToby Isaac PetscInt q, qSize; 2309566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(identityRef, p, &q)); 2319566063dSJacob Faibussowitsch PetscCall(DMLabelGetStratumSize(identityRef, q, &qSize)); 232*9371c9d4SSatish Balay if (qSize > 1) { PetscCall(PetscSectionSetDof(unionSection, p - pRefStart + (pEnd - pStart), 1)); } 233da43764aSToby Isaac } 2349566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEnd - pStart + pRefEnd - pRefStart, &permvals)); 235da43764aSToby Isaac offset = 0; 236da43764aSToby Isaac /* stratify points in the union by topological dimension */ 237da43764aSToby Isaac for (d = 0; d <= dim; d++) { 238da43764aSToby Isaac PetscInt cStart, cEnd, c; 239da43764aSToby Isaac 2409566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &cStart, &cEnd)); 241*9371c9d4SSatish Balay for (c = cStart; c < cEnd; c++) { permvals[offset++] = c; } 242da43764aSToby Isaac 2439566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(Kref, d, &cStart, &cEnd)); 244*9371c9d4SSatish Balay for (c = cStart; c < cEnd; c++) { permvals[offset++] = c + (pEnd - pStart); } 245da43764aSToby Isaac } 2469566063dSJacob Faibussowitsch PetscCall(ISCreateGeneral(comm, (pEnd - pStart) + (pRefEnd - pRefStart), permvals, PETSC_OWN_POINTER, &perm)); 2479566063dSJacob Faibussowitsch PetscCall(PetscSectionSetPermutation(unionSection, perm)); 2489566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(unionSection)); 2499566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(unionSection, &numUnionPoints)); 2509566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numUnionPoints, &coneSizes, dim + 1, &numDimPoints)); 251da43764aSToby Isaac /* count dimension points */ 252da43764aSToby Isaac for (d = 0; d <= dim; d++) { 253da43764aSToby Isaac PetscInt cStart, cOff, cOff2; 2549566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &cStart, NULL)); 2559566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, cStart - pStart, &cOff)); 256da43764aSToby Isaac if (d < dim) { 2579566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d + 1, &cStart, NULL)); 2589566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, cStart - pStart, &cOff2)); 259*9371c9d4SSatish Balay } else { 260da43764aSToby Isaac cOff2 = numUnionPoints; 261da43764aSToby Isaac } 262da43764aSToby Isaac numDimPoints[dim - d] = cOff2 - cOff; 263da43764aSToby Isaac } 2649566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(comm, &unionConeSection)); 2659566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(unionConeSection, 0, numUnionPoints)); 266da43764aSToby Isaac /* count the cones in the union */ 267da43764aSToby Isaac for (p = pStart; p < pEnd; p++) { 268da43764aSToby Isaac PetscInt dof, uOff; 269da43764aSToby Isaac 2709566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, p, &dof)); 2719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pStart, &uOff)); 2729566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(unionConeSection, uOff, dof)); 273da43764aSToby Isaac coneSizes[uOff] = dof; 274da43764aSToby Isaac } 275da43764aSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 276da43764aSToby Isaac PetscInt dof, uDof, uOff; 277da43764aSToby Isaac 2789566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(Kref, p, &dof)); 2799566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart), &uDof)); 2809566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart), &uOff)); 281da43764aSToby Isaac if (uDof) { 2829566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(unionConeSection, uOff, dof)); 283da43764aSToby Isaac coneSizes[uOff] = dof; 284da43764aSToby Isaac } 285da43764aSToby Isaac } 2869566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(unionConeSection)); 2879566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(unionConeSection, &numCones)); 2889566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numCones, &unionCones, numCones, &unionOrientations)); 289da43764aSToby Isaac /* write the cones in the union */ 290da43764aSToby Isaac for (p = pStart; p < pEnd; p++) { 291da43764aSToby Isaac PetscInt dof, uOff, c, cOff; 292da43764aSToby Isaac const PetscInt *cone, *orientation; 293da43764aSToby Isaac 2949566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, p, &dof)); 2959566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(K, p, &cone)); 2969566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(K, p, &orientation)); 2979566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pStart, &uOff)); 2989566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionConeSection, uOff, &cOff)); 299da43764aSToby Isaac for (c = 0; c < dof; c++) { 300da43764aSToby Isaac PetscInt e, eOff; 301da43764aSToby Isaac e = cone[c]; 3029566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, e - pStart, &eOff)); 303da43764aSToby Isaac unionCones[cOff + c] = eOff; 304da43764aSToby Isaac unionOrientations[cOff + c] = orientation[c]; 305da43764aSToby Isaac } 306da43764aSToby Isaac } 307da43764aSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 308da43764aSToby Isaac PetscInt dof, uDof, uOff, c, cOff; 309da43764aSToby Isaac const PetscInt *cone, *orientation; 310da43764aSToby Isaac 3119566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(Kref, p, &dof)); 3129566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(Kref, p, &cone)); 3139566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(Kref, p, &orientation)); 3149566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart), &uDof)); 3159566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart), &uOff)); 316da43764aSToby Isaac if (uDof) { 3179566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionConeSection, uOff, &cOff)); 318da43764aSToby Isaac for (c = 0; c < dof; c++) { 319da43764aSToby Isaac PetscInt e, eOff, eDof; 320da43764aSToby Isaac 321da43764aSToby Isaac e = cone[c]; 3229566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, e - pRefStart + (pEnd - pStart), &eDof)); 323da43764aSToby Isaac if (eDof) { 3249566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, e - pRefStart + (pEnd - pStart), &eOff)); 325*9371c9d4SSatish Balay } else { 3269566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(identityRef, e, &e)); 3279566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, e - pStart, &eOff)); 328da43764aSToby Isaac } 329da43764aSToby Isaac unionCones[cOff + c] = eOff; 330da43764aSToby Isaac unionOrientations[cOff + c] = orientation[c]; 331da43764aSToby Isaac } 332da43764aSToby Isaac } 333da43764aSToby Isaac } 334da43764aSToby Isaac /* get the coordinates */ 335da43764aSToby Isaac { 336da43764aSToby Isaac PetscInt vStart, vEnd, vRefStart, vRefEnd, v, vDof, vOff; 337da43764aSToby Isaac PetscSection KcoordsSec, KrefCoordsSec; 338da43764aSToby Isaac Vec KcoordsVec, KrefCoordsVec; 339da43764aSToby Isaac PetscScalar *Kcoords; 340da43764aSToby Isaac 3419566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateSection(K, &KcoordsSec)); 3429566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(K, &KcoordsVec)); 3439566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateSection(Kref, &KrefCoordsSec)); 3449566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(Kref, &KrefCoordsVec)); 345da43764aSToby Isaac 346da43764aSToby Isaac numVerts = numDimPoints[0]; 3479566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numVerts * dim, &unionCoords)); 3489566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(K, 0, &vStart, &vEnd)); 349da43764aSToby Isaac 350da43764aSToby Isaac offset = 0; 351da43764aSToby Isaac for (v = vStart; v < vEnd; v++) { 3529566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, v - pStart, &vOff)); 3539566063dSJacob Faibussowitsch PetscCall(VecGetValuesSection(KcoordsVec, KcoordsSec, v, &Kcoords)); 354*9371c9d4SSatish Balay for (d = 0; d < dim; d++) { unionCoords[offset * dim + d] = Kcoords[d]; } 355da43764aSToby Isaac offset++; 356da43764aSToby Isaac } 3579566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(Kref, 0, &vRefStart, &vRefEnd)); 358da43764aSToby Isaac for (v = vRefStart; v < vRefEnd; v++) { 3599566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, v - pRefStart + (pEnd - pStart), &vDof)); 3609566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, v - pRefStart + (pEnd - pStart), &vOff)); 3619566063dSJacob Faibussowitsch PetscCall(VecGetValuesSection(KrefCoordsVec, KrefCoordsSec, v, &Kcoords)); 362da43764aSToby Isaac if (vDof) { 363*9371c9d4SSatish Balay for (d = 0; d < dim; d++) { unionCoords[offset * dim + d] = Kcoords[d]; } 364da43764aSToby Isaac offset++; 365da43764aSToby Isaac } 366da43764aSToby Isaac } 367da43764aSToby Isaac } 3689566063dSJacob Faibussowitsch PetscCall(DMCreate(comm, ref)); 3699566063dSJacob Faibussowitsch PetscCall(DMSetType(*ref, DMPLEX)); 3709566063dSJacob Faibussowitsch PetscCall(DMSetDimension(*ref, dim)); 3719566063dSJacob Faibussowitsch PetscCall(DMPlexCreateFromDAG(*ref, dim, numDimPoints, coneSizes, unionCones, unionOrientations, unionCoords)); 37210f7e118SToby Isaac /* set the tree */ 3739566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(comm, &parentSection)); 3749566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(parentSection, 0, numUnionPoints)); 37510f7e118SToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 37610f7e118SToby Isaac PetscInt uDof, uOff; 37710f7e118SToby Isaac 3789566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart), &uDof)); 3799566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart), &uOff)); 3801baa6e33SBarry Smith if (uDof) PetscCall(PetscSectionSetDof(parentSection, uOff, 1)); 38110f7e118SToby Isaac } 3829566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(parentSection)); 3839566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(parentSection, &parentSize)); 3849566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(parentSize, &parents, parentSize, &childIDs)); 38510f7e118SToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 38610f7e118SToby Isaac PetscInt uDof, uOff; 38710f7e118SToby Isaac 3889566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart), &uDof)); 3899566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart), &uOff)); 39010f7e118SToby Isaac if (uDof) { 39110f7e118SToby Isaac PetscInt pOff, parent, parentU; 3929566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(parentSection, uOff, &pOff)); 3939566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(identityRef, p, &parent)); 3949566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, parent - pStart, &parentU)); 39510f7e118SToby Isaac parents[pOff] = parentU; 39610f7e118SToby Isaac childIDs[pOff] = uOff; 39710f7e118SToby Isaac } 39810f7e118SToby Isaac } 3999566063dSJacob Faibussowitsch PetscCall(DMPlexCreateReferenceTree_SetTree(*ref, parentSection, parents, childIDs)); 4009566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&parentSection)); 4019566063dSJacob Faibussowitsch PetscCall(PetscFree2(parents, childIDs)); 40210f7e118SToby Isaac 403da43764aSToby Isaac /* clean up */ 4049566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&unionSection)); 4059566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&unionConeSection)); 4069566063dSJacob Faibussowitsch PetscCall(ISDestroy(&perm)); 4079566063dSJacob Faibussowitsch PetscCall(PetscFree(unionCoords)); 4089566063dSJacob Faibussowitsch PetscCall(PetscFree2(unionCones, unionOrientations)); 4099566063dSJacob Faibussowitsch PetscCall(PetscFree2(coneSizes, numDimPoints)); 4100e2cc29aSToby Isaac PetscFunctionReturn(0); 4110e2cc29aSToby Isaac } 4120e2cc29aSToby Isaac 4130e2cc29aSToby Isaac /*@ 4140e2cc29aSToby Isaac DMPlexCreateDefaultReferenceTree - create a reference tree for isotropic hierarchical mesh refinement. 4150e2cc29aSToby Isaac 416d083f849SBarry Smith Collective 4170e2cc29aSToby Isaac 4180e2cc29aSToby Isaac Input Parameters: 4190e2cc29aSToby Isaac + comm - the MPI communicator 4200e2cc29aSToby Isaac . dim - the spatial dimension 4210e2cc29aSToby Isaac - simplex - Flag for simplex, otherwise use a tensor-product cell 4220e2cc29aSToby Isaac 4230e2cc29aSToby Isaac Output Parameters: 4240e2cc29aSToby Isaac . ref - the reference tree DMPlex object 4250e2cc29aSToby Isaac 4260e2cc29aSToby Isaac Level: intermediate 4270e2cc29aSToby Isaac 428db781477SPatrick Sanan .seealso: `DMPlexSetReferenceTree()`, `DMPlexGetReferenceTree()` 4290e2cc29aSToby Isaac @*/ 430*9371c9d4SSatish Balay PetscErrorCode DMPlexCreateDefaultReferenceTree(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *ref) { 4310e2cc29aSToby Isaac DM_Plex *mesh; 4320e2cc29aSToby Isaac DM K, Kref; 4330e2cc29aSToby Isaac PetscInt p, pStart, pEnd; 4340e2cc29aSToby Isaac DMLabel identity; 4350e2cc29aSToby Isaac 4360e2cc29aSToby Isaac PetscFunctionBegin; 4370e2cc29aSToby Isaac #if 1 4380e2cc29aSToby Isaac comm = PETSC_COMM_SELF; 4390e2cc29aSToby Isaac #endif 4400e2cc29aSToby Isaac /* create a reference element */ 4419566063dSJacob Faibussowitsch PetscCall(DMPlexCreateReferenceCell(comm, DMPolytopeTypeSimpleShape(dim, simplex), &K)); 4429566063dSJacob Faibussowitsch PetscCall(DMCreateLabel(K, "identity")); 4439566063dSJacob Faibussowitsch PetscCall(DMGetLabel(K, "identity", &identity)); 4449566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(K, &pStart, &pEnd)); 445*9371c9d4SSatish Balay for (p = pStart; p < pEnd; p++) { PetscCall(DMLabelSetValue(identity, p, p)); } 4460e2cc29aSToby Isaac /* refine it */ 4479566063dSJacob Faibussowitsch PetscCall(DMRefine(K, comm, &Kref)); 4480e2cc29aSToby Isaac 4490e2cc29aSToby Isaac /* the reference tree is the union of these two, without duplicating 4500e2cc29aSToby Isaac * points that appear in both */ 4519566063dSJacob Faibussowitsch PetscCall(DMPlexCreateReferenceTree_Union(K, Kref, "identity", ref)); 4520e2cc29aSToby Isaac mesh = (DM_Plex *)(*ref)->data; 4530e2cc29aSToby Isaac mesh->getchildsymmetry = DMPlexReferenceTreeGetChildSymmetry_Default; 4549566063dSJacob Faibussowitsch PetscCall(DMDestroy(&K)); 4559566063dSJacob Faibussowitsch PetscCall(DMDestroy(&Kref)); 456da43764aSToby Isaac PetscFunctionReturn(0); 457da43764aSToby Isaac } 458da43764aSToby Isaac 459*9371c9d4SSatish Balay static PetscErrorCode DMPlexTreeSymmetrize(DM dm) { 460878b19aaSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 461878b19aaSToby Isaac PetscSection childSec, pSec; 462878b19aaSToby Isaac PetscInt p, pSize, cSize, parMax = PETSC_MIN_INT, parMin = PETSC_MAX_INT; 463878b19aaSToby Isaac PetscInt *offsets, *children, pStart, pEnd; 464878b19aaSToby Isaac 465878b19aaSToby Isaac PetscFunctionBegin; 466878b19aaSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 4679566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&mesh->childSection)); 4689566063dSJacob Faibussowitsch PetscCall(PetscFree(mesh->children)); 469878b19aaSToby Isaac pSec = mesh->parentSection; 470878b19aaSToby Isaac if (!pSec) PetscFunctionReturn(0); 4719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(pSec, &pSize)); 472878b19aaSToby Isaac for (p = 0; p < pSize; p++) { 473878b19aaSToby Isaac PetscInt par = mesh->parents[p]; 474878b19aaSToby Isaac 475878b19aaSToby Isaac parMax = PetscMax(parMax, par + 1); 476878b19aaSToby Isaac parMin = PetscMin(parMin, par); 477878b19aaSToby Isaac } 478878b19aaSToby Isaac if (parMin > parMax) { 479878b19aaSToby Isaac parMin = -1; 480878b19aaSToby Isaac parMax = -1; 481878b19aaSToby Isaac } 4829566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)pSec), &childSec)); 4839566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(childSec, parMin, parMax)); 484878b19aaSToby Isaac for (p = 0; p < pSize; p++) { 485878b19aaSToby Isaac PetscInt par = mesh->parents[p]; 486878b19aaSToby Isaac 4879566063dSJacob Faibussowitsch PetscCall(PetscSectionAddDof(childSec, par, 1)); 488878b19aaSToby Isaac } 4899566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(childSec)); 4909566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(childSec, &cSize)); 4919566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(cSize, &children)); 4929566063dSJacob Faibussowitsch PetscCall(PetscCalloc1(parMax - parMin, &offsets)); 4939566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd)); 494878b19aaSToby Isaac for (p = pStart; p < pEnd; p++) { 495878b19aaSToby Isaac PetscInt dof, off, i; 496878b19aaSToby Isaac 4979566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(pSec, p, &dof)); 4989566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(pSec, p, &off)); 499878b19aaSToby Isaac for (i = 0; i < dof; i++) { 500878b19aaSToby Isaac PetscInt par = mesh->parents[off + i], cOff; 501878b19aaSToby Isaac 5029566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(childSec, par, &cOff)); 503878b19aaSToby Isaac children[cOff + offsets[par - parMin]++] = p; 504878b19aaSToby Isaac } 505878b19aaSToby Isaac } 506878b19aaSToby Isaac mesh->childSection = childSec; 507878b19aaSToby Isaac mesh->children = children; 5089566063dSJacob Faibussowitsch PetscCall(PetscFree(offsets)); 509878b19aaSToby Isaac PetscFunctionReturn(0); 510878b19aaSToby Isaac } 511878b19aaSToby Isaac 512*9371c9d4SSatish Balay static PetscErrorCode AnchorsFlatten(PetscSection section, IS is, PetscSection *sectionNew, IS *isNew) { 5136dd5a8c8SToby Isaac PetscInt pStart, pEnd, size, sizeNew, i, p, *valsNew = NULL; 5146dd5a8c8SToby Isaac const PetscInt *vals; 5156dd5a8c8SToby Isaac PetscSection secNew; 5166dd5a8c8SToby Isaac PetscBool anyNew, globalAnyNew; 5176dd5a8c8SToby Isaac PetscBool compress; 5186dd5a8c8SToby Isaac 5196dd5a8c8SToby Isaac PetscFunctionBegin; 5209566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(section, &pStart, &pEnd)); 5219566063dSJacob Faibussowitsch PetscCall(ISGetLocalSize(is, &size)); 5229566063dSJacob Faibussowitsch PetscCall(ISGetIndices(is, &vals)); 5239566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)section), &secNew)); 5249566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(secNew, pStart, pEnd)); 5256dd5a8c8SToby Isaac for (i = 0; i < size; i++) { 5266dd5a8c8SToby Isaac PetscInt dof; 5276dd5a8c8SToby Isaac 5286dd5a8c8SToby Isaac p = vals[i]; 5296dd5a8c8SToby Isaac if (p < pStart || p >= pEnd) continue; 5309566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 5316dd5a8c8SToby Isaac if (dof) break; 5326dd5a8c8SToby Isaac } 5336dd5a8c8SToby Isaac if (i == size) { 5349566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(secNew)); 5356dd5a8c8SToby Isaac anyNew = PETSC_FALSE; 5366dd5a8c8SToby Isaac compress = PETSC_FALSE; 5376dd5a8c8SToby Isaac sizeNew = 0; 538*9371c9d4SSatish Balay } else { 5396dd5a8c8SToby Isaac anyNew = PETSC_TRUE; 5406dd5a8c8SToby Isaac for (p = pStart; p < pEnd; p++) { 5416dd5a8c8SToby Isaac PetscInt dof, off; 5426dd5a8c8SToby Isaac 5439566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 5449566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, p, &off)); 5456dd5a8c8SToby Isaac for (i = 0; i < dof; i++) { 5466dd5a8c8SToby Isaac PetscInt q = vals[off + i], qDof = 0; 5476dd5a8c8SToby Isaac 548*9371c9d4SSatish Balay if (q >= pStart && q < pEnd) { PetscCall(PetscSectionGetDof(section, q, &qDof)); } 5491baa6e33SBarry Smith if (qDof) PetscCall(PetscSectionAddDof(secNew, p, qDof)); 550*9371c9d4SSatish Balay else { PetscCall(PetscSectionAddDof(secNew, p, 1)); } 5516dd5a8c8SToby Isaac } 5526dd5a8c8SToby Isaac } 5539566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(secNew)); 5549566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(secNew, &sizeNew)); 5559566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(sizeNew, &valsNew)); 5566dd5a8c8SToby Isaac compress = PETSC_FALSE; 5576dd5a8c8SToby Isaac for (p = pStart; p < pEnd; p++) { 5586dd5a8c8SToby Isaac PetscInt dof, off, count, offNew, dofNew; 5596dd5a8c8SToby Isaac 5609566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 5619566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, p, &off)); 5629566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(secNew, p, &dofNew)); 5639566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(secNew, p, &offNew)); 5646dd5a8c8SToby Isaac count = 0; 5656dd5a8c8SToby Isaac for (i = 0; i < dof; i++) { 5666dd5a8c8SToby Isaac PetscInt q = vals[off + i], qDof = 0, qOff = 0, j; 5676dd5a8c8SToby Isaac 5686dd5a8c8SToby Isaac if (q >= pStart && q < pEnd) { 5699566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, q, &qDof)); 5709566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, q, &qOff)); 5716dd5a8c8SToby Isaac } 5726dd5a8c8SToby Isaac if (qDof) { 5736dd5a8c8SToby Isaac PetscInt oldCount = count; 5746dd5a8c8SToby Isaac 5756dd5a8c8SToby Isaac for (j = 0; j < qDof; j++) { 5766dd5a8c8SToby Isaac PetscInt k, r = vals[qOff + j]; 5776dd5a8c8SToby Isaac 5786dd5a8c8SToby Isaac for (k = 0; k < oldCount; k++) { 579*9371c9d4SSatish Balay if (valsNew[offNew + k] == r) { break; } 5806dd5a8c8SToby Isaac } 581*9371c9d4SSatish Balay if (k == oldCount) { valsNew[offNew + count++] = r; } 5826dd5a8c8SToby Isaac } 583*9371c9d4SSatish Balay } else { 5846dd5a8c8SToby Isaac PetscInt k, oldCount = count; 5856dd5a8c8SToby Isaac 5866dd5a8c8SToby Isaac for (k = 0; k < oldCount; k++) { 587*9371c9d4SSatish Balay if (valsNew[offNew + k] == q) { break; } 5886dd5a8c8SToby Isaac } 589*9371c9d4SSatish Balay if (k == oldCount) { valsNew[offNew + count++] = q; } 5906dd5a8c8SToby Isaac } 5916dd5a8c8SToby Isaac } 5926dd5a8c8SToby Isaac if (count < dofNew) { 5939566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(secNew, p, count)); 5946dd5a8c8SToby Isaac compress = PETSC_TRUE; 5956dd5a8c8SToby Isaac } 5966dd5a8c8SToby Isaac } 5976dd5a8c8SToby Isaac } 5989566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(is, &vals)); 5991c2dc1cbSBarry Smith PetscCall(MPIU_Allreduce(&anyNew, &globalAnyNew, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)secNew))); 6006dd5a8c8SToby Isaac if (!globalAnyNew) { 6019566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&secNew)); 6026dd5a8c8SToby Isaac *sectionNew = NULL; 6036dd5a8c8SToby Isaac *isNew = NULL; 604*9371c9d4SSatish Balay } else { 6056dd5a8c8SToby Isaac PetscBool globalCompress; 6066dd5a8c8SToby Isaac 6071c2dc1cbSBarry Smith PetscCall(MPIU_Allreduce(&compress, &globalCompress, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)secNew))); 6086dd5a8c8SToby Isaac if (compress) { 6096dd5a8c8SToby Isaac PetscSection secComp; 6106dd5a8c8SToby Isaac PetscInt *valsComp = NULL; 6116dd5a8c8SToby Isaac 6129566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)section), &secComp)); 6139566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(secComp, pStart, pEnd)); 6146dd5a8c8SToby Isaac for (p = pStart; p < pEnd; p++) { 6156dd5a8c8SToby Isaac PetscInt dof; 6166dd5a8c8SToby Isaac 6179566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(secNew, p, &dof)); 6189566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(secComp, p, dof)); 6196dd5a8c8SToby Isaac } 6209566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(secComp)); 6219566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(secComp, &sizeNew)); 6229566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(sizeNew, &valsComp)); 6236dd5a8c8SToby Isaac for (p = pStart; p < pEnd; p++) { 6246dd5a8c8SToby Isaac PetscInt dof, off, offNew, j; 6256dd5a8c8SToby Isaac 6269566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(secNew, p, &dof)); 6279566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(secNew, p, &off)); 6289566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(secComp, p, &offNew)); 629*9371c9d4SSatish Balay for (j = 0; j < dof; j++) { valsComp[offNew + j] = valsNew[off + j]; } 6306dd5a8c8SToby Isaac } 6319566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&secNew)); 6326dd5a8c8SToby Isaac secNew = secComp; 6339566063dSJacob Faibussowitsch PetscCall(PetscFree(valsNew)); 6346dd5a8c8SToby Isaac valsNew = valsComp; 6356dd5a8c8SToby Isaac } 6369566063dSJacob Faibussowitsch PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)is), sizeNew, valsNew, PETSC_OWN_POINTER, isNew)); 6376dd5a8c8SToby Isaac } 6386dd5a8c8SToby Isaac PetscFunctionReturn(0); 6396dd5a8c8SToby Isaac } 6406dd5a8c8SToby Isaac 641*9371c9d4SSatish Balay static PetscErrorCode DMPlexCreateAnchors_Tree(DM dm) { 64266af876cSToby Isaac PetscInt p, pStart, pEnd, *anchors, size; 64366af876cSToby Isaac PetscInt aMin = PETSC_MAX_INT, aMax = PETSC_MIN_INT; 64466af876cSToby Isaac PetscSection aSec; 645f9f063d4SToby Isaac DMLabel canonLabel; 64666af876cSToby Isaac IS aIS; 64766af876cSToby Isaac 64866af876cSToby Isaac PetscFunctionBegin; 64966af876cSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 6509566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 6519566063dSJacob Faibussowitsch PetscCall(DMGetLabel(dm, "canonical", &canonLabel)); 65266af876cSToby Isaac for (p = pStart; p < pEnd; p++) { 65366af876cSToby Isaac PetscInt parent; 65466af876cSToby Isaac 655f9f063d4SToby Isaac if (canonLabel) { 656f9f063d4SToby Isaac PetscInt canon; 657f9f063d4SToby Isaac 6589566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonLabel, p, &canon)); 659f9f063d4SToby Isaac if (p != canon) continue; 660f9f063d4SToby Isaac } 6619566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, p, &parent, NULL)); 66266af876cSToby Isaac if (parent != p) { 66366af876cSToby Isaac aMin = PetscMin(aMin, p); 66466af876cSToby Isaac aMax = PetscMax(aMax, p + 1); 66566af876cSToby Isaac } 66666af876cSToby Isaac } 66766af876cSToby Isaac if (aMin > aMax) { 66866af876cSToby Isaac aMin = -1; 66966af876cSToby Isaac aMax = -1; 67066af876cSToby Isaac } 6719566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PETSC_COMM_SELF, &aSec)); 6729566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(aSec, aMin, aMax)); 67366af876cSToby Isaac for (p = aMin; p < aMax; p++) { 67466af876cSToby Isaac PetscInt parent, ancestor = p; 67566af876cSToby Isaac 676f9f063d4SToby Isaac if (canonLabel) { 677f9f063d4SToby Isaac PetscInt canon; 678f9f063d4SToby Isaac 6799566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonLabel, p, &canon)); 680f9f063d4SToby Isaac if (p != canon) continue; 681f9f063d4SToby Isaac } 6829566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, p, &parent, NULL)); 68366af876cSToby Isaac while (parent != ancestor) { 68466af876cSToby Isaac ancestor = parent; 6859566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, ancestor, &parent, NULL)); 68666af876cSToby Isaac } 68766af876cSToby Isaac if (ancestor != p) { 68866af876cSToby Isaac PetscInt closureSize, *closure = NULL; 68966af876cSToby Isaac 6909566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, ancestor, PETSC_TRUE, &closureSize, &closure)); 6919566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(aSec, p, closureSize)); 6929566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, ancestor, PETSC_TRUE, &closureSize, &closure)); 69366af876cSToby Isaac } 69466af876cSToby Isaac } 6959566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(aSec)); 6969566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(aSec, &size)); 6979566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(size, &anchors)); 69866af876cSToby Isaac for (p = aMin; p < aMax; p++) { 69966af876cSToby Isaac PetscInt parent, ancestor = p; 70066af876cSToby Isaac 701f9f063d4SToby Isaac if (canonLabel) { 702f9f063d4SToby Isaac PetscInt canon; 703f9f063d4SToby Isaac 7049566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonLabel, p, &canon)); 705f9f063d4SToby Isaac if (p != canon) continue; 706f9f063d4SToby Isaac } 7079566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, p, &parent, NULL)); 70866af876cSToby Isaac while (parent != ancestor) { 70966af876cSToby Isaac ancestor = parent; 7109566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, ancestor, &parent, NULL)); 71166af876cSToby Isaac } 71266af876cSToby Isaac if (ancestor != p) { 71366af876cSToby Isaac PetscInt j, closureSize, *closure = NULL, aOff; 71466af876cSToby Isaac 7159566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(aSec, p, &aOff)); 71666af876cSToby Isaac 7179566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, ancestor, PETSC_TRUE, &closureSize, &closure)); 718*9371c9d4SSatish Balay for (j = 0; j < closureSize; j++) { anchors[aOff + j] = closure[2 * j]; } 7199566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, ancestor, PETSC_TRUE, &closureSize, &closure)); 72066af876cSToby Isaac } 72166af876cSToby Isaac } 7229566063dSJacob Faibussowitsch PetscCall(ISCreateGeneral(PETSC_COMM_SELF, size, anchors, PETSC_OWN_POINTER, &aIS)); 7236dd5a8c8SToby Isaac { 7246dd5a8c8SToby Isaac PetscSection aSecNew = aSec; 7256dd5a8c8SToby Isaac IS aISNew = aIS; 7266dd5a8c8SToby Isaac 7279566063dSJacob Faibussowitsch PetscCall(PetscObjectReference((PetscObject)aSec)); 7289566063dSJacob Faibussowitsch PetscCall(PetscObjectReference((PetscObject)aIS)); 7296dd5a8c8SToby Isaac while (aSecNew) { 7309566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&aSec)); 7319566063dSJacob Faibussowitsch PetscCall(ISDestroy(&aIS)); 7326dd5a8c8SToby Isaac aSec = aSecNew; 7336dd5a8c8SToby Isaac aIS = aISNew; 7346dd5a8c8SToby Isaac aSecNew = NULL; 7356dd5a8c8SToby Isaac aISNew = NULL; 7369566063dSJacob Faibussowitsch PetscCall(AnchorsFlatten(aSec, aIS, &aSecNew, &aISNew)); 7376dd5a8c8SToby Isaac } 7386dd5a8c8SToby Isaac } 7399566063dSJacob Faibussowitsch PetscCall(DMPlexSetAnchors(dm, aSec, aIS)); 7409566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&aSec)); 7419566063dSJacob Faibussowitsch PetscCall(ISDestroy(&aIS)); 74266af876cSToby Isaac PetscFunctionReturn(0); 74366af876cSToby Isaac } 74466af876cSToby Isaac 745*9371c9d4SSatish Balay static PetscErrorCode DMPlexGetTrueSupportSize(DM dm, PetscInt p, PetscInt *dof, PetscInt *numTrueSupp) { 7466461c1adSToby Isaac PetscFunctionBegin; 7476461c1adSToby Isaac if (numTrueSupp[p] == -1) { 7486461c1adSToby Isaac PetscInt i, alldof; 7496461c1adSToby Isaac const PetscInt *supp; 7506461c1adSToby Isaac PetscInt count = 0; 7516461c1adSToby Isaac 7529566063dSJacob Faibussowitsch PetscCall(DMPlexGetSupportSize(dm, p, &alldof)); 7539566063dSJacob Faibussowitsch PetscCall(DMPlexGetSupport(dm, p, &supp)); 7546461c1adSToby Isaac for (i = 0; i < alldof; i++) { 7556461c1adSToby Isaac PetscInt q = supp[i], numCones, j; 7566461c1adSToby Isaac const PetscInt *cone; 7576461c1adSToby Isaac 7589566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, q, &numCones)); 7599566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, q, &cone)); 7606461c1adSToby Isaac for (j = 0; j < numCones; j++) { 7616461c1adSToby Isaac if (cone[j] == p) break; 7626461c1adSToby Isaac } 7636461c1adSToby Isaac if (j < numCones) count++; 7646461c1adSToby Isaac } 7656461c1adSToby Isaac numTrueSupp[p] = count; 7666461c1adSToby Isaac } 7676461c1adSToby Isaac *dof = numTrueSupp[p]; 7686461c1adSToby Isaac PetscFunctionReturn(0); 7696461c1adSToby Isaac } 7706461c1adSToby Isaac 771*9371c9d4SSatish Balay static PetscErrorCode DMPlexTreeExchangeSupports(DM dm) { 772776742edSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 773776742edSToby Isaac PetscSection newSupportSection; 774776742edSToby Isaac PetscInt newSize, *newSupports, pStart, pEnd, p, d, depth; 7756461c1adSToby Isaac PetscInt *numTrueSupp; 776776742edSToby Isaac PetscInt *offsets; 777776742edSToby Isaac 778776742edSToby Isaac PetscFunctionBegin; 779776742edSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 780776742edSToby Isaac /* symmetrize the hierarchy */ 7819566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepth(dm, &depth)); 7829566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)(mesh->supportSection)), &newSupportSection)); 7839566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 7849566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(newSupportSection, pStart, pEnd)); 7859566063dSJacob Faibussowitsch PetscCall(PetscCalloc1(pEnd, &offsets)); 7869566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEnd, &numTrueSupp)); 7876461c1adSToby Isaac for (p = 0; p < pEnd; p++) numTrueSupp[p] = -1; 7886461c1adSToby Isaac /* if a point is in the (true) support of q, it should be in the support of 789776742edSToby Isaac * parent(q) */ 790776742edSToby Isaac for (d = 0; d <= depth; d++) { 7919566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(dm, d, &pStart, &pEnd)); 792776742edSToby Isaac for (p = pStart; p < pEnd; ++p) { 793776742edSToby Isaac PetscInt dof, q, qdof, parent; 794776742edSToby Isaac 7959566063dSJacob Faibussowitsch PetscCall(DMPlexGetTrueSupportSize(dm, p, &dof, numTrueSupp)); 7969566063dSJacob Faibussowitsch PetscCall(PetscSectionAddDof(newSupportSection, p, dof)); 797776742edSToby Isaac q = p; 7989566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, q, &parent, NULL)); 799776742edSToby Isaac while (parent != q && parent >= pStart && parent < pEnd) { 800776742edSToby Isaac q = parent; 801776742edSToby Isaac 8029566063dSJacob Faibussowitsch PetscCall(DMPlexGetTrueSupportSize(dm, q, &qdof, numTrueSupp)); 8039566063dSJacob Faibussowitsch PetscCall(PetscSectionAddDof(newSupportSection, p, qdof)); 8049566063dSJacob Faibussowitsch PetscCall(PetscSectionAddDof(newSupportSection, q, dof)); 8059566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, q, &parent, NULL)); 806776742edSToby Isaac } 807776742edSToby Isaac } 808776742edSToby Isaac } 8099566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(newSupportSection)); 8109566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(newSupportSection, &newSize)); 8119566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(newSize, &newSupports)); 812776742edSToby Isaac for (d = 0; d <= depth; d++) { 8139566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(dm, d, &pStart, &pEnd)); 814776742edSToby Isaac for (p = pStart; p < pEnd; p++) { 815776742edSToby Isaac PetscInt dof, off, q, qdof, qoff, newDof, newOff, newqOff, i, parent; 816776742edSToby Isaac 8179566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(mesh->supportSection, p, &dof)); 8189566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off)); 8199566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(newSupportSection, p, &newDof)); 8209566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(newSupportSection, p, &newOff)); 821776742edSToby Isaac for (i = 0; i < dof; i++) { 8226461c1adSToby Isaac PetscInt numCones, j; 8236461c1adSToby Isaac const PetscInt *cone; 8246461c1adSToby Isaac PetscInt q = mesh->supports[off + i]; 8256461c1adSToby Isaac 8269566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, q, &numCones)); 8279566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, q, &cone)); 8286461c1adSToby Isaac for (j = 0; j < numCones; j++) { 8296461c1adSToby Isaac if (cone[j] == p) break; 8306461c1adSToby Isaac } 8316461c1adSToby Isaac if (j < numCones) newSupports[newOff + offsets[p]++] = q; 832776742edSToby Isaac } 833776742edSToby Isaac 834776742edSToby Isaac q = p; 8359566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, q, &parent, NULL)); 836776742edSToby Isaac while (parent != q && parent >= pStart && parent < pEnd) { 837776742edSToby Isaac q = parent; 8389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(mesh->supportSection, q, &qdof)); 8399566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(mesh->supportSection, q, &qoff)); 8409566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(newSupportSection, q, &newqOff)); 841776742edSToby Isaac for (i = 0; i < qdof; i++) { 8426461c1adSToby Isaac PetscInt numCones, j; 8436461c1adSToby Isaac const PetscInt *cone; 8446461c1adSToby Isaac PetscInt r = mesh->supports[qoff + i]; 8456461c1adSToby Isaac 8469566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, r, &numCones)); 8479566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, r, &cone)); 8486461c1adSToby Isaac for (j = 0; j < numCones; j++) { 8496461c1adSToby Isaac if (cone[j] == q) break; 8506461c1adSToby Isaac } 8516461c1adSToby Isaac if (j < numCones) newSupports[newOff + offsets[p]++] = r; 852776742edSToby Isaac } 853776742edSToby Isaac for (i = 0; i < dof; i++) { 8546461c1adSToby Isaac PetscInt numCones, j; 8556461c1adSToby Isaac const PetscInt *cone; 8566461c1adSToby Isaac PetscInt r = mesh->supports[off + i]; 8576461c1adSToby Isaac 8589566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, r, &numCones)); 8599566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, r, &cone)); 8606461c1adSToby Isaac for (j = 0; j < numCones; j++) { 8616461c1adSToby Isaac if (cone[j] == p) break; 8626461c1adSToby Isaac } 8636461c1adSToby Isaac if (j < numCones) newSupports[newqOff + offsets[q]++] = r; 864776742edSToby Isaac } 8659566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, q, &parent, NULL)); 866776742edSToby Isaac } 867776742edSToby Isaac } 868776742edSToby Isaac } 8699566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&mesh->supportSection)); 870776742edSToby Isaac mesh->supportSection = newSupportSection; 8719566063dSJacob Faibussowitsch PetscCall(PetscFree(mesh->supports)); 872776742edSToby Isaac mesh->supports = newSupports; 8739566063dSJacob Faibussowitsch PetscCall(PetscFree(offsets)); 8749566063dSJacob Faibussowitsch PetscCall(PetscFree(numTrueSupp)); 875776742edSToby Isaac 876776742edSToby Isaac PetscFunctionReturn(0); 877776742edSToby Isaac } 878776742edSToby Isaac 879f7c74593SToby Isaac static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_Direct(DM, PetscSection, PetscSection, Mat); 880f7c74593SToby Isaac static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_FromReference(DM, PetscSection, PetscSection, Mat); 881f7c74593SToby Isaac 882*9371c9d4SSatish Balay static PetscErrorCode DMPlexSetTree_Internal(DM dm, PetscSection parentSection, PetscInt *parents, PetscInt *childIDs, PetscBool computeCanonical, PetscBool exchangeSupports) { 883f9f063d4SToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 884f9f063d4SToby Isaac DM refTree; 885f9f063d4SToby Isaac PetscInt size; 886f9f063d4SToby Isaac 887f9f063d4SToby Isaac PetscFunctionBegin; 888f9f063d4SToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 889f9f063d4SToby Isaac PetscValidHeaderSpecific(parentSection, PETSC_SECTION_CLASSID, 2); 8909566063dSJacob Faibussowitsch PetscCall(PetscObjectReference((PetscObject)parentSection)); 8919566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&mesh->parentSection)); 892f9f063d4SToby Isaac mesh->parentSection = parentSection; 8939566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(parentSection, &size)); 894f9f063d4SToby Isaac if (parents != mesh->parents) { 8959566063dSJacob Faibussowitsch PetscCall(PetscFree(mesh->parents)); 8969566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(size, &mesh->parents)); 8979566063dSJacob Faibussowitsch PetscCall(PetscArraycpy(mesh->parents, parents, size)); 898f9f063d4SToby Isaac } 899f9f063d4SToby Isaac if (childIDs != mesh->childIDs) { 9009566063dSJacob Faibussowitsch PetscCall(PetscFree(mesh->childIDs)); 9019566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(size, &mesh->childIDs)); 9029566063dSJacob Faibussowitsch PetscCall(PetscArraycpy(mesh->childIDs, childIDs, size)); 903f9f063d4SToby Isaac } 9049566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(dm, &refTree)); 905f9f063d4SToby Isaac if (refTree) { 906f9f063d4SToby Isaac DMLabel canonLabel; 907f9f063d4SToby Isaac 9089566063dSJacob Faibussowitsch PetscCall(DMGetLabel(refTree, "canonical", &canonLabel)); 909f9f063d4SToby Isaac if (canonLabel) { 910f9f063d4SToby Isaac PetscInt i; 911f9f063d4SToby Isaac 912f9f063d4SToby Isaac for (i = 0; i < size; i++) { 913f9f063d4SToby Isaac PetscInt canon; 9149566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonLabel, mesh->childIDs[i], &canon)); 915*9371c9d4SSatish Balay if (canon >= 0) { mesh->childIDs[i] = canon; } 916f9f063d4SToby Isaac } 917f9f063d4SToby Isaac } 918f7c74593SToby Isaac mesh->computeanchormatrix = DMPlexComputeAnchorMatrix_Tree_FromReference; 9196e0288c8SStefano Zampini } else { 920f7c74593SToby Isaac mesh->computeanchormatrix = DMPlexComputeAnchorMatrix_Tree_Direct; 921f9f063d4SToby Isaac } 9229566063dSJacob Faibussowitsch PetscCall(DMPlexTreeSymmetrize(dm)); 923f9f063d4SToby Isaac if (computeCanonical) { 924f9f063d4SToby Isaac PetscInt d, dim; 925f9f063d4SToby Isaac 926f9f063d4SToby Isaac /* add the canonical label */ 9279566063dSJacob Faibussowitsch PetscCall(DMGetDimension(dm, &dim)); 9289566063dSJacob Faibussowitsch PetscCall(DMCreateLabel(dm, "canonical")); 929f9f063d4SToby Isaac for (d = 0; d <= dim; d++) { 930f9f063d4SToby Isaac PetscInt p, dStart, dEnd, canon = -1, cNumChildren; 931f9f063d4SToby Isaac const PetscInt *cChildren; 932f9f063d4SToby Isaac 9339566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(dm, d, &dStart, &dEnd)); 934f9f063d4SToby Isaac for (p = dStart; p < dEnd; p++) { 9359566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(dm, p, &cNumChildren, &cChildren)); 936f9f063d4SToby Isaac if (cNumChildren) { 937f9f063d4SToby Isaac canon = p; 938f9f063d4SToby Isaac break; 939f9f063d4SToby Isaac } 940f9f063d4SToby Isaac } 941f9f063d4SToby Isaac if (canon == -1) continue; 942f9f063d4SToby Isaac for (p = dStart; p < dEnd; p++) { 943f9f063d4SToby Isaac PetscInt numChildren, i; 944f9f063d4SToby Isaac const PetscInt *children; 945f9f063d4SToby Isaac 9469566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(dm, p, &numChildren, &children)); 947f9f063d4SToby Isaac if (numChildren) { 94863a3b9bcSJacob Faibussowitsch PetscCheck(numChildren == cNumChildren, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "All parent points in a stratum should have the same number of children: %" PetscInt_FMT " != %" PetscInt_FMT, numChildren, cNumChildren); 9499566063dSJacob Faibussowitsch PetscCall(DMSetLabelValue(dm, "canonical", p, canon)); 950*9371c9d4SSatish Balay for (i = 0; i < numChildren; i++) { PetscCall(DMSetLabelValue(dm, "canonical", children[i], cChildren[i])); } 951f9f063d4SToby Isaac } 952f9f063d4SToby Isaac } 953f9f063d4SToby Isaac } 954f9f063d4SToby Isaac } 9551baa6e33SBarry Smith if (exchangeSupports) PetscCall(DMPlexTreeExchangeSupports(dm)); 956f7c74593SToby Isaac mesh->createanchors = DMPlexCreateAnchors_Tree; 957f7c74593SToby Isaac /* reset anchors */ 9589566063dSJacob Faibussowitsch PetscCall(DMPlexSetAnchors(dm, NULL, NULL)); 959f9f063d4SToby Isaac PetscFunctionReturn(0); 960f9f063d4SToby Isaac } 961f9f063d4SToby Isaac 9620b7167a0SToby Isaac /*@ 9630b7167a0SToby Isaac DMPlexSetTree - set the tree that describes the hierarchy of non-conforming mesh points. This routine also creates 9640b7167a0SToby Isaac the point-to-point constraints determined by the tree: a point is constained to the points in the closure of its 9650b7167a0SToby Isaac tree root. 9660b7167a0SToby Isaac 9670b7167a0SToby Isaac Collective on dm 9680b7167a0SToby Isaac 9690b7167a0SToby Isaac Input Parameters: 9700b7167a0SToby Isaac + dm - the DMPlex object 9710b7167a0SToby Isaac . parentSection - a section describing the tree: a point has a parent if it has 1 dof in the section; the section 9720b7167a0SToby Isaac offset indexes the parent and childID list; the reference count of parentSection is incremented 9730b7167a0SToby Isaac . parents - a list of the point parents; copied, can be destroyed 9740b7167a0SToby Isaac - childIDs - identifies the relationship of the child point to the parent point; if there is a reference tree, then 9750b7167a0SToby Isaac the child corresponds to the point in the reference tree with index childIDs; copied, can be destroyed 9760b7167a0SToby Isaac 9770b7167a0SToby Isaac Level: intermediate 9780b7167a0SToby Isaac 979db781477SPatrick Sanan .seealso: `DMPlexGetTree()`, `DMPlexSetReferenceTree()`, `DMPlexSetAnchors()`, `DMPlexGetTreeParent()`, `DMPlexGetTreeChildren()` 9800b7167a0SToby Isaac @*/ 981*9371c9d4SSatish Balay PetscErrorCode DMPlexSetTree(DM dm, PetscSection parentSection, PetscInt parents[], PetscInt childIDs[]) { 9820b7167a0SToby Isaac PetscFunctionBegin; 9839566063dSJacob Faibussowitsch PetscCall(DMPlexSetTree_Internal(dm, parentSection, parents, childIDs, PETSC_FALSE, PETSC_TRUE)); 9840b7167a0SToby Isaac PetscFunctionReturn(0); 9850b7167a0SToby Isaac } 9860b7167a0SToby Isaac 987b2f41788SToby Isaac /*@ 988b2f41788SToby Isaac DMPlexGetTree - get the tree that describes the hierarchy of non-conforming mesh points. 989b2f41788SToby Isaac Collective on dm 990b2f41788SToby Isaac 991f899ff85SJose E. Roman Input Parameter: 992b2f41788SToby Isaac . dm - the DMPlex object 993b2f41788SToby Isaac 994b2f41788SToby Isaac Output Parameters: 995b2f41788SToby Isaac + parentSection - a section describing the tree: a point has a parent if it has 1 dof in the section; the section 996b2f41788SToby Isaac offset indexes the parent and childID list 997b2f41788SToby Isaac . parents - a list of the point parents 998b2f41788SToby Isaac . childIDs - identifies the relationship of the child point to the parent point; if there is a reference tree, then 999b2f41788SToby Isaac the child corresponds to the point in the reference tree with index childID 1000b2f41788SToby Isaac . childSection - the inverse of the parent section 1001b2f41788SToby Isaac - children - a list of the point children 1002b2f41788SToby Isaac 1003b2f41788SToby Isaac Level: intermediate 1004b2f41788SToby Isaac 1005db781477SPatrick Sanan .seealso: `DMPlexSetTree()`, `DMPlexSetReferenceTree()`, `DMPlexSetAnchors()`, `DMPlexGetTreeParent()`, `DMPlexGetTreeChildren()` 1006b2f41788SToby Isaac @*/ 1007*9371c9d4SSatish Balay PetscErrorCode DMPlexGetTree(DM dm, PetscSection *parentSection, PetscInt *parents[], PetscInt *childIDs[], PetscSection *childSection, PetscInt *children[]) { 1008b2f41788SToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 1009b2f41788SToby Isaac 1010b2f41788SToby Isaac PetscFunctionBegin; 1011b2f41788SToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1012b2f41788SToby Isaac if (parentSection) *parentSection = mesh->parentSection; 1013b2f41788SToby Isaac if (parents) *parents = mesh->parents; 1014b2f41788SToby Isaac if (childIDs) *childIDs = mesh->childIDs; 1015b2f41788SToby Isaac if (childSection) *childSection = mesh->childSection; 1016b2f41788SToby Isaac if (children) *children = mesh->children; 1017b2f41788SToby Isaac PetscFunctionReturn(0); 1018b2f41788SToby Isaac } 1019b2f41788SToby Isaac 1020d961a43aSToby Isaac /*@ 1021eaf898f9SPatrick Sanan DMPlexGetTreeParent - get the parent of a point in the tree describing the point hierarchy (not the DAG) 1022d961a43aSToby Isaac 1023d961a43aSToby Isaac Input Parameters: 1024d961a43aSToby Isaac + dm - the DMPlex object 1025d961a43aSToby Isaac - point - the query point 1026d961a43aSToby Isaac 1027d961a43aSToby Isaac Output Parameters: 1028d961a43aSToby Isaac + parent - if not NULL, set to the parent of the point, or the point itself if the point does not have a parent 1029d961a43aSToby Isaac - childID - if not NULL, set to the child ID of the point with respect to its parent, or 0 if the point 1030d961a43aSToby Isaac does not have a parent 1031d961a43aSToby Isaac 1032d961a43aSToby Isaac Level: intermediate 1033d961a43aSToby Isaac 1034db781477SPatrick Sanan .seealso: `DMPlexSetTree()`, `DMPlexGetTree()`, `DMPlexGetTreeChildren()` 1035d961a43aSToby Isaac @*/ 1036*9371c9d4SSatish Balay PetscErrorCode DMPlexGetTreeParent(DM dm, PetscInt point, PetscInt *parent, PetscInt *childID) { 1037d961a43aSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 1038d961a43aSToby Isaac PetscSection pSec; 1039d961a43aSToby Isaac 1040d961a43aSToby Isaac PetscFunctionBegin; 1041d961a43aSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1042d961a43aSToby Isaac pSec = mesh->parentSection; 1043d961a43aSToby Isaac if (pSec && point >= pSec->pStart && point < pSec->pEnd) { 1044d961a43aSToby Isaac PetscInt dof; 1045d961a43aSToby Isaac 10469566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(pSec, point, &dof)); 1047d961a43aSToby Isaac if (dof) { 1048d961a43aSToby Isaac PetscInt off; 1049d961a43aSToby Isaac 10509566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(pSec, point, &off)); 1051d961a43aSToby Isaac if (parent) *parent = mesh->parents[off]; 1052d961a43aSToby Isaac if (childID) *childID = mesh->childIDs[off]; 1053d961a43aSToby Isaac PetscFunctionReturn(0); 1054d961a43aSToby Isaac } 1055d961a43aSToby Isaac } 1056*9371c9d4SSatish Balay if (parent) { *parent = point; } 1057*9371c9d4SSatish Balay if (childID) { *childID = 0; } 1058d961a43aSToby Isaac PetscFunctionReturn(0); 1059d961a43aSToby Isaac } 1060d961a43aSToby Isaac 1061d961a43aSToby Isaac /*@C 1062eaf898f9SPatrick Sanan DMPlexGetTreeChildren - get the children of a point in the tree describing the point hierarchy (not the DAG) 1063d961a43aSToby Isaac 1064d961a43aSToby Isaac Input Parameters: 1065d961a43aSToby Isaac + dm - the DMPlex object 1066d961a43aSToby Isaac - point - the query point 1067d961a43aSToby Isaac 1068d961a43aSToby Isaac Output Parameters: 1069d961a43aSToby Isaac + numChildren - if not NULL, set to the number of children 1070d961a43aSToby Isaac - children - if not NULL, set to a list children, or set to NULL if the point has no children 1071d961a43aSToby Isaac 1072d961a43aSToby Isaac Level: intermediate 1073d961a43aSToby Isaac 1074d961a43aSToby Isaac Fortran Notes: 1075d961a43aSToby Isaac Since it returns an array, this routine is only available in Fortran 90, and you must 1076d961a43aSToby Isaac include petsc.h90 in your code. 1077d961a43aSToby Isaac 1078db781477SPatrick Sanan .seealso: `DMPlexSetTree()`, `DMPlexGetTree()`, `DMPlexGetTreeParent()` 1079d961a43aSToby Isaac @*/ 1080*9371c9d4SSatish Balay PetscErrorCode DMPlexGetTreeChildren(DM dm, PetscInt point, PetscInt *numChildren, const PetscInt *children[]) { 1081d961a43aSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 1082d961a43aSToby Isaac PetscSection childSec; 1083d961a43aSToby Isaac PetscInt dof = 0; 1084d961a43aSToby Isaac 1085d961a43aSToby Isaac PetscFunctionBegin; 1086d961a43aSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1087d961a43aSToby Isaac childSec = mesh->childSection; 1088*9371c9d4SSatish Balay if (childSec && point >= childSec->pStart && point < childSec->pEnd) { PetscCall(PetscSectionGetDof(childSec, point, &dof)); } 1089d961a43aSToby Isaac if (numChildren) *numChildren = dof; 1090d961a43aSToby Isaac if (children) { 1091d961a43aSToby Isaac if (dof) { 1092d961a43aSToby Isaac PetscInt off; 1093d961a43aSToby Isaac 10949566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(childSec, point, &off)); 1095d961a43aSToby Isaac *children = &mesh->children[off]; 1096*9371c9d4SSatish Balay } else { 1097d961a43aSToby Isaac *children = NULL; 1098d961a43aSToby Isaac } 1099d961a43aSToby Isaac } 1100d961a43aSToby Isaac PetscFunctionReturn(0); 1101d961a43aSToby Isaac } 11020c37af3bSToby Isaac 1103*9371c9d4SSatish Balay static PetscErrorCode EvaluateBasis(PetscSpace space, PetscInt nBasis, PetscInt nFunctionals, PetscInt nComps, PetscInt nPoints, const PetscInt *pointsPerFn, const PetscReal *points, const PetscReal *weights, PetscReal *work, Mat basisAtPoints) { 110452a3aeb4SToby Isaac PetscInt f, b, p, c, offset, qPoints; 1105b3a4bf2aSToby Isaac 1106b3a4bf2aSToby Isaac PetscFunctionBegin; 11079566063dSJacob Faibussowitsch PetscCall(PetscSpaceEvaluate(space, nPoints, points, work, NULL, NULL)); 110852a3aeb4SToby Isaac for (f = 0, offset = 0; f < nFunctionals; f++) { 110952a3aeb4SToby Isaac qPoints = pointsPerFn[f]; 111052a3aeb4SToby Isaac for (b = 0; b < nBasis; b++) { 1111b3a4bf2aSToby Isaac PetscScalar val = 0.; 1112b3a4bf2aSToby Isaac 111352a3aeb4SToby Isaac for (p = 0; p < qPoints; p++) { 1114*9371c9d4SSatish Balay for (c = 0; c < nComps; c++) { val += work[((offset + p) * nBasis + b) * nComps + c] * weights[(offset + p) * nComps + c]; } 111552a3aeb4SToby Isaac } 11169566063dSJacob Faibussowitsch PetscCall(MatSetValue(basisAtPoints, b, f, val, INSERT_VALUES)); 1117b3a4bf2aSToby Isaac } 1118b3a4bf2aSToby Isaac offset += qPoints; 1119b3a4bf2aSToby Isaac } 11209566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(basisAtPoints, MAT_FINAL_ASSEMBLY)); 11219566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(basisAtPoints, MAT_FINAL_ASSEMBLY)); 1122b3a4bf2aSToby Isaac PetscFunctionReturn(0); 1123b3a4bf2aSToby Isaac } 1124b3a4bf2aSToby Isaac 1125*9371c9d4SSatish Balay static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_Direct(DM dm, PetscSection section, PetscSection cSec, Mat cMat) { 11260c37af3bSToby Isaac PetscDS ds; 11270c37af3bSToby Isaac PetscInt spdim; 11280c37af3bSToby Isaac PetscInt numFields, f, c, cStart, cEnd, pStart, pEnd, conStart, conEnd; 11290c37af3bSToby Isaac const PetscInt *anchors; 1130f7c74593SToby Isaac PetscSection aSec; 11310c37af3bSToby Isaac PetscReal *v0, *v0parent, *vtmp, *J, *Jparent, *invJparent, detJ, detJparent; 11320c37af3bSToby Isaac IS aIS; 11330c37af3bSToby Isaac 11340c37af3bSToby Isaac PetscFunctionBegin; 11359566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 11369566063dSJacob Faibussowitsch PetscCall(DMGetDS(dm, &ds)); 11379566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 11389566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd)); 11399566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(dm, &aSec, &aIS)); 11409566063dSJacob Faibussowitsch PetscCall(ISGetIndices(aIS, &anchors)); 11419566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSec, &conStart, &conEnd)); 11429566063dSJacob Faibussowitsch PetscCall(DMGetDimension(dm, &spdim)); 11439566063dSJacob Faibussowitsch PetscCall(PetscMalloc6(spdim, &v0, spdim, &v0parent, spdim, &vtmp, spdim * spdim, &J, spdim * spdim, &Jparent, spdim * spdim, &invJparent)); 11440c37af3bSToby Isaac 11450c37af3bSToby Isaac for (f = 0; f < numFields; f++) { 11460dd1b1feSToby Isaac PetscObject disc; 11470dd1b1feSToby Isaac PetscClassId id; 1148b3a4bf2aSToby Isaac PetscSpace bspace; 1149b3a4bf2aSToby Isaac PetscDualSpace dspace; 11509c3cf19fSMatthew G. Knepley PetscInt i, j, k, nPoints, Nc, offset; 115152a3aeb4SToby Isaac PetscInt fSize, maxDof; 1152b3a4bf2aSToby Isaac PetscReal *weights, *pointsRef, *pointsReal, *work; 11531683a169SBarry Smith PetscScalar *scwork; 11541683a169SBarry Smith const PetscScalar *X; 11552c44ad04SToby Isaac PetscInt *sizes, *workIndRow, *workIndCol; 11560c37af3bSToby Isaac Mat Amat, Bmat, Xmat; 11572c44ad04SToby Isaac const PetscInt *numDof = NULL; 1158085f0adfSToby Isaac const PetscInt ***perms = NULL; 1159085f0adfSToby Isaac const PetscScalar ***flips = NULL; 11600c37af3bSToby Isaac 11619566063dSJacob Faibussowitsch PetscCall(PetscDSGetDiscretization(ds, f, &disc)); 11629566063dSJacob Faibussowitsch PetscCall(PetscObjectGetClassId(disc, &id)); 11630dd1b1feSToby Isaac if (id == PETSCFE_CLASSID) { 1164b3a4bf2aSToby Isaac PetscFE fe = (PetscFE)disc; 1165b3a4bf2aSToby Isaac 11669566063dSJacob Faibussowitsch PetscCall(PetscFEGetBasisSpace(fe, &bspace)); 11679566063dSJacob Faibussowitsch PetscCall(PetscFEGetDualSpace(fe, &dspace)); 11689566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetDimension(dspace, &fSize)); 11699566063dSJacob Faibussowitsch PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1170*9371c9d4SSatish Balay } else if (id == PETSCFV_CLASSID) { 1171b3a4bf2aSToby Isaac PetscFV fv = (PetscFV)disc; 1172b3a4bf2aSToby Isaac 11739566063dSJacob Faibussowitsch PetscCall(PetscFVGetNumComponents(fv, &Nc)); 11749566063dSJacob Faibussowitsch PetscCall(PetscSpaceCreate(PetscObjectComm((PetscObject)fv), &bspace)); 11759566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetType(bspace, PETSCSPACEPOLYNOMIAL)); 11769566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetDegree(bspace, 0, PETSC_DETERMINE)); 11779566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetNumComponents(bspace, Nc)); 11789566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetNumVariables(bspace, spdim)); 11799566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetUp(bspace)); 11809566063dSJacob Faibussowitsch PetscCall(PetscFVGetDualSpace(fv, &dspace)); 11819566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetDimension(dspace, &fSize)); 1182*9371c9d4SSatish Balay } else SETERRQ(PetscObjectComm(disc), PETSC_ERR_ARG_UNKNOWN_TYPE, "PetscDS discretization id %d not recognized.", id); 11839566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetNumDof(dspace, &numDof)); 11842c44ad04SToby Isaac for (i = 0, maxDof = 0; i <= spdim; i++) { maxDof = PetscMax(maxDof, numDof[i]); } 11859566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetSymmetries(dspace, &perms, &flips)); 11860dd1b1feSToby Isaac 11879566063dSJacob Faibussowitsch PetscCall(MatCreate(PETSC_COMM_SELF, &Amat)); 11889566063dSJacob Faibussowitsch PetscCall(MatSetSizes(Amat, fSize, fSize, fSize, fSize)); 11899566063dSJacob Faibussowitsch PetscCall(MatSetType(Amat, MATSEQDENSE)); 11909566063dSJacob Faibussowitsch PetscCall(MatSetUp(Amat)); 11919566063dSJacob Faibussowitsch PetscCall(MatDuplicate(Amat, MAT_DO_NOT_COPY_VALUES, &Bmat)); 11929566063dSJacob Faibussowitsch PetscCall(MatDuplicate(Amat, MAT_DO_NOT_COPY_VALUES, &Xmat)); 11930c37af3bSToby Isaac nPoints = 0; 11940c37af3bSToby Isaac for (i = 0; i < fSize; i++) { 119552a3aeb4SToby Isaac PetscInt qPoints, thisNc; 11960c37af3bSToby Isaac PetscQuadrature quad; 11970c37af3bSToby Isaac 11989566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetFunctional(dspace, i, &quad)); 11999566063dSJacob Faibussowitsch PetscCall(PetscQuadratureGetData(quad, NULL, &thisNc, &qPoints, NULL, NULL)); 120063a3b9bcSJacob Faibussowitsch PetscCheck(thisNc == Nc, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Functional dim %" PetscInt_FMT " does not much basis dim %" PetscInt_FMT, thisNc, Nc); 12010c37af3bSToby Isaac nPoints += qPoints; 12020c37af3bSToby Isaac } 12039566063dSJacob Faibussowitsch PetscCall(PetscMalloc7(fSize, &sizes, nPoints * Nc, &weights, spdim * nPoints, &pointsRef, spdim * nPoints, &pointsReal, nPoints * fSize * Nc, &work, maxDof, &workIndRow, maxDof, &workIndCol)); 12049566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof * maxDof, &scwork)); 12050c37af3bSToby Isaac offset = 0; 12060c37af3bSToby Isaac for (i = 0; i < fSize; i++) { 12070c37af3bSToby Isaac PetscInt qPoints; 12080c37af3bSToby Isaac const PetscReal *p, *w; 12090c37af3bSToby Isaac PetscQuadrature quad; 12100c37af3bSToby Isaac 12119566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetFunctional(dspace, i, &quad)); 12129566063dSJacob Faibussowitsch PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &qPoints, &p, &w)); 12139566063dSJacob Faibussowitsch PetscCall(PetscArraycpy(weights + Nc * offset, w, Nc * qPoints)); 12149566063dSJacob Faibussowitsch PetscCall(PetscArraycpy(pointsRef + spdim * offset, p, spdim * qPoints)); 1215b3a4bf2aSToby Isaac sizes[i] = qPoints; 12160c37af3bSToby Isaac offset += qPoints; 12170c37af3bSToby Isaac } 12189566063dSJacob Faibussowitsch PetscCall(EvaluateBasis(bspace, fSize, fSize, Nc, nPoints, sizes, pointsRef, weights, work, Amat)); 12199566063dSJacob Faibussowitsch PetscCall(MatLUFactor(Amat, NULL, NULL, NULL)); 12200c37af3bSToby Isaac for (c = cStart; c < cEnd; c++) { 12210c37af3bSToby Isaac PetscInt parent; 12220c37af3bSToby Isaac PetscInt closureSize, closureSizeP, *closure = NULL, *closureP = NULL; 12230c37af3bSToby Isaac PetscInt *childOffsets, *parentOffsets; 12240c37af3bSToby Isaac 12259566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, c, &parent, NULL)); 12260c37af3bSToby Isaac if (parent == c) continue; 12279566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure)); 12280c37af3bSToby Isaac for (i = 0; i < closureSize; i++) { 12290c37af3bSToby Isaac PetscInt p = closure[2 * i]; 12300c37af3bSToby Isaac PetscInt conDof; 12310c37af3bSToby Isaac 12320c37af3bSToby Isaac if (p < conStart || p >= conEnd) continue; 1233085f0adfSToby Isaac if (numFields) { 12349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSec, p, f, &conDof)); 1235*9371c9d4SSatish Balay } else { 12369566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSec, p, &conDof)); 12370c37af3bSToby Isaac } 12380c37af3bSToby Isaac if (conDof) break; 12390c37af3bSToby Isaac } 12400c37af3bSToby Isaac if (i == closureSize) { 12419566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure)); 12420c37af3bSToby Isaac continue; 12430c37af3bSToby Isaac } 12440c37af3bSToby Isaac 12459566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(dm, c, NULL, v0, J, NULL, &detJ)); 12469566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(dm, parent, NULL, v0parent, Jparent, invJparent, &detJparent)); 12470c37af3bSToby Isaac for (i = 0; i < nPoints; i++) { 1248c330f8ffSToby Isaac const PetscReal xi0[3] = {-1., -1., -1.}; 1249c330f8ffSToby Isaac 1250c330f8ffSToby Isaac CoordinatesRefToReal(spdim, spdim, xi0, v0, J, &pointsRef[i * spdim], vtmp); 1251c330f8ffSToby Isaac CoordinatesRealToRef(spdim, spdim, xi0, v0parent, invJparent, vtmp, &pointsReal[i * spdim]); 12520c37af3bSToby Isaac } 12539566063dSJacob Faibussowitsch PetscCall(EvaluateBasis(bspace, fSize, fSize, Nc, nPoints, sizes, pointsReal, weights, work, Bmat)); 12549566063dSJacob Faibussowitsch PetscCall(MatMatSolve(Amat, Bmat, Xmat)); 12559566063dSJacob Faibussowitsch PetscCall(MatDenseGetArrayRead(Xmat, &X)); 12569566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, parent, PETSC_TRUE, &closureSizeP, &closureP)); 12579566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(closureSize + 1, &childOffsets, closureSizeP + 1, &parentOffsets)); 12580c37af3bSToby Isaac childOffsets[0] = 0; 12590c37af3bSToby Isaac for (i = 0; i < closureSize; i++) { 12600c37af3bSToby Isaac PetscInt p = closure[2 * i]; 12610c37af3bSToby Isaac PetscInt dof; 12620c37af3bSToby Isaac 1263085f0adfSToby Isaac if (numFields) { 12649566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, p, f, &dof)); 1265*9371c9d4SSatish Balay } else { 12669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 12670c37af3bSToby Isaac } 126852a3aeb4SToby Isaac childOffsets[i + 1] = childOffsets[i] + dof; 12690c37af3bSToby Isaac } 12700c37af3bSToby Isaac parentOffsets[0] = 0; 12710c37af3bSToby Isaac for (i = 0; i < closureSizeP; i++) { 12720c37af3bSToby Isaac PetscInt p = closureP[2 * i]; 12730c37af3bSToby Isaac PetscInt dof; 12740c37af3bSToby Isaac 1275085f0adfSToby Isaac if (numFields) { 12769566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, p, f, &dof)); 1277*9371c9d4SSatish Balay } else { 12789566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 12790c37af3bSToby Isaac } 128052a3aeb4SToby Isaac parentOffsets[i + 1] = parentOffsets[i] + dof; 12810c37af3bSToby Isaac } 12820c37af3bSToby Isaac for (i = 0; i < closureSize; i++) { 12832c44ad04SToby Isaac PetscInt conDof, conOff, aDof, aOff, nWork; 12840c37af3bSToby Isaac PetscInt p = closure[2 * i]; 12850c37af3bSToby Isaac PetscInt o = closure[2 * i + 1]; 1286085f0adfSToby Isaac const PetscInt *perm; 1287085f0adfSToby Isaac const PetscScalar *flip; 12880c37af3bSToby Isaac 12890c37af3bSToby Isaac if (p < conStart || p >= conEnd) continue; 1290085f0adfSToby Isaac if (numFields) { 12919566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSec, p, f, &conDof)); 12929566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(cSec, p, f, &conOff)); 1293*9371c9d4SSatish Balay } else { 12949566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSec, p, &conDof)); 12959566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(cSec, p, &conOff)); 12960c37af3bSToby Isaac } 12970c37af3bSToby Isaac if (!conDof) continue; 1298085f0adfSToby Isaac perm = (perms && perms[i]) ? perms[i][o] : NULL; 1299085f0adfSToby Isaac flip = (flips && flips[i]) ? flips[i][o] : NULL; 13009566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(aSec, p, &aDof)); 13019566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(aSec, p, &aOff)); 13022c44ad04SToby Isaac nWork = childOffsets[i + 1] - childOffsets[i]; 13030c37af3bSToby Isaac for (k = 0; k < aDof; k++) { 13040c37af3bSToby Isaac PetscInt a = anchors[aOff + k]; 13050c37af3bSToby Isaac PetscInt aSecDof, aSecOff; 13060c37af3bSToby Isaac 1307085f0adfSToby Isaac if (numFields) { 13089566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, a, f, &aSecDof)); 13099566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(section, a, f, &aSecOff)); 1310*9371c9d4SSatish Balay } else { 13119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, a, &aSecDof)); 13129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, a, &aSecOff)); 13130c37af3bSToby Isaac } 13140c37af3bSToby Isaac if (!aSecDof) continue; 13150c37af3bSToby Isaac 13160c37af3bSToby Isaac for (j = 0; j < closureSizeP; j++) { 13170c37af3bSToby Isaac PetscInt q = closureP[2 * j]; 13180c37af3bSToby Isaac PetscInt oq = closureP[2 * j + 1]; 13192c44ad04SToby Isaac 13202c44ad04SToby Isaac if (q == a) { 132152a3aeb4SToby Isaac PetscInt r, s, nWorkP; 1322085f0adfSToby Isaac const PetscInt *permP; 1323085f0adfSToby Isaac const PetscScalar *flipP; 1324085f0adfSToby Isaac 1325085f0adfSToby Isaac permP = (perms && perms[j]) ? perms[j][oq] : NULL; 1326085f0adfSToby Isaac flipP = (flips && flips[j]) ? flips[j][oq] : NULL; 13272c44ad04SToby Isaac nWorkP = parentOffsets[j + 1] - parentOffsets[j]; 13282c44ad04SToby Isaac /* get a copy of the child-to-anchor portion of the matrix, and transpose so that rows correspond to the 13291683a169SBarry Smith * child and columns correspond to the anchor: BUT the maxrix returned by MatDenseGetArrayRead() is 13302c44ad04SToby Isaac * column-major, so transpose-transpose = do nothing */ 13312c44ad04SToby Isaac for (r = 0; r < nWork; r++) { 1332*9371c9d4SSatish Balay for (s = 0; s < nWorkP; s++) { scwork[r * nWorkP + s] = X[fSize * (r + childOffsets[i]) + (s + parentOffsets[j])]; } 13332c44ad04SToby Isaac } 133452a3aeb4SToby Isaac for (r = 0; r < nWork; r++) { workIndRow[perm ? perm[r] : r] = conOff + r; } 133552a3aeb4SToby Isaac for (s = 0; s < nWorkP; s++) { workIndCol[permP ? permP[s] : s] = aSecOff + s; } 13362c44ad04SToby Isaac if (flip) { 13372c44ad04SToby Isaac for (r = 0; r < nWork; r++) { 1338*9371c9d4SSatish Balay for (s = 0; s < nWorkP; s++) { scwork[r * nWorkP + s] *= flip[r]; } 13392c44ad04SToby Isaac } 13402c44ad04SToby Isaac } 13412c44ad04SToby Isaac if (flipP) { 13422c44ad04SToby Isaac for (r = 0; r < nWork; r++) { 1343*9371c9d4SSatish Balay for (s = 0; s < nWorkP; s++) { scwork[r * nWorkP + s] *= flipP[s]; } 13442c44ad04SToby Isaac } 13452c44ad04SToby Isaac } 13469566063dSJacob Faibussowitsch PetscCall(MatSetValues(cMat, nWork, workIndRow, nWorkP, workIndCol, scwork, INSERT_VALUES)); 13472c44ad04SToby Isaac break; 13480c37af3bSToby Isaac } 13490c37af3bSToby Isaac } 13500c37af3bSToby Isaac } 13510c37af3bSToby Isaac } 13529566063dSJacob Faibussowitsch PetscCall(MatDenseRestoreArrayRead(Xmat, &X)); 13539566063dSJacob Faibussowitsch PetscCall(PetscFree2(childOffsets, parentOffsets)); 13549566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure)); 13559566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, parent, PETSC_TRUE, &closureSizeP, &closureP)); 13560c37af3bSToby Isaac } 13579566063dSJacob Faibussowitsch PetscCall(MatDestroy(&Amat)); 13589566063dSJacob Faibussowitsch PetscCall(MatDestroy(&Bmat)); 13599566063dSJacob Faibussowitsch PetscCall(MatDestroy(&Xmat)); 13609566063dSJacob Faibussowitsch PetscCall(PetscFree(scwork)); 13619566063dSJacob Faibussowitsch PetscCall(PetscFree7(sizes, weights, pointsRef, pointsReal, work, workIndRow, workIndCol)); 1362*9371c9d4SSatish Balay if (id == PETSCFV_CLASSID) { PetscCall(PetscSpaceDestroy(&bspace)); } 13630c37af3bSToby Isaac } 13649566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(cMat, MAT_FINAL_ASSEMBLY)); 13659566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(cMat, MAT_FINAL_ASSEMBLY)); 13669566063dSJacob Faibussowitsch PetscCall(PetscFree6(v0, v0parent, vtmp, J, Jparent, invJparent)); 13679566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(aIS, &anchors)); 13680c37af3bSToby Isaac 13690c37af3bSToby Isaac PetscFunctionReturn(0); 13700c37af3bSToby Isaac } 137195a0b26dSToby Isaac 1372*9371c9d4SSatish Balay static PetscErrorCode DMPlexReferenceTreeGetChildrenMatrices(DM refTree, PetscScalar ****childrenMats, PetscInt ***childrenN) { 1373f7c74593SToby Isaac Mat refCmat; 137421968bf8SToby Isaac PetscDS ds; 1375085f0adfSToby Isaac PetscInt numFields, maxFields, f, pRefStart, pRefEnd, p, *rows, *cols, maxDof, maxAnDof, **refPointFieldN; 137621968bf8SToby Isaac PetscScalar ***refPointFieldMats; 137721968bf8SToby Isaac PetscSection refConSec, refAnSec, refSection; 137821968bf8SToby Isaac IS refAnIS; 137921968bf8SToby Isaac const PetscInt *refAnchors; 1380085f0adfSToby Isaac const PetscInt **perms; 1381085f0adfSToby Isaac const PetscScalar **flips; 138295a0b26dSToby Isaac 138395a0b26dSToby Isaac PetscFunctionBegin; 13849566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 13859566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 1386085f0adfSToby Isaac maxFields = PetscMax(1, numFields); 13879566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, &refCmat, NULL)); 13889566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(refTree, &refAnSec, &refAnIS)); 13899566063dSJacob Faibussowitsch PetscCall(ISGetIndices(refAnIS, &refAnchors)); 13909566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(refTree, &refSection)); 13919566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 13929566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pRefEnd - pRefStart, &refPointFieldMats)); 13939566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pRefEnd - pRefStart, &refPointFieldN)); 13949566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refConSec, &maxDof)); 13959566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refAnSec, &maxAnDof)); 13969566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof, &rows)); 13979566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof * maxAnDof, &cols)); 139895a0b26dSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 139995a0b26dSToby Isaac PetscInt parent, closureSize, *closure = NULL, pDof; 140095a0b26dSToby Isaac 14019566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, p, &parent, NULL)); 14029566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &pDof)); 140395a0b26dSToby Isaac if (!pDof || parent == p) continue; 140495a0b26dSToby Isaac 14059566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxFields, &refPointFieldMats[p - pRefStart])); 14069566063dSJacob Faibussowitsch PetscCall(PetscCalloc1(maxFields, &refPointFieldN[p - pRefStart])); 14079566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, parent, PETSC_TRUE, &closureSize, &closure)); 1408085f0adfSToby Isaac for (f = 0; f < maxFields; f++) { 1409085f0adfSToby Isaac PetscInt cDof, cOff, numCols, r, i; 141095a0b26dSToby Isaac 1411085f0adfSToby Isaac if (f < numFields) { 14129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refConSec, p, f, &cDof)); 14139566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refConSec, p, f, &cOff)); 14149566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldPointSyms(refSection, f, closureSize, closure, &perms, &flips)); 1415085f0adfSToby Isaac } else { 14169566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &cDof)); 14179566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refConSec, p, &cOff)); 14189566063dSJacob Faibussowitsch PetscCall(PetscSectionGetPointSyms(refSection, closureSize, closure, &perms, &flips)); 141995a0b26dSToby Isaac } 142095a0b26dSToby Isaac 1421*9371c9d4SSatish Balay for (r = 0; r < cDof; r++) { rows[r] = cOff + r; } 142295a0b26dSToby Isaac numCols = 0; 142395a0b26dSToby Isaac for (i = 0; i < closureSize; i++) { 142495a0b26dSToby Isaac PetscInt q = closure[2 * i]; 142595a0b26dSToby Isaac PetscInt aDof, aOff, j; 1426085f0adfSToby Isaac const PetscInt *perm = perms ? perms[i] : NULL; 142795a0b26dSToby Isaac 1428085f0adfSToby Isaac if (numFields) { 14299566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refSection, q, f, &aDof)); 14309566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refSection, q, f, &aOff)); 1431*9371c9d4SSatish Balay } else { 14329566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, q, &aDof)); 14339566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refSection, q, &aOff)); 143495a0b26dSToby Isaac } 143595a0b26dSToby Isaac 1436*9371c9d4SSatish Balay for (j = 0; j < aDof; j++) { cols[numCols++] = aOff + (perm ? perm[j] : j); } 143795a0b26dSToby Isaac } 143895a0b26dSToby Isaac refPointFieldN[p - pRefStart][f] = numCols; 14399566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(cDof * numCols, &refPointFieldMats[p - pRefStart][f])); 14409566063dSJacob Faibussowitsch PetscCall(MatGetValues(refCmat, cDof, rows, numCols, cols, refPointFieldMats[p - pRefStart][f])); 1441085f0adfSToby Isaac if (flips) { 1442085f0adfSToby Isaac PetscInt colOff = 0; 1443085f0adfSToby Isaac 1444085f0adfSToby Isaac for (i = 0; i < closureSize; i++) { 1445085f0adfSToby Isaac PetscInt q = closure[2 * i]; 1446085f0adfSToby Isaac PetscInt aDof, aOff, j; 1447085f0adfSToby Isaac const PetscScalar *flip = flips ? flips[i] : NULL; 1448085f0adfSToby Isaac 1449085f0adfSToby Isaac if (numFields) { 14509566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refSection, q, f, &aDof)); 14519566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refSection, q, f, &aOff)); 1452*9371c9d4SSatish Balay } else { 14539566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, q, &aDof)); 14549566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refSection, q, &aOff)); 1455085f0adfSToby Isaac } 1456085f0adfSToby Isaac if (flip) { 1457085f0adfSToby Isaac PetscInt k; 1458085f0adfSToby Isaac for (k = 0; k < cDof; k++) { 1459*9371c9d4SSatish Balay for (j = 0; j < aDof; j++) { refPointFieldMats[p - pRefStart][f][k * numCols + colOff + j] *= flip[j]; } 1460085f0adfSToby Isaac } 1461085f0adfSToby Isaac } 1462085f0adfSToby Isaac colOff += aDof; 1463085f0adfSToby Isaac } 1464085f0adfSToby Isaac } 1465085f0adfSToby Isaac if (numFields) { 14669566063dSJacob Faibussowitsch PetscCall(PetscSectionRestoreFieldPointSyms(refSection, f, closureSize, closure, &perms, &flips)); 1467085f0adfSToby Isaac } else { 14689566063dSJacob Faibussowitsch PetscCall(PetscSectionRestorePointSyms(refSection, closureSize, closure, &perms, &flips)); 1469085f0adfSToby Isaac } 147095a0b26dSToby Isaac } 14719566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, parent, PETSC_TRUE, &closureSize, &closure)); 147295a0b26dSToby Isaac } 147321968bf8SToby Isaac *childrenMats = refPointFieldMats; 147421968bf8SToby Isaac *childrenN = refPointFieldN; 14759566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(refAnIS, &refAnchors)); 14769566063dSJacob Faibussowitsch PetscCall(PetscFree(rows)); 14779566063dSJacob Faibussowitsch PetscCall(PetscFree(cols)); 147821968bf8SToby Isaac PetscFunctionReturn(0); 147921968bf8SToby Isaac } 148021968bf8SToby Isaac 1481*9371c9d4SSatish Balay static PetscErrorCode DMPlexReferenceTreeRestoreChildrenMatrices(DM refTree, PetscScalar ****childrenMats, PetscInt ***childrenN) { 148221968bf8SToby Isaac PetscDS ds; 148321968bf8SToby Isaac PetscInt **refPointFieldN; 148421968bf8SToby Isaac PetscScalar ***refPointFieldMats; 1485085f0adfSToby Isaac PetscInt numFields, maxFields, pRefStart, pRefEnd, p, f; 148621968bf8SToby Isaac PetscSection refConSec; 148721968bf8SToby Isaac 148821968bf8SToby Isaac PetscFunctionBegin; 148921968bf8SToby Isaac refPointFieldN = *childrenN; 149021968bf8SToby Isaac *childrenN = NULL; 149121968bf8SToby Isaac refPointFieldMats = *childrenMats; 149221968bf8SToby Isaac *childrenMats = NULL; 14939566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 14949566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 1495367003a6SStefano Zampini maxFields = PetscMax(1, numFields); 14969566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 14979566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 149821968bf8SToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 149921968bf8SToby Isaac PetscInt parent, pDof; 150021968bf8SToby Isaac 15019566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, p, &parent, NULL)); 15029566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &pDof)); 150321968bf8SToby Isaac if (!pDof || parent == p) continue; 150421968bf8SToby Isaac 1505085f0adfSToby Isaac for (f = 0; f < maxFields; f++) { 150621968bf8SToby Isaac PetscInt cDof; 150721968bf8SToby Isaac 1508085f0adfSToby Isaac if (numFields) { 15099566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refConSec, p, f, &cDof)); 1510*9371c9d4SSatish Balay } else { 15119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &cDof)); 151221968bf8SToby Isaac } 151321968bf8SToby Isaac 15149566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats[p - pRefStart][f])); 151521968bf8SToby Isaac } 15169566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats[p - pRefStart])); 15179566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldN[p - pRefStart])); 151821968bf8SToby Isaac } 15199566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats)); 15209566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldN)); 152121968bf8SToby Isaac PetscFunctionReturn(0); 152221968bf8SToby Isaac } 152321968bf8SToby Isaac 1524*9371c9d4SSatish Balay static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_FromReference(DM dm, PetscSection section, PetscSection conSec, Mat cMat) { 152521968bf8SToby Isaac DM refTree; 152621968bf8SToby Isaac PetscDS ds; 152721968bf8SToby Isaac Mat refCmat; 1528085f0adfSToby Isaac PetscInt numFields, maxFields, f, pRefStart, pRefEnd, p, maxDof, maxAnDof, *perm, *iperm, pStart, pEnd, conStart, conEnd, **refPointFieldN; 152921968bf8SToby Isaac PetscScalar ***refPointFieldMats, *pointWork; 153021968bf8SToby Isaac PetscSection refConSec, refAnSec, anSec; 153121968bf8SToby Isaac IS refAnIS, anIS; 153221968bf8SToby Isaac const PetscInt *anchors; 153321968bf8SToby Isaac 153421968bf8SToby Isaac PetscFunctionBegin; 153521968bf8SToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 15369566063dSJacob Faibussowitsch PetscCall(DMGetDS(dm, &ds)); 15379566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 1538085f0adfSToby Isaac maxFields = PetscMax(1, numFields); 15399566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(dm, &refTree)); 15409566063dSJacob Faibussowitsch PetscCall(DMCopyDisc(dm, refTree)); 15419566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, &refCmat, NULL)); 15429566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(refTree, &refAnSec, &refAnIS)); 15439566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(dm, &anSec, &anIS)); 15449566063dSJacob Faibussowitsch PetscCall(ISGetIndices(anIS, &anchors)); 15459566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 15469566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(conSec, &conStart, &conEnd)); 15479566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refConSec, &maxDof)); 15489566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refAnSec, &maxAnDof)); 15499566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof * maxDof * maxAnDof, &pointWork)); 155021968bf8SToby Isaac 155121968bf8SToby Isaac /* step 1: get submats for every constrained point in the reference tree */ 15529566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 155395a0b26dSToby Isaac 155495a0b26dSToby Isaac /* step 2: compute the preorder */ 15559566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 15569566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(pEnd - pStart, &perm, pEnd - pStart, &iperm)); 155795a0b26dSToby Isaac for (p = pStart; p < pEnd; p++) { 155895a0b26dSToby Isaac perm[p - pStart] = p; 155995a0b26dSToby Isaac iperm[p - pStart] = p - pStart; 156095a0b26dSToby Isaac } 156195a0b26dSToby Isaac for (p = 0; p < pEnd - pStart;) { 156295a0b26dSToby Isaac PetscInt point = perm[p]; 156395a0b26dSToby Isaac PetscInt parent; 156495a0b26dSToby Isaac 15659566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, point, &parent, NULL)); 156695a0b26dSToby Isaac if (parent == point) { 156795a0b26dSToby Isaac p++; 1568*9371c9d4SSatish Balay } else { 156995a0b26dSToby Isaac PetscInt size, closureSize, *closure = NULL, i; 157095a0b26dSToby Isaac 15719566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, parent, PETSC_TRUE, &closureSize, &closure)); 157295a0b26dSToby Isaac for (i = 0; i < closureSize; i++) { 157395a0b26dSToby Isaac PetscInt q = closure[2 * i]; 157495a0b26dSToby Isaac if (iperm[q - pStart] > iperm[point - pStart]) { 157595a0b26dSToby Isaac /* swap */ 157695a0b26dSToby Isaac perm[p] = q; 157795a0b26dSToby Isaac perm[iperm[q - pStart]] = point; 157895a0b26dSToby Isaac iperm[point - pStart] = iperm[q - pStart]; 157995a0b26dSToby Isaac iperm[q - pStart] = p; 158095a0b26dSToby Isaac break; 158195a0b26dSToby Isaac } 158295a0b26dSToby Isaac } 158395a0b26dSToby Isaac size = closureSize; 15849566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, parent, PETSC_TRUE, &closureSize, &closure)); 1585*9371c9d4SSatish Balay if (i == size) { p++; } 158695a0b26dSToby Isaac } 158795a0b26dSToby Isaac } 158895a0b26dSToby Isaac 158995a0b26dSToby Isaac /* step 3: fill the constraint matrix */ 159095a0b26dSToby Isaac /* we are going to use a preorder progressive fill strategy. Mat doesn't 159195a0b26dSToby Isaac * allow progressive fill without assembly, so we are going to set up the 159295a0b26dSToby Isaac * values outside of the Mat first. 159395a0b26dSToby Isaac */ 159495a0b26dSToby Isaac { 159595a0b26dSToby Isaac PetscInt nRows, row, nnz; 159695a0b26dSToby Isaac PetscBool done; 159795a0b26dSToby Isaac const PetscInt *ia, *ja; 159895a0b26dSToby Isaac PetscScalar *vals; 159995a0b26dSToby Isaac 16009566063dSJacob Faibussowitsch PetscCall(MatGetRowIJ(cMat, 0, PETSC_FALSE, PETSC_FALSE, &nRows, &ia, &ja, &done)); 160128b400f6SJacob Faibussowitsch PetscCheck(done, PetscObjectComm((PetscObject)cMat), PETSC_ERR_PLIB, "Could not get RowIJ of constraint matrix"); 160295a0b26dSToby Isaac nnz = ia[nRows]; 160395a0b26dSToby Isaac /* malloc and then zero rows right before we fill them: this way valgrind 160495a0b26dSToby Isaac * can tell if we are doing progressive fill in the wrong order */ 16059566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(nnz, &vals)); 160695a0b26dSToby Isaac for (p = 0; p < pEnd - pStart; p++) { 160795a0b26dSToby Isaac PetscInt parent, childid, closureSize, *closure = NULL; 160895a0b26dSToby Isaac PetscInt point = perm[p], pointDof; 160995a0b26dSToby Isaac 16109566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, point, &parent, &childid)); 161195a0b26dSToby Isaac if ((point < conStart) || (point >= conEnd) || (parent == point)) continue; 16129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(conSec, point, &pointDof)); 161395a0b26dSToby Isaac if (!pointDof) continue; 16149566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, parent, PETSC_TRUE, &closureSize, &closure)); 1615085f0adfSToby Isaac for (f = 0; f < maxFields; f++) { 1616085f0adfSToby Isaac PetscInt cDof, cOff, numCols, numFillCols, i, r, matOffset, offset; 161795a0b26dSToby Isaac PetscScalar *pointMat; 1618085f0adfSToby Isaac const PetscInt **perms; 1619085f0adfSToby Isaac const PetscScalar **flips; 162095a0b26dSToby Isaac 1621085f0adfSToby Isaac if (numFields) { 16229566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(conSec, point, f, &cDof)); 16239566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(conSec, point, f, &cOff)); 1624*9371c9d4SSatish Balay } else { 16259566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(conSec, point, &cDof)); 16269566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(conSec, point, &cOff)); 162795a0b26dSToby Isaac } 162895a0b26dSToby Isaac if (!cDof) continue; 16299566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionGetFieldPointSyms(section, f, closureSize, closure, &perms, &flips)); 16309566063dSJacob Faibussowitsch else PetscCall(PetscSectionGetPointSyms(section, closureSize, closure, &perms, &flips)); 163195a0b26dSToby Isaac 163295a0b26dSToby Isaac /* make sure that every row for this point is the same size */ 163376bd3646SJed Brown if (PetscDefined(USE_DEBUG)) { 163495a0b26dSToby Isaac for (r = 0; r < cDof; r++) { 163595a0b26dSToby Isaac if (cDof > 1 && r) { 163663a3b9bcSJacob Faibussowitsch PetscCheck((ia[cOff + r + 1] - ia[cOff + r]) == (ia[cOff + r] - ia[cOff + r - 1]), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two point rows have different nnz: %" PetscInt_FMT " vs. %" PetscInt_FMT, (ia[cOff + r + 1] - ia[cOff + r]), (ia[cOff + r] - ia[cOff + r - 1])); 163795a0b26dSToby Isaac } 163895a0b26dSToby Isaac } 163976bd3646SJed Brown } 164095a0b26dSToby Isaac /* zero rows */ 1641*9371c9d4SSatish Balay for (i = ia[cOff]; i < ia[cOff + cDof]; i++) { vals[i] = 0.; } 164295a0b26dSToby Isaac matOffset = ia[cOff]; 164395a0b26dSToby Isaac numFillCols = ia[cOff + 1] - matOffset; 164495a0b26dSToby Isaac pointMat = refPointFieldMats[childid - pRefStart][f]; 164595a0b26dSToby Isaac numCols = refPointFieldN[childid - pRefStart][f]; 164695a0b26dSToby Isaac offset = 0; 164795a0b26dSToby Isaac for (i = 0; i < closureSize; i++) { 164895a0b26dSToby Isaac PetscInt q = closure[2 * i]; 164995a0b26dSToby Isaac PetscInt aDof, aOff, j, k, qConDof, qConOff; 1650085f0adfSToby Isaac const PetscInt *perm = perms ? perms[i] : NULL; 1651085f0adfSToby Isaac const PetscScalar *flip = flips ? flips[i] : NULL; 165295a0b26dSToby Isaac 165395a0b26dSToby Isaac qConDof = qConOff = 0; 1654085f0adfSToby Isaac if (numFields) { 16559566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, q, f, &aDof)); 16569566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(section, q, f, &aOff)); 165795a0b26dSToby Isaac if (q >= conStart && q < conEnd) { 16589566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(conSec, q, f, &qConDof)); 16599566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(conSec, q, f, &qConOff)); 166095a0b26dSToby Isaac } 1661*9371c9d4SSatish Balay } else { 16629566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, q, &aDof)); 16639566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, q, &aOff)); 166495a0b26dSToby Isaac if (q >= conStart && q < conEnd) { 16659566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(conSec, q, &qConDof)); 16669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(conSec, q, &qConOff)); 166795a0b26dSToby Isaac } 166895a0b26dSToby Isaac } 166995a0b26dSToby Isaac if (!aDof) continue; 167095a0b26dSToby Isaac if (qConDof) { 167195a0b26dSToby Isaac /* this point has anchors: its rows of the matrix should already 167295a0b26dSToby Isaac * be filled, thanks to preordering */ 167395a0b26dSToby Isaac /* first multiply into pointWork, then set in matrix */ 167495a0b26dSToby Isaac PetscInt aMatOffset = ia[qConOff]; 167595a0b26dSToby Isaac PetscInt aNumFillCols = ia[qConOff + 1] - aMatOffset; 167695a0b26dSToby Isaac for (r = 0; r < cDof; r++) { 167795a0b26dSToby Isaac for (j = 0; j < aNumFillCols; j++) { 167895a0b26dSToby Isaac PetscScalar inVal = 0; 167995a0b26dSToby Isaac for (k = 0; k < aDof; k++) { 1680085f0adfSToby Isaac PetscInt col = perm ? perm[k] : k; 168195a0b26dSToby Isaac 1682085f0adfSToby Isaac inVal += pointMat[r * numCols + offset + col] * vals[aMatOffset + aNumFillCols * k + j] * (flip ? flip[col] : 1.); 168395a0b26dSToby Isaac } 168495a0b26dSToby Isaac pointWork[r * aNumFillCols + j] = inVal; 168595a0b26dSToby Isaac } 168695a0b26dSToby Isaac } 168795a0b26dSToby Isaac /* assume that the columns are sorted, spend less time searching */ 168895a0b26dSToby Isaac for (j = 0, k = 0; j < aNumFillCols; j++) { 168995a0b26dSToby Isaac PetscInt col = ja[aMatOffset + j]; 169095a0b26dSToby Isaac for (; k < numFillCols; k++) { 1691*9371c9d4SSatish Balay if (ja[matOffset + k] == col) { break; } 169295a0b26dSToby Isaac } 169363a3b9bcSJacob Faibussowitsch PetscCheck(k != numFillCols, PETSC_COMM_SELF, PETSC_ERR_PLIB, "No nonzero space for (%" PetscInt_FMT ", %" PetscInt_FMT ")", cOff, col); 1694*9371c9d4SSatish Balay for (r = 0; r < cDof; r++) { vals[matOffset + numFillCols * r + k] = pointWork[r * aNumFillCols + j]; } 169595a0b26dSToby Isaac } 1696*9371c9d4SSatish Balay } else { 169795a0b26dSToby Isaac /* find where to put this portion of pointMat into the matrix */ 169895a0b26dSToby Isaac for (k = 0; k < numFillCols; k++) { 1699*9371c9d4SSatish Balay if (ja[matOffset + k] == aOff) { break; } 170095a0b26dSToby Isaac } 170163a3b9bcSJacob Faibussowitsch PetscCheck(k != numFillCols, PETSC_COMM_SELF, PETSC_ERR_PLIB, "No nonzero space for (%" PetscInt_FMT ", %" PetscInt_FMT ")", cOff, aOff); 170295a0b26dSToby Isaac for (r = 0; r < cDof; r++) { 1703085f0adfSToby Isaac for (j = 0; j < aDof; j++) { 1704085f0adfSToby Isaac PetscInt col = perm ? perm[j] : j; 1705085f0adfSToby Isaac 1706085f0adfSToby Isaac vals[matOffset + numFillCols * r + k + col] += pointMat[r * numCols + offset + j] * (flip ? flip[col] : 1.); 170795a0b26dSToby Isaac } 170895a0b26dSToby Isaac } 170995a0b26dSToby Isaac } 171095a0b26dSToby Isaac offset += aDof; 171195a0b26dSToby Isaac } 1712085f0adfSToby Isaac if (numFields) { 17139566063dSJacob Faibussowitsch PetscCall(PetscSectionRestoreFieldPointSyms(section, f, closureSize, closure, &perms, &flips)); 1714085f0adfSToby Isaac } else { 17159566063dSJacob Faibussowitsch PetscCall(PetscSectionRestorePointSyms(section, closureSize, closure, &perms, &flips)); 1716085f0adfSToby Isaac } 171795a0b26dSToby Isaac } 17189566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, parent, PETSC_TRUE, &closureSize, &closure)); 171995a0b26dSToby Isaac } 1720*9371c9d4SSatish Balay for (row = 0; row < nRows; row++) { PetscCall(MatSetValues(cMat, 1, &row, ia[row + 1] - ia[row], &ja[ia[row]], &vals[ia[row]], INSERT_VALUES)); } 17219566063dSJacob Faibussowitsch PetscCall(MatRestoreRowIJ(cMat, 0, PETSC_FALSE, PETSC_FALSE, &nRows, &ia, &ja, &done)); 172228b400f6SJacob Faibussowitsch PetscCheck(done, PetscObjectComm((PetscObject)cMat), PETSC_ERR_PLIB, "Could not restore RowIJ of constraint matrix"); 17239566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(cMat, MAT_FINAL_ASSEMBLY)); 17249566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(cMat, MAT_FINAL_ASSEMBLY)); 17259566063dSJacob Faibussowitsch PetscCall(PetscFree(vals)); 172695a0b26dSToby Isaac } 172795a0b26dSToby Isaac 172895a0b26dSToby Isaac /* clean up */ 17299566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(anIS, &anchors)); 17309566063dSJacob Faibussowitsch PetscCall(PetscFree2(perm, iperm)); 17319566063dSJacob Faibussowitsch PetscCall(PetscFree(pointWork)); 17329566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 173395a0b26dSToby Isaac PetscFunctionReturn(0); 173495a0b26dSToby Isaac } 173595a0b26dSToby Isaac 17366f5f1567SToby Isaac /* refine a single cell on rank 0: this is not intended to provide good local refinement, only to create an example of 17376f5f1567SToby Isaac * a non-conforming mesh. Local refinement comes later */ 1738*9371c9d4SSatish Balay PetscErrorCode DMPlexTreeRefineCell(DM dm, PetscInt cell, DM *ncdm) { 17396f5f1567SToby Isaac DM K; 1740420f55faSMatthew G. Knepley PetscMPIInt rank; 17416f5f1567SToby Isaac PetscInt dim, *pNewStart, *pNewEnd, *pNewCount, *pOldStart, *pOldEnd, offset, d, pStart, pEnd; 17426f5f1567SToby Isaac PetscInt numNewCones, *newConeSizes, *newCones, *newOrientations; 17436f5f1567SToby Isaac PetscInt *Kembedding; 17446f5f1567SToby Isaac PetscInt *cellClosure = NULL, nc; 17456f5f1567SToby Isaac PetscScalar *newVertexCoords; 17466f5f1567SToby Isaac PetscInt numPointsWithParents, *parents, *childIDs, *perm, *iperm, *preOrient, pOffset; 17476f5f1567SToby Isaac PetscSection parentSection; 17486f5f1567SToby Isaac 17496f5f1567SToby Isaac PetscFunctionBegin; 17509566063dSJacob Faibussowitsch PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank)); 17519566063dSJacob Faibussowitsch PetscCall(DMGetDimension(dm, &dim)); 17529566063dSJacob Faibussowitsch PetscCall(DMPlexCreate(PetscObjectComm((PetscObject)dm), ncdm)); 17539566063dSJacob Faibussowitsch PetscCall(DMSetDimension(*ncdm, dim)); 17546f5f1567SToby Isaac 17559566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 17569566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &parentSection)); 17579566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(dm, &K)); 17586858538eSMatthew G. Knepley PetscCall(DMGetCoordinatesLocalSetUp(dm)); 1759dd400576SPatrick Sanan if (rank == 0) { 17606f5f1567SToby Isaac /* compute the new charts */ 17619566063dSJacob Faibussowitsch PetscCall(PetscMalloc5(dim + 1, &pNewCount, dim + 1, &pNewStart, dim + 1, &pNewEnd, dim + 1, &pOldStart, dim + 1, &pOldEnd)); 17626f5f1567SToby Isaac offset = 0; 17636f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 17646f5f1567SToby Isaac PetscInt pOldCount, kStart, kEnd, k; 17656f5f1567SToby Isaac 17666f5f1567SToby Isaac pNewStart[d] = offset; 17679566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(dm, d, &pOldStart[d], &pOldEnd[d])); 17689566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &kStart, &kEnd)); 17696f5f1567SToby Isaac pOldCount = pOldEnd[d] - pOldStart[d]; 17706f5f1567SToby Isaac /* adding the new points */ 17716f5f1567SToby Isaac pNewCount[d] = pOldCount + kEnd - kStart; 17726f5f1567SToby Isaac if (!d) { 17736f5f1567SToby Isaac /* removing the cell */ 17746f5f1567SToby Isaac pNewCount[d]--; 17756f5f1567SToby Isaac } 17766f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 17776f5f1567SToby Isaac PetscInt parent; 17789566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(K, k, &parent, NULL)); 17796f5f1567SToby Isaac if (parent == k) { 17806f5f1567SToby Isaac /* avoid double counting points that won't actually be new */ 17816f5f1567SToby Isaac pNewCount[d]--; 17826f5f1567SToby Isaac } 17836f5f1567SToby Isaac } 17846f5f1567SToby Isaac pNewEnd[d] = pNewStart[d] + pNewCount[d]; 17856f5f1567SToby Isaac offset = pNewEnd[d]; 17866f5f1567SToby Isaac } 17871dca8a05SBarry Smith PetscCheck(cell >= pOldStart[0] && cell < pOldEnd[0], PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "%" PetscInt_FMT " not in cell range [%" PetscInt_FMT ", %" PetscInt_FMT ")", cell, pOldStart[0], pOldEnd[0]); 17886f5f1567SToby Isaac /* get the current closure of the cell that we are removing */ 17899566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &nc, &cellClosure)); 17906f5f1567SToby Isaac 17919566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pNewEnd[dim], &newConeSizes)); 17926f5f1567SToby Isaac { 1793b5a892a1SMatthew G. Knepley DMPolytopeType pct, qct; 17946f5f1567SToby Isaac PetscInt kStart, kEnd, k, closureSizeK, *closureK = NULL, j; 17956f5f1567SToby Isaac 17969566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(K, &kStart, &kEnd)); 17979566063dSJacob Faibussowitsch PetscCall(PetscMalloc4(kEnd - kStart, &Kembedding, kEnd - kStart, &perm, kEnd - kStart, &iperm, kEnd - kStart, &preOrient)); 17986f5f1567SToby Isaac 17996f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 18006f5f1567SToby Isaac perm[k - kStart] = k; 18016f5f1567SToby Isaac iperm[k - kStart] = k - kStart; 18026f5f1567SToby Isaac preOrient[k - kStart] = 0; 18036f5f1567SToby Isaac } 18046f5f1567SToby Isaac 18059566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(K, 0, PETSC_TRUE, &closureSizeK, &closureK)); 18066f5f1567SToby Isaac for (j = 1; j < closureSizeK; j++) { 18076f5f1567SToby Isaac PetscInt parentOrientA = closureK[2 * j + 1]; 18086f5f1567SToby Isaac PetscInt parentOrientB = cellClosure[2 * j + 1]; 18096f5f1567SToby Isaac PetscInt p, q; 18106f5f1567SToby Isaac 18116f5f1567SToby Isaac p = closureK[2 * j]; 18126f5f1567SToby Isaac q = cellClosure[2 * j]; 18139566063dSJacob Faibussowitsch PetscCall(DMPlexGetCellType(K, p, &pct)); 18149566063dSJacob Faibussowitsch PetscCall(DMPlexGetCellType(dm, q, &qct)); 18156f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 1816*9371c9d4SSatish Balay if (q >= pOldStart[d] && q < pOldEnd[d]) { Kembedding[p] = (q - pOldStart[d]) + pNewStart[d]; } 18176f5f1567SToby Isaac } 1818b5a892a1SMatthew G. Knepley parentOrientA = DMPolytopeConvertNewOrientation_Internal(pct, parentOrientA); 1819b5a892a1SMatthew G. Knepley parentOrientB = DMPolytopeConvertNewOrientation_Internal(qct, parentOrientB); 18206f5f1567SToby Isaac if (parentOrientA != parentOrientB) { 18216f5f1567SToby Isaac PetscInt numChildren, i; 18226f5f1567SToby Isaac const PetscInt *children; 18236f5f1567SToby Isaac 18249566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(K, p, &numChildren, &children)); 18256f5f1567SToby Isaac for (i = 0; i < numChildren; i++) { 18266f5f1567SToby Isaac PetscInt kPerm, oPerm; 18276f5f1567SToby Isaac 18286f5f1567SToby Isaac k = children[i]; 18299566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildSymmetry(K, p, parentOrientA, 0, k, parentOrientB, &oPerm, &kPerm)); 18306f5f1567SToby Isaac /* perm = what refTree position I'm in */ 18316f5f1567SToby Isaac perm[kPerm - kStart] = k; 18326f5f1567SToby Isaac /* iperm = who is at this position */ 18336f5f1567SToby Isaac iperm[k - kStart] = kPerm - kStart; 18346f5f1567SToby Isaac preOrient[kPerm - kStart] = oPerm; 18356f5f1567SToby Isaac } 18366f5f1567SToby Isaac } 18376f5f1567SToby Isaac } 18389566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(K, 0, PETSC_TRUE, &closureSizeK, &closureK)); 18396f5f1567SToby Isaac } 18409566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(parentSection, 0, pNewEnd[dim])); 18416f5f1567SToby Isaac offset = 0; 18426f5f1567SToby Isaac numNewCones = 0; 18436f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 18446f5f1567SToby Isaac PetscInt kStart, kEnd, k; 18456f5f1567SToby Isaac PetscInt p; 18466f5f1567SToby Isaac PetscInt size; 18476f5f1567SToby Isaac 18486f5f1567SToby Isaac for (p = pOldStart[d]; p < pOldEnd[d]; p++) { 18496f5f1567SToby Isaac /* skip cell 0 */ 18506f5f1567SToby Isaac if (p == cell) continue; 18516f5f1567SToby Isaac /* old cones to new cones */ 18529566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, p, &size)); 18536f5f1567SToby Isaac newConeSizes[offset++] = size; 18546f5f1567SToby Isaac numNewCones += size; 18556f5f1567SToby Isaac } 18566f5f1567SToby Isaac 18579566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &kStart, &kEnd)); 18586f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 18596f5f1567SToby Isaac PetscInt kParent; 18606f5f1567SToby Isaac 18619566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(K, k, &kParent, NULL)); 18626f5f1567SToby Isaac if (kParent != k) { 18636f5f1567SToby Isaac Kembedding[k] = offset; 18649566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, k, &size)); 18656f5f1567SToby Isaac newConeSizes[offset++] = size; 18666f5f1567SToby Isaac numNewCones += size; 1867*9371c9d4SSatish Balay if (kParent != 0) { PetscCall(PetscSectionSetDof(parentSection, Kembedding[k], 1)); } 18686f5f1567SToby Isaac } 18696f5f1567SToby Isaac } 18706f5f1567SToby Isaac } 18716f5f1567SToby Isaac 18729566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(parentSection)); 18739566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(parentSection, &numPointsWithParents)); 18749566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numNewCones, &newCones, numNewCones, &newOrientations)); 18759566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numPointsWithParents, &parents, numPointsWithParents, &childIDs)); 18766f5f1567SToby Isaac 18776f5f1567SToby Isaac /* fill new cones */ 18786f5f1567SToby Isaac offset = 0; 18796f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 18806f5f1567SToby Isaac PetscInt kStart, kEnd, k, l; 18816f5f1567SToby Isaac PetscInt p; 18826f5f1567SToby Isaac PetscInt size; 18836f5f1567SToby Isaac const PetscInt *cone, *orientation; 18846f5f1567SToby Isaac 18856f5f1567SToby Isaac for (p = pOldStart[d]; p < pOldEnd[d]; p++) { 18866f5f1567SToby Isaac /* skip cell 0 */ 18876f5f1567SToby Isaac if (p == cell) continue; 18886f5f1567SToby Isaac /* old cones to new cones */ 18899566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, p, &size)); 18909566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, p, &cone)); 18919566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(dm, p, &orientation)); 18926f5f1567SToby Isaac for (l = 0; l < size; l++) { 18936f5f1567SToby Isaac newCones[offset] = (cone[l] - pOldStart[d + 1]) + pNewStart[d + 1]; 18946f5f1567SToby Isaac newOrientations[offset++] = orientation[l]; 18956f5f1567SToby Isaac } 18966f5f1567SToby Isaac } 18976f5f1567SToby Isaac 18989566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &kStart, &kEnd)); 18996f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 19006f5f1567SToby Isaac PetscInt kPerm = perm[k], kParent; 19016f5f1567SToby Isaac PetscInt preO = preOrient[k]; 19026f5f1567SToby Isaac 19039566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(K, k, &kParent, NULL)); 19046f5f1567SToby Isaac if (kParent != k) { 19056f5f1567SToby Isaac /* embed new cones */ 19069566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, k, &size)); 19079566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(K, kPerm, &cone)); 19089566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(K, kPerm, &orientation)); 19096f5f1567SToby Isaac for (l = 0; l < size; l++) { 19106f5f1567SToby Isaac PetscInt q, m = (preO >= 0) ? ((preO + l) % size) : ((size - (preO + 1) - l) % size); 19116f5f1567SToby Isaac PetscInt newO, lSize, oTrue; 1912b5a892a1SMatthew G. Knepley DMPolytopeType ct = DM_NUM_POLYTOPES; 19136f5f1567SToby Isaac 19146f5f1567SToby Isaac q = iperm[cone[m]]; 19156f5f1567SToby Isaac newCones[offset] = Kembedding[q]; 19169566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, q, &lSize)); 1917b5a892a1SMatthew G. Knepley if (lSize == 2) ct = DM_POLYTOPE_SEGMENT; 1918b5a892a1SMatthew G. Knepley else if (lSize == 4) ct = DM_POLYTOPE_QUADRILATERAL; 1919b5a892a1SMatthew G. Knepley oTrue = DMPolytopeConvertNewOrientation_Internal(ct, orientation[m]); 19206f5f1567SToby Isaac oTrue = ((!lSize) || (preOrient[k] >= 0)) ? oTrue : -(oTrue + 2); 19216f5f1567SToby Isaac newO = DihedralCompose(lSize, oTrue, preOrient[q]); 1922b5a892a1SMatthew G. Knepley newOrientations[offset++] = DMPolytopeConvertOldOrientation_Internal(ct, newO); 19236f5f1567SToby Isaac } 19246f5f1567SToby Isaac if (kParent != 0) { 19256f5f1567SToby Isaac PetscInt newPoint = Kembedding[kParent]; 19269566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(parentSection, Kembedding[k], &pOffset)); 19276f5f1567SToby Isaac parents[pOffset] = newPoint; 19286f5f1567SToby Isaac childIDs[pOffset] = k; 19296f5f1567SToby Isaac } 19306f5f1567SToby Isaac } 19316f5f1567SToby Isaac } 19326f5f1567SToby Isaac } 19336f5f1567SToby Isaac 19349566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(dim * (pNewEnd[dim] - pNewStart[dim]), &newVertexCoords)); 19356f5f1567SToby Isaac 19366f5f1567SToby Isaac /* fill coordinates */ 19376f5f1567SToby Isaac offset = 0; 19386f5f1567SToby Isaac { 1939d90620a3SMatthew G. Knepley PetscInt kStart, kEnd, l; 19406f5f1567SToby Isaac PetscSection vSection; 19416f5f1567SToby Isaac PetscInt v; 19426f5f1567SToby Isaac Vec coords; 19436f5f1567SToby Isaac PetscScalar *coordvals; 19446f5f1567SToby Isaac PetscInt dof, off; 1945c111c6b7SMatthew G. Knepley PetscReal v0[3], J[9], detJ; 19466f5f1567SToby Isaac 194776bd3646SJed Brown if (PetscDefined(USE_DEBUG)) { 1948d90620a3SMatthew G. Knepley PetscInt k; 19499566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, 0, &kStart, &kEnd)); 19506f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 19519566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(K, k, NULL, v0, J, NULL, &detJ)); 195263a3b9bcSJacob Faibussowitsch PetscCheck(detJ > 0., PETSC_COMM_SELF, PETSC_ERR_PLIB, "reference tree cell %" PetscInt_FMT " has bad determinant", k); 19536f5f1567SToby Isaac } 1954d90620a3SMatthew G. Knepley } 19559566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, NULL, v0, J, NULL, &detJ)); 19569566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateSection(dm, &vSection)); 19579566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(dm, &coords)); 19589566063dSJacob Faibussowitsch PetscCall(VecGetArray(coords, &coordvals)); 19596f5f1567SToby Isaac for (v = pOldStart[dim]; v < pOldEnd[dim]; v++) { 19609566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(vSection, v, &dof)); 19619566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(vSection, v, &off)); 1962*9371c9d4SSatish Balay for (l = 0; l < dof; l++) { newVertexCoords[offset++] = coordvals[off + l]; } 19636f5f1567SToby Isaac } 19649566063dSJacob Faibussowitsch PetscCall(VecRestoreArray(coords, &coordvals)); 19656f5f1567SToby Isaac 19669566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateSection(K, &vSection)); 19679566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(K, &coords)); 19689566063dSJacob Faibussowitsch PetscCall(VecGetArray(coords, &coordvals)); 19699566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(K, 0, &kStart, &kEnd)); 19706f5f1567SToby Isaac for (v = kStart; v < kEnd; v++) { 19719bc368c7SMatthew G. Knepley PetscReal coord[3], newCoord[3]; 19726f5f1567SToby Isaac PetscInt vPerm = perm[v]; 19736f5f1567SToby Isaac PetscInt kParent; 1974c330f8ffSToby Isaac const PetscReal xi0[3] = {-1., -1., -1.}; 19756f5f1567SToby Isaac 19769566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(K, v, &kParent, NULL)); 19776f5f1567SToby Isaac if (kParent != v) { 19786f5f1567SToby Isaac /* this is a new vertex */ 19799566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(vSection, vPerm, &off)); 19809bc368c7SMatthew G. Knepley for (l = 0; l < dim; ++l) coord[l] = PetscRealPart(coordvals[off + l]); 1981367003a6SStefano Zampini CoordinatesRefToReal(dim, dim, xi0, v0, J, coord, newCoord); 19829bc368c7SMatthew G. Knepley for (l = 0; l < dim; ++l) newVertexCoords[offset + l] = newCoord[l]; 19836f5f1567SToby Isaac offset += dim; 19846f5f1567SToby Isaac } 19856f5f1567SToby Isaac } 19869566063dSJacob Faibussowitsch PetscCall(VecRestoreArray(coords, &coordvals)); 19876f5f1567SToby Isaac } 19886f5f1567SToby Isaac 19896f5f1567SToby Isaac /* need to reverse the order of pNewCount: vertices first, cells last */ 19906f5f1567SToby Isaac for (d = 0; d < (dim + 1) / 2; d++) { 19916f5f1567SToby Isaac PetscInt tmp; 19926f5f1567SToby Isaac 19936f5f1567SToby Isaac tmp = pNewCount[d]; 19946f5f1567SToby Isaac pNewCount[d] = pNewCount[dim - d]; 19956f5f1567SToby Isaac pNewCount[dim - d] = tmp; 19966f5f1567SToby Isaac } 19976f5f1567SToby Isaac 19989566063dSJacob Faibussowitsch PetscCall(DMPlexCreateFromDAG(*ncdm, dim, pNewCount, newConeSizes, newCones, newOrientations, newVertexCoords)); 19999566063dSJacob Faibussowitsch PetscCall(DMPlexSetReferenceTree(*ncdm, K)); 20009566063dSJacob Faibussowitsch PetscCall(DMPlexSetTree(*ncdm, parentSection, parents, childIDs)); 20016f5f1567SToby Isaac 20026f5f1567SToby Isaac /* clean up */ 20039566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &nc, &cellClosure)); 20049566063dSJacob Faibussowitsch PetscCall(PetscFree5(pNewCount, pNewStart, pNewEnd, pOldStart, pOldEnd)); 20059566063dSJacob Faibussowitsch PetscCall(PetscFree(newConeSizes)); 20069566063dSJacob Faibussowitsch PetscCall(PetscFree2(newCones, newOrientations)); 20079566063dSJacob Faibussowitsch PetscCall(PetscFree(newVertexCoords)); 20089566063dSJacob Faibussowitsch PetscCall(PetscFree2(parents, childIDs)); 20099566063dSJacob Faibussowitsch PetscCall(PetscFree4(Kembedding, perm, iperm, preOrient)); 2010*9371c9d4SSatish Balay } else { 20116f5f1567SToby Isaac PetscInt p, counts[4]; 20126f5f1567SToby Isaac PetscInt *coneSizes, *cones, *orientations; 20136f5f1567SToby Isaac Vec coordVec; 20146f5f1567SToby Isaac PetscScalar *coords; 20156f5f1567SToby Isaac 20166f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 20176f5f1567SToby Isaac PetscInt dStart, dEnd; 20186f5f1567SToby Isaac 20199566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(dm, d, &dStart, &dEnd)); 20206f5f1567SToby Isaac counts[d] = dEnd - dStart; 20216f5f1567SToby Isaac } 20229566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEnd - pStart, &coneSizes)); 2023*9371c9d4SSatish Balay for (p = pStart; p < pEnd; p++) { PetscCall(DMPlexGetConeSize(dm, p, &coneSizes[p - pStart])); } 20249566063dSJacob Faibussowitsch PetscCall(DMPlexGetCones(dm, &cones)); 20259566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientations(dm, &orientations)); 20269566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(dm, &coordVec)); 20279566063dSJacob Faibussowitsch PetscCall(VecGetArray(coordVec, &coords)); 20286f5f1567SToby Isaac 20299566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(parentSection, pStart, pEnd)); 20309566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(parentSection)); 20319566063dSJacob Faibussowitsch PetscCall(DMPlexCreateFromDAG(*ncdm, dim, counts, coneSizes, cones, orientations, NULL)); 20329566063dSJacob Faibussowitsch PetscCall(DMPlexSetReferenceTree(*ncdm, K)); 20339566063dSJacob Faibussowitsch PetscCall(DMPlexSetTree(*ncdm, parentSection, NULL, NULL)); 20349566063dSJacob Faibussowitsch PetscCall(VecRestoreArray(coordVec, &coords)); 20356f5f1567SToby Isaac } 20369566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&parentSection)); 20376f5f1567SToby Isaac 20386f5f1567SToby Isaac PetscFunctionReturn(0); 20396f5f1567SToby Isaac } 20406ecaa68aSToby Isaac 2041*9371c9d4SSatish Balay PetscErrorCode DMPlexComputeInterpolatorTree(DM coarse, DM fine, PetscSF coarseToFine, PetscInt *childIds, Mat mat) { 20426ecaa68aSToby Isaac PetscSF coarseToFineEmbedded; 20436ecaa68aSToby Isaac PetscSection globalCoarse, globalFine; 20446ecaa68aSToby Isaac PetscSection localCoarse, localFine; 20456ecaa68aSToby Isaac PetscSection aSec, cSec; 20466ecaa68aSToby Isaac PetscSection rootIndicesSec, rootMatricesSec; 204746bdb399SToby Isaac PetscSection leafIndicesSec, leafMatricesSec; 204846bdb399SToby Isaac PetscInt *rootIndices, *leafIndices; 204946bdb399SToby Isaac PetscScalar *rootMatrices, *leafMatrices; 20506ecaa68aSToby Isaac IS aIS; 20516ecaa68aSToby Isaac const PetscInt *anchors; 20526ecaa68aSToby Isaac Mat cMat; 20534acb8e1eSToby Isaac PetscInt numFields, maxFields; 20546ecaa68aSToby Isaac PetscInt pStartC, pEndC, pStartF, pEndF, p; 20556ecaa68aSToby Isaac PetscInt aStart, aEnd, cStart, cEnd; 20561c58ffc4SToby Isaac PetscInt *maxChildIds; 2057e44e4e7fSToby Isaac PetscInt *offsets, *newOffsets, *offsetsCopy, *newOffsetsCopy, *rowOffsets, *numD, *numO; 20584acb8e1eSToby Isaac const PetscInt ***perms; 20594acb8e1eSToby Isaac const PetscScalar ***flips; 20606ecaa68aSToby Isaac 20616ecaa68aSToby Isaac PetscFunctionBegin; 20629566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 20639566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 20649566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 20656ecaa68aSToby Isaac { /* winnow fine points that don't have global dofs out of the sf */ 206689698031SToby Isaac PetscInt dof, cdof, numPointsWithDofs, offset, *pointsWithDofs, nleaves, l; 206789698031SToby Isaac const PetscInt *leaves; 20686ecaa68aSToby Isaac 20699566063dSJacob Faibussowitsch PetscCall(PetscSFGetGraph(coarseToFine, NULL, &nleaves, &leaves, NULL)); 207089698031SToby Isaac for (l = 0, numPointsWithDofs = 0; l < nleaves; l++) { 207189698031SToby Isaac p = leaves ? leaves[l] : l; 20729566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 20739566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 2074*9371c9d4SSatish Balay if ((dof - cdof) > 0) { numPointsWithDofs++; } 20756ecaa68aSToby Isaac } 20769566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numPointsWithDofs, &pointsWithDofs)); 20777cc7abc7SToby Isaac for (l = 0, offset = 0; l < nleaves; l++) { 207889698031SToby Isaac p = leaves ? leaves[l] : l; 20799566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 20809566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 2081*9371c9d4SSatish Balay if ((dof - cdof) > 0) { pointsWithDofs[offset++] = l; } 20826ecaa68aSToby Isaac } 20839566063dSJacob Faibussowitsch PetscCall(PetscSFCreateEmbeddedLeafSF(coarseToFine, numPointsWithDofs, pointsWithDofs, &coarseToFineEmbedded)); 20849566063dSJacob Faibussowitsch PetscCall(PetscFree(pointsWithDofs)); 20856ecaa68aSToby Isaac } 20866ecaa68aSToby Isaac /* communicate back to the coarse mesh which coarse points have children (that may require interpolation) */ 20879566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEndC - pStartC, &maxChildIds)); 2088*9371c9d4SSatish Balay for (p = pStartC; p < pEndC; p++) { maxChildIds[p - pStartC] = -2; } 208957168dbeSPierre Jolivet PetscCall(PetscSFReduceBegin(coarseToFineEmbedded, MPIU_INT, childIds, maxChildIds, MPI_MAX)); 209057168dbeSPierre Jolivet PetscCall(PetscSFReduceEnd(coarseToFineEmbedded, MPIU_INT, childIds, maxChildIds, MPI_MAX)); 209146bdb399SToby Isaac 20929566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 20939566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 209446bdb399SToby Isaac 20959566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(coarse, &aSec, &aIS)); 20969566063dSJacob Faibussowitsch PetscCall(ISGetIndices(aIS, &anchors)); 20979566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(aSec, &aStart, &aEnd)); 209846bdb399SToby Isaac 20999566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(coarse, &cSec, &cMat, NULL)); 21009566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSec, &cStart, &cEnd)); 210146bdb399SToby Isaac 210246bdb399SToby Isaac /* create sections that will send to children the indices and matrices they will need to construct the interpolator */ 21039566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &rootIndicesSec)); 21049566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &rootMatricesSec)); 21059566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(rootIndicesSec, pStartC, pEndC)); 21069566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(rootMatricesSec, pStartC, pEndC)); 21079566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(localCoarse, &numFields)); 2108713c1c5dSToby Isaac maxFields = PetscMax(1, numFields); 21099566063dSJacob Faibussowitsch PetscCall(PetscMalloc7(maxFields + 1, &offsets, maxFields + 1, &offsetsCopy, maxFields + 1, &newOffsets, maxFields + 1, &newOffsetsCopy, maxFields + 1, &rowOffsets, maxFields + 1, &numD, maxFields + 1, &numO)); 21109566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(maxFields + 1, (PetscInt ****)&perms, maxFields + 1, (PetscScalar ****)&flips)); 21119566063dSJacob Faibussowitsch PetscCall(PetscMemzero((void *)perms, (maxFields + 1) * sizeof(const PetscInt **))); 21129566063dSJacob Faibussowitsch PetscCall(PetscMemzero((void *)flips, (maxFields + 1) * sizeof(const PetscScalar **))); 211346bdb399SToby Isaac 211446bdb399SToby Isaac for (p = pStartC; p < pEndC; p++) { /* count the sizes of the indices and matrices */ 21158d2f55e7SToby Isaac PetscInt dof, matSize = 0; 21166ecaa68aSToby Isaac PetscInt aDof = 0; 21176ecaa68aSToby Isaac PetscInt cDof = 0; 21186ecaa68aSToby Isaac PetscInt maxChildId = maxChildIds[p - pStartC]; 21196ecaa68aSToby Isaac PetscInt numRowIndices = 0; 21206ecaa68aSToby Isaac PetscInt numColIndices = 0; 2121f13f9184SToby Isaac PetscInt f; 21226ecaa68aSToby Isaac 21239566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 2124*9371c9d4SSatish Balay if (dof < 0) { dof = -(dof + 1); } 2125*9371c9d4SSatish Balay if (p >= aStart && p < aEnd) { PetscCall(PetscSectionGetDof(aSec, p, &aDof)); } 2126*9371c9d4SSatish Balay if (p >= cStart && p < cEnd) { PetscCall(PetscSectionGetDof(cSec, p, &cDof)); } 2127f13f9184SToby Isaac for (f = 0; f <= numFields; f++) offsets[f] = 0; 2128f13f9184SToby Isaac for (f = 0; f <= numFields; f++) newOffsets[f] = 0; 21296ecaa68aSToby Isaac if (maxChildId >= 0) { /* this point has children (with dofs) that will need to be interpolated from the closure of p */ 2130f13f9184SToby Isaac PetscInt *closure = NULL, closureSize, cl; 21316ecaa68aSToby Isaac 21329566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 213346bdb399SToby Isaac for (cl = 0; cl < closureSize; cl++) { /* get the closure */ 21346ecaa68aSToby Isaac PetscInt c = closure[2 * cl], clDof; 21356ecaa68aSToby Isaac 21369566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, c, &clDof)); 21376ecaa68aSToby Isaac numRowIndices += clDof; 21386ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 21399566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, c, f, &clDof)); 21406ecaa68aSToby Isaac offsets[f + 1] += clDof; 21416ecaa68aSToby Isaac } 21426ecaa68aSToby Isaac } 21436ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 21446ecaa68aSToby Isaac offsets[f + 1] += offsets[f]; 21456ecaa68aSToby Isaac newOffsets[f + 1] = offsets[f + 1]; 21466ecaa68aSToby Isaac } 214746bdb399SToby Isaac /* get the number of indices needed and their field offsets */ 21489566063dSJacob Faibussowitsch PetscCall(DMPlexAnchorsModifyMat(coarse, localCoarse, closureSize, numRowIndices, closure, NULL, NULL, NULL, &numColIndices, NULL, NULL, newOffsets, PETSC_FALSE)); 21499566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 21506ecaa68aSToby Isaac if (!numColIndices) { /* there are no hanging constraint modifications, so the matrix is just the identity: do not send it */ 21516ecaa68aSToby Isaac numColIndices = numRowIndices; 21526ecaa68aSToby Isaac matSize = 0; 2153*9371c9d4SSatish Balay } else if (numFields) { /* we send one submat for each field: sum their sizes */ 21546ecaa68aSToby Isaac matSize = 0; 21556ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 21566ecaa68aSToby Isaac PetscInt numRow, numCol; 21576ecaa68aSToby Isaac 21586ecaa68aSToby Isaac numRow = offsets[f + 1] - offsets[f]; 2159f13f9184SToby Isaac numCol = newOffsets[f + 1] - newOffsets[f]; 21606ecaa68aSToby Isaac matSize += numRow * numCol; 21616ecaa68aSToby Isaac } 2162*9371c9d4SSatish Balay } else { 21636ecaa68aSToby Isaac matSize = numRowIndices * numColIndices; 21646ecaa68aSToby Isaac } 2165f13f9184SToby Isaac } else if (maxChildId == -1) { 21668d2f55e7SToby Isaac if (cDof > 0) { /* this point's dofs are interpolated via cMat: get the submatrix of cMat */ 2167f13f9184SToby Isaac PetscInt aOff, a; 21686ecaa68aSToby Isaac 21699566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(aSec, p, &aOff)); 21706ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 21716ecaa68aSToby Isaac PetscInt fDof; 21726ecaa68aSToby Isaac 21739566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 217421968bf8SToby Isaac offsets[f + 1] = fDof; 21756ecaa68aSToby Isaac } 21766ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 21776ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff], aLocalDof; 21786ecaa68aSToby Isaac 21799566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, anchor, &aLocalDof)); 21806ecaa68aSToby Isaac numColIndices += aLocalDof; 21816ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 21826ecaa68aSToby Isaac PetscInt fDof; 21836ecaa68aSToby Isaac 21849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, anchor, f, &fDof)); 218521968bf8SToby Isaac newOffsets[f + 1] += fDof; 21866ecaa68aSToby Isaac } 21876ecaa68aSToby Isaac } 21886ecaa68aSToby Isaac if (numFields) { 21896ecaa68aSToby Isaac matSize = 0; 2190*9371c9d4SSatish Balay for (f = 0; f < numFields; f++) { matSize += offsets[f + 1] * newOffsets[f + 1]; } 2191*9371c9d4SSatish Balay } else { 21926ecaa68aSToby Isaac matSize = numColIndices * dof; 21936ecaa68aSToby Isaac } 2194*9371c9d4SSatish Balay } else { /* no children, and no constraints on dofs: just get the global indices */ 21956ecaa68aSToby Isaac numColIndices = dof; 21966ecaa68aSToby Isaac matSize = 0; 21976ecaa68aSToby Isaac } 21988d2f55e7SToby Isaac } 219946bdb399SToby Isaac /* we will pack the column indices with the field offsets */ 22009566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(rootIndicesSec, p, numColIndices ? numColIndices + 2 * numFields : 0)); 22019566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(rootMatricesSec, p, matSize)); 22026ecaa68aSToby Isaac } 22039566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(rootIndicesSec)); 22049566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(rootMatricesSec)); 22056ecaa68aSToby Isaac { 22066ecaa68aSToby Isaac PetscInt numRootIndices, numRootMatrices; 22076ecaa68aSToby Isaac 22089566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(rootIndicesSec, &numRootIndices)); 22099566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(rootMatricesSec, &numRootMatrices)); 22109566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numRootIndices, &rootIndices, numRootMatrices, &rootMatrices)); 22116ecaa68aSToby Isaac for (p = pStartC; p < pEndC; p++) { 22126ecaa68aSToby Isaac PetscInt numRowIndices, numColIndices, matSize, dof; 2213f13f9184SToby Isaac PetscInt pIndOff, pMatOff, f; 22146ecaa68aSToby Isaac PetscInt *pInd; 22156ecaa68aSToby Isaac PetscInt maxChildId = maxChildIds[p - pStartC]; 22166ecaa68aSToby Isaac PetscScalar *pMat = NULL; 22176ecaa68aSToby Isaac 22189566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootIndicesSec, p, &numColIndices)); 2219*9371c9d4SSatish Balay if (!numColIndices) { continue; } 2220f13f9184SToby Isaac for (f = 0; f <= numFields; f++) { 2221f13f9184SToby Isaac offsets[f] = 0; 2222f13f9184SToby Isaac newOffsets[f] = 0; 2223f13f9184SToby Isaac offsetsCopy[f] = 0; 2224f13f9184SToby Isaac newOffsetsCopy[f] = 0; 2225f13f9184SToby Isaac } 22266ecaa68aSToby Isaac numColIndices -= 2 * numFields; 22279566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootIndicesSec, p, &pIndOff)); 22286ecaa68aSToby Isaac pInd = &(rootIndices[pIndOff]); 22299566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootMatricesSec, p, &matSize)); 22306ecaa68aSToby Isaac if (matSize) { 22319566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootMatricesSec, p, &pMatOff)); 22326ecaa68aSToby Isaac pMat = &rootMatrices[pMatOff]; 22336ecaa68aSToby Isaac } 22349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 2235*9371c9d4SSatish Balay if (dof < 0) { dof = -(dof + 1); } 22366ecaa68aSToby Isaac if (maxChildId >= 0) { /* build an identity matrix, apply matrix constraints on the right */ 22376ecaa68aSToby Isaac PetscInt i, j; 22386ecaa68aSToby Isaac PetscInt numRowIndices = matSize / numColIndices; 22396ecaa68aSToby Isaac 22406ecaa68aSToby Isaac if (!numRowIndices) { /* don't need to calculate the mat, just the indices */ 22416ecaa68aSToby Isaac PetscInt numIndices, *indices; 22429566063dSJacob Faibussowitsch PetscCall(DMPlexGetClosureIndices(coarse, localCoarse, globalCoarse, p, PETSC_TRUE, &numIndices, &indices, offsets, NULL)); 224308401ef6SPierre Jolivet PetscCheck(numIndices == numColIndices, PETSC_COMM_SELF, PETSC_ERR_PLIB, "mismatching constraint indices calculations"); 2244*9371c9d4SSatish Balay for (i = 0; i < numColIndices; i++) { pInd[i] = indices[i]; } 22456ecaa68aSToby Isaac for (i = 0; i < numFields; i++) { 224646bdb399SToby Isaac pInd[numColIndices + i] = offsets[i + 1]; 224746bdb399SToby Isaac pInd[numColIndices + numFields + i] = offsets[i + 1]; 22486ecaa68aSToby Isaac } 22499566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreClosureIndices(coarse, localCoarse, globalCoarse, p, PETSC_TRUE, &numIndices, &indices, offsets, NULL)); 2250*9371c9d4SSatish Balay } else { 22516ecaa68aSToby Isaac PetscInt closureSize, *closure = NULL, cl; 22526ecaa68aSToby Isaac PetscScalar *pMatIn, *pMatModified; 22536ecaa68aSToby Isaac PetscInt numPoints, *points; 22546ecaa68aSToby Isaac 22559566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(coarse, numRowIndices * numRowIndices, MPIU_SCALAR, &pMatIn)); 22566ecaa68aSToby Isaac for (i = 0; i < numRowIndices; i++) { /* initialize to the identity */ 2257*9371c9d4SSatish Balay for (j = 0; j < numRowIndices; j++) { pMatIn[i * numRowIndices + j] = (i == j) ? 1. : 0.; } 22586ecaa68aSToby Isaac } 22599566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 22604acb8e1eSToby Isaac for (f = 0; f < maxFields; f++) { 22619566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionGetFieldPointSyms(localCoarse, f, closureSize, closure, &perms[f], &flips[f])); 22629566063dSJacob Faibussowitsch else PetscCall(PetscSectionGetPointSyms(localCoarse, closureSize, closure, &perms[f], &flips[f])); 22634acb8e1eSToby Isaac } 22646ecaa68aSToby Isaac if (numFields) { 22656ecaa68aSToby Isaac for (cl = 0; cl < closureSize; cl++) { 22666ecaa68aSToby Isaac PetscInt c = closure[2 * cl]; 22676ecaa68aSToby Isaac 22686ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 22696ecaa68aSToby Isaac PetscInt fDof; 22706ecaa68aSToby Isaac 22719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, c, f, &fDof)); 22726ecaa68aSToby Isaac offsets[f + 1] += fDof; 22736ecaa68aSToby Isaac } 22746ecaa68aSToby Isaac } 22756ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 22766ecaa68aSToby Isaac offsets[f + 1] += offsets[f]; 22776ecaa68aSToby Isaac newOffsets[f + 1] = offsets[f + 1]; 22786ecaa68aSToby Isaac } 22796ecaa68aSToby Isaac } 22804acb8e1eSToby Isaac /* TODO : flips here ? */ 22816ecaa68aSToby Isaac /* apply hanging node constraints on the right, get the new points and the new offsets */ 22829566063dSJacob Faibussowitsch PetscCall(DMPlexAnchorsModifyMat(coarse, localCoarse, closureSize, numRowIndices, closure, perms, pMatIn, &numPoints, NULL, &points, &pMatModified, newOffsets, PETSC_FALSE)); 22834acb8e1eSToby Isaac for (f = 0; f < maxFields; f++) { 22849566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionRestoreFieldPointSyms(localCoarse, f, closureSize, closure, &perms[f], &flips[f])); 22859566063dSJacob Faibussowitsch else PetscCall(PetscSectionRestorePointSyms(localCoarse, closureSize, closure, &perms[f], &flips[f])); 22864acb8e1eSToby Isaac } 22874acb8e1eSToby Isaac for (f = 0; f < maxFields; f++) { 22889566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionGetFieldPointSyms(localCoarse, f, numPoints, points, &perms[f], &flips[f])); 22899566063dSJacob Faibussowitsch else PetscCall(PetscSectionGetPointSyms(localCoarse, numPoints, points, &perms[f], &flips[f])); 22904acb8e1eSToby Isaac } 22916ecaa68aSToby Isaac if (!numFields) { 2292*9371c9d4SSatish Balay for (i = 0; i < numRowIndices * numColIndices; i++) { pMat[i] = pMatModified[i]; } 2293*9371c9d4SSatish Balay } else { 2294f13f9184SToby Isaac PetscInt i, j, count; 22956ecaa68aSToby Isaac for (f = 0, count = 0; f < numFields; f++) { 22966ecaa68aSToby Isaac for (i = offsets[f]; i < offsets[f + 1]; i++) { 2297*9371c9d4SSatish Balay for (j = newOffsets[f]; j < newOffsets[f + 1]; j++, count++) { pMat[count] = pMatModified[i * numColIndices + j]; } 22986ecaa68aSToby Isaac } 22996ecaa68aSToby Isaac } 23006ecaa68aSToby Isaac } 23019566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numRowIndices * numColIndices, MPIU_SCALAR, &pMatModified)); 23029566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 23039566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numRowIndices * numColIndices, MPIU_SCALAR, &pMatIn)); 23046ecaa68aSToby Isaac if (numFields) { 230546bdb399SToby Isaac for (f = 0; f < numFields; f++) { 230646bdb399SToby Isaac pInd[numColIndices + f] = offsets[f + 1]; 230746bdb399SToby Isaac pInd[numColIndices + numFields + f] = newOffsets[f + 1]; 23086ecaa68aSToby Isaac } 23094acb8e1eSToby Isaac for (cl = 0; cl < numPoints; cl++) { 23104acb8e1eSToby Isaac PetscInt globalOff, c = points[2 * cl]; 23119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, c, &globalOff)); 23129566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, c, globalOff < 0 ? -(globalOff + 1) : globalOff, newOffsets, PETSC_FALSE, perms, cl, NULL, pInd)); 23136ecaa68aSToby Isaac } 23146ecaa68aSToby Isaac } else { 23154acb8e1eSToby Isaac for (cl = 0; cl < numPoints; cl++) { 23164acb8e1eSToby Isaac PetscInt c = points[2 * cl], globalOff; 23174acb8e1eSToby Isaac const PetscInt *perm = perms[0] ? perms[0][cl] : NULL; 23184acb8e1eSToby Isaac 23199566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, c, &globalOff)); 23209566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, c, globalOff < 0 ? -(globalOff + 1) : globalOff, newOffsets, PETSC_FALSE, perm, NULL, pInd)); 23216ecaa68aSToby Isaac } 23226ecaa68aSToby Isaac } 23234acb8e1eSToby Isaac for (f = 0; f < maxFields; f++) { 23249566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionRestoreFieldPointSyms(localCoarse, f, numPoints, points, &perms[f], &flips[f])); 23259566063dSJacob Faibussowitsch else PetscCall(PetscSectionRestorePointSyms(localCoarse, numPoints, points, &perms[f], &flips[f])); 23264acb8e1eSToby Isaac } 23279566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numPoints, MPIU_SCALAR, &points)); 23286ecaa68aSToby Isaac } 2329*9371c9d4SSatish Balay } else if (matSize) { 23306ecaa68aSToby Isaac PetscInt cOff; 23316ecaa68aSToby Isaac PetscInt *rowIndices, *colIndices, a, aDof, aOff; 23326ecaa68aSToby Isaac 23336ecaa68aSToby Isaac numRowIndices = matSize / numColIndices; 233408401ef6SPierre Jolivet PetscCheck(numRowIndices == dof, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Miscounted dofs"); 23359566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(coarse, numRowIndices, MPIU_INT, &rowIndices)); 23369566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(coarse, numColIndices, MPIU_INT, &colIndices)); 23379566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(cSec, p, &cOff)); 23389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(aSec, p, &aDof)); 23399566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(aSec, p, &aOff)); 23406ecaa68aSToby Isaac if (numFields) { 23416ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 23426ecaa68aSToby Isaac PetscInt fDof; 2343f13f9184SToby Isaac 23449566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSec, p, f, &fDof)); 23456ecaa68aSToby Isaac offsets[f + 1] = fDof; 23466ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 23476ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff]; 23489566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, anchor, f, &fDof)); 23496ecaa68aSToby Isaac newOffsets[f + 1] += fDof; 23506ecaa68aSToby Isaac } 23516ecaa68aSToby Isaac } 23526ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 23536ecaa68aSToby Isaac offsets[f + 1] += offsets[f]; 23546ecaa68aSToby Isaac offsetsCopy[f + 1] = offsets[f + 1]; 23556ecaa68aSToby Isaac newOffsets[f + 1] += newOffsets[f]; 23566ecaa68aSToby Isaac newOffsetsCopy[f + 1] = newOffsets[f + 1]; 23576ecaa68aSToby Isaac } 23589566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, p, cOff, offsetsCopy, PETSC_TRUE, NULL, -1, NULL, rowIndices)); 23596ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 23606ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff], lOff; 23619566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(localCoarse, anchor, &lOff)); 23629566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_TRUE, anchor, lOff, newOffsetsCopy, PETSC_TRUE, NULL, -1, NULL, colIndices)); 23636ecaa68aSToby Isaac } 2364*9371c9d4SSatish Balay } else { 23659566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, p, cOff, offsetsCopy, PETSC_TRUE, NULL, NULL, rowIndices)); 23666ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 23676ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff], lOff; 23689566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(localCoarse, anchor, &lOff)); 23699566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_TRUE, anchor, lOff, newOffsetsCopy, PETSC_TRUE, NULL, NULL, colIndices)); 23706ecaa68aSToby Isaac } 23716ecaa68aSToby Isaac } 23726ecaa68aSToby Isaac if (numFields) { 2373f13f9184SToby Isaac PetscInt count, a; 2374f13f9184SToby Isaac 23756ecaa68aSToby Isaac for (f = 0, count = 0; f < numFields; f++) { 23766ecaa68aSToby Isaac PetscInt iSize = offsets[f + 1] - offsets[f]; 23776ecaa68aSToby Isaac PetscInt jSize = newOffsets[f + 1] - newOffsets[f]; 23789566063dSJacob Faibussowitsch PetscCall(MatGetValues(cMat, iSize, &rowIndices[offsets[f]], jSize, &colIndices[newOffsets[f]], &pMat[count])); 23796ecaa68aSToby Isaac count += iSize * jSize; 238046bdb399SToby Isaac pInd[numColIndices + f] = offsets[f + 1]; 238146bdb399SToby Isaac pInd[numColIndices + numFields + f] = newOffsets[f + 1]; 23826ecaa68aSToby Isaac } 23836ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 23846ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff]; 23856ecaa68aSToby Isaac PetscInt gOff; 23869566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, anchor, &gOff)); 23879566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, anchor, gOff < 0 ? -(gOff + 1) : gOff, newOffsets, PETSC_FALSE, NULL, -1, NULL, pInd)); 23886ecaa68aSToby Isaac } 2389*9371c9d4SSatish Balay } else { 23906ecaa68aSToby Isaac PetscInt a; 23919566063dSJacob Faibussowitsch PetscCall(MatGetValues(cMat, numRowIndices, rowIndices, numColIndices, colIndices, pMat)); 23926ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 23936ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff]; 23946ecaa68aSToby Isaac PetscInt gOff; 23959566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, anchor, &gOff)); 23969566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, anchor, gOff < 0 ? -(gOff + 1) : gOff, newOffsets, PETSC_FALSE, NULL, NULL, pInd)); 23976ecaa68aSToby Isaac } 23986ecaa68aSToby Isaac } 23999566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numColIndices, MPIU_INT, &colIndices)); 24009566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numRowIndices, MPIU_INT, &rowIndices)); 2401*9371c9d4SSatish Balay } else { 24026ecaa68aSToby Isaac PetscInt gOff; 24036ecaa68aSToby Isaac 24049566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, p, &gOff)); 24056ecaa68aSToby Isaac if (numFields) { 24066ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 24076ecaa68aSToby Isaac PetscInt fDof; 24089566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 24096ecaa68aSToby Isaac offsets[f + 1] = fDof + offsets[f]; 24106ecaa68aSToby Isaac } 24116ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 241246bdb399SToby Isaac pInd[numColIndices + f] = offsets[f + 1]; 241346bdb399SToby Isaac pInd[numColIndices + numFields + f] = offsets[f + 1]; 24146ecaa68aSToby Isaac } 24159566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsets, PETSC_FALSE, NULL, -1, NULL, pInd)); 2416367003a6SStefano Zampini } else { 24179566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsets, PETSC_FALSE, NULL, NULL, pInd)); 24186ecaa68aSToby Isaac } 24196ecaa68aSToby Isaac } 24206ecaa68aSToby Isaac } 24219566063dSJacob Faibussowitsch PetscCall(PetscFree(maxChildIds)); 24226ecaa68aSToby Isaac } 242346bdb399SToby Isaac { 242446bdb399SToby Isaac PetscSF indicesSF, matricesSF; 242546bdb399SToby Isaac PetscInt *remoteOffsetsIndices, *remoteOffsetsMatrices, numLeafIndices, numLeafMatrices; 242646bdb399SToby Isaac 24279566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)fine), &leafIndicesSec)); 24289566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)fine), &leafMatricesSec)); 24299566063dSJacob Faibussowitsch PetscCall(PetscSFDistributeSection(coarseToFineEmbedded, rootIndicesSec, &remoteOffsetsIndices, leafIndicesSec)); 24309566063dSJacob Faibussowitsch PetscCall(PetscSFDistributeSection(coarseToFineEmbedded, rootMatricesSec, &remoteOffsetsMatrices, leafMatricesSec)); 24319566063dSJacob Faibussowitsch PetscCall(PetscSFCreateSectionSF(coarseToFineEmbedded, rootIndicesSec, remoteOffsetsIndices, leafIndicesSec, &indicesSF)); 24329566063dSJacob Faibussowitsch PetscCall(PetscSFCreateSectionSF(coarseToFineEmbedded, rootMatricesSec, remoteOffsetsMatrices, leafMatricesSec, &matricesSF)); 24339566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 24349566063dSJacob Faibussowitsch PetscCall(PetscFree(remoteOffsetsIndices)); 24359566063dSJacob Faibussowitsch PetscCall(PetscFree(remoteOffsetsMatrices)); 24369566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(leafIndicesSec, &numLeafIndices)); 24379566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(leafMatricesSec, &numLeafMatrices)); 24389566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numLeafIndices, &leafIndices, numLeafMatrices, &leafMatrices)); 24399566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(indicesSF, MPIU_INT, rootIndices, leafIndices, MPI_REPLACE)); 24409566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(matricesSF, MPIU_SCALAR, rootMatrices, leafMatrices, MPI_REPLACE)); 24419566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(indicesSF, MPIU_INT, rootIndices, leafIndices, MPI_REPLACE)); 24429566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(matricesSF, MPIU_SCALAR, rootMatrices, leafMatrices, MPI_REPLACE)); 24439566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&matricesSF)); 24449566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&indicesSF)); 24459566063dSJacob Faibussowitsch PetscCall(PetscFree2(rootIndices, rootMatrices)); 24469566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootIndicesSec)); 24479566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootMatricesSec)); 244846bdb399SToby Isaac } 244946bdb399SToby Isaac /* count to preallocate */ 24509566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 245146bdb399SToby Isaac { 245246bdb399SToby Isaac PetscInt nGlobal; 245346bdb399SToby Isaac PetscInt *dnnz, *onnz; 2454b9a5774bSToby Isaac PetscLayout rowMap, colMap; 2455b9a5774bSToby Isaac PetscInt rowStart, rowEnd, colStart, colEnd; 24561c58ffc4SToby Isaac PetscInt maxDof; 24571c58ffc4SToby Isaac PetscInt *rowIndices; 24581c58ffc4SToby Isaac DM refTree; 24591c58ffc4SToby Isaac PetscInt **refPointFieldN; 24601c58ffc4SToby Isaac PetscScalar ***refPointFieldMats; 24611c58ffc4SToby Isaac PetscSection refConSec, refAnSec; 24620eb7e1eaSToby Isaac PetscInt pRefStart, pRefEnd, maxConDof, maxColumns, leafStart, leafEnd; 24631c58ffc4SToby Isaac PetscScalar *pointWork; 246446bdb399SToby Isaac 24659566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstrainedStorageSize(globalFine, &nGlobal)); 24669566063dSJacob Faibussowitsch PetscCall(PetscCalloc2(nGlobal, &dnnz, nGlobal, &onnz)); 24679566063dSJacob Faibussowitsch PetscCall(MatGetLayouts(mat, &rowMap, &colMap)); 24689566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(rowMap)); 24699566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(colMap)); 24709566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(rowMap, &rowStart, &rowEnd)); 24719566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(colMap, &colStart, &colEnd)); 24729566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localFine, &maxDof)); 24739566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(leafIndicesSec, &leafStart, &leafEnd)); 24749566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(fine, maxDof, MPIU_INT, &rowIndices)); 24750eb7e1eaSToby Isaac for (p = leafStart; p < leafEnd; p++) { 247646bdb399SToby Isaac PetscInt gDof, gcDof, gOff; 247746bdb399SToby Isaac PetscInt numColIndices, pIndOff, *pInd; 247846bdb399SToby Isaac PetscInt matSize; 247921968bf8SToby Isaac PetscInt i; 248046bdb399SToby Isaac 24819566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &gDof)); 24829566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &gcDof)); 2483*9371c9d4SSatish Balay if ((gDof - gcDof) <= 0) { continue; } 24849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalFine, p, &gOff)); 248508401ef6SPierre Jolivet PetscCheck(gOff >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "I though having global dofs meant a non-negative offset"); 24861dca8a05SBarry Smith PetscCheck(gOff >= rowStart && (gOff + gDof - gcDof) <= rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "I thought the row map would constrain the global dofs"); 24879566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafIndicesSec, p, &numColIndices)); 24889566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafIndicesSec, p, &pIndOff)); 248946bdb399SToby Isaac numColIndices -= 2 * numFields; 249008401ef6SPierre Jolivet PetscCheck(numColIndices > 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "global fine dof with no dofs to interpolate from"); 249146bdb399SToby Isaac pInd = &leafIndices[pIndOff]; 249221968bf8SToby Isaac offsets[0] = 0; 249321968bf8SToby Isaac offsetsCopy[0] = 0; 249421968bf8SToby Isaac newOffsets[0] = 0; 249521968bf8SToby Isaac newOffsetsCopy[0] = 0; 249646bdb399SToby Isaac if (numFields) { 249721968bf8SToby Isaac PetscInt f; 249846bdb399SToby Isaac for (f = 0; f < numFields; f++) { 249946bdb399SToby Isaac PetscInt rowDof; 250046bdb399SToby Isaac 25019566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localFine, p, f, &rowDof)); 250221968bf8SToby Isaac offsets[f + 1] = offsets[f] + rowDof; 250321968bf8SToby Isaac offsetsCopy[f + 1] = offsets[f + 1]; 250421968bf8SToby Isaac newOffsets[f + 1] = pInd[numColIndices + numFields + f]; 250521968bf8SToby Isaac numD[f] = 0; 250621968bf8SToby Isaac numO[f] = 0; 250746bdb399SToby Isaac } 25089566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, rowIndices)); 250946bdb399SToby Isaac for (f = 0; f < numFields; f++) { 251021968bf8SToby Isaac PetscInt colOffset = newOffsets[f]; 251121968bf8SToby Isaac PetscInt numFieldCols = newOffsets[f + 1] - newOffsets[f]; 251246bdb399SToby Isaac 251346bdb399SToby Isaac for (i = 0; i < numFieldCols; i++) { 251446bdb399SToby Isaac PetscInt gInd = pInd[i + colOffset]; 251546bdb399SToby Isaac 251646bdb399SToby Isaac if (gInd >= colStart && gInd < colEnd) { 251721968bf8SToby Isaac numD[f]++; 2518*9371c9d4SSatish Balay } else if (gInd >= 0) { /* negative means non-entry */ 251921968bf8SToby Isaac numO[f]++; 252046bdb399SToby Isaac } 252146bdb399SToby Isaac } 252246bdb399SToby Isaac } 2523*9371c9d4SSatish Balay } else { 25249566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, rowIndices)); 252521968bf8SToby Isaac numD[0] = 0; 252621968bf8SToby Isaac numO[0] = 0; 252746bdb399SToby Isaac for (i = 0; i < numColIndices; i++) { 252846bdb399SToby Isaac PetscInt gInd = pInd[i]; 252946bdb399SToby Isaac 253046bdb399SToby Isaac if (gInd >= colStart && gInd < colEnd) { 253121968bf8SToby Isaac numD[0]++; 2532*9371c9d4SSatish Balay } else if (gInd >= 0) { /* negative means non-entry */ 253321968bf8SToby Isaac numO[0]++; 253446bdb399SToby Isaac } 253546bdb399SToby Isaac } 253646bdb399SToby Isaac } 25379566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafMatricesSec, p, &matSize)); 253846bdb399SToby Isaac if (!matSize) { /* incoming matrix is identity */ 253946bdb399SToby Isaac PetscInt childId; 254046bdb399SToby Isaac 254146bdb399SToby Isaac childId = childIds[p - pStartF]; 254221968bf8SToby Isaac if (childId < 0) { /* no child interpolation: one nnz per */ 254346bdb399SToby Isaac if (numFields) { 2544b9a5774bSToby Isaac PetscInt f; 2545b9a5774bSToby Isaac for (f = 0; f < numFields; f++) { 254621968bf8SToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f], row; 254746bdb399SToby Isaac for (row = 0; row < numRows; row++) { 254821968bf8SToby Isaac PetscInt gIndCoarse = pInd[newOffsets[f] + row]; 254921968bf8SToby Isaac PetscInt gIndFine = rowIndices[offsets[f] + row]; 255046bdb399SToby Isaac if (gIndCoarse >= colStart && gIndCoarse < colEnd) { /* local */ 25511dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2552b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = 1; 2553*9371c9d4SSatish Balay } else if (gIndCoarse >= 0) { /* remote */ 25541dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2555b9a5774bSToby Isaac onnz[gIndFine - rowStart] = 1; 2556*9371c9d4SSatish Balay } else { /* constrained */ 255708401ef6SPierre Jolivet PetscCheck(gIndFine < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 255846bdb399SToby Isaac } 255946bdb399SToby Isaac } 256046bdb399SToby Isaac } 2561*9371c9d4SSatish Balay } else { 2562b9a5774bSToby Isaac PetscInt i; 2563b9a5774bSToby Isaac for (i = 0; i < gDof; i++) { 256446bdb399SToby Isaac PetscInt gIndCoarse = pInd[i]; 256546bdb399SToby Isaac PetscInt gIndFine = rowIndices[i]; 256646bdb399SToby Isaac if (gIndCoarse >= colStart && gIndCoarse < colEnd) { /* local */ 25671dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2568b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = 1; 2569*9371c9d4SSatish Balay } else if (gIndCoarse >= 0) { /* remote */ 25701dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2571b9a5774bSToby Isaac onnz[gIndFine - rowStart] = 1; 2572*9371c9d4SSatish Balay } else { /* constrained */ 257308401ef6SPierre Jolivet PetscCheck(gIndFine < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 257446bdb399SToby Isaac } 257546bdb399SToby Isaac } 257646bdb399SToby Isaac } 2577*9371c9d4SSatish Balay } else { /* interpolate from all */ 257846bdb399SToby Isaac if (numFields) { 2579b9a5774bSToby Isaac PetscInt f; 2580b9a5774bSToby Isaac for (f = 0; f < numFields; f++) { 258121968bf8SToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f], row; 258246bdb399SToby Isaac for (row = 0; row < numRows; row++) { 258321968bf8SToby Isaac PetscInt gIndFine = rowIndices[offsets[f] + row]; 258446bdb399SToby Isaac if (gIndFine >= 0) { 25851dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2586b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = numD[f]; 2587b9a5774bSToby Isaac onnz[gIndFine - rowStart] = numO[f]; 258846bdb399SToby Isaac } 258946bdb399SToby Isaac } 259046bdb399SToby Isaac } 2591*9371c9d4SSatish Balay } else { 2592b9a5774bSToby Isaac PetscInt i; 2593b9a5774bSToby Isaac for (i = 0; i < gDof; i++) { 259446bdb399SToby Isaac PetscInt gIndFine = rowIndices[i]; 259546bdb399SToby Isaac if (gIndFine >= 0) { 25961dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2597b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = numD[0]; 2598b9a5774bSToby Isaac onnz[gIndFine - rowStart] = numO[0]; 259946bdb399SToby Isaac } 260046bdb399SToby Isaac } 260146bdb399SToby Isaac } 260246bdb399SToby Isaac } 2603*9371c9d4SSatish Balay } else { /* interpolate from all */ 260446bdb399SToby Isaac if (numFields) { 2605b9a5774bSToby Isaac PetscInt f; 2606b9a5774bSToby Isaac for (f = 0; f < numFields; f++) { 260721968bf8SToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f], row; 260846bdb399SToby Isaac for (row = 0; row < numRows; row++) { 260921968bf8SToby Isaac PetscInt gIndFine = rowIndices[offsets[f] + row]; 261046bdb399SToby Isaac if (gIndFine >= 0) { 26111dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2612b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = numD[f]; 2613b9a5774bSToby Isaac onnz[gIndFine - rowStart] = numO[f]; 261446bdb399SToby Isaac } 261546bdb399SToby Isaac } 261646bdb399SToby Isaac } 2617*9371c9d4SSatish Balay } else { /* every dof get a full row */ 2618b9a5774bSToby Isaac PetscInt i; 2619b9a5774bSToby Isaac for (i = 0; i < gDof; i++) { 262046bdb399SToby Isaac PetscInt gIndFine = rowIndices[i]; 262146bdb399SToby Isaac if (gIndFine >= 0) { 26221dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2623b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = numD[0]; 2624b9a5774bSToby Isaac onnz[gIndFine - rowStart] = numO[0]; 262546bdb399SToby Isaac } 262646bdb399SToby Isaac } 262746bdb399SToby Isaac } 262846bdb399SToby Isaac } 262946bdb399SToby Isaac } 26309566063dSJacob Faibussowitsch PetscCall(MatXAIJSetPreallocation(mat, 1, dnnz, onnz, NULL, NULL)); 26319566063dSJacob Faibussowitsch PetscCall(PetscFree2(dnnz, onnz)); 263221968bf8SToby Isaac 26339566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(fine, &refTree)); 26349566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 26359566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 26369566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(refTree, &refAnSec, NULL)); 26379566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 26389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refConSec, &maxConDof)); 26399566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(leafIndicesSec, &maxColumns)); 26409566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxConDof * maxColumns, &pointWork)); 26410eb7e1eaSToby Isaac for (p = leafStart; p < leafEnd; p++) { 2642e44e4e7fSToby Isaac PetscInt gDof, gcDof, gOff; 2643e44e4e7fSToby Isaac PetscInt numColIndices, pIndOff, *pInd; 2644e44e4e7fSToby Isaac PetscInt matSize; 2645e44e4e7fSToby Isaac PetscInt childId; 2646e44e4e7fSToby Isaac 26479566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &gDof)); 26489566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &gcDof)); 2649*9371c9d4SSatish Balay if ((gDof - gcDof) <= 0) { continue; } 2650e44e4e7fSToby Isaac childId = childIds[p - pStartF]; 26519566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalFine, p, &gOff)); 26529566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafIndicesSec, p, &numColIndices)); 26539566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafIndicesSec, p, &pIndOff)); 2654e44e4e7fSToby Isaac numColIndices -= 2 * numFields; 2655e44e4e7fSToby Isaac pInd = &leafIndices[pIndOff]; 2656e44e4e7fSToby Isaac offsets[0] = 0; 2657e44e4e7fSToby Isaac offsetsCopy[0] = 0; 2658e44e4e7fSToby Isaac newOffsets[0] = 0; 2659e44e4e7fSToby Isaac newOffsetsCopy[0] = 0; 2660e44e4e7fSToby Isaac rowOffsets[0] = 0; 2661e44e4e7fSToby Isaac if (numFields) { 2662e44e4e7fSToby Isaac PetscInt f; 2663e44e4e7fSToby Isaac for (f = 0; f < numFields; f++) { 2664e44e4e7fSToby Isaac PetscInt rowDof; 2665e44e4e7fSToby Isaac 26669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localFine, p, f, &rowDof)); 2667e44e4e7fSToby Isaac offsets[f + 1] = offsets[f] + rowDof; 2668e44e4e7fSToby Isaac offsetsCopy[f + 1] = offsets[f + 1]; 2669e44e4e7fSToby Isaac rowOffsets[f + 1] = pInd[numColIndices + f]; 2670e44e4e7fSToby Isaac newOffsets[f + 1] = pInd[numColIndices + numFields + f]; 2671e44e4e7fSToby Isaac } 26729566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, rowIndices)); 2673*9371c9d4SSatish Balay } else { 26749566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, rowIndices)); 26751c58ffc4SToby Isaac } 26769566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafMatricesSec, p, &matSize)); 2677e44e4e7fSToby Isaac if (!matSize) { /* incoming matrix is identity */ 2678e44e4e7fSToby Isaac if (childId < 0) { /* no child interpolation: scatter */ 2679e44e4e7fSToby Isaac if (numFields) { 2680e44e4e7fSToby Isaac PetscInt f; 2681e44e4e7fSToby Isaac for (f = 0; f < numFields; f++) { 2682e44e4e7fSToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f], row; 2683*9371c9d4SSatish Balay for (row = 0; row < numRows; row++) { PetscCall(MatSetValue(mat, rowIndices[offsets[f] + row], pInd[newOffsets[f] + row], 1., INSERT_VALUES)); } 268421968bf8SToby Isaac } 2685*9371c9d4SSatish Balay } else { 2686e44e4e7fSToby Isaac PetscInt numRows = gDof, row; 2687*9371c9d4SSatish Balay for (row = 0; row < numRows; row++) { PetscCall(MatSetValue(mat, rowIndices[row], pInd[row], 1., INSERT_VALUES)); } 2688e44e4e7fSToby Isaac } 2689*9371c9d4SSatish Balay } else { /* interpolate from all */ 2690e44e4e7fSToby Isaac if (numFields) { 2691e44e4e7fSToby Isaac PetscInt f; 2692e44e4e7fSToby Isaac for (f = 0; f < numFields; f++) { 2693e44e4e7fSToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f]; 2694e44e4e7fSToby Isaac PetscInt numCols = newOffsets[f + 1] - newOffsets[f]; 26959566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numRows, &rowIndices[offsets[f]], numCols, &pInd[newOffsets[f]], refPointFieldMats[childId - pRefStart][f], INSERT_VALUES)); 2696e44e4e7fSToby Isaac } 2697*9371c9d4SSatish Balay } else { 26989566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, gDof, rowIndices, numColIndices, pInd, refPointFieldMats[childId - pRefStart][0], INSERT_VALUES)); 2699e44e4e7fSToby Isaac } 2700e44e4e7fSToby Isaac } 2701*9371c9d4SSatish Balay } else { /* interpolate from all */ 2702e44e4e7fSToby Isaac PetscInt pMatOff; 2703e44e4e7fSToby Isaac PetscScalar *pMat; 2704e44e4e7fSToby Isaac 27059566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafMatricesSec, p, &pMatOff)); 2706e44e4e7fSToby Isaac pMat = &leafMatrices[pMatOff]; 2707e44e4e7fSToby Isaac if (childId < 0) { /* copy the incoming matrix */ 2708e44e4e7fSToby Isaac if (numFields) { 2709e44e4e7fSToby Isaac PetscInt f, count; 2710e44e4e7fSToby Isaac for (f = 0, count = 0; f < numFields; f++) { 2711e44e4e7fSToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f]; 2712e44e4e7fSToby Isaac PetscInt numCols = newOffsets[f + 1] - newOffsets[f]; 2713e44e4e7fSToby Isaac PetscInt numInRows = rowOffsets[f + 1] - rowOffsets[f]; 2714e44e4e7fSToby Isaac PetscScalar *inMat = &pMat[count]; 2715e44e4e7fSToby Isaac 27169566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numRows, &rowIndices[offsets[f]], numCols, &pInd[newOffsets[f]], inMat, INSERT_VALUES)); 2717e44e4e7fSToby Isaac count += numCols * numInRows; 2718e44e4e7fSToby Isaac } 2719*9371c9d4SSatish Balay } else { 27209566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, gDof, rowIndices, numColIndices, pInd, pMat, INSERT_VALUES)); 2721e44e4e7fSToby Isaac } 2722*9371c9d4SSatish Balay } else { /* multiply the incoming matrix by the child interpolation */ 2723e44e4e7fSToby Isaac if (numFields) { 2724e44e4e7fSToby Isaac PetscInt f, count; 2725e44e4e7fSToby Isaac for (f = 0, count = 0; f < numFields; f++) { 2726e44e4e7fSToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f]; 2727e44e4e7fSToby Isaac PetscInt numCols = newOffsets[f + 1] - newOffsets[f]; 2728e44e4e7fSToby Isaac PetscInt numInRows = rowOffsets[f + 1] - rowOffsets[f]; 2729e44e4e7fSToby Isaac PetscScalar *inMat = &pMat[count]; 2730e44e4e7fSToby Isaac PetscInt i, j, k; 273108401ef6SPierre Jolivet PetscCheck(refPointFieldN[childId - pRefStart][f] == numInRows, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point constraint matrix multiply dimension mismatch"); 2732e44e4e7fSToby Isaac for (i = 0; i < numRows; i++) { 2733e44e4e7fSToby Isaac for (j = 0; j < numCols; j++) { 2734e44e4e7fSToby Isaac PetscScalar val = 0.; 2735*9371c9d4SSatish Balay for (k = 0; k < numInRows; k++) { val += refPointFieldMats[childId - pRefStart][f][i * numInRows + k] * inMat[k * numCols + j]; } 2736e44e4e7fSToby Isaac pointWork[i * numCols + j] = val; 2737e44e4e7fSToby Isaac } 2738e44e4e7fSToby Isaac } 27399566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numRows, &rowIndices[offsets[f]], numCols, &pInd[newOffsets[f]], pointWork, INSERT_VALUES)); 2740e44e4e7fSToby Isaac count += numCols * numInRows; 2741e44e4e7fSToby Isaac } 2742*9371c9d4SSatish Balay } else { /* every dof gets a full row */ 2743e44e4e7fSToby Isaac PetscInt numRows = gDof; 2744e44e4e7fSToby Isaac PetscInt numCols = numColIndices; 2745e44e4e7fSToby Isaac PetscInt numInRows = matSize / numColIndices; 2746e44e4e7fSToby Isaac PetscInt i, j, k; 274708401ef6SPierre Jolivet PetscCheck(refPointFieldN[childId - pRefStart][0] == numInRows, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point constraint matrix multiply dimension mismatch"); 2748e44e4e7fSToby Isaac for (i = 0; i < numRows; i++) { 2749e44e4e7fSToby Isaac for (j = 0; j < numCols; j++) { 2750e44e4e7fSToby Isaac PetscScalar val = 0.; 2751*9371c9d4SSatish Balay for (k = 0; k < numInRows; k++) { val += refPointFieldMats[childId - pRefStart][0][i * numInRows + k] * pMat[k * numCols + j]; } 2752e44e4e7fSToby Isaac pointWork[i * numCols + j] = val; 2753e44e4e7fSToby Isaac } 2754e44e4e7fSToby Isaac } 27559566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numRows, rowIndices, numCols, pInd, pointWork, INSERT_VALUES)); 2756e44e4e7fSToby Isaac } 2757e44e4e7fSToby Isaac } 2758e44e4e7fSToby Isaac } 2759e44e4e7fSToby Isaac } 27609566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 27619566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(fine, maxDof, MPIU_INT, &rowIndices)); 27629566063dSJacob Faibussowitsch PetscCall(PetscFree(pointWork)); 2763e44e4e7fSToby Isaac } 27649566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(mat, MAT_FINAL_ASSEMBLY)); 27659566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(mat, MAT_FINAL_ASSEMBLY)); 27669566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&leafIndicesSec)); 27679566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&leafMatricesSec)); 27689566063dSJacob Faibussowitsch PetscCall(PetscFree2(leafIndices, leafMatrices)); 27699566063dSJacob Faibussowitsch PetscCall(PetscFree2(*(PetscInt ****)&perms, *(PetscScalar ****)&flips)); 27709566063dSJacob Faibussowitsch PetscCall(PetscFree7(offsets, offsetsCopy, newOffsets, newOffsetsCopy, rowOffsets, numD, numO)); 27719566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(aIS, &anchors)); 27726ecaa68aSToby Isaac PetscFunctionReturn(0); 27736ecaa68aSToby Isaac } 2774154bca37SToby Isaac 27758d2f55e7SToby Isaac /* 27768d2f55e7SToby Isaac * Assuming a nodal basis (w.r.t. the dual basis) basis: 27778d2f55e7SToby Isaac * 27788d2f55e7SToby Isaac * for each coarse dof \phi^c_i: 27798d2f55e7SToby Isaac * for each quadrature point (w_l,x_l) in the dual basis definition of \phi^c_i: 27808d2f55e7SToby Isaac * for each fine dof \phi^f_j; 27818d2f55e7SToby Isaac * a_{i,j} = 0; 27828d2f55e7SToby Isaac * for each fine dof \phi^f_k: 27838d2f55e7SToby Isaac * a_{i,j} += interp_{i,k} * \phi^f_k(x_l) * \phi^f_j(x_l) * w_l 27848d2f55e7SToby Isaac * [^^^ this is = \phi^c_i ^^^] 27858d2f55e7SToby Isaac */ 2786*9371c9d4SSatish Balay PetscErrorCode DMPlexComputeInjectorReferenceTree(DM refTree, Mat *inj) { 27878d2f55e7SToby Isaac PetscDS ds; 27888d2f55e7SToby Isaac PetscSection section, cSection; 27898d2f55e7SToby Isaac DMLabel canonical, depth; 27908d2f55e7SToby Isaac Mat cMat, mat; 27918d2f55e7SToby Isaac PetscInt *nnz; 27928d2f55e7SToby Isaac PetscInt f, dim, numFields, numSecFields, p, pStart, pEnd, cStart, cEnd; 27938d2f55e7SToby Isaac PetscInt m, n; 27948d2f55e7SToby Isaac PetscScalar *pointScalar; 27958d2f55e7SToby Isaac PetscReal *v0, *v0parent, *vtmp, *J, *Jparent, *invJ, *pointRef, detJ, detJparent; 27968d2f55e7SToby Isaac 27978d2f55e7SToby Isaac PetscFunctionBegin; 27989566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(refTree, §ion)); 27999566063dSJacob Faibussowitsch PetscCall(DMGetDimension(refTree, &dim)); 28009566063dSJacob Faibussowitsch PetscCall(PetscMalloc6(dim, &v0, dim, &v0parent, dim, &vtmp, dim * dim, &J, dim * dim, &Jparent, dim * dim, &invJ)); 28019566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(dim, &pointScalar, dim, &pointRef)); 28029566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 28039566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 28049566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(section, &numSecFields)); 28059566063dSJacob Faibussowitsch PetscCall(DMGetLabel(refTree, "canonical", &canonical)); 28069566063dSJacob Faibussowitsch PetscCall(DMGetLabel(refTree, "depth", &depth)); 28079566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &cSection, &cMat, NULL)); 28089566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(refTree, &pStart, &pEnd)); 28099566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(refTree, 0, &cStart, &cEnd)); 28109566063dSJacob Faibussowitsch PetscCall(MatGetSize(cMat, &n, &m)); /* the injector has transpose sizes from the constraint matrix */ 28118d2f55e7SToby Isaac /* Step 1: compute non-zero pattern. A proper subset of constraint matrix non-zero */ 28129566063dSJacob Faibussowitsch PetscCall(PetscCalloc1(m, &nnz)); 28138d2f55e7SToby Isaac for (p = pStart; p < pEnd; p++) { /* a point will have non-zeros if it is canonical, it has dofs, and its children have dofs */ 28148d2f55e7SToby Isaac const PetscInt *children; 28158d2f55e7SToby Isaac PetscInt numChildren; 28168d2f55e7SToby Isaac PetscInt i, numChildDof, numSelfDof; 28178d2f55e7SToby Isaac 28188d2f55e7SToby Isaac if (canonical) { 28198d2f55e7SToby Isaac PetscInt pCanonical; 28209566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonical, p, &pCanonical)); 28218d2f55e7SToby Isaac if (p != pCanonical) continue; 28228d2f55e7SToby Isaac } 28239566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(refTree, p, &numChildren, &children)); 28248d2f55e7SToby Isaac if (!numChildren) continue; 28258d2f55e7SToby Isaac for (i = 0, numChildDof = 0; i < numChildren; i++) { 28268d2f55e7SToby Isaac PetscInt child = children[i]; 28278d2f55e7SToby Isaac PetscInt dof; 28288d2f55e7SToby Isaac 28299566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, child, &dof)); 28308d2f55e7SToby Isaac numChildDof += dof; 28318d2f55e7SToby Isaac } 28329566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &numSelfDof)); 28338d2f55e7SToby Isaac if (!numChildDof || !numSelfDof) continue; 28348d2f55e7SToby Isaac for (f = 0; f < numFields; f++) { 28358d2f55e7SToby Isaac PetscInt selfOff; 28368d2f55e7SToby Isaac 28378d2f55e7SToby Isaac if (numSecFields) { /* count the dofs for just this field */ 28388d2f55e7SToby Isaac for (i = 0, numChildDof = 0; i < numChildren; i++) { 28398d2f55e7SToby Isaac PetscInt child = children[i]; 28408d2f55e7SToby Isaac PetscInt dof; 28418d2f55e7SToby Isaac 28429566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, child, f, &dof)); 28438d2f55e7SToby Isaac numChildDof += dof; 28448d2f55e7SToby Isaac } 28459566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, p, f, &numSelfDof)); 28469566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(section, p, f, &selfOff)); 2847*9371c9d4SSatish Balay } else { 28489566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, p, &selfOff)); 28498d2f55e7SToby Isaac } 2850*9371c9d4SSatish Balay for (i = 0; i < numSelfDof; i++) { nnz[selfOff + i] = numChildDof; } 28518d2f55e7SToby Isaac } 28528d2f55e7SToby Isaac } 28539566063dSJacob Faibussowitsch PetscCall(MatCreateAIJ(PETSC_COMM_SELF, m, n, m, n, -1, nnz, -1, NULL, &mat)); 28549566063dSJacob Faibussowitsch PetscCall(PetscFree(nnz)); 28558d2f55e7SToby Isaac /* Setp 2: compute entries */ 28568d2f55e7SToby Isaac for (p = pStart; p < pEnd; p++) { 28578d2f55e7SToby Isaac const PetscInt *children; 28588d2f55e7SToby Isaac PetscInt numChildren; 28598d2f55e7SToby Isaac PetscInt i, numChildDof, numSelfDof; 28608d2f55e7SToby Isaac 28618d2f55e7SToby Isaac /* same conditions about when entries occur */ 28628d2f55e7SToby Isaac if (canonical) { 28638d2f55e7SToby Isaac PetscInt pCanonical; 28649566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonical, p, &pCanonical)); 28658d2f55e7SToby Isaac if (p != pCanonical) continue; 28668d2f55e7SToby Isaac } 28679566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(refTree, p, &numChildren, &children)); 28688d2f55e7SToby Isaac if (!numChildren) continue; 28698d2f55e7SToby Isaac for (i = 0, numChildDof = 0; i < numChildren; i++) { 28708d2f55e7SToby Isaac PetscInt child = children[i]; 28718d2f55e7SToby Isaac PetscInt dof; 28728d2f55e7SToby Isaac 28739566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, child, &dof)); 28748d2f55e7SToby Isaac numChildDof += dof; 28758d2f55e7SToby Isaac } 28769566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &numSelfDof)); 28778d2f55e7SToby Isaac if (!numChildDof || !numSelfDof) continue; 28788d2f55e7SToby Isaac 28798d2f55e7SToby Isaac for (f = 0; f < numFields; f++) { 288059fc6756SToby Isaac PetscInt pI = -1, cI = -1; 288152a3aeb4SToby Isaac PetscInt selfOff, Nc, parentCell; 28828d2f55e7SToby Isaac PetscInt cellShapeOff; 28838d2f55e7SToby Isaac PetscObject disc; 28848d2f55e7SToby Isaac PetscDualSpace dsp; 28858d2f55e7SToby Isaac PetscClassId classId; 28868d2f55e7SToby Isaac PetscScalar *pointMat; 28873b1c2a6aSToby Isaac PetscInt *matRows, *matCols; 28888d2f55e7SToby Isaac PetscInt pO = PETSC_MIN_INT; 28898d2f55e7SToby Isaac const PetscInt *depthNumDof; 28908d2f55e7SToby Isaac 28918d2f55e7SToby Isaac if (numSecFields) { 28928d2f55e7SToby Isaac for (i = 0, numChildDof = 0; i < numChildren; i++) { 28938d2f55e7SToby Isaac PetscInt child = children[i]; 28948d2f55e7SToby Isaac PetscInt dof; 28958d2f55e7SToby Isaac 28969566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, child, f, &dof)); 28978d2f55e7SToby Isaac numChildDof += dof; 28988d2f55e7SToby Isaac } 28999566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, p, f, &numSelfDof)); 29009566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(section, p, f, &selfOff)); 2901*9371c9d4SSatish Balay } else { 29029566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, p, &selfOff)); 29038d2f55e7SToby Isaac } 29048d2f55e7SToby Isaac 29053b1c2a6aSToby Isaac /* find a cell whose closure contains p */ 29068d2f55e7SToby Isaac if (p >= cStart && p < cEnd) { 29078d2f55e7SToby Isaac parentCell = p; 2908*9371c9d4SSatish Balay } else { 29098d2f55e7SToby Isaac PetscInt *star = NULL; 29108d2f55e7SToby Isaac PetscInt numStar; 29118d2f55e7SToby Isaac 29128d2f55e7SToby Isaac parentCell = -1; 29139566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, p, PETSC_FALSE, &numStar, &star)); 29148d2f55e7SToby Isaac for (i = numStar - 1; i >= 0; i--) { 29158d2f55e7SToby Isaac PetscInt c = star[2 * i]; 29168d2f55e7SToby Isaac 29178d2f55e7SToby Isaac if (c >= cStart && c < cEnd) { 29188d2f55e7SToby Isaac parentCell = c; 29198d2f55e7SToby Isaac break; 29208d2f55e7SToby Isaac } 29218d2f55e7SToby Isaac } 29229566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, p, PETSC_FALSE, &numStar, &star)); 29238d2f55e7SToby Isaac } 2924a5b23f4aSJose E. Roman /* determine the offset of p's shape functions within parentCell's shape functions */ 29259566063dSJacob Faibussowitsch PetscCall(PetscDSGetDiscretization(ds, f, &disc)); 29269566063dSJacob Faibussowitsch PetscCall(PetscObjectGetClassId(disc, &classId)); 2927c5356c36SToby Isaac if (classId == PETSCFE_CLASSID) { 29289566063dSJacob Faibussowitsch PetscCall(PetscFEGetDualSpace((PetscFE)disc, &dsp)); 2929*9371c9d4SSatish Balay } else if (classId == PETSCFV_CLASSID) { 29309566063dSJacob Faibussowitsch PetscCall(PetscFVGetDualSpace((PetscFV)disc, &dsp)); 2931*9371c9d4SSatish Balay } else { 29329b90b7cdSMatthew G. Knepley SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Unsupported discretization object"); 2933c5356c36SToby Isaac } 29349566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetNumDof(dsp, &depthNumDof)); 29359566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetNumComponents(dsp, &Nc)); 29368d2f55e7SToby Isaac { 29378d2f55e7SToby Isaac PetscInt *closure = NULL; 29388d2f55e7SToby Isaac PetscInt numClosure; 29398d2f55e7SToby Isaac 29409566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, parentCell, PETSC_TRUE, &numClosure, &closure)); 294159fc6756SToby Isaac for (i = 0, pI = -1, cellShapeOff = 0; i < numClosure; i++) { 29428d2f55e7SToby Isaac PetscInt point = closure[2 * i], pointDepth; 29438d2f55e7SToby Isaac 29448d2f55e7SToby Isaac pO = closure[2 * i + 1]; 294559fc6756SToby Isaac if (point == p) { 294659fc6756SToby Isaac pI = i; 294759fc6756SToby Isaac break; 294859fc6756SToby Isaac } 29499566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(depth, point, &pointDepth)); 29508d2f55e7SToby Isaac cellShapeOff += depthNumDof[pointDepth]; 29518d2f55e7SToby Isaac } 29529566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, parentCell, PETSC_TRUE, &numClosure, &closure)); 29538d2f55e7SToby Isaac } 29548d2f55e7SToby Isaac 29559566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(refTree, numSelfDof * numChildDof, MPIU_SCALAR, &pointMat)); 29569566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(refTree, numSelfDof + numChildDof, MPIU_INT, &matRows)); 295752a3aeb4SToby Isaac matCols = matRows + numSelfDof; 2958*9371c9d4SSatish Balay for (i = 0; i < numSelfDof; i++) { matRows[i] = selfOff + i; } 295952a3aeb4SToby Isaac for (i = 0; i < numSelfDof * numChildDof; i++) pointMat[i] = 0.; 29603b1c2a6aSToby Isaac { 29613b1c2a6aSToby Isaac PetscInt colOff = 0; 29623b1c2a6aSToby Isaac 29633b1c2a6aSToby Isaac for (i = 0; i < numChildren; i++) { 29643b1c2a6aSToby Isaac PetscInt child = children[i]; 29653b1c2a6aSToby Isaac PetscInt dof, off, j; 29663b1c2a6aSToby Isaac 29673b1c2a6aSToby Isaac if (numSecFields) { 29689566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSection, child, f, &dof)); 29699566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(cSection, child, f, &off)); 2970*9371c9d4SSatish Balay } else { 29719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSection, child, &dof)); 29729566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(cSection, child, &off)); 29733b1c2a6aSToby Isaac } 29743b1c2a6aSToby Isaac 2975*9371c9d4SSatish Balay for (j = 0; j < dof; j++) { matCols[colOff++] = off + j; } 29763b1c2a6aSToby Isaac } 29773b1c2a6aSToby Isaac } 29788d2f55e7SToby Isaac if (classId == PETSCFE_CLASSID) { 29798d2f55e7SToby Isaac PetscFE fe = (PetscFE)disc; 29808d2f55e7SToby Isaac PetscInt fSize; 298159fc6756SToby Isaac const PetscInt ***perms; 298259fc6756SToby Isaac const PetscScalar ***flips; 298359fc6756SToby Isaac const PetscInt *pperms; 298459fc6756SToby Isaac 29859566063dSJacob Faibussowitsch PetscCall(PetscFEGetDualSpace(fe, &dsp)); 29869566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetDimension(dsp, &fSize)); 29879566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetSymmetries(dsp, &perms, &flips)); 298859fc6756SToby Isaac pperms = perms ? perms[pI] ? perms[pI][pO] : NULL : NULL; 298952a3aeb4SToby Isaac for (i = 0; i < numSelfDof; i++) { /* for every shape function */ 29908d2f55e7SToby Isaac PetscQuadrature q; 299152a3aeb4SToby Isaac PetscInt dim, thisNc, numPoints, j, k; 29928d2f55e7SToby Isaac const PetscReal *points; 29938d2f55e7SToby Isaac const PetscReal *weights; 29948d2f55e7SToby Isaac PetscInt *closure = NULL; 29958d2f55e7SToby Isaac PetscInt numClosure; 299659fc6756SToby Isaac PetscInt iCell = pperms ? pperms[i] : i; 299759fc6756SToby Isaac PetscInt parentCellShapeDof = cellShapeOff + iCell; 2998ef0bb6c7SMatthew G. Knepley PetscTabulation Tparent; 29998d2f55e7SToby Isaac 30009566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetFunctional(dsp, parentCellShapeDof, &q)); 30019566063dSJacob Faibussowitsch PetscCall(PetscQuadratureGetData(q, &dim, &thisNc, &numPoints, &points, &weights)); 300263a3b9bcSJacob Faibussowitsch PetscCheck(thisNc == Nc, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Functional dim %" PetscInt_FMT " does not much basis dim %" PetscInt_FMT, thisNc, Nc); 30039566063dSJacob Faibussowitsch PetscCall(PetscFECreateTabulation(fe, 1, numPoints, points, 0, &Tparent)); /* I'm expecting a nodal basis: weights[:]' * Bparent[:,cellShapeDof] = 1. */ 30043b1c2a6aSToby Isaac for (j = 0; j < numPoints; j++) { 30058d2f55e7SToby Isaac PetscInt childCell = -1; 300652a3aeb4SToby Isaac PetscReal *parentValAtPoint; 3007c330f8ffSToby Isaac const PetscReal xi0[3] = {-1., -1., -1.}; 30088d2f55e7SToby Isaac const PetscReal *pointReal = &points[dim * j]; 30098d2f55e7SToby Isaac const PetscScalar *point; 3010ef0bb6c7SMatthew G. Knepley PetscTabulation Tchild; 30118d2f55e7SToby Isaac PetscInt childCellShapeOff, pointMatOff; 30128d2f55e7SToby Isaac #if defined(PETSC_USE_COMPLEX) 30138d2f55e7SToby Isaac PetscInt d; 30148d2f55e7SToby Isaac 3015*9371c9d4SSatish Balay for (d = 0; d < dim; d++) { pointScalar[d] = points[dim * j + d]; } 30168d2f55e7SToby Isaac point = pointScalar; 30178d2f55e7SToby Isaac #else 30188d2f55e7SToby Isaac point = pointReal; 30198d2f55e7SToby Isaac #endif 30208d2f55e7SToby Isaac 3021ef0bb6c7SMatthew G. Knepley parentValAtPoint = &Tparent->T[0][(fSize * j + parentCellShapeDof) * Nc]; 30223b1c2a6aSToby Isaac 30233b1c2a6aSToby Isaac for (k = 0; k < numChildren; k++) { /* locate the point in a child's star cell*/ 30248d2f55e7SToby Isaac PetscInt child = children[k]; 30258d2f55e7SToby Isaac PetscInt *star = NULL; 30268d2f55e7SToby Isaac PetscInt numStar, s; 30278d2f55e7SToby Isaac 30289566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, child, PETSC_FALSE, &numStar, &star)); 30298d2f55e7SToby Isaac for (s = numStar - 1; s >= 0; s--) { 30308d2f55e7SToby Isaac PetscInt c = star[2 * s]; 30318d2f55e7SToby Isaac 30328d2f55e7SToby Isaac if (c < cStart || c >= cEnd) continue; 30339566063dSJacob Faibussowitsch PetscCall(DMPlexLocatePoint_Internal(refTree, dim, point, c, &childCell)); 30348d2f55e7SToby Isaac if (childCell >= 0) break; 30358d2f55e7SToby Isaac } 30369566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, child, PETSC_FALSE, &numStar, &star)); 30378d2f55e7SToby Isaac if (childCell >= 0) break; 30388d2f55e7SToby Isaac } 303908401ef6SPierre Jolivet PetscCheck(childCell >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not locate quadrature point"); 30409566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(refTree, childCell, NULL, v0, J, invJ, &detJ)); 30419566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(refTree, parentCell, NULL, v0parent, Jparent, NULL, &detJparent)); 3042c330f8ffSToby Isaac CoordinatesRefToReal(dim, dim, xi0, v0parent, Jparent, pointReal, vtmp); 3043c330f8ffSToby Isaac CoordinatesRealToRef(dim, dim, xi0, v0, invJ, vtmp, pointRef); 30448d2f55e7SToby Isaac 30459566063dSJacob Faibussowitsch PetscCall(PetscFECreateTabulation(fe, 1, 1, pointRef, 0, &Tchild)); 30469566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, childCell, PETSC_TRUE, &numClosure, &closure)); 30473b1c2a6aSToby Isaac for (k = 0, pointMatOff = 0; k < numChildren; k++) { /* point is located in cell => child dofs support at point are in closure of cell */ 3048c5356c36SToby Isaac PetscInt child = children[k], childDepth, childDof, childO = PETSC_MIN_INT; 30498d2f55e7SToby Isaac PetscInt l; 305059fc6756SToby Isaac const PetscInt *cperms; 30518d2f55e7SToby Isaac 30529566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(depth, child, &childDepth)); 30538d2f55e7SToby Isaac childDof = depthNumDof[childDepth]; 305459fc6756SToby Isaac for (l = 0, cI = -1, childCellShapeOff = 0; l < numClosure; l++) { 30558d2f55e7SToby Isaac PetscInt point = closure[2 * l]; 30568d2f55e7SToby Isaac PetscInt pointDepth; 30578d2f55e7SToby Isaac 30588d2f55e7SToby Isaac childO = closure[2 * l + 1]; 305959fc6756SToby Isaac if (point == child) { 306059fc6756SToby Isaac cI = l; 306159fc6756SToby Isaac break; 306259fc6756SToby Isaac } 30639566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(depth, point, &pointDepth)); 30648d2f55e7SToby Isaac childCellShapeOff += depthNumDof[pointDepth]; 30658d2f55e7SToby Isaac } 30668d2f55e7SToby Isaac if (l == numClosure) { 30678d2f55e7SToby Isaac pointMatOff += childDof; 30688d2f55e7SToby Isaac continue; /* child is not in the closure of the cell: has nothing to contribute to this point */ 30698d2f55e7SToby Isaac } 307059fc6756SToby Isaac cperms = perms ? perms[cI] ? perms[cI][childO] : NULL : NULL; 30718d2f55e7SToby Isaac for (l = 0; l < childDof; l++) { 307259fc6756SToby Isaac PetscInt lCell = cperms ? cperms[l] : l; 307359fc6756SToby Isaac PetscInt childCellDof = childCellShapeOff + lCell; 307452a3aeb4SToby Isaac PetscReal *childValAtPoint; 307552a3aeb4SToby Isaac PetscReal val = 0.; 30768d2f55e7SToby Isaac 3077ef0bb6c7SMatthew G. Knepley childValAtPoint = &Tchild->T[0][childCellDof * Nc]; 3078*9371c9d4SSatish Balay for (m = 0; m < Nc; m++) { val += weights[j * Nc + m] * parentValAtPoint[m] * childValAtPoint[m]; } 307952a3aeb4SToby Isaac 308052a3aeb4SToby Isaac pointMat[i * numChildDof + pointMatOff + l] += val; 30818d2f55e7SToby Isaac } 30828d2f55e7SToby Isaac pointMatOff += childDof; 30838d2f55e7SToby Isaac } 30849566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, childCell, PETSC_TRUE, &numClosure, &closure)); 30859566063dSJacob Faibussowitsch PetscCall(PetscTabulationDestroy(&Tchild)); 30868d2f55e7SToby Isaac } 30879566063dSJacob Faibussowitsch PetscCall(PetscTabulationDestroy(&Tparent)); 30888d2f55e7SToby Isaac } 3089*9371c9d4SSatish Balay } else { /* just the volume-weighted averages of the children */ 30903b1c2a6aSToby Isaac PetscReal parentVol; 3091bfaa5bdcSToby Isaac PetscInt childCell; 30923b1c2a6aSToby Isaac 30939566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFVM(refTree, p, &parentVol, NULL, NULL)); 3094bfaa5bdcSToby Isaac for (i = 0, childCell = 0; i < numChildren; i++) { 309552a3aeb4SToby Isaac PetscInt child = children[i], j; 30963b1c2a6aSToby Isaac PetscReal childVol; 30973b1c2a6aSToby Isaac 30983b1c2a6aSToby Isaac if (child < cStart || child >= cEnd) continue; 30999566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFVM(refTree, child, &childVol, NULL, NULL)); 3100*9371c9d4SSatish Balay for (j = 0; j < Nc; j++) { pointMat[j * numChildDof + Nc * childCell + j] = childVol / parentVol; } 3101bfaa5bdcSToby Isaac childCell++; 31023b1c2a6aSToby Isaac } 31038d2f55e7SToby Isaac } 31043b1c2a6aSToby Isaac /* Insert pointMat into mat */ 31059566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numSelfDof, matRows, numChildDof, matCols, pointMat, INSERT_VALUES)); 31069566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(refTree, numSelfDof + numChildDof, MPIU_INT, &matRows)); 31079566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(refTree, numSelfDof * numChildDof, MPIU_SCALAR, &pointMat)); 31088d2f55e7SToby Isaac } 31098d2f55e7SToby Isaac } 31109566063dSJacob Faibussowitsch PetscCall(PetscFree6(v0, v0parent, vtmp, J, Jparent, invJ)); 31119566063dSJacob Faibussowitsch PetscCall(PetscFree2(pointScalar, pointRef)); 31129566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(mat, MAT_FINAL_ASSEMBLY)); 31139566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(mat, MAT_FINAL_ASSEMBLY)); 31148d2f55e7SToby Isaac *inj = mat; 31158d2f55e7SToby Isaac PetscFunctionReturn(0); 31168d2f55e7SToby Isaac } 31178d2f55e7SToby Isaac 3118*9371c9d4SSatish Balay static PetscErrorCode DMPlexReferenceTreeGetChildrenMatrices_Injection(DM refTree, Mat inj, PetscScalar ****childrenMats) { 3119f30e825dSToby Isaac PetscDS ds; 3120f30e825dSToby Isaac PetscInt numFields, f, pRefStart, pRefEnd, p, *rows, *cols, maxDof; 3121f30e825dSToby Isaac PetscScalar ***refPointFieldMats; 3122f30e825dSToby Isaac PetscSection refConSec, refSection; 3123f30e825dSToby Isaac 3124f30e825dSToby Isaac PetscFunctionBegin; 31259566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 31269566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 31279566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 31289566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(refTree, &refSection)); 31299566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 31309566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pRefEnd - pRefStart, &refPointFieldMats)); 31319566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refConSec, &maxDof)); 31329566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof, &rows)); 31339566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof * maxDof, &cols)); 3134f30e825dSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 3135f30e825dSToby Isaac PetscInt parent, pDof, parentDof; 3136f30e825dSToby Isaac 31379566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, p, &parent, NULL)); 31389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &pDof)); 31399566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, parent, &parentDof)); 3140f30e825dSToby Isaac if (!pDof || !parentDof || parent == p) continue; 3141f30e825dSToby Isaac 31429566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numFields, &refPointFieldMats[p - pRefStart])); 3143f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 314452a3aeb4SToby Isaac PetscInt cDof, cOff, numCols, r; 3145f30e825dSToby Isaac 3146f30e825dSToby Isaac if (numFields > 1) { 31479566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refConSec, p, f, &cDof)); 31489566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refConSec, p, f, &cOff)); 3149*9371c9d4SSatish Balay } else { 31509566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &cDof)); 31519566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refConSec, p, &cOff)); 3152f30e825dSToby Isaac } 3153f30e825dSToby Isaac 3154*9371c9d4SSatish Balay for (r = 0; r < cDof; r++) { rows[r] = cOff + r; } 3155f30e825dSToby Isaac numCols = 0; 3156f30e825dSToby Isaac { 3157f30e825dSToby Isaac PetscInt aDof, aOff, j; 3158f30e825dSToby Isaac 3159f30e825dSToby Isaac if (numFields > 1) { 31609566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refSection, parent, f, &aDof)); 31619566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refSection, parent, f, &aOff)); 3162*9371c9d4SSatish Balay } else { 31639566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, parent, &aDof)); 31649566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refSection, parent, &aOff)); 3165f30e825dSToby Isaac } 3166f30e825dSToby Isaac 3167*9371c9d4SSatish Balay for (j = 0; j < aDof; j++) { cols[numCols++] = aOff + j; } 3168f30e825dSToby Isaac } 31699566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(cDof * numCols, &refPointFieldMats[p - pRefStart][f])); 3170f30e825dSToby Isaac /* transpose of constraint matrix */ 31719566063dSJacob Faibussowitsch PetscCall(MatGetValues(inj, numCols, cols, cDof, rows, refPointFieldMats[p - pRefStart][f])); 3172f30e825dSToby Isaac } 3173f30e825dSToby Isaac } 3174f30e825dSToby Isaac *childrenMats = refPointFieldMats; 31759566063dSJacob Faibussowitsch PetscCall(PetscFree(rows)); 31769566063dSJacob Faibussowitsch PetscCall(PetscFree(cols)); 3177f30e825dSToby Isaac PetscFunctionReturn(0); 3178f30e825dSToby Isaac } 3179f30e825dSToby Isaac 3180*9371c9d4SSatish Balay static PetscErrorCode DMPlexReferenceTreeRestoreChildrenMatrices_Injection(DM refTree, Mat inj, PetscScalar ****childrenMats) { 3181f30e825dSToby Isaac PetscDS ds; 3182f30e825dSToby Isaac PetscScalar ***refPointFieldMats; 3183f30e825dSToby Isaac PetscInt numFields, pRefStart, pRefEnd, p, f; 3184c6154584SToby Isaac PetscSection refConSec, refSection; 3185f30e825dSToby Isaac 3186f30e825dSToby Isaac PetscFunctionBegin; 3187f30e825dSToby Isaac refPointFieldMats = *childrenMats; 3188f30e825dSToby Isaac *childrenMats = NULL; 31899566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 31909566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(refTree, &refSection)); 31919566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 31929566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 31939566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 3194f30e825dSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 3195f30e825dSToby Isaac PetscInt parent, pDof, parentDof; 3196f30e825dSToby Isaac 31979566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, p, &parent, NULL)); 31989566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &pDof)); 31999566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, parent, &parentDof)); 3200f30e825dSToby Isaac if (!pDof || !parentDof || parent == p) continue; 3201f30e825dSToby Isaac 3202f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 3203f30e825dSToby Isaac PetscInt cDof; 3204f30e825dSToby Isaac 3205f30e825dSToby Isaac if (numFields > 1) { 32069566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refConSec, p, f, &cDof)); 3207*9371c9d4SSatish Balay } else { 32089566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &cDof)); 3209f30e825dSToby Isaac } 3210f30e825dSToby Isaac 32119566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats[p - pRefStart][f])); 3212f30e825dSToby Isaac } 32139566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats[p - pRefStart])); 3214f30e825dSToby Isaac } 32159566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats)); 3216f30e825dSToby Isaac PetscFunctionReturn(0); 3217f30e825dSToby Isaac } 3218f30e825dSToby Isaac 3219*9371c9d4SSatish Balay static PetscErrorCode DMPlexReferenceTreeGetInjector(DM refTree, Mat *injRef) { 3220ebf164c7SToby Isaac Mat cMatRef; 32216148253fSToby Isaac PetscObject injRefObj; 32228d2f55e7SToby Isaac 3223154bca37SToby Isaac PetscFunctionBegin; 32249566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, NULL, &cMatRef, NULL)); 32259566063dSJacob Faibussowitsch PetscCall(PetscObjectQuery((PetscObject)cMatRef, "DMPlexComputeInjectorTree_refTree", &injRefObj)); 3226ebf164c7SToby Isaac *injRef = (Mat)injRefObj; 3227ebf164c7SToby Isaac if (!*injRef) { 32289566063dSJacob Faibussowitsch PetscCall(DMPlexComputeInjectorReferenceTree(refTree, injRef)); 32299566063dSJacob Faibussowitsch PetscCall(PetscObjectCompose((PetscObject)cMatRef, "DMPlexComputeInjectorTree_refTree", (PetscObject)*injRef)); 3230ec92bd66SToby Isaac /* there is now a reference in cMatRef, which should be the only one for symmetry with the above case */ 32319566063dSJacob Faibussowitsch PetscCall(PetscObjectDereference((PetscObject)*injRef)); 3232ebf164c7SToby Isaac } 3233ebf164c7SToby Isaac PetscFunctionReturn(0); 32346148253fSToby Isaac } 3235f30e825dSToby Isaac 3236*9371c9d4SSatish Balay static PetscErrorCode DMPlexTransferInjectorTree(DM coarse, DM fine, PetscSF coarseToFine, const PetscInt *childIds, Vec fineVec, PetscInt numFields, PetscInt *offsets, PetscSection *rootMultiSec, PetscSection *multiLeafSec, PetscInt **gatheredIndices, PetscScalar **gatheredValues) { 3237c921d74cSToby Isaac PetscInt pStartF, pEndF, pStartC, pEndC, p, maxDof, numMulti; 3238ebf164c7SToby Isaac PetscSection globalCoarse, globalFine; 3239ebf164c7SToby Isaac PetscSection localCoarse, localFine, leafIndicesSec; 3240c921d74cSToby Isaac PetscSection multiRootSec, rootIndicesSec; 3241c921d74cSToby Isaac PetscInt *leafInds, *rootInds = NULL; 3242c921d74cSToby Isaac const PetscInt *rootDegrees; 3243c921d74cSToby Isaac PetscScalar *leafVals = NULL, *rootVals = NULL; 3244ebf164c7SToby Isaac PetscSF coarseToFineEmbedded; 3245ebf164c7SToby Isaac 3246ebf164c7SToby Isaac PetscFunctionBegin; 32479566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 32489566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 32499566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 32509566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 32519566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)fine), &leafIndicesSec)); 32529566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(leafIndicesSec, pStartF, pEndF)); 32539566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localFine, &maxDof)); 32548d2f55e7SToby Isaac { /* winnow fine points that don't have global dofs out of the sf */ 32557e96bdafSToby Isaac PetscInt l, nleaves, dof, cdof, numPointsWithDofs, offset, *pointsWithDofs, numIndices; 32567e96bdafSToby Isaac const PetscInt *leaves; 32578d2f55e7SToby Isaac 32589566063dSJacob Faibussowitsch PetscCall(PetscSFGetGraph(coarseToFine, NULL, &nleaves, &leaves, NULL)); 32597e96bdafSToby Isaac for (l = 0, numPointsWithDofs = 0; l < nleaves; l++) { 32607e96bdafSToby Isaac p = leaves ? leaves[l] : l; 32619566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 32629566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 32638d2f55e7SToby Isaac if ((dof - cdof) > 0) { 32648d2f55e7SToby Isaac numPointsWithDofs++; 3265f30e825dSToby Isaac 32669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localFine, p, &dof)); 32679566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(leafIndicesSec, p, dof + 1)); 32688d2f55e7SToby Isaac } 32698d2f55e7SToby Isaac } 32709566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numPointsWithDofs, &pointsWithDofs)); 32719566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(leafIndicesSec)); 32729566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(leafIndicesSec, &numIndices)); 32739566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(gatheredIndices ? numIndices : (maxDof + 1), &leafInds)); 32749566063dSJacob Faibussowitsch if (gatheredValues) PetscCall(PetscMalloc1(numIndices, &leafVals)); 32757e96bdafSToby Isaac for (l = 0, offset = 0; l < nleaves; l++) { 32767e96bdafSToby Isaac p = leaves ? leaves[l] : l; 32779566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 32789566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 32798d2f55e7SToby Isaac if ((dof - cdof) > 0) { 3280f30e825dSToby Isaac PetscInt off, gOff; 3281f30e825dSToby Isaac PetscInt *pInd; 3282c921d74cSToby Isaac PetscScalar *pVal = NULL; 3283f30e825dSToby Isaac 32847e96bdafSToby Isaac pointsWithDofs[offset++] = l; 3285f30e825dSToby Isaac 32869566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafIndicesSec, p, &off)); 3287f30e825dSToby Isaac 3288c921d74cSToby Isaac pInd = gatheredIndices ? (&leafInds[off + 1]) : leafInds; 3289c921d74cSToby Isaac if (gatheredValues) { 3290c921d74cSToby Isaac PetscInt i; 3291c921d74cSToby Isaac 3292c921d74cSToby Isaac pVal = &leafVals[off + 1]; 3293c921d74cSToby Isaac for (i = 0; i < dof; i++) pVal[i] = 0.; 3294c921d74cSToby Isaac } 32959566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalFine, p, &gOff)); 3296f30e825dSToby Isaac 3297f30e825dSToby Isaac offsets[0] = 0; 3298f30e825dSToby Isaac if (numFields) { 3299f30e825dSToby Isaac PetscInt f; 3300f30e825dSToby Isaac 3301f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 3302f30e825dSToby Isaac PetscInt fDof; 33039566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localFine, p, f, &fDof)); 3304f30e825dSToby Isaac offsets[f + 1] = fDof + offsets[f]; 3305f30e825dSToby Isaac } 33069566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localFine, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsets, PETSC_FALSE, NULL, -1, NULL, pInd)); 3307367003a6SStefano Zampini } else { 33089566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localFine, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsets, PETSC_FALSE, NULL, NULL, pInd)); 3309f30e825dSToby Isaac } 33109566063dSJacob Faibussowitsch if (gatheredValues) PetscCall(VecGetValues(fineVec, dof, pInd, pVal)); 33118d2f55e7SToby Isaac } 33128d2f55e7SToby Isaac } 33139566063dSJacob Faibussowitsch PetscCall(PetscSFCreateEmbeddedLeafSF(coarseToFine, numPointsWithDofs, pointsWithDofs, &coarseToFineEmbedded)); 33149566063dSJacob Faibussowitsch PetscCall(PetscFree(pointsWithDofs)); 33158d2f55e7SToby Isaac } 3316f30e825dSToby Isaac 33179566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 33189566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 33199566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 3320f30e825dSToby Isaac 33216148253fSToby Isaac { /* there may be the case where an sf root has a parent: broadcast parents back to children */ 33226148253fSToby Isaac MPI_Datatype threeInt; 33236148253fSToby Isaac PetscMPIInt rank; 33246148253fSToby Isaac PetscInt(*parentNodeAndIdCoarse)[3]; 33256148253fSToby Isaac PetscInt(*parentNodeAndIdFine)[3]; 33266148253fSToby Isaac PetscInt p, nleaves, nleavesToParents; 33276148253fSToby Isaac PetscSF pointSF, sfToParents; 33286148253fSToby Isaac const PetscInt *ilocal; 33296148253fSToby Isaac const PetscSFNode *iremote; 33306148253fSToby Isaac PetscSFNode *iremoteToParents; 33316148253fSToby Isaac PetscInt *ilocalToParents; 33326148253fSToby Isaac 33339566063dSJacob Faibussowitsch PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)coarse), &rank)); 33349566063dSJacob Faibussowitsch PetscCallMPI(MPI_Type_contiguous(3, MPIU_INT, &threeInt)); 33359566063dSJacob Faibussowitsch PetscCallMPI(MPI_Type_commit(&threeInt)); 33369566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(pEndC - pStartC, &parentNodeAndIdCoarse, pEndF - pStartF, &parentNodeAndIdFine)); 33379566063dSJacob Faibussowitsch PetscCall(DMGetPointSF(coarse, &pointSF)); 33389566063dSJacob Faibussowitsch PetscCall(PetscSFGetGraph(pointSF, NULL, &nleaves, &ilocal, &iremote)); 33396148253fSToby Isaac for (p = pStartC; p < pEndC; p++) { 33406148253fSToby Isaac PetscInt parent, childId; 33419566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(coarse, p, &parent, &childId)); 33426148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][0] = rank; 33436148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][1] = parent - pStartC; 33446148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][2] = (p == parent) ? -1 : childId; 33456148253fSToby Isaac if (nleaves > 0) { 33466148253fSToby Isaac PetscInt leaf = -1; 33476148253fSToby Isaac 33486148253fSToby Isaac if (ilocal) { 33499566063dSJacob Faibussowitsch PetscCall(PetscFindInt(parent, nleaves, ilocal, &leaf)); 3350*9371c9d4SSatish Balay } else { 33516148253fSToby Isaac leaf = p - pStartC; 33526148253fSToby Isaac } 33536148253fSToby Isaac if (leaf >= 0) { 33546148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][0] = iremote[leaf].rank; 33556148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][1] = iremote[leaf].index; 33566148253fSToby Isaac } 33576148253fSToby Isaac } 33586148253fSToby Isaac } 33596148253fSToby Isaac for (p = pStartF; p < pEndF; p++) { 33606148253fSToby Isaac parentNodeAndIdFine[p - pStartF][0] = -1; 33616148253fSToby Isaac parentNodeAndIdFine[p - pStartF][1] = -1; 33626148253fSToby Isaac parentNodeAndIdFine[p - pStartF][2] = -1; 33636148253fSToby Isaac } 33649566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(coarseToFineEmbedded, threeInt, parentNodeAndIdCoarse, parentNodeAndIdFine, MPI_REPLACE)); 33659566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(coarseToFineEmbedded, threeInt, parentNodeAndIdCoarse, parentNodeAndIdFine, MPI_REPLACE)); 33666148253fSToby Isaac for (p = pStartF, nleavesToParents = 0; p < pEndF; p++) { 3367f30e825dSToby Isaac PetscInt dof; 3368f30e825dSToby Isaac 33699566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafIndicesSec, p, &dof)); 3370f30e825dSToby Isaac if (dof) { 3371f30e825dSToby Isaac PetscInt off; 3372f30e825dSToby Isaac 33739566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafIndicesSec, p, &off)); 3374c921d74cSToby Isaac if (gatheredIndices) { 3375c921d74cSToby Isaac leafInds[off] = PetscMax(childIds[p - pStartF], parentNodeAndIdFine[p - pStartF][2]); 3376c921d74cSToby Isaac } else if (gatheredValues) { 3377c921d74cSToby Isaac leafVals[off] = (PetscScalar)PetscMax(childIds[p - pStartF], parentNodeAndIdFine[p - pStartF][2]); 3378c921d74cSToby Isaac } 3379f30e825dSToby Isaac } 3380*9371c9d4SSatish Balay if (parentNodeAndIdFine[p - pStartF][0] >= 0) { nleavesToParents++; } 33816148253fSToby Isaac } 33829566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(nleavesToParents, &ilocalToParents)); 33839566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(nleavesToParents, &iremoteToParents)); 33846148253fSToby Isaac for (p = pStartF, nleavesToParents = 0; p < pEndF; p++) { 33856148253fSToby Isaac if (parentNodeAndIdFine[p - pStartF][0] >= 0) { 33866148253fSToby Isaac ilocalToParents[nleavesToParents] = p - pStartF; 33876148253fSToby Isaac iremoteToParents[nleavesToParents].rank = parentNodeAndIdFine[p - pStartF][0]; 33886148253fSToby Isaac iremoteToParents[nleavesToParents].index = parentNodeAndIdFine[p - pStartF][1]; 33896148253fSToby Isaac nleavesToParents++; 33906148253fSToby Isaac } 33916148253fSToby Isaac } 33929566063dSJacob Faibussowitsch PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)coarse), &sfToParents)); 33939566063dSJacob Faibussowitsch PetscCall(PetscSFSetGraph(sfToParents, pEndC - pStartC, nleavesToParents, ilocalToParents, PETSC_OWN_POINTER, iremoteToParents, PETSC_OWN_POINTER)); 33949566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 33956148253fSToby Isaac 33966148253fSToby Isaac coarseToFineEmbedded = sfToParents; 33976148253fSToby Isaac 33989566063dSJacob Faibussowitsch PetscCall(PetscFree2(parentNodeAndIdCoarse, parentNodeAndIdFine)); 33999566063dSJacob Faibussowitsch PetscCallMPI(MPI_Type_free(&threeInt)); 34006148253fSToby Isaac } 3401f30e825dSToby Isaac 34026148253fSToby Isaac { /* winnow out coarse points that don't have dofs */ 34036148253fSToby Isaac PetscInt dof, cdof, numPointsWithDofs, offset, *pointsWithDofs; 34046148253fSToby Isaac PetscSF sfDofsOnly; 34056148253fSToby Isaac 34066148253fSToby Isaac for (p = pStartC, numPointsWithDofs = 0; p < pEndC; p++) { 34079566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 34089566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 3409*9371c9d4SSatish Balay if ((dof - cdof) > 0) { numPointsWithDofs++; } 34106148253fSToby Isaac } 34119566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numPointsWithDofs, &pointsWithDofs)); 34126148253fSToby Isaac for (p = pStartC, offset = 0; p < pEndC; p++) { 34139566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 34149566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 3415*9371c9d4SSatish Balay if ((dof - cdof) > 0) { pointsWithDofs[offset++] = p - pStartC; } 34166148253fSToby Isaac } 34179566063dSJacob Faibussowitsch PetscCall(PetscSFCreateEmbeddedRootSF(coarseToFineEmbedded, numPointsWithDofs, pointsWithDofs, &sfDofsOnly)); 34189566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 34199566063dSJacob Faibussowitsch PetscCall(PetscFree(pointsWithDofs)); 34206148253fSToby Isaac coarseToFineEmbedded = sfDofsOnly; 34216148253fSToby Isaac } 3422f30e825dSToby Isaac 34236148253fSToby Isaac /* communicate back to the coarse mesh which coarse points have children (that may require injection) */ 34249566063dSJacob Faibussowitsch PetscCall(PetscSFComputeDegreeBegin(coarseToFineEmbedded, &rootDegrees)); 34259566063dSJacob Faibussowitsch PetscCall(PetscSFComputeDegreeEnd(coarseToFineEmbedded, &rootDegrees)); 34269566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &multiRootSec)); 34279566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(multiRootSec, pStartC, pEndC)); 3428*9371c9d4SSatish Balay for (p = pStartC; p < pEndC; p++) { PetscCall(PetscSectionSetDof(multiRootSec, p, rootDegrees[p - pStartC])); } 34299566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(multiRootSec)); 34309566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(multiRootSec, &numMulti)); 34319566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &rootIndicesSec)); 3432f30e825dSToby Isaac { /* distribute the leaf section */ 3433f30e825dSToby Isaac PetscSF multi, multiInv, indicesSF; 3434f30e825dSToby Isaac PetscInt *remoteOffsets, numRootIndices; 34358d2f55e7SToby Isaac 34369566063dSJacob Faibussowitsch PetscCall(PetscSFGetMultiSF(coarseToFineEmbedded, &multi)); 34379566063dSJacob Faibussowitsch PetscCall(PetscSFCreateInverseSF(multi, &multiInv)); 34389566063dSJacob Faibussowitsch PetscCall(PetscSFDistributeSection(multiInv, leafIndicesSec, &remoteOffsets, rootIndicesSec)); 34399566063dSJacob Faibussowitsch PetscCall(PetscSFCreateSectionSF(multiInv, leafIndicesSec, remoteOffsets, rootIndicesSec, &indicesSF)); 34409566063dSJacob Faibussowitsch PetscCall(PetscFree(remoteOffsets)); 34419566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&multiInv)); 34429566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(rootIndicesSec, &numRootIndices)); 3443c921d74cSToby Isaac if (gatheredIndices) { 34449566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numRootIndices, &rootInds)); 34459566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(indicesSF, MPIU_INT, leafInds, rootInds, MPI_REPLACE)); 34469566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(indicesSF, MPIU_INT, leafInds, rootInds, MPI_REPLACE)); 3447c921d74cSToby Isaac } 3448c921d74cSToby Isaac if (gatheredValues) { 34499566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numRootIndices, &rootVals)); 34509566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(indicesSF, MPIU_SCALAR, leafVals, rootVals, MPI_REPLACE)); 34519566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(indicesSF, MPIU_SCALAR, leafVals, rootVals, MPI_REPLACE)); 3452c921d74cSToby Isaac } 34539566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&indicesSF)); 34548d2f55e7SToby Isaac } 34559566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&leafIndicesSec)); 34569566063dSJacob Faibussowitsch PetscCall(PetscFree(leafInds)); 34579566063dSJacob Faibussowitsch PetscCall(PetscFree(leafVals)); 34589566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 3459c921d74cSToby Isaac *rootMultiSec = multiRootSec; 3460c921d74cSToby Isaac *multiLeafSec = rootIndicesSec; 3461c921d74cSToby Isaac if (gatheredIndices) *gatheredIndices = rootInds; 3462c921d74cSToby Isaac if (gatheredValues) *gatheredValues = rootVals; 3463ebf164c7SToby Isaac PetscFunctionReturn(0); 3464ebf164c7SToby Isaac } 3465ebf164c7SToby Isaac 3466*9371c9d4SSatish Balay PetscErrorCode DMPlexComputeInjectorTree(DM coarse, DM fine, PetscSF coarseToFine, PetscInt *childIds, Mat mat) { 3467ebf164c7SToby Isaac DM refTree; 3468c921d74cSToby Isaac PetscSection multiRootSec, rootIndicesSec; 3469ebf164c7SToby Isaac PetscSection globalCoarse, globalFine; 3470ebf164c7SToby Isaac PetscSection localCoarse, localFine; 3471ebf164c7SToby Isaac PetscSection cSecRef; 3472277f51e8SBarry Smith PetscInt *rootIndices = NULL, *parentIndices, pRefStart, pRefEnd; 3473ebf164c7SToby Isaac Mat injRef; 3474c921d74cSToby Isaac PetscInt numFields, maxDof; 3475ebf164c7SToby Isaac PetscInt pStartC, pEndC, pStartF, pEndF, p; 3476ebf164c7SToby Isaac PetscInt *offsets, *offsetsCopy, *rowOffsets; 3477ebf164c7SToby Isaac PetscLayout rowMap, colMap; 3478ebf164c7SToby Isaac PetscInt rowStart, rowEnd, colStart, colEnd, *nnzD, *nnzO; 3479ebf164c7SToby Isaac PetscScalar ***childrenMats = NULL; /* gcc -O gives 'may be used uninitialized' warning'. Initializing to suppress this warning */ 3480ebf164c7SToby Isaac 3481ebf164c7SToby Isaac PetscFunctionBegin; 3482ebf164c7SToby Isaac 3483ebf164c7SToby Isaac /* get the templates for the fine-to-coarse injection from the reference tree */ 34849566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(coarse, &refTree)); 34859566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &cSecRef, NULL, NULL)); 34869566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSecRef, &pRefStart, &pRefEnd)); 34879566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetInjector(refTree, &injRef)); 3488ebf164c7SToby Isaac 34899566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 34909566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 34919566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 34929566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(localFine, &numFields)); 34939566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 34949566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 34959566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 34969566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localCoarse, &maxDof)); 3497ebf164c7SToby Isaac { 3498ebf164c7SToby Isaac PetscInt maxFields = PetscMax(1, numFields) + 1; 34999566063dSJacob Faibussowitsch PetscCall(PetscMalloc3(maxFields, &offsets, maxFields, &offsetsCopy, maxFields, &rowOffsets)); 3500ebf164c7SToby Isaac } 3501ebf164c7SToby Isaac 35029566063dSJacob Faibussowitsch PetscCall(DMPlexTransferInjectorTree(coarse, fine, coarseToFine, childIds, NULL, numFields, offsets, &multiRootSec, &rootIndicesSec, &rootIndices, NULL)); 35038d2f55e7SToby Isaac 35049566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof, &parentIndices)); 3505f30e825dSToby Isaac 3506f30e825dSToby Isaac /* count indices */ 35079566063dSJacob Faibussowitsch PetscCall(MatGetLayouts(mat, &rowMap, &colMap)); 35089566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(rowMap)); 35099566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(colMap)); 35109566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(rowMap, &rowStart, &rowEnd)); 35119566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(colMap, &colStart, &colEnd)); 35129566063dSJacob Faibussowitsch PetscCall(PetscCalloc2(rowEnd - rowStart, &nnzD, rowEnd - rowStart, &nnzO)); 3513f30e825dSToby Isaac for (p = pStartC; p < pEndC; p++) { 3514f30e825dSToby Isaac PetscInt numLeaves, leafStart, leafEnd, l, dof, cdof, gOff; 35158d2f55e7SToby Isaac 35169566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 35179566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 3518f30e825dSToby Isaac if ((dof - cdof) <= 0) continue; 35199566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, p, &gOff)); 35208d2f55e7SToby Isaac 35218d2f55e7SToby Isaac rowOffsets[0] = 0; 3522f30e825dSToby Isaac offsetsCopy[0] = 0; 35238d2f55e7SToby Isaac if (numFields) { 35248d2f55e7SToby Isaac PetscInt f; 35258d2f55e7SToby Isaac 3526f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 3527f30e825dSToby Isaac PetscInt fDof; 35289566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 3529f30e825dSToby Isaac rowOffsets[f + 1] = offsetsCopy[f + 1] = fDof + rowOffsets[f]; 35308d2f55e7SToby Isaac } 35319566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, parentIndices)); 3532367003a6SStefano Zampini } else { 35339566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, parentIndices)); 3534f30e825dSToby Isaac rowOffsets[1] = offsetsCopy[0]; 35358d2f55e7SToby Isaac } 3536f30e825dSToby Isaac 35379566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(multiRootSec, p, &numLeaves)); 35389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(multiRootSec, p, &leafStart)); 3539f30e825dSToby Isaac leafEnd = leafStart + numLeaves; 3540f30e825dSToby Isaac for (l = leafStart; l < leafEnd; l++) { 3541f30e825dSToby Isaac PetscInt numIndices, childId, offset; 3542f30e825dSToby Isaac const PetscInt *childIndices; 3543f30e825dSToby Isaac 35449566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootIndicesSec, l, &numIndices)); 35459566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootIndicesSec, l, &offset)); 3546f30e825dSToby Isaac childId = rootIndices[offset++]; 3547f30e825dSToby Isaac childIndices = &rootIndices[offset]; 3548f30e825dSToby Isaac numIndices--; 3549f30e825dSToby Isaac 3550f30e825dSToby Isaac if (childId == -1) { /* equivalent points: scatter */ 3551f30e825dSToby Isaac PetscInt i; 3552f30e825dSToby Isaac 3553f30e825dSToby Isaac for (i = 0; i < numIndices; i++) { 3554f30e825dSToby Isaac PetscInt colIndex = childIndices[i]; 3555f30e825dSToby Isaac PetscInt rowIndex = parentIndices[i]; 3556f30e825dSToby Isaac if (rowIndex < 0) continue; 355708401ef6SPierre Jolivet PetscCheck(colIndex >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unconstrained fine and constrained coarse"); 3558a47f92cbSToby Isaac if (colIndex >= colStart && colIndex < colEnd) { 3559f30e825dSToby Isaac nnzD[rowIndex - rowStart] = 1; 3560*9371c9d4SSatish Balay } else { 3561f30e825dSToby Isaac nnzO[rowIndex - rowStart] = 1; 3562f30e825dSToby Isaac } 3563f30e825dSToby Isaac } 3564*9371c9d4SSatish Balay } else { 3565f30e825dSToby Isaac PetscInt parentId, f, lim; 3566f30e825dSToby Isaac 35679566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, childId, &parentId, NULL)); 3568f30e825dSToby Isaac 3569f30e825dSToby Isaac lim = PetscMax(1, numFields); 3570f30e825dSToby Isaac offsets[0] = 0; 35718d2f55e7SToby Isaac if (numFields) { 35728d2f55e7SToby Isaac PetscInt f; 3573f30e825dSToby Isaac 35748d2f55e7SToby Isaac for (f = 0; f < numFields; f++) { 3575f30e825dSToby Isaac PetscInt fDof; 35769566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSecRef, childId, f, &fDof)); 3577f30e825dSToby Isaac 3578f30e825dSToby Isaac offsets[f + 1] = fDof + offsets[f]; 35798d2f55e7SToby Isaac } 3580*9371c9d4SSatish Balay } else { 3581f30e825dSToby Isaac PetscInt cDof; 3582f30e825dSToby Isaac 35839566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSecRef, childId, &cDof)); 3584f30e825dSToby Isaac offsets[1] = cDof; 3585f30e825dSToby Isaac } 3586f30e825dSToby Isaac for (f = 0; f < lim; f++) { 3587f30e825dSToby Isaac PetscInt parentStart = rowOffsets[f], parentEnd = rowOffsets[f + 1]; 3588f30e825dSToby Isaac PetscInt childStart = offsets[f], childEnd = offsets[f + 1]; 3589f30e825dSToby Isaac PetscInt i, numD = 0, numO = 0; 3590f30e825dSToby Isaac 3591f30e825dSToby Isaac for (i = childStart; i < childEnd; i++) { 3592f30e825dSToby Isaac PetscInt colIndex = childIndices[i]; 3593f30e825dSToby Isaac 3594f30e825dSToby Isaac if (colIndex < 0) continue; 3595f30e825dSToby Isaac if (colIndex >= colStart && colIndex < colEnd) { 3596f30e825dSToby Isaac numD++; 3597*9371c9d4SSatish Balay } else { 3598f30e825dSToby Isaac numO++; 3599f30e825dSToby Isaac } 3600f30e825dSToby Isaac } 3601f30e825dSToby Isaac for (i = parentStart; i < parentEnd; i++) { 3602f30e825dSToby Isaac PetscInt rowIndex = parentIndices[i]; 3603f30e825dSToby Isaac 3604f30e825dSToby Isaac if (rowIndex < 0) continue; 3605f30e825dSToby Isaac nnzD[rowIndex - rowStart] += numD; 3606f30e825dSToby Isaac nnzO[rowIndex - rowStart] += numO; 36078d2f55e7SToby Isaac } 36088d2f55e7SToby Isaac } 36098d2f55e7SToby Isaac } 3610f30e825dSToby Isaac } 3611f30e825dSToby Isaac } 3612f30e825dSToby Isaac /* preallocate */ 36139566063dSJacob Faibussowitsch PetscCall(MatXAIJSetPreallocation(mat, 1, nnzD, nnzO, NULL, NULL)); 36149566063dSJacob Faibussowitsch PetscCall(PetscFree2(nnzD, nnzO)); 3615f30e825dSToby Isaac /* insert values */ 36169566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices_Injection(refTree, injRef, &childrenMats)); 3617f30e825dSToby Isaac for (p = pStartC; p < pEndC; p++) { 3618f30e825dSToby Isaac PetscInt numLeaves, leafStart, leafEnd, l, dof, cdof, gOff; 3619f30e825dSToby Isaac 36209566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 36219566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 3622f30e825dSToby Isaac if ((dof - cdof) <= 0) continue; 36239566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, p, &gOff)); 3624f30e825dSToby Isaac 3625f30e825dSToby Isaac rowOffsets[0] = 0; 3626f30e825dSToby Isaac offsetsCopy[0] = 0; 36278d2f55e7SToby Isaac if (numFields) { 36288d2f55e7SToby Isaac PetscInt f; 3629f30e825dSToby Isaac 36308d2f55e7SToby Isaac for (f = 0; f < numFields; f++) { 3631f30e825dSToby Isaac PetscInt fDof; 36329566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 3633f30e825dSToby Isaac rowOffsets[f + 1] = offsetsCopy[f + 1] = fDof + rowOffsets[f]; 3634f30e825dSToby Isaac } 36359566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, parentIndices)); 3636367003a6SStefano Zampini } else { 36379566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, parentIndices)); 3638f30e825dSToby Isaac rowOffsets[1] = offsetsCopy[0]; 3639f30e825dSToby Isaac } 3640f30e825dSToby Isaac 36419566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(multiRootSec, p, &numLeaves)); 36429566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(multiRootSec, p, &leafStart)); 3643f30e825dSToby Isaac leafEnd = leafStart + numLeaves; 3644f30e825dSToby Isaac for (l = leafStart; l < leafEnd; l++) { 3645f30e825dSToby Isaac PetscInt numIndices, childId, offset; 3646f30e825dSToby Isaac const PetscInt *childIndices; 3647f30e825dSToby Isaac 36489566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootIndicesSec, l, &numIndices)); 36499566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootIndicesSec, l, &offset)); 3650f30e825dSToby Isaac childId = rootIndices[offset++]; 3651f30e825dSToby Isaac childIndices = &rootIndices[offset]; 3652f30e825dSToby Isaac numIndices--; 3653f30e825dSToby Isaac 3654f30e825dSToby Isaac if (childId == -1) { /* equivalent points: scatter */ 3655f30e825dSToby Isaac PetscInt i; 3656f30e825dSToby Isaac 3657*9371c9d4SSatish Balay for (i = 0; i < numIndices; i++) { PetscCall(MatSetValue(mat, parentIndices[i], childIndices[i], 1., INSERT_VALUES)); } 3658*9371c9d4SSatish Balay } else { 3659f30e825dSToby Isaac PetscInt parentId, f, lim; 36608d2f55e7SToby Isaac 36619566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, childId, &parentId, NULL)); 3662f30e825dSToby Isaac 3663f30e825dSToby Isaac lim = PetscMax(1, numFields); 3664f30e825dSToby Isaac offsets[0] = 0; 36658d2f55e7SToby Isaac if (numFields) { 3666f30e825dSToby Isaac PetscInt f; 36678d2f55e7SToby Isaac 3668f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 3669f30e825dSToby Isaac PetscInt fDof; 36709566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSecRef, childId, f, &fDof)); 3671f30e825dSToby Isaac 3672f30e825dSToby Isaac offsets[f + 1] = fDof + offsets[f]; 36738d2f55e7SToby Isaac } 3674*9371c9d4SSatish Balay } else { 3675f30e825dSToby Isaac PetscInt cDof; 3676f30e825dSToby Isaac 36779566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSecRef, childId, &cDof)); 3678f30e825dSToby Isaac offsets[1] = cDof; 36798d2f55e7SToby Isaac } 3680f30e825dSToby Isaac for (f = 0; f < lim; f++) { 3681f30e825dSToby Isaac PetscScalar *childMat = &childrenMats[childId - pRefStart][f][0]; 3682f30e825dSToby Isaac PetscInt *rowIndices = &parentIndices[rowOffsets[f]]; 3683f30e825dSToby Isaac const PetscInt *colIndices = &childIndices[offsets[f]]; 3684f30e825dSToby Isaac 36859566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, rowOffsets[f + 1] - rowOffsets[f], rowIndices, offsets[f + 1] - offsets[f], colIndices, childMat, INSERT_VALUES)); 36868d2f55e7SToby Isaac } 36878d2f55e7SToby Isaac } 36888d2f55e7SToby Isaac } 36898d2f55e7SToby Isaac } 36909566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&multiRootSec)); 36919566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootIndicesSec)); 36929566063dSJacob Faibussowitsch PetscCall(PetscFree(parentIndices)); 36939566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices_Injection(refTree, injRef, &childrenMats)); 36949566063dSJacob Faibussowitsch PetscCall(PetscFree(rootIndices)); 36959566063dSJacob Faibussowitsch PetscCall(PetscFree3(offsets, offsetsCopy, rowOffsets)); 3696f30e825dSToby Isaac 36979566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(mat, MAT_FINAL_ASSEMBLY)); 36989566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(mat, MAT_FINAL_ASSEMBLY)); 3699154bca37SToby Isaac PetscFunctionReturn(0); 3700154bca37SToby Isaac } 370138fc2455SToby Isaac 3702*9371c9d4SSatish Balay static PetscErrorCode DMPlexTransferVecTree_Interpolate(DM coarse, Vec vecCoarseLocal, DM fine, Vec vecFine, PetscSF coarseToFine, PetscInt *cids, Vec grad, Vec cellGeom) { 370362095d54SToby Isaac PetscSF coarseToFineEmbedded; 370462095d54SToby Isaac PetscSection globalCoarse, globalFine; 370562095d54SToby Isaac PetscSection localCoarse, localFine; 370662095d54SToby Isaac PetscSection aSec, cSec; 370762095d54SToby Isaac PetscSection rootValuesSec; 370862095d54SToby Isaac PetscSection leafValuesSec; 370962095d54SToby Isaac PetscScalar *rootValues, *leafValues; 371062095d54SToby Isaac IS aIS; 371162095d54SToby Isaac const PetscInt *anchors; 371262095d54SToby Isaac Mat cMat; 371362095d54SToby Isaac PetscInt numFields; 3714412e9a14SMatthew G. Knepley PetscInt pStartC, pEndC, pStartF, pEndF, p, cellStart, cellEnd; 371562095d54SToby Isaac PetscInt aStart, aEnd, cStart, cEnd; 371662095d54SToby Isaac PetscInt *maxChildIds; 371762095d54SToby Isaac PetscInt *offsets, *newOffsets, *offsetsCopy, *newOffsetsCopy, *rowOffsets, *numD, *numO; 37180eb7e1eaSToby Isaac PetscFV fv = NULL; 37190eb7e1eaSToby Isaac PetscInt dim, numFVcomps = -1, fvField = -1; 37200eb7e1eaSToby Isaac DM cellDM = NULL, gradDM = NULL; 37210eb7e1eaSToby Isaac const PetscScalar *cellGeomArray = NULL; 37220eb7e1eaSToby Isaac const PetscScalar *gradArray = NULL; 372362095d54SToby Isaac 3724ebf164c7SToby Isaac PetscFunctionBegin; 37259566063dSJacob Faibussowitsch PetscCall(VecSetOption(vecFine, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE)); 37269566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 37279566063dSJacob Faibussowitsch PetscCall(DMPlexGetSimplexOrBoxCells(coarse, 0, &cellStart, &cellEnd)); 37289566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 37299566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 37309566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateDim(coarse, &dim)); 373162095d54SToby Isaac { /* winnow fine points that don't have global dofs out of the sf */ 3732e4a60869SToby Isaac PetscInt nleaves, l; 3733e4a60869SToby Isaac const PetscInt *leaves; 373462095d54SToby Isaac PetscInt dof, cdof, numPointsWithDofs, offset, *pointsWithDofs; 373562095d54SToby Isaac 37369566063dSJacob Faibussowitsch PetscCall(PetscSFGetGraph(coarseToFine, NULL, &nleaves, &leaves, NULL)); 3737e4a60869SToby Isaac 3738e4a60869SToby Isaac for (l = 0, numPointsWithDofs = 0; l < nleaves; l++) { 3739e4a60869SToby Isaac PetscInt p = leaves ? leaves[l] : l; 3740e4a60869SToby Isaac 37419566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 37429566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 3743*9371c9d4SSatish Balay if ((dof - cdof) > 0) { numPointsWithDofs++; } 374462095d54SToby Isaac } 37459566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numPointsWithDofs, &pointsWithDofs)); 37464833aeb0SToby Isaac for (l = 0, offset = 0; l < nleaves; l++) { 3747e4a60869SToby Isaac PetscInt p = leaves ? leaves[l] : l; 3748e4a60869SToby Isaac 37499566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 37509566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 3751*9371c9d4SSatish Balay if ((dof - cdof) > 0) { pointsWithDofs[offset++] = l; } 375262095d54SToby Isaac } 37539566063dSJacob Faibussowitsch PetscCall(PetscSFCreateEmbeddedLeafSF(coarseToFine, numPointsWithDofs, pointsWithDofs, &coarseToFineEmbedded)); 37549566063dSJacob Faibussowitsch PetscCall(PetscFree(pointsWithDofs)); 375562095d54SToby Isaac } 375662095d54SToby Isaac /* communicate back to the coarse mesh which coarse points have children (that may require interpolation) */ 37579566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEndC - pStartC, &maxChildIds)); 3758*9371c9d4SSatish Balay for (p = pStartC; p < pEndC; p++) { maxChildIds[p - pStartC] = -2; } 37599566063dSJacob Faibussowitsch PetscCall(PetscSFReduceBegin(coarseToFineEmbedded, MPIU_INT, cids, maxChildIds, MPIU_MAX)); 37609566063dSJacob Faibussowitsch PetscCall(PetscSFReduceEnd(coarseToFineEmbedded, MPIU_INT, cids, maxChildIds, MPIU_MAX)); 376162095d54SToby Isaac 37629566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 37639566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 376462095d54SToby Isaac 37659566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(coarse, &aSec, &aIS)); 37669566063dSJacob Faibussowitsch PetscCall(ISGetIndices(aIS, &anchors)); 37679566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(aSec, &aStart, &aEnd)); 376862095d54SToby Isaac 37699566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(coarse, &cSec, &cMat, NULL)); 37709566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSec, &cStart, &cEnd)); 377162095d54SToby Isaac 377262095d54SToby Isaac /* create sections that will send to children the indices and matrices they will need to construct the interpolator */ 37739566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &rootValuesSec)); 37749566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(rootValuesSec, pStartC, pEndC)); 37759566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(localCoarse, &numFields)); 377662095d54SToby Isaac { 377762095d54SToby Isaac PetscInt maxFields = PetscMax(1, numFields) + 1; 37789566063dSJacob Faibussowitsch PetscCall(PetscMalloc7(maxFields, &offsets, maxFields, &offsetsCopy, maxFields, &newOffsets, maxFields, &newOffsetsCopy, maxFields, &rowOffsets, maxFields, &numD, maxFields, &numO)); 377962095d54SToby Isaac } 37800eb7e1eaSToby Isaac if (grad) { 37810eb7e1eaSToby Isaac PetscInt i; 37820eb7e1eaSToby Isaac 37839566063dSJacob Faibussowitsch PetscCall(VecGetDM(cellGeom, &cellDM)); 37849566063dSJacob Faibussowitsch PetscCall(VecGetArrayRead(cellGeom, &cellGeomArray)); 37859566063dSJacob Faibussowitsch PetscCall(VecGetDM(grad, &gradDM)); 37869566063dSJacob Faibussowitsch PetscCall(VecGetArrayRead(grad, &gradArray)); 37870eb7e1eaSToby Isaac for (i = 0; i < PetscMax(1, numFields); i++) { 37880eb7e1eaSToby Isaac PetscObject obj; 37890eb7e1eaSToby Isaac PetscClassId id; 37900eb7e1eaSToby Isaac 37919566063dSJacob Faibussowitsch PetscCall(DMGetField(coarse, i, NULL, &obj)); 37929566063dSJacob Faibussowitsch PetscCall(PetscObjectGetClassId(obj, &id)); 37930eb7e1eaSToby Isaac if (id == PETSCFV_CLASSID) { 37940eb7e1eaSToby Isaac fv = (PetscFV)obj; 37959566063dSJacob Faibussowitsch PetscCall(PetscFVGetNumComponents(fv, &numFVcomps)); 37960eb7e1eaSToby Isaac fvField = i; 37970eb7e1eaSToby Isaac break; 37980eb7e1eaSToby Isaac } 37990eb7e1eaSToby Isaac } 38000eb7e1eaSToby Isaac } 380162095d54SToby Isaac 380262095d54SToby Isaac for (p = pStartC; p < pEndC; p++) { /* count the sizes of the indices and matrices */ 380362095d54SToby Isaac PetscInt dof; 380462095d54SToby Isaac PetscInt maxChildId = maxChildIds[p - pStartC]; 380562095d54SToby Isaac PetscInt numValues = 0; 380662095d54SToby Isaac 38079566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 3808*9371c9d4SSatish Balay if (dof < 0) { dof = -(dof + 1); } 380962095d54SToby Isaac offsets[0] = 0; 381062095d54SToby Isaac newOffsets[0] = 0; 381162095d54SToby Isaac if (maxChildId >= 0) { /* this point has children (with dofs) that will need to be interpolated from the closure of p */ 381262095d54SToby Isaac PetscInt *closure = NULL, closureSize, cl; 381362095d54SToby Isaac 38149566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 381562095d54SToby Isaac for (cl = 0; cl < closureSize; cl++) { /* get the closure */ 381662095d54SToby Isaac PetscInt c = closure[2 * cl], clDof; 381762095d54SToby Isaac 38189566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, c, &clDof)); 381962095d54SToby Isaac numValues += clDof; 382062095d54SToby Isaac } 38219566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 3822*9371c9d4SSatish Balay } else if (maxChildId == -1) { 38239566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, p, &numValues)); 382462095d54SToby Isaac } 382562095d54SToby Isaac /* we will pack the column indices with the field offsets */ 382678b7adb5SToby Isaac if (maxChildId >= 0 && grad && p >= cellStart && p < cellEnd) { 38270eb7e1eaSToby Isaac /* also send the centroid, and the gradient */ 38280eb7e1eaSToby Isaac numValues += dim * (1 + numFVcomps); 38290eb7e1eaSToby Isaac } 38309566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(rootValuesSec, p, numValues)); 383162095d54SToby Isaac } 38329566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(rootValuesSec)); 383362095d54SToby Isaac { 383462095d54SToby Isaac PetscInt numRootValues; 383562095d54SToby Isaac const PetscScalar *coarseArray; 383662095d54SToby Isaac 38379566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(rootValuesSec, &numRootValues)); 38389566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numRootValues, &rootValues)); 38399566063dSJacob Faibussowitsch PetscCall(VecGetArrayRead(vecCoarseLocal, &coarseArray)); 384062095d54SToby Isaac for (p = pStartC; p < pEndC; p++) { 384162095d54SToby Isaac PetscInt numValues; 384262095d54SToby Isaac PetscInt pValOff; 384362095d54SToby Isaac PetscScalar *pVal; 384462095d54SToby Isaac PetscInt maxChildId = maxChildIds[p - pStartC]; 384562095d54SToby Isaac 38469566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootValuesSec, p, &numValues)); 3847*9371c9d4SSatish Balay if (!numValues) { continue; } 38489566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootValuesSec, p, &pValOff)); 384962095d54SToby Isaac pVal = &(rootValues[pValOff]); 385062095d54SToby Isaac if (maxChildId >= 0) { /* build an identity matrix, apply matrix constraints on the right */ 38510eb7e1eaSToby Isaac PetscInt closureSize = numValues; 38529566063dSJacob Faibussowitsch PetscCall(DMPlexVecGetClosure(coarse, NULL, vecCoarseLocal, p, &closureSize, &pVal)); 38530eb7e1eaSToby Isaac if (grad && p >= cellStart && p < cellEnd) { 3854193eb951SToby Isaac PetscFVCellGeom *cg; 38556dd00756SToby Isaac PetscScalar *gradVals = NULL; 38560eb7e1eaSToby Isaac PetscInt i; 38570eb7e1eaSToby Isaac 38580eb7e1eaSToby Isaac pVal += (numValues - dim * (1 + numFVcomps)); 38590eb7e1eaSToby Isaac 38609566063dSJacob Faibussowitsch PetscCall(DMPlexPointLocalRead(cellDM, p, cellGeomArray, (void *)&cg)); 38610eb7e1eaSToby Isaac for (i = 0; i < dim; i++) pVal[i] = cg->centroid[i]; 38620eb7e1eaSToby Isaac pVal += dim; 38639566063dSJacob Faibussowitsch PetscCall(DMPlexPointGlobalRead(gradDM, p, gradArray, (void *)&gradVals)); 38640eb7e1eaSToby Isaac for (i = 0; i < dim * numFVcomps; i++) pVal[i] = gradVals[i]; 38650eb7e1eaSToby Isaac } 3866*9371c9d4SSatish Balay } else if (maxChildId == -1) { 386778b7adb5SToby Isaac PetscInt lDof, lOff, i; 386878b7adb5SToby Isaac 38699566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, p, &lDof)); 38709566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(localCoarse, p, &lOff)); 387178b7adb5SToby Isaac for (i = 0; i < lDof; i++) pVal[i] = coarseArray[lOff + i]; 387278b7adb5SToby Isaac } 387378b7adb5SToby Isaac } 38749566063dSJacob Faibussowitsch PetscCall(VecRestoreArrayRead(vecCoarseLocal, &coarseArray)); 38759566063dSJacob Faibussowitsch PetscCall(PetscFree(maxChildIds)); 387662095d54SToby Isaac } 387762095d54SToby Isaac { 387862095d54SToby Isaac PetscSF valuesSF; 387962095d54SToby Isaac PetscInt *remoteOffsetsValues, numLeafValues; 388062095d54SToby Isaac 38819566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)fine), &leafValuesSec)); 38829566063dSJacob Faibussowitsch PetscCall(PetscSFDistributeSection(coarseToFineEmbedded, rootValuesSec, &remoteOffsetsValues, leafValuesSec)); 38839566063dSJacob Faibussowitsch PetscCall(PetscSFCreateSectionSF(coarseToFineEmbedded, rootValuesSec, remoteOffsetsValues, leafValuesSec, &valuesSF)); 38849566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 38859566063dSJacob Faibussowitsch PetscCall(PetscFree(remoteOffsetsValues)); 38869566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(leafValuesSec, &numLeafValues)); 38879566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numLeafValues, &leafValues)); 38889566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(valuesSF, MPIU_SCALAR, rootValues, leafValues, MPI_REPLACE)); 38899566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(valuesSF, MPIU_SCALAR, rootValues, leafValues, MPI_REPLACE)); 38909566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&valuesSF)); 38919566063dSJacob Faibussowitsch PetscCall(PetscFree(rootValues)); 38929566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootValuesSec)); 389362095d54SToby Isaac } 38949566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 389562095d54SToby Isaac { 389662095d54SToby Isaac PetscInt maxDof; 389762095d54SToby Isaac PetscInt *rowIndices; 389862095d54SToby Isaac DM refTree; 389962095d54SToby Isaac PetscInt **refPointFieldN; 390062095d54SToby Isaac PetscScalar ***refPointFieldMats; 390162095d54SToby Isaac PetscSection refConSec, refAnSec; 39020eb7e1eaSToby Isaac PetscInt pRefStart, pRefEnd, leafStart, leafEnd; 390362095d54SToby Isaac PetscScalar *pointWork; 390462095d54SToby Isaac 39059566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localFine, &maxDof)); 39069566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(fine, maxDof, MPIU_INT, &rowIndices)); 39079566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(fine, maxDof, MPIU_SCALAR, &pointWork)); 39089566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(fine, &refTree)); 39099566063dSJacob Faibussowitsch PetscCall(DMCopyDisc(fine, refTree)); 39109566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 39119566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 39129566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(refTree, &refAnSec, NULL)); 39139566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 39149566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(leafValuesSec, &leafStart, &leafEnd)); 39159566063dSJacob Faibussowitsch PetscCall(DMPlexGetSimplexOrBoxCells(fine, 0, &cellStart, &cellEnd)); 39160eb7e1eaSToby Isaac for (p = leafStart; p < leafEnd; p++) { 391762095d54SToby Isaac PetscInt gDof, gcDof, gOff, lDof; 391862095d54SToby Isaac PetscInt numValues, pValOff; 391962095d54SToby Isaac PetscInt childId; 392062095d54SToby Isaac const PetscScalar *pVal; 39210eb7e1eaSToby Isaac const PetscScalar *fvGradData = NULL; 392262095d54SToby Isaac 39239566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &gDof)); 39249566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localFine, p, &lDof)); 39259566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &gcDof)); 3926*9371c9d4SSatish Balay if ((gDof - gcDof) <= 0) { continue; } 39279566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalFine, p, &gOff)); 39289566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafValuesSec, p, &numValues)); 392962095d54SToby Isaac if (!numValues) continue; 39309566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafValuesSec, p, &pValOff)); 393162095d54SToby Isaac pVal = &leafValues[pValOff]; 393262095d54SToby Isaac offsets[0] = 0; 393362095d54SToby Isaac offsetsCopy[0] = 0; 393462095d54SToby Isaac newOffsets[0] = 0; 393562095d54SToby Isaac newOffsetsCopy[0] = 0; 39364833aeb0SToby Isaac childId = cids[p - pStartF]; 393762095d54SToby Isaac if (numFields) { 393862095d54SToby Isaac PetscInt f; 393962095d54SToby Isaac for (f = 0; f < numFields; f++) { 394062095d54SToby Isaac PetscInt rowDof; 394162095d54SToby Isaac 39429566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localFine, p, f, &rowDof)); 394362095d54SToby Isaac offsets[f + 1] = offsets[f] + rowDof; 394462095d54SToby Isaac offsetsCopy[f + 1] = offsets[f + 1]; 394562095d54SToby Isaac /* TODO: closure indices */ 39469f4e70e1SToby Isaac newOffsets[f + 1] = newOffsets[f] + ((childId == -1) ? rowDof : refPointFieldN[childId - pRefStart][f]); 394762095d54SToby Isaac } 39489566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, rowIndices)); 3949*9371c9d4SSatish Balay } else { 39504833aeb0SToby Isaac offsets[0] = 0; 39514833aeb0SToby Isaac offsets[1] = lDof; 39524833aeb0SToby Isaac newOffsets[0] = 0; 39534833aeb0SToby Isaac newOffsets[1] = (childId == -1) ? lDof : refPointFieldN[childId - pRefStart][0]; 39549566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, rowIndices)); 395562095d54SToby Isaac } 395662095d54SToby Isaac if (childId == -1) { /* no child interpolation: one nnz per */ 39579566063dSJacob Faibussowitsch PetscCall(VecSetValues(vecFine, numValues, rowIndices, pVal, INSERT_VALUES)); 395862095d54SToby Isaac } else { 395962095d54SToby Isaac PetscInt f; 396062095d54SToby Isaac 396178b7adb5SToby Isaac if (grad && p >= cellStart && p < cellEnd) { 396278b7adb5SToby Isaac numValues -= (dim * (1 + numFVcomps)); 396378b7adb5SToby Isaac fvGradData = &pVal[numValues]; 396478b7adb5SToby Isaac } 396562095d54SToby Isaac for (f = 0; f < PetscMax(1, numFields); f++) { 396662095d54SToby Isaac const PetscScalar *childMat = refPointFieldMats[childId - pRefStart][f]; 396762095d54SToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f]; 396862095d54SToby Isaac PetscInt numCols = newOffsets[f + 1] - newOffsets[f]; 396962095d54SToby Isaac const PetscScalar *cVal = &pVal[newOffsets[f]]; 397062095d54SToby Isaac PetscScalar *rVal = &pointWork[offsets[f]]; 397162095d54SToby Isaac PetscInt i, j; 397262095d54SToby Isaac 3973708c7f19SToby Isaac #if 0 397463a3b9bcSJacob Faibussowitsch PetscCall(PetscInfo(coarse,"childId %" PetscInt_FMT ", numRows %" PetscInt_FMT ", numCols %" PetscInt_FMT ", refPointFieldN %" PetscInt_FMT " maxDof %" PetscInt_FMT "\n",childId,numRows,numCols,refPointFieldN[childId - pRefStart][f], maxDof)); 3975708c7f19SToby Isaac #endif 397662095d54SToby Isaac for (i = 0; i < numRows; i++) { 397762095d54SToby Isaac PetscScalar val = 0.; 3978*9371c9d4SSatish Balay for (j = 0; j < numCols; j++) { val += childMat[i * numCols + j] * cVal[j]; } 397962095d54SToby Isaac rVal[i] = val; 398062095d54SToby Isaac } 39810eb7e1eaSToby Isaac if (f == fvField && p >= cellStart && p < cellEnd) { 39820eb7e1eaSToby Isaac PetscReal centroid[3]; 39830eb7e1eaSToby Isaac PetscScalar diff[3]; 39840eb7e1eaSToby Isaac const PetscScalar *parentCentroid = &fvGradData[0]; 39850eb7e1eaSToby Isaac const PetscScalar *gradient = &fvGradData[dim]; 39860eb7e1eaSToby Isaac 39879566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFVM(fine, p, NULL, centroid, NULL)); 3988*9371c9d4SSatish Balay for (i = 0; i < dim; i++) { diff[i] = centroid[i] - parentCentroid[i]; } 39890eb7e1eaSToby Isaac for (i = 0; i < numFVcomps; i++) { 39900eb7e1eaSToby Isaac PetscScalar val = 0.; 39910eb7e1eaSToby Isaac 3992*9371c9d4SSatish Balay for (j = 0; j < dim; j++) { val += gradient[dim * i + j] * diff[j]; } 39930eb7e1eaSToby Isaac rVal[i] += val; 39940eb7e1eaSToby Isaac } 39950eb7e1eaSToby Isaac } 39969566063dSJacob Faibussowitsch PetscCall(VecSetValues(vecFine, numRows, &rowIndices[offsets[f]], rVal, INSERT_VALUES)); 399762095d54SToby Isaac } 399862095d54SToby Isaac } 399962095d54SToby Isaac } 40009566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 40019566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(fine, maxDof, MPIU_SCALAR, &pointWork)); 40029566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(fine, maxDof, MPIU_INT, &rowIndices)); 400362095d54SToby Isaac } 40049566063dSJacob Faibussowitsch PetscCall(PetscFree(leafValues)); 40059566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&leafValuesSec)); 40069566063dSJacob Faibussowitsch PetscCall(PetscFree7(offsets, offsetsCopy, newOffsets, newOffsetsCopy, rowOffsets, numD, numO)); 40079566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(aIS, &anchors)); 4008ebf164c7SToby Isaac PetscFunctionReturn(0); 4009ebf164c7SToby Isaac } 4010ebf164c7SToby Isaac 4011*9371c9d4SSatish Balay static PetscErrorCode DMPlexTransferVecTree_Inject(DM fine, Vec vecFine, DM coarse, Vec vecCoarse, PetscSF coarseToFine, PetscInt *cids) { 4012c921d74cSToby Isaac DM refTree; 4013c921d74cSToby Isaac PetscSection multiRootSec, rootIndicesSec; 4014c921d74cSToby Isaac PetscSection globalCoarse, globalFine; 4015c921d74cSToby Isaac PetscSection localCoarse, localFine; 4016c921d74cSToby Isaac PetscSection cSecRef; 4017c921d74cSToby Isaac PetscInt *parentIndices, pRefStart, pRefEnd; 4018d3bc4906SToby Isaac PetscScalar *rootValues, *parentValues; 4019c921d74cSToby Isaac Mat injRef; 4020c921d74cSToby Isaac PetscInt numFields, maxDof; 4021c921d74cSToby Isaac PetscInt pStartC, pEndC, pStartF, pEndF, p; 4022c921d74cSToby Isaac PetscInt *offsets, *offsetsCopy, *rowOffsets; 4023c921d74cSToby Isaac PetscLayout rowMap, colMap; 4024c921d74cSToby Isaac PetscInt rowStart, rowEnd, colStart, colEnd; 4025c921d74cSToby Isaac PetscScalar ***childrenMats = NULL; /* gcc -O gives 'may be used uninitialized' warning'. Initializing to suppress this warning */ 4026c921d74cSToby Isaac 4027ebf164c7SToby Isaac PetscFunctionBegin; 4028c921d74cSToby Isaac 4029c921d74cSToby Isaac /* get the templates for the fine-to-coarse injection from the reference tree */ 40309566063dSJacob Faibussowitsch PetscCall(VecSetOption(vecFine, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE)); 40319566063dSJacob Faibussowitsch PetscCall(VecSetOption(vecCoarse, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE)); 40329566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(coarse, &refTree)); 40339566063dSJacob Faibussowitsch PetscCall(DMCopyDisc(coarse, refTree)); 40349566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &cSecRef, NULL, NULL)); 40359566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSecRef, &pRefStart, &pRefEnd)); 40369566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetInjector(refTree, &injRef)); 4037c921d74cSToby Isaac 40389566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 40399566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 40409566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 40419566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(localFine, &numFields)); 40429566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 40439566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 40449566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 40459566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localCoarse, &maxDof)); 4046c921d74cSToby Isaac { 4047c921d74cSToby Isaac PetscInt maxFields = PetscMax(1, numFields) + 1; 40489566063dSJacob Faibussowitsch PetscCall(PetscMalloc3(maxFields, &offsets, maxFields, &offsetsCopy, maxFields, &rowOffsets)); 4049c921d74cSToby Isaac } 4050c921d74cSToby Isaac 40519566063dSJacob Faibussowitsch PetscCall(DMPlexTransferInjectorTree(coarse, fine, coarseToFine, cids, vecFine, numFields, offsets, &multiRootSec, &rootIndicesSec, NULL, &rootValues)); 4052c921d74cSToby Isaac 40539566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(maxDof, &parentIndices, maxDof, &parentValues)); 4054c921d74cSToby Isaac 4055c921d74cSToby Isaac /* count indices */ 40569566063dSJacob Faibussowitsch PetscCall(VecGetLayout(vecFine, &colMap)); 40579566063dSJacob Faibussowitsch PetscCall(VecGetLayout(vecCoarse, &rowMap)); 40589566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(rowMap)); 40599566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(colMap)); 40609566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(rowMap, &rowStart, &rowEnd)); 40619566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(colMap, &colStart, &colEnd)); 4062c921d74cSToby Isaac /* insert values */ 40639566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices_Injection(refTree, injRef, &childrenMats)); 4064c921d74cSToby Isaac for (p = pStartC; p < pEndC; p++) { 4065c921d74cSToby Isaac PetscInt numLeaves, leafStart, leafEnd, l, dof, cdof, gOff; 406678b7adb5SToby Isaac PetscBool contribute = PETSC_FALSE; 4067c921d74cSToby Isaac 40689566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 40699566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 4070c921d74cSToby Isaac if ((dof - cdof) <= 0) continue; 40719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, p, &dof)); 40729566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, p, &gOff)); 4073c921d74cSToby Isaac 4074c921d74cSToby Isaac rowOffsets[0] = 0; 4075c921d74cSToby Isaac offsetsCopy[0] = 0; 4076c921d74cSToby Isaac if (numFields) { 4077c921d74cSToby Isaac PetscInt f; 4078c921d74cSToby Isaac 4079c921d74cSToby Isaac for (f = 0; f < numFields; f++) { 4080c921d74cSToby Isaac PetscInt fDof; 40819566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 4082c921d74cSToby Isaac rowOffsets[f + 1] = offsetsCopy[f + 1] = fDof + rowOffsets[f]; 4083c921d74cSToby Isaac } 40849566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, parentIndices)); 4085367003a6SStefano Zampini } else { 40869566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, parentIndices)); 4087c921d74cSToby Isaac rowOffsets[1] = offsetsCopy[0]; 4088c921d74cSToby Isaac } 4089c921d74cSToby Isaac 40909566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(multiRootSec, p, &numLeaves)); 40919566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(multiRootSec, p, &leafStart)); 4092c921d74cSToby Isaac leafEnd = leafStart + numLeaves; 40932f65e181SToby Isaac for (l = 0; l < dof; l++) parentValues[l] = 0.; 4094c921d74cSToby Isaac for (l = leafStart; l < leafEnd; l++) { 4095c921d74cSToby Isaac PetscInt numIndices, childId, offset; 4096c921d74cSToby Isaac const PetscScalar *childValues; 4097c921d74cSToby Isaac 40989566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootIndicesSec, l, &numIndices)); 40999566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootIndicesSec, l, &offset)); 4100c921d74cSToby Isaac childId = (PetscInt)PetscRealPart(rootValues[offset++]); 4101c921d74cSToby Isaac childValues = &rootValues[offset]; 4102c921d74cSToby Isaac numIndices--; 4103c921d74cSToby Isaac 4104c921d74cSToby Isaac if (childId == -2) { /* skip */ 4105c921d74cSToby Isaac continue; 4106c921d74cSToby Isaac } else if (childId == -1) { /* equivalent points: scatter */ 41072f65e181SToby Isaac PetscInt m; 41082f65e181SToby Isaac 410978b7adb5SToby Isaac contribute = PETSC_TRUE; 41102f65e181SToby Isaac for (m = 0; m < numIndices; m++) parentValues[m] = childValues[m]; 4111beedf8abSToby Isaac } else { /* contributions from children: sum with injectors from reference tree */ 4112d3bc4906SToby Isaac PetscInt parentId, f, lim; 4113d3bc4906SToby Isaac 411478b7adb5SToby Isaac contribute = PETSC_TRUE; 41159566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, childId, &parentId, NULL)); 4116d3bc4906SToby Isaac 4117d3bc4906SToby Isaac lim = PetscMax(1, numFields); 4118d3bc4906SToby Isaac offsets[0] = 0; 4119d3bc4906SToby Isaac if (numFields) { 4120d3bc4906SToby Isaac PetscInt f; 4121d3bc4906SToby Isaac 4122d3bc4906SToby Isaac for (f = 0; f < numFields; f++) { 4123d3bc4906SToby Isaac PetscInt fDof; 41249566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSecRef, childId, f, &fDof)); 4125d3bc4906SToby Isaac 4126d3bc4906SToby Isaac offsets[f + 1] = fDof + offsets[f]; 4127d3bc4906SToby Isaac } 4128*9371c9d4SSatish Balay } else { 4129d3bc4906SToby Isaac PetscInt cDof; 4130d3bc4906SToby Isaac 41319566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSecRef, childId, &cDof)); 4132d3bc4906SToby Isaac offsets[1] = cDof; 4133d3bc4906SToby Isaac } 4134d3bc4906SToby Isaac for (f = 0; f < lim; f++) { 4135d3bc4906SToby Isaac PetscScalar *childMat = &childrenMats[childId - pRefStart][f][0]; 4136d3bc4906SToby Isaac PetscInt n = offsets[f + 1] - offsets[f]; 4137e328ff09SToby Isaac PetscInt m = rowOffsets[f + 1] - rowOffsets[f]; 4138d3bc4906SToby Isaac PetscInt i, j; 4139d3bc4906SToby Isaac const PetscScalar *colValues = &childValues[offsets[f]]; 4140d3bc4906SToby Isaac 4141e328ff09SToby Isaac for (i = 0; i < m; i++) { 4142d3bc4906SToby Isaac PetscScalar val = 0.; 4143*9371c9d4SSatish Balay for (j = 0; j < n; j++) { val += childMat[n * i + j] * colValues[j]; } 4144e328ff09SToby Isaac parentValues[rowOffsets[f] + i] += val; 4145d3bc4906SToby Isaac } 4146d3bc4906SToby Isaac } 4147c921d74cSToby Isaac } 4148c921d74cSToby Isaac } 41499566063dSJacob Faibussowitsch if (contribute) PetscCall(VecSetValues(vecCoarse, dof, parentIndices, parentValues, INSERT_VALUES)); 4150c921d74cSToby Isaac } 41519566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&multiRootSec)); 41529566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootIndicesSec)); 41539566063dSJacob Faibussowitsch PetscCall(PetscFree2(parentIndices, parentValues)); 41549566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices_Injection(refTree, injRef, &childrenMats)); 41559566063dSJacob Faibussowitsch PetscCall(PetscFree(rootValues)); 41569566063dSJacob Faibussowitsch PetscCall(PetscFree3(offsets, offsetsCopy, rowOffsets)); 4157ebf164c7SToby Isaac PetscFunctionReturn(0); 4158ebf164c7SToby Isaac } 4159ebf164c7SToby Isaac 4160ff1f73f7SToby Isaac /*@ 4161ff1f73f7SToby Isaac DMPlexTransferVecTree - transfer a vector between two meshes that differ from each other by refinement/coarsening 4162ff1f73f7SToby Isaac that can be represented by a common reference tree used by both. This routine can be used for a combination of 4163ff1f73f7SToby Isaac coarsening and refinement at the same time. 4164ff1f73f7SToby Isaac 4165ff1f73f7SToby Isaac collective 4166ff1f73f7SToby Isaac 4167ff1f73f7SToby Isaac Input Parameters: 4168ff1f73f7SToby Isaac + dmIn - The DMPlex mesh for the input vector 4169ff1f73f7SToby Isaac . vecIn - The input vector 4170ff1f73f7SToby Isaac . sfRefine - A star forest indicating points in the mesh dmIn (roots in the star forest) that are parents to points in 4171ff1f73f7SToby Isaac the mesh dmOut (leaves in the star forest), i.e. where dmOut is more refined than dmIn 4172ff1f73f7SToby Isaac . sfCoarsen - A star forest indicating points in the mesh dmOut (roots in the star forest) that are parents to points in 4173ff1f73f7SToby Isaac the mesh dmIn (leaves in the star forest), i.e. where dmOut is more coarsened than dmIn 4174ff1f73f7SToby Isaac . cidsRefine - The childIds of the points in dmOut. These childIds relate back to the reference tree: childid[j] = k implies 4175ff1f73f7SToby Isaac that mesh point j of dmOut was refined from a point in dmIn just as the mesh point k in the reference 4176ff1f73f7SToby Isaac tree was refined from its parent. childid[j] = -1 indicates that the point j in dmOut is exactly 4177ff1f73f7SToby Isaac equivalent to its root in dmIn, so no interpolation is necessary. childid[j] = -2 indicates that this 4178ff1f73f7SToby Isaac point j in dmOut is not a leaf of sfRefine. 4179ff1f73f7SToby Isaac . cidsCoarsen - The childIds of the points in dmIn. These childIds relate back to the reference tree: childid[j] = k implies 4180ff1f73f7SToby Isaac that mesh point j of dmIn coarsens to a point in dmOut just as the mesh point k in the reference 4181ff1f73f7SToby Isaac tree coarsens to its parent. childid[j] = -2 indicates that point j in dmOut is not a leaf in sfCoarsen. 4182ff1f73f7SToby Isaac . useBCs - PETSC_TRUE indicates that boundary values should be inserted into vecIn before transfer. 4183ff1f73f7SToby Isaac - time - Used if boundary values are time dependent. 4184ff1f73f7SToby Isaac 4185ff1f73f7SToby Isaac Output Parameters: 41868966356dSPierre Jolivet . vecOut - Using interpolation and injection operators calculated on the reference tree, the transferred 4187ff1f73f7SToby Isaac projection of vecIn from dmIn to dmOut. Note that any field discretized with a PetscFV finite volume 4188ff1f73f7SToby Isaac method that uses gradient reconstruction will use reconstructed gradients when interpolating from 4189ff1f73f7SToby Isaac coarse points to fine points. 4190ff1f73f7SToby Isaac 4191ff1f73f7SToby Isaac Level: developer 4192ff1f73f7SToby Isaac 4193db781477SPatrick Sanan .seealso: `DMPlexSetReferenceTree()`, `DMPlexGetReferenceTree()`, `PetscFVGetComputeGradients()` 4194ff1f73f7SToby Isaac @*/ 4195*9371c9d4SSatish Balay PetscErrorCode DMPlexTransferVecTree(DM dmIn, Vec vecIn, DM dmOut, Vec vecOut, PetscSF sfRefine, PetscSF sfCoarsen, PetscInt *cidsRefine, PetscInt *cidsCoarsen, PetscBool useBCs, PetscReal time) { 419638fc2455SToby Isaac PetscFunctionBegin; 41979566063dSJacob Faibussowitsch PetscCall(VecSet(vecOut, 0.0)); 4198ff1f73f7SToby Isaac if (sfRefine) { 4199fbfa57b9SToby Isaac Vec vecInLocal; 42000eb7e1eaSToby Isaac DM dmGrad = NULL; 42010eb7e1eaSToby Isaac Vec faceGeom = NULL, cellGeom = NULL, grad = NULL; 4202fbfa57b9SToby Isaac 42039566063dSJacob Faibussowitsch PetscCall(DMGetLocalVector(dmIn, &vecInLocal)); 42049566063dSJacob Faibussowitsch PetscCall(VecSet(vecInLocal, 0.0)); 42050eb7e1eaSToby Isaac { 42060eb7e1eaSToby Isaac PetscInt numFields, i; 42070eb7e1eaSToby Isaac 42089566063dSJacob Faibussowitsch PetscCall(DMGetNumFields(dmIn, &numFields)); 42090eb7e1eaSToby Isaac for (i = 0; i < numFields; i++) { 42100eb7e1eaSToby Isaac PetscObject obj; 42110eb7e1eaSToby Isaac PetscClassId classid; 42120eb7e1eaSToby Isaac 42139566063dSJacob Faibussowitsch PetscCall(DMGetField(dmIn, i, NULL, &obj)); 42149566063dSJacob Faibussowitsch PetscCall(PetscObjectGetClassId(obj, &classid)); 42150eb7e1eaSToby Isaac if (classid == PETSCFV_CLASSID) { 42169566063dSJacob Faibussowitsch PetscCall(DMPlexGetDataFVM(dmIn, (PetscFV)obj, &cellGeom, &faceGeom, &dmGrad)); 42170eb7e1eaSToby Isaac break; 42180eb7e1eaSToby Isaac } 42190eb7e1eaSToby Isaac } 42200eb7e1eaSToby Isaac } 42211baa6e33SBarry Smith if (useBCs) PetscCall(DMPlexInsertBoundaryValues(dmIn, PETSC_TRUE, vecInLocal, time, faceGeom, cellGeom, NULL)); 42229566063dSJacob Faibussowitsch PetscCall(DMGlobalToLocalBegin(dmIn, vecIn, INSERT_VALUES, vecInLocal)); 42239566063dSJacob Faibussowitsch PetscCall(DMGlobalToLocalEnd(dmIn, vecIn, INSERT_VALUES, vecInLocal)); 42240eb7e1eaSToby Isaac if (dmGrad) { 42259566063dSJacob Faibussowitsch PetscCall(DMGetGlobalVector(dmGrad, &grad)); 42269566063dSJacob Faibussowitsch PetscCall(DMPlexReconstructGradientsFVM(dmIn, vecInLocal, grad)); 42270eb7e1eaSToby Isaac } 42289566063dSJacob Faibussowitsch PetscCall(DMPlexTransferVecTree_Interpolate(dmIn, vecInLocal, dmOut, vecOut, sfRefine, cidsRefine, grad, cellGeom)); 42299566063dSJacob Faibussowitsch PetscCall(DMRestoreLocalVector(dmIn, &vecInLocal)); 4230*9371c9d4SSatish Balay if (dmGrad) { PetscCall(DMRestoreGlobalVector(dmGrad, &grad)); } 4231ebf164c7SToby Isaac } 42321baa6e33SBarry Smith if (sfCoarsen) PetscCall(DMPlexTransferVecTree_Inject(dmIn, vecIn, dmOut, vecOut, sfCoarsen, cidsCoarsen)); 42339566063dSJacob Faibussowitsch PetscCall(VecAssemblyBegin(vecOut)); 42349566063dSJacob Faibussowitsch PetscCall(VecAssemblyEnd(vecOut)); 423538fc2455SToby Isaac PetscFunctionReturn(0); 423638fc2455SToby Isaac } 4237