1af0996ceSBarry Smith #include <petsc/private/dmpleximpl.h> /*I "petscdmplex.h" I*/ 2af0996ceSBarry Smith #include <petsc/private/isimpl.h> 3af0996ceSBarry Smith #include <petsc/private/petscfeimpl.h> 4d6a7ad0dSToby Isaac #include <petscsf.h> 50c37af3bSToby Isaac #include <petscds.h> 6d6a7ad0dSToby Isaac 70e3d61c9SBarry Smith /* hierarchy routines */ 8d6a7ad0dSToby Isaac 9d6a7ad0dSToby Isaac /*@ 10d6a7ad0dSToby Isaac DMPlexSetReferenceTree - set the reference tree for hierarchically non-conforming meshes. 11d6a7ad0dSToby Isaac 1220f4b53cSBarry Smith Not Collective 13d6a7ad0dSToby Isaac 14d6a7ad0dSToby Isaac Input Parameters: 15a1cb98faSBarry Smith + dm - The `DMPLEX` object 16a1cb98faSBarry Smith - ref - The reference tree `DMPLEX` object 17d6a7ad0dSToby Isaac 180b7167a0SToby Isaac Level: intermediate 19d6a7ad0dSToby Isaac 201cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`,`DMPlexGetReferenceTree()`, `DMPlexCreateDefaultReferenceTree()` 21d6a7ad0dSToby Isaac @*/ 22d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexSetReferenceTree(DM dm, DM ref) 23d71ae5a4SJacob Faibussowitsch { 24d6a7ad0dSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 25d6a7ad0dSToby Isaac 26d6a7ad0dSToby Isaac PetscFunctionBegin; 27d6a7ad0dSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 28ad540459SPierre Jolivet if (ref) PetscValidHeaderSpecific(ref, DM_CLASSID, 2); 299566063dSJacob Faibussowitsch PetscCall(PetscObjectReference((PetscObject)ref)); 309566063dSJacob Faibussowitsch PetscCall(DMDestroy(&mesh->referenceTree)); 31d6a7ad0dSToby Isaac mesh->referenceTree = ref; 323ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 33d6a7ad0dSToby Isaac } 34d6a7ad0dSToby Isaac 35d6a7ad0dSToby Isaac /*@ 36d6a7ad0dSToby Isaac DMPlexGetReferenceTree - get the reference tree for hierarchically non-conforming meshes. 37d6a7ad0dSToby Isaac 3820f4b53cSBarry Smith Not Collective 39d6a7ad0dSToby Isaac 402fe279fdSBarry Smith Input Parameter: 41a1cb98faSBarry Smith . dm - The `DMPLEX` object 42d6a7ad0dSToby Isaac 432fe279fdSBarry Smith Output Parameter: 44a1cb98faSBarry Smith . ref - The reference tree `DMPLEX` object 45d6a7ad0dSToby Isaac 460b7167a0SToby Isaac Level: intermediate 47d6a7ad0dSToby Isaac 48*d3a532e9SStefano Zampini Developer Notes: 49*d3a532e9SStefano Zampini The reference tree is shallow copied during `DMClone()`, thus it is may be shared by different `DM`s. 50*d3a532e9SStefano Zampini It is not a topological-only object, since some parts of the library use its local section to compute 51*d3a532e9SStefano Zampini interpolation and injection matrices. This may lead to unexpected failures during those calls. 52*d3a532e9SStefano Zampini 531cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetReferenceTree()`, `DMPlexCreateDefaultReferenceTree()` 54d6a7ad0dSToby Isaac @*/ 55d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetReferenceTree(DM dm, DM *ref) 56d71ae5a4SJacob Faibussowitsch { 57d6a7ad0dSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 58d6a7ad0dSToby Isaac 59d6a7ad0dSToby Isaac PetscFunctionBegin; 60d6a7ad0dSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 614f572ea9SToby Isaac PetscAssertPointer(ref, 2); 62d6a7ad0dSToby Isaac *ref = mesh->referenceTree; 633ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 64d6a7ad0dSToby Isaac } 65d6a7ad0dSToby Isaac 66d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexReferenceTreeGetChildSymmetry_Default(DM dm, PetscInt parent, PetscInt parentOrientA, PetscInt childOrientA, PetscInt childA, PetscInt parentOrientB, PetscInt *childOrientB, PetscInt *childB) 67d71ae5a4SJacob Faibussowitsch { 68dcbd3bf7SToby Isaac PetscInt coneSize, dStart, dEnd, dim, ABswap, oAvert, oBvert, ABswapVert; 69dcbd3bf7SToby Isaac 70dcbd3bf7SToby Isaac PetscFunctionBegin; 71dcbd3bf7SToby Isaac if (parentOrientA == parentOrientB) { 72dcbd3bf7SToby Isaac if (childOrientB) *childOrientB = childOrientA; 73dcbd3bf7SToby Isaac if (childB) *childB = childA; 743ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 75dcbd3bf7SToby Isaac } 76dcbd3bf7SToby Isaac for (dim = 0; dim < 3; dim++) { 779566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(dm, dim, &dStart, &dEnd)); 78ad540459SPierre Jolivet if (parent >= dStart && parent <= dEnd) break; 79dcbd3bf7SToby Isaac } 8063a3b9bcSJacob Faibussowitsch PetscCheck(dim <= 2, PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot perform child symmetry for %" PetscInt_FMT "-cells", dim); 8128b400f6SJacob Faibussowitsch PetscCheck(dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "A vertex has no children"); 82dcbd3bf7SToby Isaac if (childA < dStart || childA >= dEnd) { 83dcbd3bf7SToby Isaac /* this is a lower-dimensional child: bootstrap */ 84dcbd3bf7SToby Isaac PetscInt size, i, sA = -1, sB, sOrientB, sConeSize; 85dcbd3bf7SToby Isaac const PetscInt *supp, *coneA, *coneB, *oA, *oB; 86dcbd3bf7SToby Isaac 879566063dSJacob Faibussowitsch PetscCall(DMPlexGetSupportSize(dm, childA, &size)); 889566063dSJacob Faibussowitsch PetscCall(DMPlexGetSupport(dm, childA, &supp)); 89dcbd3bf7SToby Isaac 90dcbd3bf7SToby Isaac /* find a point sA in supp(childA) that has the same parent */ 91dcbd3bf7SToby Isaac for (i = 0; i < size; i++) { 92dcbd3bf7SToby Isaac PetscInt sParent; 93dcbd3bf7SToby Isaac 94dcbd3bf7SToby Isaac sA = supp[i]; 95dcbd3bf7SToby Isaac if (sA == parent) continue; 969566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, sA, &sParent, NULL)); 97ad540459SPierre Jolivet if (sParent == parent) break; 98dcbd3bf7SToby Isaac } 9908401ef6SPierre Jolivet PetscCheck(i != size, PETSC_COMM_SELF, PETSC_ERR_PLIB, "could not find support in children"); 100dcbd3bf7SToby Isaac /* find out which point sB is in an equivalent position to sA under 101dcbd3bf7SToby Isaac * parentOrientB */ 1029566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildSymmetry_Default(dm, parent, parentOrientA, 0, sA, parentOrientB, &sOrientB, &sB)); 1039566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, sA, &sConeSize)); 1049566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, sA, &coneA)); 1059566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, sB, &coneB)); 1069566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(dm, sA, &oA)); 1079566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(dm, sB, &oB)); 108dcbd3bf7SToby Isaac /* step through the cone of sA in natural order */ 109dcbd3bf7SToby Isaac for (i = 0; i < sConeSize; i++) { 110dcbd3bf7SToby Isaac if (coneA[i] == childA) { 111dcbd3bf7SToby Isaac /* if childA is at position i in coneA, 112dcbd3bf7SToby Isaac * then we want the point that is at sOrientB*i in coneB */ 113dcbd3bf7SToby Isaac PetscInt j = (sOrientB >= 0) ? ((sOrientB + i) % sConeSize) : ((sConeSize - (sOrientB + 1) - i) % sConeSize); 114dcbd3bf7SToby Isaac if (childB) *childB = coneB[j]; 115dcbd3bf7SToby Isaac if (childOrientB) { 116b5a892a1SMatthew G. Knepley DMPolytopeType ct; 117dcbd3bf7SToby Isaac PetscInt oBtrue; 118dcbd3bf7SToby Isaac 1199566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, childA, &coneSize)); 120dcbd3bf7SToby Isaac /* compose sOrientB and oB[j] */ 1211dca8a05SBarry Smith PetscCheck(coneSize == 0 || coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected a vertex or an edge"); 122b5a892a1SMatthew G. Knepley ct = coneSize ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT; 123dcbd3bf7SToby Isaac /* we may have to flip an edge */ 124b5a892a1SMatthew G. Knepley oBtrue = (sOrientB >= 0) ? oB[j] : DMPolytopeTypeComposeOrientation(ct, -1, oB[j]); 125b5a892a1SMatthew G. Knepley oBtrue = DMPolytopeConvertNewOrientation_Internal(ct, oBtrue); 126b5a892a1SMatthew G. Knepley ABswap = DihedralSwap(coneSize, DMPolytopeConvertNewOrientation_Internal(ct, oA[i]), oBtrue); 127dcbd3bf7SToby Isaac *childOrientB = DihedralCompose(coneSize, childOrientA, ABswap); 128dcbd3bf7SToby Isaac } 129dcbd3bf7SToby Isaac break; 130dcbd3bf7SToby Isaac } 131dcbd3bf7SToby Isaac } 13208401ef6SPierre Jolivet PetscCheck(i != sConeSize, PETSC_COMM_SELF, PETSC_ERR_PLIB, "support cone mismatch"); 1333ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 134dcbd3bf7SToby Isaac } 135dcbd3bf7SToby Isaac /* get the cone size and symmetry swap */ 1369566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, parent, &coneSize)); 137dcbd3bf7SToby Isaac ABswap = DihedralSwap(coneSize, parentOrientA, parentOrientB); 138dcbd3bf7SToby Isaac if (dim == 2) { 139dcbd3bf7SToby Isaac /* orientations refer to cones: we want them to refer to vertices: 140dcbd3bf7SToby Isaac * if it's a rotation, they are the same, but if the order is reversed, a 141dcbd3bf7SToby Isaac * permutation that puts side i first does *not* put vertex i first */ 142dcbd3bf7SToby Isaac oAvert = (parentOrientA >= 0) ? parentOrientA : -((-parentOrientA % coneSize) + 1); 143dcbd3bf7SToby Isaac oBvert = (parentOrientB >= 0) ? parentOrientB : -((-parentOrientB % coneSize) + 1); 144dcbd3bf7SToby Isaac ABswapVert = DihedralSwap(coneSize, oAvert, oBvert); 145947b95d8SBarry Smith } else { 146dcbd3bf7SToby Isaac ABswapVert = ABswap; 147dcbd3bf7SToby Isaac } 148dcbd3bf7SToby Isaac if (childB) { 149dcbd3bf7SToby Isaac /* assume that each child corresponds to a vertex, in the same order */ 150dcbd3bf7SToby Isaac PetscInt p, posA = -1, numChildren, i; 151dcbd3bf7SToby Isaac const PetscInt *children; 152dcbd3bf7SToby Isaac 153dcbd3bf7SToby Isaac /* count which position the child is in */ 1549566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(dm, parent, &numChildren, &children)); 155dcbd3bf7SToby Isaac for (i = 0; i < numChildren; i++) { 156dcbd3bf7SToby Isaac p = children[i]; 157dcbd3bf7SToby Isaac if (p == childA) { 158dcbd3bf7SToby Isaac posA = i; 159dcbd3bf7SToby Isaac break; 160dcbd3bf7SToby Isaac } 161dcbd3bf7SToby Isaac } 162dcbd3bf7SToby Isaac if (posA >= coneSize) { 163dcbd3bf7SToby Isaac /* this is the triangle in the middle of a uniformly refined triangle: it 164dcbd3bf7SToby Isaac * is invariant */ 1651dca8a05SBarry Smith PetscCheck(dim == 2 && posA == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Expected a middle triangle, got something else"); 166dcbd3bf7SToby Isaac *childB = childA; 1679371c9d4SSatish Balay } else { 168dcbd3bf7SToby Isaac /* figure out position B by applying ABswapVert */ 169dcbd3bf7SToby Isaac PetscInt posB; 170dcbd3bf7SToby Isaac 171dcbd3bf7SToby Isaac posB = (ABswapVert >= 0) ? ((ABswapVert + posA) % coneSize) : ((coneSize - (ABswapVert + 1) - posA) % coneSize); 172dcbd3bf7SToby Isaac if (childB) *childB = children[posB]; 173dcbd3bf7SToby Isaac } 174dcbd3bf7SToby Isaac } 175dcbd3bf7SToby Isaac if (childOrientB) *childOrientB = DihedralCompose(coneSize, childOrientA, ABswap); 1763ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 177dcbd3bf7SToby Isaac } 178dcbd3bf7SToby Isaac 179dcbd3bf7SToby Isaac /*@ 180dcbd3bf7SToby Isaac DMPlexReferenceTreeGetChildSymmetry - Given a reference tree, transform a childid and orientation from one parent frame to another 181dcbd3bf7SToby Isaac 182dcbd3bf7SToby Isaac Input Parameters: 183a1cb98faSBarry Smith + dm - the reference tree `DMPLEX` object 184dcbd3bf7SToby Isaac . parent - the parent point 185dcbd3bf7SToby Isaac . parentOrientA - the reference orientation for describing the parent 186dcbd3bf7SToby Isaac . childOrientA - the reference orientation for describing the child 187dcbd3bf7SToby Isaac . childA - the reference childID for describing the child 188dcbd3bf7SToby Isaac - parentOrientB - the new orientation for describing the parent 189dcbd3bf7SToby Isaac 190dcbd3bf7SToby Isaac Output Parameters: 19120f4b53cSBarry Smith + childOrientB - if not `NULL`, set to the new orientation for describing the child 19220f4b53cSBarry Smith - childB - if not `NULL`, the new childID for describing the child 193dcbd3bf7SToby Isaac 194dcbd3bf7SToby Isaac Level: developer 195dcbd3bf7SToby Isaac 1961cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetReferenceTree()`, `DMPlexSetReferenceTree()`, `DMPlexSetTree()` 197dcbd3bf7SToby Isaac @*/ 198d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexReferenceTreeGetChildSymmetry(DM dm, PetscInt parent, PetscInt parentOrientA, PetscInt childOrientA, PetscInt childA, PetscInt parentOrientB, PetscInt *childOrientB, PetscInt *childB) 199d71ae5a4SJacob Faibussowitsch { 200dcbd3bf7SToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 201dcbd3bf7SToby Isaac 202dcbd3bf7SToby Isaac PetscFunctionBegin; 203dcbd3bf7SToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 20428b400f6SJacob Faibussowitsch PetscCheck(mesh->getchildsymmetry, PETSC_COMM_SELF, PETSC_ERR_SUP, "DMPlexReferenceTreeGetChildSymmetry not implemented"); 2059566063dSJacob Faibussowitsch PetscCall(mesh->getchildsymmetry(dm, parent, parentOrientA, childOrientA, childA, parentOrientB, childOrientB, childB)); 2063ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 207dcbd3bf7SToby Isaac } 208dcbd3bf7SToby Isaac 209776742edSToby Isaac static PetscErrorCode DMPlexSetTree_Internal(DM, PetscSection, PetscInt *, PetscInt *, PetscBool, PetscBool); 210f9f063d4SToby Isaac 211d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexCreateReferenceTree_SetTree(DM dm, PetscSection parentSection, PetscInt parents[], PetscInt childIDs[]) 212d71ae5a4SJacob Faibussowitsch { 213f2c1aa1dSLisandro Dalcin PetscFunctionBegin; 2149566063dSJacob Faibussowitsch PetscCall(DMPlexSetTree_Internal(dm, parentSection, parents, childIDs, PETSC_TRUE, PETSC_FALSE)); 2153ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 216f2c1aa1dSLisandro Dalcin } 217f2c1aa1dSLisandro Dalcin 218d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexCreateReferenceTree_Union(DM K, DM Kref, const char *labelName, DM *ref) 219d71ae5a4SJacob Faibussowitsch { 2200e2cc29aSToby Isaac MPI_Comm comm; 2210e2cc29aSToby Isaac PetscInt dim, p, pStart, pEnd, pRefStart, pRefEnd, d, offset, parentSize, *parents, *childIDs; 222da43764aSToby Isaac PetscInt *permvals, *unionCones, *coneSizes, *unionOrientations, numUnionPoints, *numDimPoints, numCones, numVerts; 223da43764aSToby Isaac DMLabel identity, identityRef; 22410f7e118SToby Isaac PetscSection unionSection, unionConeSection, parentSection; 225da43764aSToby Isaac PetscScalar *unionCoords; 226da43764aSToby Isaac IS perm; 227da43764aSToby Isaac 228da43764aSToby Isaac PetscFunctionBegin; 2290e2cc29aSToby Isaac comm = PetscObjectComm((PetscObject)K); 2309566063dSJacob Faibussowitsch PetscCall(DMGetDimension(K, &dim)); 2319566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(K, &pStart, &pEnd)); 2329566063dSJacob Faibussowitsch PetscCall(DMGetLabel(K, labelName, &identity)); 2339566063dSJacob Faibussowitsch PetscCall(DMGetLabel(Kref, labelName, &identityRef)); 2349566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(Kref, &pRefStart, &pRefEnd)); 2359566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(comm, &unionSection)); 2369566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(unionSection, 0, (pEnd - pStart) + (pRefEnd - pRefStart))); 237da43764aSToby Isaac /* count points that will go in the union */ 23848a46eb9SPierre Jolivet for (p = pStart; p < pEnd; p++) PetscCall(PetscSectionSetDof(unionSection, p - pStart, 1)); 239da43764aSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 240da43764aSToby Isaac PetscInt q, qSize; 2419566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(identityRef, p, &q)); 2429566063dSJacob Faibussowitsch PetscCall(DMLabelGetStratumSize(identityRef, q, &qSize)); 24348a46eb9SPierre Jolivet if (qSize > 1) PetscCall(PetscSectionSetDof(unionSection, p - pRefStart + (pEnd - pStart), 1)); 244da43764aSToby Isaac } 2459566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEnd - pStart + pRefEnd - pRefStart, &permvals)); 246da43764aSToby Isaac offset = 0; 247da43764aSToby Isaac /* stratify points in the union by topological dimension */ 248da43764aSToby Isaac for (d = 0; d <= dim; d++) { 249da43764aSToby Isaac PetscInt cStart, cEnd, c; 250da43764aSToby Isaac 2519566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &cStart, &cEnd)); 252ad540459SPierre Jolivet for (c = cStart; c < cEnd; c++) permvals[offset++] = c; 253da43764aSToby Isaac 2549566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(Kref, d, &cStart, &cEnd)); 255ad540459SPierre Jolivet for (c = cStart; c < cEnd; c++) permvals[offset++] = c + (pEnd - pStart); 256da43764aSToby Isaac } 2579566063dSJacob Faibussowitsch PetscCall(ISCreateGeneral(comm, (pEnd - pStart) + (pRefEnd - pRefStart), permvals, PETSC_OWN_POINTER, &perm)); 2589566063dSJacob Faibussowitsch PetscCall(PetscSectionSetPermutation(unionSection, perm)); 2599566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(unionSection)); 2609566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(unionSection, &numUnionPoints)); 2619566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numUnionPoints, &coneSizes, dim + 1, &numDimPoints)); 262da43764aSToby Isaac /* count dimension points */ 263da43764aSToby Isaac for (d = 0; d <= dim; d++) { 264da43764aSToby Isaac PetscInt cStart, cOff, cOff2; 2659566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &cStart, NULL)); 2669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, cStart - pStart, &cOff)); 267da43764aSToby Isaac if (d < dim) { 2689566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d + 1, &cStart, NULL)); 2699566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, cStart - pStart, &cOff2)); 2709371c9d4SSatish Balay } else { 271da43764aSToby Isaac cOff2 = numUnionPoints; 272da43764aSToby Isaac } 273da43764aSToby Isaac numDimPoints[dim - d] = cOff2 - cOff; 274da43764aSToby Isaac } 2759566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(comm, &unionConeSection)); 2769566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(unionConeSection, 0, numUnionPoints)); 277da43764aSToby Isaac /* count the cones in the union */ 278da43764aSToby Isaac for (p = pStart; p < pEnd; p++) { 279da43764aSToby Isaac PetscInt dof, uOff; 280da43764aSToby Isaac 2819566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, p, &dof)); 2829566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pStart, &uOff)); 2839566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(unionConeSection, uOff, dof)); 284da43764aSToby Isaac coneSizes[uOff] = dof; 285da43764aSToby Isaac } 286da43764aSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 287da43764aSToby Isaac PetscInt dof, uDof, uOff; 288da43764aSToby Isaac 2899566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(Kref, p, &dof)); 2909566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart), &uDof)); 2919566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart), &uOff)); 292da43764aSToby Isaac if (uDof) { 2939566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(unionConeSection, uOff, dof)); 294da43764aSToby Isaac coneSizes[uOff] = dof; 295da43764aSToby Isaac } 296da43764aSToby Isaac } 2979566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(unionConeSection)); 2989566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(unionConeSection, &numCones)); 2999566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numCones, &unionCones, numCones, &unionOrientations)); 300da43764aSToby Isaac /* write the cones in the union */ 301da43764aSToby Isaac for (p = pStart; p < pEnd; p++) { 302da43764aSToby Isaac PetscInt dof, uOff, c, cOff; 303da43764aSToby Isaac const PetscInt *cone, *orientation; 304da43764aSToby Isaac 3059566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, p, &dof)); 3069566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(K, p, &cone)); 3079566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(K, p, &orientation)); 3089566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pStart, &uOff)); 3099566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionConeSection, uOff, &cOff)); 310da43764aSToby Isaac for (c = 0; c < dof; c++) { 311da43764aSToby Isaac PetscInt e, eOff; 312da43764aSToby Isaac e = cone[c]; 3139566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, e - pStart, &eOff)); 314da43764aSToby Isaac unionCones[cOff + c] = eOff; 315da43764aSToby Isaac unionOrientations[cOff + c] = orientation[c]; 316da43764aSToby Isaac } 317da43764aSToby Isaac } 318da43764aSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 319da43764aSToby Isaac PetscInt dof, uDof, uOff, c, cOff; 320da43764aSToby Isaac const PetscInt *cone, *orientation; 321da43764aSToby Isaac 3229566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(Kref, p, &dof)); 3239566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(Kref, p, &cone)); 3249566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(Kref, p, &orientation)); 3259566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart), &uDof)); 3269566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart), &uOff)); 327da43764aSToby Isaac if (uDof) { 3289566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionConeSection, uOff, &cOff)); 329da43764aSToby Isaac for (c = 0; c < dof; c++) { 330da43764aSToby Isaac PetscInt e, eOff, eDof; 331da43764aSToby Isaac 332da43764aSToby Isaac e = cone[c]; 3339566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, e - pRefStart + (pEnd - pStart), &eDof)); 334da43764aSToby Isaac if (eDof) { 3359566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, e - pRefStart + (pEnd - pStart), &eOff)); 3369371c9d4SSatish Balay } else { 3379566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(identityRef, e, &e)); 3389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, e - pStart, &eOff)); 339da43764aSToby Isaac } 340da43764aSToby Isaac unionCones[cOff + c] = eOff; 341da43764aSToby Isaac unionOrientations[cOff + c] = orientation[c]; 342da43764aSToby Isaac } 343da43764aSToby Isaac } 344da43764aSToby Isaac } 345da43764aSToby Isaac /* get the coordinates */ 346da43764aSToby Isaac { 347da43764aSToby Isaac PetscInt vStart, vEnd, vRefStart, vRefEnd, v, vDof, vOff; 348da43764aSToby Isaac PetscSection KcoordsSec, KrefCoordsSec; 349da43764aSToby Isaac Vec KcoordsVec, KrefCoordsVec; 350da43764aSToby Isaac PetscScalar *Kcoords; 351da43764aSToby Isaac 3529566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateSection(K, &KcoordsSec)); 3539566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(K, &KcoordsVec)); 3549566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateSection(Kref, &KrefCoordsSec)); 3559566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(Kref, &KrefCoordsVec)); 356da43764aSToby Isaac 357da43764aSToby Isaac numVerts = numDimPoints[0]; 3589566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numVerts * dim, &unionCoords)); 3599566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(K, 0, &vStart, &vEnd)); 360da43764aSToby Isaac 361da43764aSToby Isaac offset = 0; 362da43764aSToby Isaac for (v = vStart; v < vEnd; v++) { 3639566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, v - pStart, &vOff)); 3649566063dSJacob Faibussowitsch PetscCall(VecGetValuesSection(KcoordsVec, KcoordsSec, v, &Kcoords)); 365ad540459SPierre Jolivet for (d = 0; d < dim; d++) unionCoords[offset * dim + d] = Kcoords[d]; 366da43764aSToby Isaac offset++; 367da43764aSToby Isaac } 3689566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(Kref, 0, &vRefStart, &vRefEnd)); 369da43764aSToby Isaac for (v = vRefStart; v < vRefEnd; v++) { 3709566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, v - pRefStart + (pEnd - pStart), &vDof)); 3719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, v - pRefStart + (pEnd - pStart), &vOff)); 3729566063dSJacob Faibussowitsch PetscCall(VecGetValuesSection(KrefCoordsVec, KrefCoordsSec, v, &Kcoords)); 373da43764aSToby Isaac if (vDof) { 374ad540459SPierre Jolivet for (d = 0; d < dim; d++) unionCoords[offset * dim + d] = Kcoords[d]; 375da43764aSToby Isaac offset++; 376da43764aSToby Isaac } 377da43764aSToby Isaac } 378da43764aSToby Isaac } 3799566063dSJacob Faibussowitsch PetscCall(DMCreate(comm, ref)); 3809566063dSJacob Faibussowitsch PetscCall(DMSetType(*ref, DMPLEX)); 3819566063dSJacob Faibussowitsch PetscCall(DMSetDimension(*ref, dim)); 3829566063dSJacob Faibussowitsch PetscCall(DMPlexCreateFromDAG(*ref, dim, numDimPoints, coneSizes, unionCones, unionOrientations, unionCoords)); 38310f7e118SToby Isaac /* set the tree */ 3849566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(comm, &parentSection)); 3859566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(parentSection, 0, numUnionPoints)); 38610f7e118SToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 38710f7e118SToby Isaac PetscInt uDof, uOff; 38810f7e118SToby Isaac 3899566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart), &uDof)); 3909566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart), &uOff)); 3911baa6e33SBarry Smith if (uDof) PetscCall(PetscSectionSetDof(parentSection, uOff, 1)); 39210f7e118SToby Isaac } 3939566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(parentSection)); 3949566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(parentSection, &parentSize)); 3959566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(parentSize, &parents, parentSize, &childIDs)); 39610f7e118SToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 39710f7e118SToby Isaac PetscInt uDof, uOff; 39810f7e118SToby Isaac 3999566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart), &uDof)); 4009566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart), &uOff)); 40110f7e118SToby Isaac if (uDof) { 40210f7e118SToby Isaac PetscInt pOff, parent, parentU; 4039566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(parentSection, uOff, &pOff)); 4049566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(identityRef, p, &parent)); 4059566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(unionSection, parent - pStart, &parentU)); 40610f7e118SToby Isaac parents[pOff] = parentU; 40710f7e118SToby Isaac childIDs[pOff] = uOff; 40810f7e118SToby Isaac } 40910f7e118SToby Isaac } 4109566063dSJacob Faibussowitsch PetscCall(DMPlexCreateReferenceTree_SetTree(*ref, parentSection, parents, childIDs)); 4119566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&parentSection)); 4129566063dSJacob Faibussowitsch PetscCall(PetscFree2(parents, childIDs)); 41310f7e118SToby Isaac 414da43764aSToby Isaac /* clean up */ 4159566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&unionSection)); 4169566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&unionConeSection)); 4179566063dSJacob Faibussowitsch PetscCall(ISDestroy(&perm)); 4189566063dSJacob Faibussowitsch PetscCall(PetscFree(unionCoords)); 4199566063dSJacob Faibussowitsch PetscCall(PetscFree2(unionCones, unionOrientations)); 4209566063dSJacob Faibussowitsch PetscCall(PetscFree2(coneSizes, numDimPoints)); 4213ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 4220e2cc29aSToby Isaac } 4230e2cc29aSToby Isaac 4240e2cc29aSToby Isaac /*@ 4250e2cc29aSToby Isaac DMPlexCreateDefaultReferenceTree - create a reference tree for isotropic hierarchical mesh refinement. 4260e2cc29aSToby Isaac 427d083f849SBarry Smith Collective 4280e2cc29aSToby Isaac 4290e2cc29aSToby Isaac Input Parameters: 4300e2cc29aSToby Isaac + comm - the MPI communicator 4310e2cc29aSToby Isaac . dim - the spatial dimension 4320e2cc29aSToby Isaac - simplex - Flag for simplex, otherwise use a tensor-product cell 4330e2cc29aSToby Isaac 4342fe279fdSBarry Smith Output Parameter: 435a1cb98faSBarry Smith . ref - the reference tree `DMPLEX` object 4360e2cc29aSToby Isaac 4370e2cc29aSToby Isaac Level: intermediate 4380e2cc29aSToby Isaac 439db781477SPatrick Sanan .seealso: `DMPlexSetReferenceTree()`, `DMPlexGetReferenceTree()` 4400e2cc29aSToby Isaac @*/ 441d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexCreateDefaultReferenceTree(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *ref) 442d71ae5a4SJacob Faibussowitsch { 4430e2cc29aSToby Isaac DM_Plex *mesh; 4440e2cc29aSToby Isaac DM K, Kref; 4450e2cc29aSToby Isaac PetscInt p, pStart, pEnd; 4460e2cc29aSToby Isaac DMLabel identity; 4470e2cc29aSToby Isaac 4480e2cc29aSToby Isaac PetscFunctionBegin; 4490e2cc29aSToby Isaac #if 1 4500e2cc29aSToby Isaac comm = PETSC_COMM_SELF; 4510e2cc29aSToby Isaac #endif 4520e2cc29aSToby Isaac /* create a reference element */ 4539566063dSJacob Faibussowitsch PetscCall(DMPlexCreateReferenceCell(comm, DMPolytopeTypeSimpleShape(dim, simplex), &K)); 4549566063dSJacob Faibussowitsch PetscCall(DMCreateLabel(K, "identity")); 4559566063dSJacob Faibussowitsch PetscCall(DMGetLabel(K, "identity", &identity)); 4569566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(K, &pStart, &pEnd)); 45748a46eb9SPierre Jolivet for (p = pStart; p < pEnd; p++) PetscCall(DMLabelSetValue(identity, p, p)); 4580e2cc29aSToby Isaac /* refine it */ 4599566063dSJacob Faibussowitsch PetscCall(DMRefine(K, comm, &Kref)); 4600e2cc29aSToby Isaac 4610e2cc29aSToby Isaac /* the reference tree is the union of these two, without duplicating 4620e2cc29aSToby Isaac * points that appear in both */ 4639566063dSJacob Faibussowitsch PetscCall(DMPlexCreateReferenceTree_Union(K, Kref, "identity", ref)); 4640e2cc29aSToby Isaac mesh = (DM_Plex *)(*ref)->data; 4650e2cc29aSToby Isaac mesh->getchildsymmetry = DMPlexReferenceTreeGetChildSymmetry_Default; 4669566063dSJacob Faibussowitsch PetscCall(DMDestroy(&K)); 4679566063dSJacob Faibussowitsch PetscCall(DMDestroy(&Kref)); 4683ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 469da43764aSToby Isaac } 470da43764aSToby Isaac 471d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexTreeSymmetrize(DM dm) 472d71ae5a4SJacob Faibussowitsch { 473878b19aaSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 474878b19aaSToby Isaac PetscSection childSec, pSec; 475878b19aaSToby Isaac PetscInt p, pSize, cSize, parMax = PETSC_MIN_INT, parMin = PETSC_MAX_INT; 476878b19aaSToby Isaac PetscInt *offsets, *children, pStart, pEnd; 477878b19aaSToby Isaac 478878b19aaSToby Isaac PetscFunctionBegin; 479878b19aaSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 4809566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&mesh->childSection)); 4819566063dSJacob Faibussowitsch PetscCall(PetscFree(mesh->children)); 482878b19aaSToby Isaac pSec = mesh->parentSection; 4833ba16761SJacob Faibussowitsch if (!pSec) PetscFunctionReturn(PETSC_SUCCESS); 4849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(pSec, &pSize)); 485878b19aaSToby Isaac for (p = 0; p < pSize; p++) { 486878b19aaSToby Isaac PetscInt par = mesh->parents[p]; 487878b19aaSToby Isaac 488878b19aaSToby Isaac parMax = PetscMax(parMax, par + 1); 489878b19aaSToby Isaac parMin = PetscMin(parMin, par); 490878b19aaSToby Isaac } 491878b19aaSToby Isaac if (parMin > parMax) { 492878b19aaSToby Isaac parMin = -1; 493878b19aaSToby Isaac parMax = -1; 494878b19aaSToby Isaac } 4959566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)pSec), &childSec)); 4969566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(childSec, parMin, parMax)); 497878b19aaSToby Isaac for (p = 0; p < pSize; p++) { 498878b19aaSToby Isaac PetscInt par = mesh->parents[p]; 499878b19aaSToby Isaac 5009566063dSJacob Faibussowitsch PetscCall(PetscSectionAddDof(childSec, par, 1)); 501878b19aaSToby Isaac } 5029566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(childSec)); 5039566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(childSec, &cSize)); 5049566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(cSize, &children)); 5059566063dSJacob Faibussowitsch PetscCall(PetscCalloc1(parMax - parMin, &offsets)); 5069566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd)); 507878b19aaSToby Isaac for (p = pStart; p < pEnd; p++) { 508878b19aaSToby Isaac PetscInt dof, off, i; 509878b19aaSToby Isaac 5109566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(pSec, p, &dof)); 5119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(pSec, p, &off)); 512878b19aaSToby Isaac for (i = 0; i < dof; i++) { 513878b19aaSToby Isaac PetscInt par = mesh->parents[off + i], cOff; 514878b19aaSToby Isaac 5159566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(childSec, par, &cOff)); 516878b19aaSToby Isaac children[cOff + offsets[par - parMin]++] = p; 517878b19aaSToby Isaac } 518878b19aaSToby Isaac } 519878b19aaSToby Isaac mesh->childSection = childSec; 520878b19aaSToby Isaac mesh->children = children; 5219566063dSJacob Faibussowitsch PetscCall(PetscFree(offsets)); 5223ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 523878b19aaSToby Isaac } 524878b19aaSToby Isaac 525d71ae5a4SJacob Faibussowitsch static PetscErrorCode AnchorsFlatten(PetscSection section, IS is, PetscSection *sectionNew, IS *isNew) 526d71ae5a4SJacob Faibussowitsch { 5276dd5a8c8SToby Isaac PetscInt pStart, pEnd, size, sizeNew, i, p, *valsNew = NULL; 5286dd5a8c8SToby Isaac const PetscInt *vals; 5296dd5a8c8SToby Isaac PetscSection secNew; 5306dd5a8c8SToby Isaac PetscBool anyNew, globalAnyNew; 5316dd5a8c8SToby Isaac PetscBool compress; 5326dd5a8c8SToby Isaac 5336dd5a8c8SToby Isaac PetscFunctionBegin; 5349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(section, &pStart, &pEnd)); 5359566063dSJacob Faibussowitsch PetscCall(ISGetLocalSize(is, &size)); 5369566063dSJacob Faibussowitsch PetscCall(ISGetIndices(is, &vals)); 5379566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)section), &secNew)); 5389566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(secNew, pStart, pEnd)); 5396dd5a8c8SToby Isaac for (i = 0; i < size; i++) { 5406dd5a8c8SToby Isaac PetscInt dof; 5416dd5a8c8SToby Isaac 5426dd5a8c8SToby Isaac p = vals[i]; 5436dd5a8c8SToby Isaac if (p < pStart || p >= pEnd) continue; 5449566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 5456dd5a8c8SToby Isaac if (dof) break; 5466dd5a8c8SToby Isaac } 5476dd5a8c8SToby Isaac if (i == size) { 5489566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(secNew)); 5496dd5a8c8SToby Isaac anyNew = PETSC_FALSE; 5506dd5a8c8SToby Isaac compress = PETSC_FALSE; 5516dd5a8c8SToby Isaac sizeNew = 0; 5529371c9d4SSatish Balay } else { 5536dd5a8c8SToby Isaac anyNew = PETSC_TRUE; 5546dd5a8c8SToby Isaac for (p = pStart; p < pEnd; p++) { 5556dd5a8c8SToby Isaac PetscInt dof, off; 5566dd5a8c8SToby Isaac 5579566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 5589566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, p, &off)); 5596dd5a8c8SToby Isaac for (i = 0; i < dof; i++) { 5606dd5a8c8SToby Isaac PetscInt q = vals[off + i], qDof = 0; 5616dd5a8c8SToby Isaac 56248a46eb9SPierre Jolivet if (q >= pStart && q < pEnd) PetscCall(PetscSectionGetDof(section, q, &qDof)); 5631baa6e33SBarry Smith if (qDof) PetscCall(PetscSectionAddDof(secNew, p, qDof)); 56448a46eb9SPierre Jolivet else PetscCall(PetscSectionAddDof(secNew, p, 1)); 5656dd5a8c8SToby Isaac } 5666dd5a8c8SToby Isaac } 5679566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(secNew)); 5689566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(secNew, &sizeNew)); 5699566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(sizeNew, &valsNew)); 5706dd5a8c8SToby Isaac compress = PETSC_FALSE; 5716dd5a8c8SToby Isaac for (p = pStart; p < pEnd; p++) { 5726dd5a8c8SToby Isaac PetscInt dof, off, count, offNew, dofNew; 5736dd5a8c8SToby Isaac 5749566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 5759566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, p, &off)); 5769566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(secNew, p, &dofNew)); 5779566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(secNew, p, &offNew)); 5786dd5a8c8SToby Isaac count = 0; 5796dd5a8c8SToby Isaac for (i = 0; i < dof; i++) { 5806dd5a8c8SToby Isaac PetscInt q = vals[off + i], qDof = 0, qOff = 0, j; 5816dd5a8c8SToby Isaac 5826dd5a8c8SToby Isaac if (q >= pStart && q < pEnd) { 5839566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, q, &qDof)); 5849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, q, &qOff)); 5856dd5a8c8SToby Isaac } 5866dd5a8c8SToby Isaac if (qDof) { 5876dd5a8c8SToby Isaac PetscInt oldCount = count; 5886dd5a8c8SToby Isaac 5896dd5a8c8SToby Isaac for (j = 0; j < qDof; j++) { 5906dd5a8c8SToby Isaac PetscInt k, r = vals[qOff + j]; 5916dd5a8c8SToby Isaac 5926dd5a8c8SToby Isaac for (k = 0; k < oldCount; k++) { 593ad540459SPierre Jolivet if (valsNew[offNew + k] == r) break; 5946dd5a8c8SToby Isaac } 595ad540459SPierre Jolivet if (k == oldCount) valsNew[offNew + count++] = r; 5966dd5a8c8SToby Isaac } 5979371c9d4SSatish Balay } else { 5986dd5a8c8SToby Isaac PetscInt k, oldCount = count; 5996dd5a8c8SToby Isaac 6006dd5a8c8SToby Isaac for (k = 0; k < oldCount; k++) { 601ad540459SPierre Jolivet if (valsNew[offNew + k] == q) break; 6026dd5a8c8SToby Isaac } 603ad540459SPierre Jolivet if (k == oldCount) valsNew[offNew + count++] = q; 6046dd5a8c8SToby Isaac } 6056dd5a8c8SToby Isaac } 6066dd5a8c8SToby Isaac if (count < dofNew) { 6079566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(secNew, p, count)); 6086dd5a8c8SToby Isaac compress = PETSC_TRUE; 6096dd5a8c8SToby Isaac } 6106dd5a8c8SToby Isaac } 6116dd5a8c8SToby Isaac } 6129566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(is, &vals)); 6131c2dc1cbSBarry Smith PetscCall(MPIU_Allreduce(&anyNew, &globalAnyNew, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)secNew))); 6146dd5a8c8SToby Isaac if (!globalAnyNew) { 6159566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&secNew)); 6166dd5a8c8SToby Isaac *sectionNew = NULL; 6176dd5a8c8SToby Isaac *isNew = NULL; 6189371c9d4SSatish Balay } else { 6196dd5a8c8SToby Isaac PetscBool globalCompress; 6206dd5a8c8SToby Isaac 6211c2dc1cbSBarry Smith PetscCall(MPIU_Allreduce(&compress, &globalCompress, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)secNew))); 6226dd5a8c8SToby Isaac if (compress) { 6236dd5a8c8SToby Isaac PetscSection secComp; 6246dd5a8c8SToby Isaac PetscInt *valsComp = NULL; 6256dd5a8c8SToby Isaac 6269566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)section), &secComp)); 6279566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(secComp, pStart, pEnd)); 6286dd5a8c8SToby Isaac for (p = pStart; p < pEnd; p++) { 6296dd5a8c8SToby Isaac PetscInt dof; 6306dd5a8c8SToby Isaac 6319566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(secNew, p, &dof)); 6329566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(secComp, p, dof)); 6336dd5a8c8SToby Isaac } 6349566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(secComp)); 6359566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(secComp, &sizeNew)); 6369566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(sizeNew, &valsComp)); 6376dd5a8c8SToby Isaac for (p = pStart; p < pEnd; p++) { 6386dd5a8c8SToby Isaac PetscInt dof, off, offNew, j; 6396dd5a8c8SToby Isaac 6409566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(secNew, p, &dof)); 6419566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(secNew, p, &off)); 6429566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(secComp, p, &offNew)); 643ad540459SPierre Jolivet for (j = 0; j < dof; j++) valsComp[offNew + j] = valsNew[off + j]; 6446dd5a8c8SToby Isaac } 6459566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&secNew)); 6466dd5a8c8SToby Isaac secNew = secComp; 6479566063dSJacob Faibussowitsch PetscCall(PetscFree(valsNew)); 6486dd5a8c8SToby Isaac valsNew = valsComp; 6496dd5a8c8SToby Isaac } 6509566063dSJacob Faibussowitsch PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)is), sizeNew, valsNew, PETSC_OWN_POINTER, isNew)); 6516dd5a8c8SToby Isaac } 6523ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 6536dd5a8c8SToby Isaac } 6546dd5a8c8SToby Isaac 655d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexCreateAnchors_Tree(DM dm) 656d71ae5a4SJacob Faibussowitsch { 65766af876cSToby Isaac PetscInt p, pStart, pEnd, *anchors, size; 65866af876cSToby Isaac PetscInt aMin = PETSC_MAX_INT, aMax = PETSC_MIN_INT; 65966af876cSToby Isaac PetscSection aSec; 660f9f063d4SToby Isaac DMLabel canonLabel; 66166af876cSToby Isaac IS aIS; 66266af876cSToby Isaac 66366af876cSToby Isaac PetscFunctionBegin; 66466af876cSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 6659566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 6669566063dSJacob Faibussowitsch PetscCall(DMGetLabel(dm, "canonical", &canonLabel)); 66766af876cSToby Isaac for (p = pStart; p < pEnd; p++) { 66866af876cSToby Isaac PetscInt parent; 66966af876cSToby Isaac 670f9f063d4SToby Isaac if (canonLabel) { 671f9f063d4SToby Isaac PetscInt canon; 672f9f063d4SToby Isaac 6739566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonLabel, p, &canon)); 674f9f063d4SToby Isaac if (p != canon) continue; 675f9f063d4SToby Isaac } 6769566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, p, &parent, NULL)); 67766af876cSToby Isaac if (parent != p) { 67866af876cSToby Isaac aMin = PetscMin(aMin, p); 67966af876cSToby Isaac aMax = PetscMax(aMax, p + 1); 68066af876cSToby Isaac } 68166af876cSToby Isaac } 68266af876cSToby Isaac if (aMin > aMax) { 68366af876cSToby Isaac aMin = -1; 68466af876cSToby Isaac aMax = -1; 68566af876cSToby Isaac } 6869566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PETSC_COMM_SELF, &aSec)); 6879566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(aSec, aMin, aMax)); 68866af876cSToby Isaac for (p = aMin; p < aMax; p++) { 68966af876cSToby Isaac PetscInt parent, ancestor = p; 69066af876cSToby Isaac 691f9f063d4SToby Isaac if (canonLabel) { 692f9f063d4SToby Isaac PetscInt canon; 693f9f063d4SToby Isaac 6949566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonLabel, p, &canon)); 695f9f063d4SToby Isaac if (p != canon) continue; 696f9f063d4SToby Isaac } 6979566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, p, &parent, NULL)); 69866af876cSToby Isaac while (parent != ancestor) { 69966af876cSToby Isaac ancestor = parent; 7009566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, ancestor, &parent, NULL)); 70166af876cSToby Isaac } 70266af876cSToby Isaac if (ancestor != p) { 70366af876cSToby Isaac PetscInt closureSize, *closure = NULL; 70466af876cSToby Isaac 7059566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, ancestor, PETSC_TRUE, &closureSize, &closure)); 7069566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(aSec, p, closureSize)); 7079566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, ancestor, PETSC_TRUE, &closureSize, &closure)); 70866af876cSToby Isaac } 70966af876cSToby Isaac } 7109566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(aSec)); 7119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(aSec, &size)); 7129566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(size, &anchors)); 71366af876cSToby Isaac for (p = aMin; p < aMax; p++) { 71466af876cSToby Isaac PetscInt parent, ancestor = p; 71566af876cSToby Isaac 716f9f063d4SToby Isaac if (canonLabel) { 717f9f063d4SToby Isaac PetscInt canon; 718f9f063d4SToby Isaac 7199566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonLabel, p, &canon)); 720f9f063d4SToby Isaac if (p != canon) continue; 721f9f063d4SToby Isaac } 7229566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, p, &parent, NULL)); 72366af876cSToby Isaac while (parent != ancestor) { 72466af876cSToby Isaac ancestor = parent; 7259566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, ancestor, &parent, NULL)); 72666af876cSToby Isaac } 72766af876cSToby Isaac if (ancestor != p) { 72866af876cSToby Isaac PetscInt j, closureSize, *closure = NULL, aOff; 72966af876cSToby Isaac 7309566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(aSec, p, &aOff)); 73166af876cSToby Isaac 7329566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, ancestor, PETSC_TRUE, &closureSize, &closure)); 733ad540459SPierre Jolivet for (j = 0; j < closureSize; j++) anchors[aOff + j] = closure[2 * j]; 7349566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, ancestor, PETSC_TRUE, &closureSize, &closure)); 73566af876cSToby Isaac } 73666af876cSToby Isaac } 7379566063dSJacob Faibussowitsch PetscCall(ISCreateGeneral(PETSC_COMM_SELF, size, anchors, PETSC_OWN_POINTER, &aIS)); 7386dd5a8c8SToby Isaac { 7396dd5a8c8SToby Isaac PetscSection aSecNew = aSec; 7406dd5a8c8SToby Isaac IS aISNew = aIS; 7416dd5a8c8SToby Isaac 7429566063dSJacob Faibussowitsch PetscCall(PetscObjectReference((PetscObject)aSec)); 7439566063dSJacob Faibussowitsch PetscCall(PetscObjectReference((PetscObject)aIS)); 7446dd5a8c8SToby Isaac while (aSecNew) { 7459566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&aSec)); 7469566063dSJacob Faibussowitsch PetscCall(ISDestroy(&aIS)); 7476dd5a8c8SToby Isaac aSec = aSecNew; 7486dd5a8c8SToby Isaac aIS = aISNew; 7496dd5a8c8SToby Isaac aSecNew = NULL; 7506dd5a8c8SToby Isaac aISNew = NULL; 7519566063dSJacob Faibussowitsch PetscCall(AnchorsFlatten(aSec, aIS, &aSecNew, &aISNew)); 7526dd5a8c8SToby Isaac } 7536dd5a8c8SToby Isaac } 7549566063dSJacob Faibussowitsch PetscCall(DMPlexSetAnchors(dm, aSec, aIS)); 7559566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&aSec)); 7569566063dSJacob Faibussowitsch PetscCall(ISDestroy(&aIS)); 7573ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 75866af876cSToby Isaac } 75966af876cSToby Isaac 760d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexGetTrueSupportSize(DM dm, PetscInt p, PetscInt *dof, PetscInt *numTrueSupp) 761d71ae5a4SJacob Faibussowitsch { 7626461c1adSToby Isaac PetscFunctionBegin; 7636461c1adSToby Isaac if (numTrueSupp[p] == -1) { 7646461c1adSToby Isaac PetscInt i, alldof; 7656461c1adSToby Isaac const PetscInt *supp; 7666461c1adSToby Isaac PetscInt count = 0; 7676461c1adSToby Isaac 7689566063dSJacob Faibussowitsch PetscCall(DMPlexGetSupportSize(dm, p, &alldof)); 7699566063dSJacob Faibussowitsch PetscCall(DMPlexGetSupport(dm, p, &supp)); 7706461c1adSToby Isaac for (i = 0; i < alldof; i++) { 7716461c1adSToby Isaac PetscInt q = supp[i], numCones, j; 7726461c1adSToby Isaac const PetscInt *cone; 7736461c1adSToby Isaac 7749566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, q, &numCones)); 7759566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, q, &cone)); 7766461c1adSToby Isaac for (j = 0; j < numCones; j++) { 7776461c1adSToby Isaac if (cone[j] == p) break; 7786461c1adSToby Isaac } 7796461c1adSToby Isaac if (j < numCones) count++; 7806461c1adSToby Isaac } 7816461c1adSToby Isaac numTrueSupp[p] = count; 7826461c1adSToby Isaac } 7836461c1adSToby Isaac *dof = numTrueSupp[p]; 7843ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 7856461c1adSToby Isaac } 7866461c1adSToby Isaac 787d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexTreeExchangeSupports(DM dm) 788d71ae5a4SJacob Faibussowitsch { 789776742edSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 790776742edSToby Isaac PetscSection newSupportSection; 791776742edSToby Isaac PetscInt newSize, *newSupports, pStart, pEnd, p, d, depth; 7926461c1adSToby Isaac PetscInt *numTrueSupp; 793776742edSToby Isaac PetscInt *offsets; 794776742edSToby Isaac 795776742edSToby Isaac PetscFunctionBegin; 796776742edSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 797776742edSToby Isaac /* symmetrize the hierarchy */ 7989566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepth(dm, &depth)); 7999566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)(mesh->supportSection)), &newSupportSection)); 8009566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 8019566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(newSupportSection, pStart, pEnd)); 8029566063dSJacob Faibussowitsch PetscCall(PetscCalloc1(pEnd, &offsets)); 8039566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEnd, &numTrueSupp)); 8046461c1adSToby Isaac for (p = 0; p < pEnd; p++) numTrueSupp[p] = -1; 8056461c1adSToby Isaac /* if a point is in the (true) support of q, it should be in the support of 806776742edSToby Isaac * parent(q) */ 807776742edSToby Isaac for (d = 0; d <= depth; d++) { 8089566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(dm, d, &pStart, &pEnd)); 809776742edSToby Isaac for (p = pStart; p < pEnd; ++p) { 810776742edSToby Isaac PetscInt dof, q, qdof, parent; 811776742edSToby Isaac 8129566063dSJacob Faibussowitsch PetscCall(DMPlexGetTrueSupportSize(dm, p, &dof, numTrueSupp)); 8139566063dSJacob Faibussowitsch PetscCall(PetscSectionAddDof(newSupportSection, p, dof)); 814776742edSToby Isaac q = p; 8159566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, q, &parent, NULL)); 816776742edSToby Isaac while (parent != q && parent >= pStart && parent < pEnd) { 817776742edSToby Isaac q = parent; 818776742edSToby Isaac 8199566063dSJacob Faibussowitsch PetscCall(DMPlexGetTrueSupportSize(dm, q, &qdof, numTrueSupp)); 8209566063dSJacob Faibussowitsch PetscCall(PetscSectionAddDof(newSupportSection, p, qdof)); 8219566063dSJacob Faibussowitsch PetscCall(PetscSectionAddDof(newSupportSection, q, dof)); 8229566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, q, &parent, NULL)); 823776742edSToby Isaac } 824776742edSToby Isaac } 825776742edSToby Isaac } 8269566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(newSupportSection)); 8279566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(newSupportSection, &newSize)); 8289566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(newSize, &newSupports)); 829776742edSToby Isaac for (d = 0; d <= depth; d++) { 8309566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(dm, d, &pStart, &pEnd)); 831776742edSToby Isaac for (p = pStart; p < pEnd; p++) { 832776742edSToby Isaac PetscInt dof, off, q, qdof, qoff, newDof, newOff, newqOff, i, parent; 833776742edSToby Isaac 8349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(mesh->supportSection, p, &dof)); 8359566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off)); 8369566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(newSupportSection, p, &newDof)); 8379566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(newSupportSection, p, &newOff)); 838776742edSToby Isaac for (i = 0; i < dof; i++) { 8396461c1adSToby Isaac PetscInt numCones, j; 8406461c1adSToby Isaac const PetscInt *cone; 8416461c1adSToby Isaac PetscInt q = mesh->supports[off + i]; 8426461c1adSToby Isaac 8439566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, q, &numCones)); 8449566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, q, &cone)); 8456461c1adSToby Isaac for (j = 0; j < numCones; j++) { 8466461c1adSToby Isaac if (cone[j] == p) break; 8476461c1adSToby Isaac } 8486461c1adSToby Isaac if (j < numCones) newSupports[newOff + offsets[p]++] = q; 849776742edSToby Isaac } 850776742edSToby Isaac 851776742edSToby Isaac q = p; 8529566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, q, &parent, NULL)); 853776742edSToby Isaac while (parent != q && parent >= pStart && parent < pEnd) { 854776742edSToby Isaac q = parent; 8559566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(mesh->supportSection, q, &qdof)); 8569566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(mesh->supportSection, q, &qoff)); 8579566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(newSupportSection, q, &newqOff)); 858776742edSToby Isaac for (i = 0; i < qdof; i++) { 8596461c1adSToby Isaac PetscInt numCones, j; 8606461c1adSToby Isaac const PetscInt *cone; 8616461c1adSToby Isaac PetscInt r = mesh->supports[qoff + i]; 8626461c1adSToby Isaac 8639566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, r, &numCones)); 8649566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, r, &cone)); 8656461c1adSToby Isaac for (j = 0; j < numCones; j++) { 8666461c1adSToby Isaac if (cone[j] == q) break; 8676461c1adSToby Isaac } 8686461c1adSToby Isaac if (j < numCones) newSupports[newOff + offsets[p]++] = r; 869776742edSToby Isaac } 870776742edSToby Isaac for (i = 0; i < dof; i++) { 8716461c1adSToby Isaac PetscInt numCones, j; 8726461c1adSToby Isaac const PetscInt *cone; 8736461c1adSToby Isaac PetscInt r = mesh->supports[off + i]; 8746461c1adSToby Isaac 8759566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, r, &numCones)); 8769566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, r, &cone)); 8776461c1adSToby Isaac for (j = 0; j < numCones; j++) { 8786461c1adSToby Isaac if (cone[j] == p) break; 8796461c1adSToby Isaac } 8806461c1adSToby Isaac if (j < numCones) newSupports[newqOff + offsets[q]++] = r; 881776742edSToby Isaac } 8829566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, q, &parent, NULL)); 883776742edSToby Isaac } 884776742edSToby Isaac } 885776742edSToby Isaac } 8869566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&mesh->supportSection)); 887776742edSToby Isaac mesh->supportSection = newSupportSection; 8889566063dSJacob Faibussowitsch PetscCall(PetscFree(mesh->supports)); 889776742edSToby Isaac mesh->supports = newSupports; 8909566063dSJacob Faibussowitsch PetscCall(PetscFree(offsets)); 8919566063dSJacob Faibussowitsch PetscCall(PetscFree(numTrueSupp)); 892776742edSToby Isaac 8933ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 894776742edSToby Isaac } 895776742edSToby Isaac 896f7c74593SToby Isaac static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_Direct(DM, PetscSection, PetscSection, Mat); 897f7c74593SToby Isaac static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_FromReference(DM, PetscSection, PetscSection, Mat); 898f7c74593SToby Isaac 899d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexSetTree_Internal(DM dm, PetscSection parentSection, PetscInt *parents, PetscInt *childIDs, PetscBool computeCanonical, PetscBool exchangeSupports) 900d71ae5a4SJacob Faibussowitsch { 901f9f063d4SToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 902f9f063d4SToby Isaac DM refTree; 903f9f063d4SToby Isaac PetscInt size; 904f9f063d4SToby Isaac 905f9f063d4SToby Isaac PetscFunctionBegin; 906f9f063d4SToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 907f9f063d4SToby Isaac PetscValidHeaderSpecific(parentSection, PETSC_SECTION_CLASSID, 2); 9089566063dSJacob Faibussowitsch PetscCall(PetscObjectReference((PetscObject)parentSection)); 9099566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&mesh->parentSection)); 910f9f063d4SToby Isaac mesh->parentSection = parentSection; 9119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(parentSection, &size)); 912f9f063d4SToby Isaac if (parents != mesh->parents) { 9139566063dSJacob Faibussowitsch PetscCall(PetscFree(mesh->parents)); 9149566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(size, &mesh->parents)); 9159566063dSJacob Faibussowitsch PetscCall(PetscArraycpy(mesh->parents, parents, size)); 916f9f063d4SToby Isaac } 917f9f063d4SToby Isaac if (childIDs != mesh->childIDs) { 9189566063dSJacob Faibussowitsch PetscCall(PetscFree(mesh->childIDs)); 9199566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(size, &mesh->childIDs)); 9209566063dSJacob Faibussowitsch PetscCall(PetscArraycpy(mesh->childIDs, childIDs, size)); 921f9f063d4SToby Isaac } 9229566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(dm, &refTree)); 923f9f063d4SToby Isaac if (refTree) { 924f9f063d4SToby Isaac DMLabel canonLabel; 925f9f063d4SToby Isaac 9269566063dSJacob Faibussowitsch PetscCall(DMGetLabel(refTree, "canonical", &canonLabel)); 927f9f063d4SToby Isaac if (canonLabel) { 928f9f063d4SToby Isaac PetscInt i; 929f9f063d4SToby Isaac 930f9f063d4SToby Isaac for (i = 0; i < size; i++) { 931f9f063d4SToby Isaac PetscInt canon; 9329566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonLabel, mesh->childIDs[i], &canon)); 933ad540459SPierre Jolivet if (canon >= 0) mesh->childIDs[i] = canon; 934f9f063d4SToby Isaac } 935f9f063d4SToby Isaac } 936f7c74593SToby Isaac mesh->computeanchormatrix = DMPlexComputeAnchorMatrix_Tree_FromReference; 9376e0288c8SStefano Zampini } else { 938f7c74593SToby Isaac mesh->computeanchormatrix = DMPlexComputeAnchorMatrix_Tree_Direct; 939f9f063d4SToby Isaac } 9409566063dSJacob Faibussowitsch PetscCall(DMPlexTreeSymmetrize(dm)); 941f9f063d4SToby Isaac if (computeCanonical) { 942f9f063d4SToby Isaac PetscInt d, dim; 943f9f063d4SToby Isaac 944f9f063d4SToby Isaac /* add the canonical label */ 9459566063dSJacob Faibussowitsch PetscCall(DMGetDimension(dm, &dim)); 9469566063dSJacob Faibussowitsch PetscCall(DMCreateLabel(dm, "canonical")); 947f9f063d4SToby Isaac for (d = 0; d <= dim; d++) { 948f9f063d4SToby Isaac PetscInt p, dStart, dEnd, canon = -1, cNumChildren; 949f9f063d4SToby Isaac const PetscInt *cChildren; 950f9f063d4SToby Isaac 9519566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(dm, d, &dStart, &dEnd)); 952f9f063d4SToby Isaac for (p = dStart; p < dEnd; p++) { 9539566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(dm, p, &cNumChildren, &cChildren)); 954f9f063d4SToby Isaac if (cNumChildren) { 955f9f063d4SToby Isaac canon = p; 956f9f063d4SToby Isaac break; 957f9f063d4SToby Isaac } 958f9f063d4SToby Isaac } 959f9f063d4SToby Isaac if (canon == -1) continue; 960f9f063d4SToby Isaac for (p = dStart; p < dEnd; p++) { 961f9f063d4SToby Isaac PetscInt numChildren, i; 962f9f063d4SToby Isaac const PetscInt *children; 963f9f063d4SToby Isaac 9649566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(dm, p, &numChildren, &children)); 965f9f063d4SToby Isaac if (numChildren) { 96663a3b9bcSJacob Faibussowitsch PetscCheck(numChildren == cNumChildren, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "All parent points in a stratum should have the same number of children: %" PetscInt_FMT " != %" PetscInt_FMT, numChildren, cNumChildren); 9679566063dSJacob Faibussowitsch PetscCall(DMSetLabelValue(dm, "canonical", p, canon)); 96848a46eb9SPierre Jolivet for (i = 0; i < numChildren; i++) PetscCall(DMSetLabelValue(dm, "canonical", children[i], cChildren[i])); 969f9f063d4SToby Isaac } 970f9f063d4SToby Isaac } 971f9f063d4SToby Isaac } 972f9f063d4SToby Isaac } 9731baa6e33SBarry Smith if (exchangeSupports) PetscCall(DMPlexTreeExchangeSupports(dm)); 974f7c74593SToby Isaac mesh->createanchors = DMPlexCreateAnchors_Tree; 975f7c74593SToby Isaac /* reset anchors */ 9769566063dSJacob Faibussowitsch PetscCall(DMPlexSetAnchors(dm, NULL, NULL)); 9773ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 978f9f063d4SToby Isaac } 979f9f063d4SToby Isaac 9800b7167a0SToby Isaac /*@ 9810b7167a0SToby Isaac DMPlexSetTree - set the tree that describes the hierarchy of non-conforming mesh points. This routine also creates 982aaa8cc7dSPierre Jolivet the point-to-point constraints determined by the tree: a point is constrained to the points in the closure of its 9830b7167a0SToby Isaac tree root. 9840b7167a0SToby Isaac 98520f4b53cSBarry Smith Collective 9860b7167a0SToby Isaac 9870b7167a0SToby Isaac Input Parameters: 988a1cb98faSBarry Smith + dm - the `DMPLEX` object 9890b7167a0SToby Isaac . parentSection - a section describing the tree: a point has a parent if it has 1 dof in the section; the section 9900b7167a0SToby Isaac offset indexes the parent and childID list; the reference count of parentSection is incremented 9910b7167a0SToby Isaac . parents - a list of the point parents; copied, can be destroyed 9920b7167a0SToby Isaac - childIDs - identifies the relationship of the child point to the parent point; if there is a reference tree, then 9930b7167a0SToby Isaac the child corresponds to the point in the reference tree with index childIDs; copied, can be destroyed 9940b7167a0SToby Isaac 9950b7167a0SToby Isaac Level: intermediate 9960b7167a0SToby Isaac 9971cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetTree()`, `DMPlexSetReferenceTree()`, `DMPlexSetAnchors()`, `DMPlexGetTreeParent()`, `DMPlexGetTreeChildren()` 9980b7167a0SToby Isaac @*/ 999d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexSetTree(DM dm, PetscSection parentSection, PetscInt parents[], PetscInt childIDs[]) 1000d71ae5a4SJacob Faibussowitsch { 10010b7167a0SToby Isaac PetscFunctionBegin; 10029566063dSJacob Faibussowitsch PetscCall(DMPlexSetTree_Internal(dm, parentSection, parents, childIDs, PETSC_FALSE, PETSC_TRUE)); 10033ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 10040b7167a0SToby Isaac } 10050b7167a0SToby Isaac 1006b2f41788SToby Isaac /*@ 1007b2f41788SToby Isaac DMPlexGetTree - get the tree that describes the hierarchy of non-conforming mesh points. 100820f4b53cSBarry Smith Collective 1009b2f41788SToby Isaac 1010f899ff85SJose E. Roman Input Parameter: 1011a1cb98faSBarry Smith . dm - the `DMPLEX` object 1012b2f41788SToby Isaac 1013b2f41788SToby Isaac Output Parameters: 1014b2f41788SToby Isaac + parentSection - a section describing the tree: a point has a parent if it has 1 dof in the section; the section 1015b2f41788SToby Isaac offset indexes the parent and childID list 1016b2f41788SToby Isaac . parents - a list of the point parents 1017b2f41788SToby Isaac . childIDs - identifies the relationship of the child point to the parent point; if there is a reference tree, then 1018b2f41788SToby Isaac the child corresponds to the point in the reference tree with index childID 1019b2f41788SToby Isaac . childSection - the inverse of the parent section 1020b2f41788SToby Isaac - children - a list of the point children 1021b2f41788SToby Isaac 1022b2f41788SToby Isaac Level: intermediate 1023b2f41788SToby Isaac 10241cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`,`DMPlexSetTree()`, `DMPlexSetReferenceTree()`, `DMPlexSetAnchors()`, `DMPlexGetTreeParent()`, `DMPlexGetTreeChildren()` 1025b2f41788SToby Isaac @*/ 1026d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetTree(DM dm, PetscSection *parentSection, PetscInt *parents[], PetscInt *childIDs[], PetscSection *childSection, PetscInt *children[]) 1027d71ae5a4SJacob Faibussowitsch { 1028b2f41788SToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 1029b2f41788SToby Isaac 1030b2f41788SToby Isaac PetscFunctionBegin; 1031b2f41788SToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1032b2f41788SToby Isaac if (parentSection) *parentSection = mesh->parentSection; 1033b2f41788SToby Isaac if (parents) *parents = mesh->parents; 1034b2f41788SToby Isaac if (childIDs) *childIDs = mesh->childIDs; 1035b2f41788SToby Isaac if (childSection) *childSection = mesh->childSection; 1036b2f41788SToby Isaac if (children) *children = mesh->children; 10373ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 1038b2f41788SToby Isaac } 1039b2f41788SToby Isaac 1040d961a43aSToby Isaac /*@ 1041eaf898f9SPatrick Sanan DMPlexGetTreeParent - get the parent of a point in the tree describing the point hierarchy (not the DAG) 1042d961a43aSToby Isaac 1043d961a43aSToby Isaac Input Parameters: 1044a1cb98faSBarry Smith + dm - the `DMPLEX` object 1045d961a43aSToby Isaac - point - the query point 1046d961a43aSToby Isaac 1047d961a43aSToby Isaac Output Parameters: 104820f4b53cSBarry Smith + parent - if not `NULL`, set to the parent of the point, or the point itself if the point does not have a parent 104920f4b53cSBarry Smith - childID - if not `NULL`, set to the child ID of the point with respect to its parent, or 0 if the point 1050d961a43aSToby Isaac does not have a parent 1051d961a43aSToby Isaac 1052d961a43aSToby Isaac Level: intermediate 1053d961a43aSToby Isaac 10541cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetTree()`, `DMPlexGetTree()`, `DMPlexGetTreeChildren()` 1055d961a43aSToby Isaac @*/ 1056d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetTreeParent(DM dm, PetscInt point, PetscInt *parent, PetscInt *childID) 1057d71ae5a4SJacob Faibussowitsch { 1058d961a43aSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 1059d961a43aSToby Isaac PetscSection pSec; 1060d961a43aSToby Isaac 1061d961a43aSToby Isaac PetscFunctionBegin; 1062d961a43aSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1063d961a43aSToby Isaac pSec = mesh->parentSection; 1064d961a43aSToby Isaac if (pSec && point >= pSec->pStart && point < pSec->pEnd) { 1065d961a43aSToby Isaac PetscInt dof; 1066d961a43aSToby Isaac 10679566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(pSec, point, &dof)); 1068d961a43aSToby Isaac if (dof) { 1069d961a43aSToby Isaac PetscInt off; 1070d961a43aSToby Isaac 10719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(pSec, point, &off)); 1072d961a43aSToby Isaac if (parent) *parent = mesh->parents[off]; 1073d961a43aSToby Isaac if (childID) *childID = mesh->childIDs[off]; 10743ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 1075d961a43aSToby Isaac } 1076d961a43aSToby Isaac } 1077ad540459SPierre Jolivet if (parent) *parent = point; 1078ad540459SPierre Jolivet if (childID) *childID = 0; 10793ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 1080d961a43aSToby Isaac } 1081d961a43aSToby Isaac 1082d961a43aSToby Isaac /*@C 1083eaf898f9SPatrick Sanan DMPlexGetTreeChildren - get the children of a point in the tree describing the point hierarchy (not the DAG) 1084d961a43aSToby Isaac 1085d961a43aSToby Isaac Input Parameters: 1086a1cb98faSBarry Smith + dm - the `DMPLEX` object 1087d961a43aSToby Isaac - point - the query point 1088d961a43aSToby Isaac 1089d961a43aSToby Isaac Output Parameters: 109020f4b53cSBarry Smith + numChildren - if not `NULL`, set to the number of children 109120f4b53cSBarry Smith - children - if not `NULL`, set to a list children, or set to `NULL` if the point has no children 1092d961a43aSToby Isaac 1093d961a43aSToby Isaac Level: intermediate 1094d961a43aSToby Isaac 10951cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetTree()`, `DMPlexGetTree()`, `DMPlexGetTreeParent()` 1096d961a43aSToby Isaac @*/ 1097d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetTreeChildren(DM dm, PetscInt point, PetscInt *numChildren, const PetscInt *children[]) 1098d71ae5a4SJacob Faibussowitsch { 1099d961a43aSToby Isaac DM_Plex *mesh = (DM_Plex *)dm->data; 1100d961a43aSToby Isaac PetscSection childSec; 1101d961a43aSToby Isaac PetscInt dof = 0; 1102d961a43aSToby Isaac 1103d961a43aSToby Isaac PetscFunctionBegin; 1104d961a43aSToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1105d961a43aSToby Isaac childSec = mesh->childSection; 110648a46eb9SPierre Jolivet if (childSec && point >= childSec->pStart && point < childSec->pEnd) PetscCall(PetscSectionGetDof(childSec, point, &dof)); 1107d961a43aSToby Isaac if (numChildren) *numChildren = dof; 1108d961a43aSToby Isaac if (children) { 1109d961a43aSToby Isaac if (dof) { 1110d961a43aSToby Isaac PetscInt off; 1111d961a43aSToby Isaac 11129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(childSec, point, &off)); 1113d961a43aSToby Isaac *children = &mesh->children[off]; 11149371c9d4SSatish Balay } else { 1115d961a43aSToby Isaac *children = NULL; 1116d961a43aSToby Isaac } 1117d961a43aSToby Isaac } 11183ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 1119d961a43aSToby Isaac } 11200c37af3bSToby Isaac 1121d71ae5a4SJacob Faibussowitsch static PetscErrorCode EvaluateBasis(PetscSpace space, PetscInt nBasis, PetscInt nFunctionals, PetscInt nComps, PetscInt nPoints, const PetscInt *pointsPerFn, const PetscReal *points, const PetscReal *weights, PetscReal *work, Mat basisAtPoints) 1122d71ae5a4SJacob Faibussowitsch { 112352a3aeb4SToby Isaac PetscInt f, b, p, c, offset, qPoints; 1124b3a4bf2aSToby Isaac 1125b3a4bf2aSToby Isaac PetscFunctionBegin; 11269566063dSJacob Faibussowitsch PetscCall(PetscSpaceEvaluate(space, nPoints, points, work, NULL, NULL)); 112752a3aeb4SToby Isaac for (f = 0, offset = 0; f < nFunctionals; f++) { 112852a3aeb4SToby Isaac qPoints = pointsPerFn[f]; 112952a3aeb4SToby Isaac for (b = 0; b < nBasis; b++) { 1130b3a4bf2aSToby Isaac PetscScalar val = 0.; 1131b3a4bf2aSToby Isaac 113252a3aeb4SToby Isaac for (p = 0; p < qPoints; p++) { 1133ad540459SPierre Jolivet for (c = 0; c < nComps; c++) val += work[((offset + p) * nBasis + b) * nComps + c] * weights[(offset + p) * nComps + c]; 113452a3aeb4SToby Isaac } 11359566063dSJacob Faibussowitsch PetscCall(MatSetValue(basisAtPoints, b, f, val, INSERT_VALUES)); 1136b3a4bf2aSToby Isaac } 1137b3a4bf2aSToby Isaac offset += qPoints; 1138b3a4bf2aSToby Isaac } 11399566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(basisAtPoints, MAT_FINAL_ASSEMBLY)); 11409566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(basisAtPoints, MAT_FINAL_ASSEMBLY)); 11413ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 1142b3a4bf2aSToby Isaac } 1143b3a4bf2aSToby Isaac 1144d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_Direct(DM dm, PetscSection section, PetscSection cSec, Mat cMat) 1145d71ae5a4SJacob Faibussowitsch { 11460c37af3bSToby Isaac PetscDS ds; 11470c37af3bSToby Isaac PetscInt spdim; 11480c37af3bSToby Isaac PetscInt numFields, f, c, cStart, cEnd, pStart, pEnd, conStart, conEnd; 11490c37af3bSToby Isaac const PetscInt *anchors; 1150f7c74593SToby Isaac PetscSection aSec; 11510c37af3bSToby Isaac PetscReal *v0, *v0parent, *vtmp, *J, *Jparent, *invJparent, detJ, detJparent; 11520c37af3bSToby Isaac IS aIS; 11530c37af3bSToby Isaac 11540c37af3bSToby Isaac PetscFunctionBegin; 11559566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 11569566063dSJacob Faibussowitsch PetscCall(DMGetDS(dm, &ds)); 11579566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 11589566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd)); 11599566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(dm, &aSec, &aIS)); 11609566063dSJacob Faibussowitsch PetscCall(ISGetIndices(aIS, &anchors)); 11619566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSec, &conStart, &conEnd)); 11629566063dSJacob Faibussowitsch PetscCall(DMGetDimension(dm, &spdim)); 11639566063dSJacob Faibussowitsch PetscCall(PetscMalloc6(spdim, &v0, spdim, &v0parent, spdim, &vtmp, spdim * spdim, &J, spdim * spdim, &Jparent, spdim * spdim, &invJparent)); 11640c37af3bSToby Isaac 11650c37af3bSToby Isaac for (f = 0; f < numFields; f++) { 11660dd1b1feSToby Isaac PetscObject disc; 11670dd1b1feSToby Isaac PetscClassId id; 1168b3a4bf2aSToby Isaac PetscSpace bspace; 1169b3a4bf2aSToby Isaac PetscDualSpace dspace; 11709c3cf19fSMatthew G. Knepley PetscInt i, j, k, nPoints, Nc, offset; 117152a3aeb4SToby Isaac PetscInt fSize, maxDof; 1172b3a4bf2aSToby Isaac PetscReal *weights, *pointsRef, *pointsReal, *work; 11731683a169SBarry Smith PetscScalar *scwork; 11741683a169SBarry Smith const PetscScalar *X; 11752c44ad04SToby Isaac PetscInt *sizes, *workIndRow, *workIndCol; 11760c37af3bSToby Isaac Mat Amat, Bmat, Xmat; 11772c44ad04SToby Isaac const PetscInt *numDof = NULL; 1178085f0adfSToby Isaac const PetscInt ***perms = NULL; 1179085f0adfSToby Isaac const PetscScalar ***flips = NULL; 11800c37af3bSToby Isaac 11819566063dSJacob Faibussowitsch PetscCall(PetscDSGetDiscretization(ds, f, &disc)); 11829566063dSJacob Faibussowitsch PetscCall(PetscObjectGetClassId(disc, &id)); 11830dd1b1feSToby Isaac if (id == PETSCFE_CLASSID) { 1184b3a4bf2aSToby Isaac PetscFE fe = (PetscFE)disc; 1185b3a4bf2aSToby Isaac 11869566063dSJacob Faibussowitsch PetscCall(PetscFEGetBasisSpace(fe, &bspace)); 11879566063dSJacob Faibussowitsch PetscCall(PetscFEGetDualSpace(fe, &dspace)); 11889566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetDimension(dspace, &fSize)); 11899566063dSJacob Faibussowitsch PetscCall(PetscFEGetNumComponents(fe, &Nc)); 11909371c9d4SSatish Balay } else if (id == PETSCFV_CLASSID) { 1191b3a4bf2aSToby Isaac PetscFV fv = (PetscFV)disc; 1192b3a4bf2aSToby Isaac 11939566063dSJacob Faibussowitsch PetscCall(PetscFVGetNumComponents(fv, &Nc)); 11949566063dSJacob Faibussowitsch PetscCall(PetscSpaceCreate(PetscObjectComm((PetscObject)fv), &bspace)); 11959566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetType(bspace, PETSCSPACEPOLYNOMIAL)); 11969566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetDegree(bspace, 0, PETSC_DETERMINE)); 11979566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetNumComponents(bspace, Nc)); 11989566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetNumVariables(bspace, spdim)); 11999566063dSJacob Faibussowitsch PetscCall(PetscSpaceSetUp(bspace)); 12009566063dSJacob Faibussowitsch PetscCall(PetscFVGetDualSpace(fv, &dspace)); 12019566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetDimension(dspace, &fSize)); 12029371c9d4SSatish Balay } else SETERRQ(PetscObjectComm(disc), PETSC_ERR_ARG_UNKNOWN_TYPE, "PetscDS discretization id %d not recognized.", id); 12039566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetNumDof(dspace, &numDof)); 1204ad540459SPierre Jolivet for (i = 0, maxDof = 0; i <= spdim; i++) maxDof = PetscMax(maxDof, numDof[i]); 12059566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetSymmetries(dspace, &perms, &flips)); 12060dd1b1feSToby Isaac 12079566063dSJacob Faibussowitsch PetscCall(MatCreate(PETSC_COMM_SELF, &Amat)); 12089566063dSJacob Faibussowitsch PetscCall(MatSetSizes(Amat, fSize, fSize, fSize, fSize)); 12099566063dSJacob Faibussowitsch PetscCall(MatSetType(Amat, MATSEQDENSE)); 12109566063dSJacob Faibussowitsch PetscCall(MatSetUp(Amat)); 12119566063dSJacob Faibussowitsch PetscCall(MatDuplicate(Amat, MAT_DO_NOT_COPY_VALUES, &Bmat)); 12129566063dSJacob Faibussowitsch PetscCall(MatDuplicate(Amat, MAT_DO_NOT_COPY_VALUES, &Xmat)); 12130c37af3bSToby Isaac nPoints = 0; 12140c37af3bSToby Isaac for (i = 0; i < fSize; i++) { 121552a3aeb4SToby Isaac PetscInt qPoints, thisNc; 12160c37af3bSToby Isaac PetscQuadrature quad; 12170c37af3bSToby Isaac 12189566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetFunctional(dspace, i, &quad)); 12199566063dSJacob Faibussowitsch PetscCall(PetscQuadratureGetData(quad, NULL, &thisNc, &qPoints, NULL, NULL)); 122063a3b9bcSJacob Faibussowitsch PetscCheck(thisNc == Nc, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Functional dim %" PetscInt_FMT " does not much basis dim %" PetscInt_FMT, thisNc, Nc); 12210c37af3bSToby Isaac nPoints += qPoints; 12220c37af3bSToby Isaac } 12239566063dSJacob Faibussowitsch PetscCall(PetscMalloc7(fSize, &sizes, nPoints * Nc, &weights, spdim * nPoints, &pointsRef, spdim * nPoints, &pointsReal, nPoints * fSize * Nc, &work, maxDof, &workIndRow, maxDof, &workIndCol)); 12249566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof * maxDof, &scwork)); 12250c37af3bSToby Isaac offset = 0; 12260c37af3bSToby Isaac for (i = 0; i < fSize; i++) { 12270c37af3bSToby Isaac PetscInt qPoints; 12280c37af3bSToby Isaac const PetscReal *p, *w; 12290c37af3bSToby Isaac PetscQuadrature quad; 12300c37af3bSToby Isaac 12319566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetFunctional(dspace, i, &quad)); 12329566063dSJacob Faibussowitsch PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &qPoints, &p, &w)); 12339566063dSJacob Faibussowitsch PetscCall(PetscArraycpy(weights + Nc * offset, w, Nc * qPoints)); 12349566063dSJacob Faibussowitsch PetscCall(PetscArraycpy(pointsRef + spdim * offset, p, spdim * qPoints)); 1235b3a4bf2aSToby Isaac sizes[i] = qPoints; 12360c37af3bSToby Isaac offset += qPoints; 12370c37af3bSToby Isaac } 12389566063dSJacob Faibussowitsch PetscCall(EvaluateBasis(bspace, fSize, fSize, Nc, nPoints, sizes, pointsRef, weights, work, Amat)); 12399566063dSJacob Faibussowitsch PetscCall(MatLUFactor(Amat, NULL, NULL, NULL)); 12400c37af3bSToby Isaac for (c = cStart; c < cEnd; c++) { 12410c37af3bSToby Isaac PetscInt parent; 12420c37af3bSToby Isaac PetscInt closureSize, closureSizeP, *closure = NULL, *closureP = NULL; 12430c37af3bSToby Isaac PetscInt *childOffsets, *parentOffsets; 12440c37af3bSToby Isaac 12459566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, c, &parent, NULL)); 12460c37af3bSToby Isaac if (parent == c) continue; 12479566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure)); 12480c37af3bSToby Isaac for (i = 0; i < closureSize; i++) { 12490c37af3bSToby Isaac PetscInt p = closure[2 * i]; 12500c37af3bSToby Isaac PetscInt conDof; 12510c37af3bSToby Isaac 12520c37af3bSToby Isaac if (p < conStart || p >= conEnd) continue; 1253085f0adfSToby Isaac if (numFields) { 12549566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSec, p, f, &conDof)); 12559371c9d4SSatish Balay } else { 12569566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSec, p, &conDof)); 12570c37af3bSToby Isaac } 12580c37af3bSToby Isaac if (conDof) break; 12590c37af3bSToby Isaac } 12600c37af3bSToby Isaac if (i == closureSize) { 12619566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure)); 12620c37af3bSToby Isaac continue; 12630c37af3bSToby Isaac } 12640c37af3bSToby Isaac 12659566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(dm, c, NULL, v0, J, NULL, &detJ)); 12669566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(dm, parent, NULL, v0parent, Jparent, invJparent, &detJparent)); 12670c37af3bSToby Isaac for (i = 0; i < nPoints; i++) { 1268c330f8ffSToby Isaac const PetscReal xi0[3] = {-1., -1., -1.}; 1269c330f8ffSToby Isaac 1270c330f8ffSToby Isaac CoordinatesRefToReal(spdim, spdim, xi0, v0, J, &pointsRef[i * spdim], vtmp); 1271c330f8ffSToby Isaac CoordinatesRealToRef(spdim, spdim, xi0, v0parent, invJparent, vtmp, &pointsReal[i * spdim]); 12720c37af3bSToby Isaac } 12739566063dSJacob Faibussowitsch PetscCall(EvaluateBasis(bspace, fSize, fSize, Nc, nPoints, sizes, pointsReal, weights, work, Bmat)); 12749566063dSJacob Faibussowitsch PetscCall(MatMatSolve(Amat, Bmat, Xmat)); 12759566063dSJacob Faibussowitsch PetscCall(MatDenseGetArrayRead(Xmat, &X)); 12769566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, parent, PETSC_TRUE, &closureSizeP, &closureP)); 12779566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(closureSize + 1, &childOffsets, closureSizeP + 1, &parentOffsets)); 12780c37af3bSToby Isaac childOffsets[0] = 0; 12790c37af3bSToby Isaac for (i = 0; i < closureSize; i++) { 12800c37af3bSToby Isaac PetscInt p = closure[2 * i]; 12810c37af3bSToby Isaac PetscInt dof; 12820c37af3bSToby Isaac 1283085f0adfSToby Isaac if (numFields) { 12849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, p, f, &dof)); 12859371c9d4SSatish Balay } else { 12869566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 12870c37af3bSToby Isaac } 128852a3aeb4SToby Isaac childOffsets[i + 1] = childOffsets[i] + dof; 12890c37af3bSToby Isaac } 12900c37af3bSToby Isaac parentOffsets[0] = 0; 12910c37af3bSToby Isaac for (i = 0; i < closureSizeP; i++) { 12920c37af3bSToby Isaac PetscInt p = closureP[2 * i]; 12930c37af3bSToby Isaac PetscInt dof; 12940c37af3bSToby Isaac 1295085f0adfSToby Isaac if (numFields) { 12969566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, p, f, &dof)); 12979371c9d4SSatish Balay } else { 12989566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &dof)); 12990c37af3bSToby Isaac } 130052a3aeb4SToby Isaac parentOffsets[i + 1] = parentOffsets[i] + dof; 13010c37af3bSToby Isaac } 13020c37af3bSToby Isaac for (i = 0; i < closureSize; i++) { 13032c44ad04SToby Isaac PetscInt conDof, conOff, aDof, aOff, nWork; 13040c37af3bSToby Isaac PetscInt p = closure[2 * i]; 13050c37af3bSToby Isaac PetscInt o = closure[2 * i + 1]; 1306085f0adfSToby Isaac const PetscInt *perm; 1307085f0adfSToby Isaac const PetscScalar *flip; 13080c37af3bSToby Isaac 13090c37af3bSToby Isaac if (p < conStart || p >= conEnd) continue; 1310085f0adfSToby Isaac if (numFields) { 13119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSec, p, f, &conDof)); 13129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(cSec, p, f, &conOff)); 13139371c9d4SSatish Balay } else { 13149566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSec, p, &conDof)); 13159566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(cSec, p, &conOff)); 13160c37af3bSToby Isaac } 13170c37af3bSToby Isaac if (!conDof) continue; 1318085f0adfSToby Isaac perm = (perms && perms[i]) ? perms[i][o] : NULL; 1319085f0adfSToby Isaac flip = (flips && flips[i]) ? flips[i][o] : NULL; 13209566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(aSec, p, &aDof)); 13219566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(aSec, p, &aOff)); 13222c44ad04SToby Isaac nWork = childOffsets[i + 1] - childOffsets[i]; 13230c37af3bSToby Isaac for (k = 0; k < aDof; k++) { 13240c37af3bSToby Isaac PetscInt a = anchors[aOff + k]; 13250c37af3bSToby Isaac PetscInt aSecDof, aSecOff; 13260c37af3bSToby Isaac 1327085f0adfSToby Isaac if (numFields) { 13289566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, a, f, &aSecDof)); 13299566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(section, a, f, &aSecOff)); 13309371c9d4SSatish Balay } else { 13319566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, a, &aSecDof)); 13329566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, a, &aSecOff)); 13330c37af3bSToby Isaac } 13340c37af3bSToby Isaac if (!aSecDof) continue; 13350c37af3bSToby Isaac 13360c37af3bSToby Isaac for (j = 0; j < closureSizeP; j++) { 13370c37af3bSToby Isaac PetscInt q = closureP[2 * j]; 13380c37af3bSToby Isaac PetscInt oq = closureP[2 * j + 1]; 13392c44ad04SToby Isaac 13402c44ad04SToby Isaac if (q == a) { 134152a3aeb4SToby Isaac PetscInt r, s, nWorkP; 1342085f0adfSToby Isaac const PetscInt *permP; 1343085f0adfSToby Isaac const PetscScalar *flipP; 1344085f0adfSToby Isaac 1345085f0adfSToby Isaac permP = (perms && perms[j]) ? perms[j][oq] : NULL; 1346085f0adfSToby Isaac flipP = (flips && flips[j]) ? flips[j][oq] : NULL; 13472c44ad04SToby Isaac nWorkP = parentOffsets[j + 1] - parentOffsets[j]; 13482c44ad04SToby Isaac /* get a copy of the child-to-anchor portion of the matrix, and transpose so that rows correspond to the 13491683a169SBarry Smith * child and columns correspond to the anchor: BUT the maxrix returned by MatDenseGetArrayRead() is 13502c44ad04SToby Isaac * column-major, so transpose-transpose = do nothing */ 13512c44ad04SToby Isaac for (r = 0; r < nWork; r++) { 1352ad540459SPierre Jolivet for (s = 0; s < nWorkP; s++) scwork[r * nWorkP + s] = X[fSize * (r + childOffsets[i]) + (s + parentOffsets[j])]; 13532c44ad04SToby Isaac } 1354ad540459SPierre Jolivet for (r = 0; r < nWork; r++) workIndRow[perm ? perm[r] : r] = conOff + r; 1355ad540459SPierre Jolivet for (s = 0; s < nWorkP; s++) workIndCol[permP ? permP[s] : s] = aSecOff + s; 13562c44ad04SToby Isaac if (flip) { 13572c44ad04SToby Isaac for (r = 0; r < nWork; r++) { 1358ad540459SPierre Jolivet for (s = 0; s < nWorkP; s++) scwork[r * nWorkP + s] *= flip[r]; 13592c44ad04SToby Isaac } 13602c44ad04SToby Isaac } 13612c44ad04SToby Isaac if (flipP) { 13622c44ad04SToby Isaac for (r = 0; r < nWork; r++) { 1363ad540459SPierre Jolivet for (s = 0; s < nWorkP; s++) scwork[r * nWorkP + s] *= flipP[s]; 13642c44ad04SToby Isaac } 13652c44ad04SToby Isaac } 13669566063dSJacob Faibussowitsch PetscCall(MatSetValues(cMat, nWork, workIndRow, nWorkP, workIndCol, scwork, INSERT_VALUES)); 13672c44ad04SToby Isaac break; 13680c37af3bSToby Isaac } 13690c37af3bSToby Isaac } 13700c37af3bSToby Isaac } 13710c37af3bSToby Isaac } 13729566063dSJacob Faibussowitsch PetscCall(MatDenseRestoreArrayRead(Xmat, &X)); 13739566063dSJacob Faibussowitsch PetscCall(PetscFree2(childOffsets, parentOffsets)); 13749566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure)); 13759566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, parent, PETSC_TRUE, &closureSizeP, &closureP)); 13760c37af3bSToby Isaac } 13779566063dSJacob Faibussowitsch PetscCall(MatDestroy(&Amat)); 13789566063dSJacob Faibussowitsch PetscCall(MatDestroy(&Bmat)); 13799566063dSJacob Faibussowitsch PetscCall(MatDestroy(&Xmat)); 13809566063dSJacob Faibussowitsch PetscCall(PetscFree(scwork)); 13819566063dSJacob Faibussowitsch PetscCall(PetscFree7(sizes, weights, pointsRef, pointsReal, work, workIndRow, workIndCol)); 138248a46eb9SPierre Jolivet if (id == PETSCFV_CLASSID) PetscCall(PetscSpaceDestroy(&bspace)); 13830c37af3bSToby Isaac } 13849566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(cMat, MAT_FINAL_ASSEMBLY)); 13859566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(cMat, MAT_FINAL_ASSEMBLY)); 13869566063dSJacob Faibussowitsch PetscCall(PetscFree6(v0, v0parent, vtmp, J, Jparent, invJparent)); 13879566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(aIS, &anchors)); 13880c37af3bSToby Isaac 13893ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 13900c37af3bSToby Isaac } 139195a0b26dSToby Isaac 1392d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexReferenceTreeGetChildrenMatrices(DM refTree, PetscScalar ****childrenMats, PetscInt ***childrenN) 1393d71ae5a4SJacob Faibussowitsch { 1394f7c74593SToby Isaac Mat refCmat; 139521968bf8SToby Isaac PetscDS ds; 1396085f0adfSToby Isaac PetscInt numFields, maxFields, f, pRefStart, pRefEnd, p, *rows, *cols, maxDof, maxAnDof, **refPointFieldN; 139721968bf8SToby Isaac PetscScalar ***refPointFieldMats; 139821968bf8SToby Isaac PetscSection refConSec, refAnSec, refSection; 139921968bf8SToby Isaac IS refAnIS; 140021968bf8SToby Isaac const PetscInt *refAnchors; 1401085f0adfSToby Isaac const PetscInt **perms; 1402085f0adfSToby Isaac const PetscScalar **flips; 140395a0b26dSToby Isaac 140495a0b26dSToby Isaac PetscFunctionBegin; 14059566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 14069566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 1407085f0adfSToby Isaac maxFields = PetscMax(1, numFields); 14089566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, &refCmat, NULL)); 14099566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(refTree, &refAnSec, &refAnIS)); 14109566063dSJacob Faibussowitsch PetscCall(ISGetIndices(refAnIS, &refAnchors)); 14119566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(refTree, &refSection)); 14129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 14139566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pRefEnd - pRefStart, &refPointFieldMats)); 14149566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pRefEnd - pRefStart, &refPointFieldN)); 14159566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refConSec, &maxDof)); 14169566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refAnSec, &maxAnDof)); 14179566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof, &rows)); 14189566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof * maxAnDof, &cols)); 141995a0b26dSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 142095a0b26dSToby Isaac PetscInt parent, closureSize, *closure = NULL, pDof; 142195a0b26dSToby Isaac 14229566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, p, &parent, NULL)); 14239566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &pDof)); 142495a0b26dSToby Isaac if (!pDof || parent == p) continue; 142595a0b26dSToby Isaac 14269566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxFields, &refPointFieldMats[p - pRefStart])); 14279566063dSJacob Faibussowitsch PetscCall(PetscCalloc1(maxFields, &refPointFieldN[p - pRefStart])); 14289566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, parent, PETSC_TRUE, &closureSize, &closure)); 1429085f0adfSToby Isaac for (f = 0; f < maxFields; f++) { 1430085f0adfSToby Isaac PetscInt cDof, cOff, numCols, r, i; 143195a0b26dSToby Isaac 1432085f0adfSToby Isaac if (f < numFields) { 14339566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refConSec, p, f, &cDof)); 14349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refConSec, p, f, &cOff)); 14359566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldPointSyms(refSection, f, closureSize, closure, &perms, &flips)); 1436085f0adfSToby Isaac } else { 14379566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &cDof)); 14389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refConSec, p, &cOff)); 14399566063dSJacob Faibussowitsch PetscCall(PetscSectionGetPointSyms(refSection, closureSize, closure, &perms, &flips)); 144095a0b26dSToby Isaac } 144195a0b26dSToby Isaac 1442ad540459SPierre Jolivet for (r = 0; r < cDof; r++) rows[r] = cOff + r; 144395a0b26dSToby Isaac numCols = 0; 144495a0b26dSToby Isaac for (i = 0; i < closureSize; i++) { 144595a0b26dSToby Isaac PetscInt q = closure[2 * i]; 144695a0b26dSToby Isaac PetscInt aDof, aOff, j; 1447085f0adfSToby Isaac const PetscInt *perm = perms ? perms[i] : NULL; 144895a0b26dSToby Isaac 1449085f0adfSToby Isaac if (numFields) { 14509566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refSection, q, f, &aDof)); 14519566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refSection, q, f, &aOff)); 14529371c9d4SSatish Balay } else { 14539566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, q, &aDof)); 14549566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refSection, q, &aOff)); 145595a0b26dSToby Isaac } 145695a0b26dSToby Isaac 1457ad540459SPierre Jolivet for (j = 0; j < aDof; j++) cols[numCols++] = aOff + (perm ? perm[j] : j); 145895a0b26dSToby Isaac } 145995a0b26dSToby Isaac refPointFieldN[p - pRefStart][f] = numCols; 14609566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(cDof * numCols, &refPointFieldMats[p - pRefStart][f])); 14619566063dSJacob Faibussowitsch PetscCall(MatGetValues(refCmat, cDof, rows, numCols, cols, refPointFieldMats[p - pRefStart][f])); 1462085f0adfSToby Isaac if (flips) { 1463085f0adfSToby Isaac PetscInt colOff = 0; 1464085f0adfSToby Isaac 1465085f0adfSToby Isaac for (i = 0; i < closureSize; i++) { 1466085f0adfSToby Isaac PetscInt q = closure[2 * i]; 1467085f0adfSToby Isaac PetscInt aDof, aOff, j; 1468085f0adfSToby Isaac const PetscScalar *flip = flips ? flips[i] : NULL; 1469085f0adfSToby Isaac 1470085f0adfSToby Isaac if (numFields) { 14719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refSection, q, f, &aDof)); 14729566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refSection, q, f, &aOff)); 14739371c9d4SSatish Balay } else { 14749566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, q, &aDof)); 14759566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refSection, q, &aOff)); 1476085f0adfSToby Isaac } 1477085f0adfSToby Isaac if (flip) { 1478085f0adfSToby Isaac PetscInt k; 1479085f0adfSToby Isaac for (k = 0; k < cDof; k++) { 1480ad540459SPierre Jolivet for (j = 0; j < aDof; j++) refPointFieldMats[p - pRefStart][f][k * numCols + colOff + j] *= flip[j]; 1481085f0adfSToby Isaac } 1482085f0adfSToby Isaac } 1483085f0adfSToby Isaac colOff += aDof; 1484085f0adfSToby Isaac } 1485085f0adfSToby Isaac } 1486085f0adfSToby Isaac if (numFields) { 14879566063dSJacob Faibussowitsch PetscCall(PetscSectionRestoreFieldPointSyms(refSection, f, closureSize, closure, &perms, &flips)); 1488085f0adfSToby Isaac } else { 14899566063dSJacob Faibussowitsch PetscCall(PetscSectionRestorePointSyms(refSection, closureSize, closure, &perms, &flips)); 1490085f0adfSToby Isaac } 149195a0b26dSToby Isaac } 14929566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, parent, PETSC_TRUE, &closureSize, &closure)); 149395a0b26dSToby Isaac } 149421968bf8SToby Isaac *childrenMats = refPointFieldMats; 149521968bf8SToby Isaac *childrenN = refPointFieldN; 14969566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(refAnIS, &refAnchors)); 14979566063dSJacob Faibussowitsch PetscCall(PetscFree(rows)); 14989566063dSJacob Faibussowitsch PetscCall(PetscFree(cols)); 14993ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 150021968bf8SToby Isaac } 150121968bf8SToby Isaac 1502d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexReferenceTreeRestoreChildrenMatrices(DM refTree, PetscScalar ****childrenMats, PetscInt ***childrenN) 1503d71ae5a4SJacob Faibussowitsch { 150421968bf8SToby Isaac PetscDS ds; 150521968bf8SToby Isaac PetscInt **refPointFieldN; 150621968bf8SToby Isaac PetscScalar ***refPointFieldMats; 1507085f0adfSToby Isaac PetscInt numFields, maxFields, pRefStart, pRefEnd, p, f; 150821968bf8SToby Isaac PetscSection refConSec; 150921968bf8SToby Isaac 151021968bf8SToby Isaac PetscFunctionBegin; 151121968bf8SToby Isaac refPointFieldN = *childrenN; 151221968bf8SToby Isaac *childrenN = NULL; 151321968bf8SToby Isaac refPointFieldMats = *childrenMats; 151421968bf8SToby Isaac *childrenMats = NULL; 15159566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 15169566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 1517367003a6SStefano Zampini maxFields = PetscMax(1, numFields); 15189566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 15199566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 152021968bf8SToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 152121968bf8SToby Isaac PetscInt parent, pDof; 152221968bf8SToby Isaac 15239566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, p, &parent, NULL)); 15249566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &pDof)); 152521968bf8SToby Isaac if (!pDof || parent == p) continue; 152621968bf8SToby Isaac 1527085f0adfSToby Isaac for (f = 0; f < maxFields; f++) { 152821968bf8SToby Isaac PetscInt cDof; 152921968bf8SToby Isaac 1530085f0adfSToby Isaac if (numFields) { 15319566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refConSec, p, f, &cDof)); 15329371c9d4SSatish Balay } else { 15339566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &cDof)); 153421968bf8SToby Isaac } 153521968bf8SToby Isaac 15369566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats[p - pRefStart][f])); 153721968bf8SToby Isaac } 15389566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats[p - pRefStart])); 15399566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldN[p - pRefStart])); 154021968bf8SToby Isaac } 15419566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats)); 15429566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldN)); 15433ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 154421968bf8SToby Isaac } 154521968bf8SToby Isaac 1546d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_FromReference(DM dm, PetscSection section, PetscSection conSec, Mat cMat) 1547d71ae5a4SJacob Faibussowitsch { 154821968bf8SToby Isaac DM refTree; 154921968bf8SToby Isaac PetscDS ds; 155021968bf8SToby Isaac Mat refCmat; 1551085f0adfSToby Isaac PetscInt numFields, maxFields, f, pRefStart, pRefEnd, p, maxDof, maxAnDof, *perm, *iperm, pStart, pEnd, conStart, conEnd, **refPointFieldN; 155221968bf8SToby Isaac PetscScalar ***refPointFieldMats, *pointWork; 155321968bf8SToby Isaac PetscSection refConSec, refAnSec, anSec; 155421968bf8SToby Isaac IS refAnIS, anIS; 155521968bf8SToby Isaac const PetscInt *anchors; 155621968bf8SToby Isaac 155721968bf8SToby Isaac PetscFunctionBegin; 155821968bf8SToby Isaac PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 15599566063dSJacob Faibussowitsch PetscCall(DMGetDS(dm, &ds)); 15609566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 1561085f0adfSToby Isaac maxFields = PetscMax(1, numFields); 15629566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(dm, &refTree)); 15639566063dSJacob Faibussowitsch PetscCall(DMCopyDisc(dm, refTree)); 1564c77c71ffSToby Isaac PetscCall(DMSetLocalSection(refTree, NULL)); 1565c77c71ffSToby Isaac PetscCall(DMSetDefaultConstraints(refTree, NULL, NULL, NULL)); 15669566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, &refCmat, NULL)); 15679566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(refTree, &refAnSec, &refAnIS)); 15689566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(dm, &anSec, &anIS)); 15699566063dSJacob Faibussowitsch PetscCall(ISGetIndices(anIS, &anchors)); 15709566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 15719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(conSec, &conStart, &conEnd)); 15729566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refConSec, &maxDof)); 15739566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refAnSec, &maxAnDof)); 15749566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof * maxDof * maxAnDof, &pointWork)); 157521968bf8SToby Isaac 157621968bf8SToby Isaac /* step 1: get submats for every constrained point in the reference tree */ 15779566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 157895a0b26dSToby Isaac 157995a0b26dSToby Isaac /* step 2: compute the preorder */ 15809566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 15819566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(pEnd - pStart, &perm, pEnd - pStart, &iperm)); 158295a0b26dSToby Isaac for (p = pStart; p < pEnd; p++) { 158395a0b26dSToby Isaac perm[p - pStart] = p; 158495a0b26dSToby Isaac iperm[p - pStart] = p - pStart; 158595a0b26dSToby Isaac } 158695a0b26dSToby Isaac for (p = 0; p < pEnd - pStart;) { 158795a0b26dSToby Isaac PetscInt point = perm[p]; 158895a0b26dSToby Isaac PetscInt parent; 158995a0b26dSToby Isaac 15909566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, point, &parent, NULL)); 159195a0b26dSToby Isaac if (parent == point) { 159295a0b26dSToby Isaac p++; 15939371c9d4SSatish Balay } else { 159495a0b26dSToby Isaac PetscInt size, closureSize, *closure = NULL, i; 159595a0b26dSToby Isaac 15969566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, parent, PETSC_TRUE, &closureSize, &closure)); 159795a0b26dSToby Isaac for (i = 0; i < closureSize; i++) { 159895a0b26dSToby Isaac PetscInt q = closure[2 * i]; 159995a0b26dSToby Isaac if (iperm[q - pStart] > iperm[point - pStart]) { 160095a0b26dSToby Isaac /* swap */ 160195a0b26dSToby Isaac perm[p] = q; 160295a0b26dSToby Isaac perm[iperm[q - pStart]] = point; 160395a0b26dSToby Isaac iperm[point - pStart] = iperm[q - pStart]; 160495a0b26dSToby Isaac iperm[q - pStart] = p; 160595a0b26dSToby Isaac break; 160695a0b26dSToby Isaac } 160795a0b26dSToby Isaac } 160895a0b26dSToby Isaac size = closureSize; 16099566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, parent, PETSC_TRUE, &closureSize, &closure)); 1610ad540459SPierre Jolivet if (i == size) p++; 161195a0b26dSToby Isaac } 161295a0b26dSToby Isaac } 161395a0b26dSToby Isaac 161495a0b26dSToby Isaac /* step 3: fill the constraint matrix */ 161595a0b26dSToby Isaac /* we are going to use a preorder progressive fill strategy. Mat doesn't 161695a0b26dSToby Isaac * allow progressive fill without assembly, so we are going to set up the 161795a0b26dSToby Isaac * values outside of the Mat first. 161895a0b26dSToby Isaac */ 161995a0b26dSToby Isaac { 162095a0b26dSToby Isaac PetscInt nRows, row, nnz; 162195a0b26dSToby Isaac PetscBool done; 1622cd6fc93eSToby Isaac PetscInt secStart, secEnd; 162395a0b26dSToby Isaac const PetscInt *ia, *ja; 162495a0b26dSToby Isaac PetscScalar *vals; 162595a0b26dSToby Isaac 1626cd6fc93eSToby Isaac PetscCall(PetscSectionGetChart(section, &secStart, &secEnd)); 16279566063dSJacob Faibussowitsch PetscCall(MatGetRowIJ(cMat, 0, PETSC_FALSE, PETSC_FALSE, &nRows, &ia, &ja, &done)); 162828b400f6SJacob Faibussowitsch PetscCheck(done, PetscObjectComm((PetscObject)cMat), PETSC_ERR_PLIB, "Could not get RowIJ of constraint matrix"); 162995a0b26dSToby Isaac nnz = ia[nRows]; 163095a0b26dSToby Isaac /* malloc and then zero rows right before we fill them: this way valgrind 163195a0b26dSToby Isaac * can tell if we are doing progressive fill in the wrong order */ 16329566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(nnz, &vals)); 163395a0b26dSToby Isaac for (p = 0; p < pEnd - pStart; p++) { 163495a0b26dSToby Isaac PetscInt parent, childid, closureSize, *closure = NULL; 163595a0b26dSToby Isaac PetscInt point = perm[p], pointDof; 163695a0b26dSToby Isaac 16379566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(dm, point, &parent, &childid)); 163895a0b26dSToby Isaac if ((point < conStart) || (point >= conEnd) || (parent == point)) continue; 16399566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(conSec, point, &pointDof)); 164095a0b26dSToby Isaac if (!pointDof) continue; 16419566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, parent, PETSC_TRUE, &closureSize, &closure)); 1642085f0adfSToby Isaac for (f = 0; f < maxFields; f++) { 1643085f0adfSToby Isaac PetscInt cDof, cOff, numCols, numFillCols, i, r, matOffset, offset; 164495a0b26dSToby Isaac PetscScalar *pointMat; 1645085f0adfSToby Isaac const PetscInt **perms; 1646085f0adfSToby Isaac const PetscScalar **flips; 164795a0b26dSToby Isaac 1648085f0adfSToby Isaac if (numFields) { 16499566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(conSec, point, f, &cDof)); 16509566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(conSec, point, f, &cOff)); 16519371c9d4SSatish Balay } else { 16529566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(conSec, point, &cDof)); 16539566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(conSec, point, &cOff)); 165495a0b26dSToby Isaac } 165595a0b26dSToby Isaac if (!cDof) continue; 16569566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionGetFieldPointSyms(section, f, closureSize, closure, &perms, &flips)); 16579566063dSJacob Faibussowitsch else PetscCall(PetscSectionGetPointSyms(section, closureSize, closure, &perms, &flips)); 165895a0b26dSToby Isaac 165995a0b26dSToby Isaac /* make sure that every row for this point is the same size */ 166076bd3646SJed Brown if (PetscDefined(USE_DEBUG)) { 166195a0b26dSToby Isaac for (r = 0; r < cDof; r++) { 166295a0b26dSToby Isaac if (cDof > 1 && r) { 166363a3b9bcSJacob Faibussowitsch PetscCheck((ia[cOff + r + 1] - ia[cOff + r]) == (ia[cOff + r] - ia[cOff + r - 1]), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two point rows have different nnz: %" PetscInt_FMT " vs. %" PetscInt_FMT, (ia[cOff + r + 1] - ia[cOff + r]), (ia[cOff + r] - ia[cOff + r - 1])); 166495a0b26dSToby Isaac } 166595a0b26dSToby Isaac } 166676bd3646SJed Brown } 166795a0b26dSToby Isaac /* zero rows */ 1668ad540459SPierre Jolivet for (i = ia[cOff]; i < ia[cOff + cDof]; i++) vals[i] = 0.; 166995a0b26dSToby Isaac matOffset = ia[cOff]; 167095a0b26dSToby Isaac numFillCols = ia[cOff + 1] - matOffset; 167195a0b26dSToby Isaac pointMat = refPointFieldMats[childid - pRefStart][f]; 167295a0b26dSToby Isaac numCols = refPointFieldN[childid - pRefStart][f]; 167395a0b26dSToby Isaac offset = 0; 167495a0b26dSToby Isaac for (i = 0; i < closureSize; i++) { 167595a0b26dSToby Isaac PetscInt q = closure[2 * i]; 167695a0b26dSToby Isaac PetscInt aDof, aOff, j, k, qConDof, qConOff; 1677085f0adfSToby Isaac const PetscInt *perm = perms ? perms[i] : NULL; 1678085f0adfSToby Isaac const PetscScalar *flip = flips ? flips[i] : NULL; 167995a0b26dSToby Isaac 168095a0b26dSToby Isaac qConDof = qConOff = 0; 1681cd6fc93eSToby Isaac if (q < secStart || q >= secEnd) continue; 1682085f0adfSToby Isaac if (numFields) { 16839566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, q, f, &aDof)); 16849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(section, q, f, &aOff)); 168595a0b26dSToby Isaac if (q >= conStart && q < conEnd) { 16869566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(conSec, q, f, &qConDof)); 16879566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(conSec, q, f, &qConOff)); 168895a0b26dSToby Isaac } 16899371c9d4SSatish Balay } else { 16909566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, q, &aDof)); 16919566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, q, &aOff)); 169295a0b26dSToby Isaac if (q >= conStart && q < conEnd) { 16939566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(conSec, q, &qConDof)); 16949566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(conSec, q, &qConOff)); 169595a0b26dSToby Isaac } 169695a0b26dSToby Isaac } 169795a0b26dSToby Isaac if (!aDof) continue; 169895a0b26dSToby Isaac if (qConDof) { 169995a0b26dSToby Isaac /* this point has anchors: its rows of the matrix should already 170095a0b26dSToby Isaac * be filled, thanks to preordering */ 170195a0b26dSToby Isaac /* first multiply into pointWork, then set in matrix */ 170295a0b26dSToby Isaac PetscInt aMatOffset = ia[qConOff]; 170395a0b26dSToby Isaac PetscInt aNumFillCols = ia[qConOff + 1] - aMatOffset; 170495a0b26dSToby Isaac for (r = 0; r < cDof; r++) { 170595a0b26dSToby Isaac for (j = 0; j < aNumFillCols; j++) { 170695a0b26dSToby Isaac PetscScalar inVal = 0; 170795a0b26dSToby Isaac for (k = 0; k < aDof; k++) { 1708085f0adfSToby Isaac PetscInt col = perm ? perm[k] : k; 170995a0b26dSToby Isaac 1710085f0adfSToby Isaac inVal += pointMat[r * numCols + offset + col] * vals[aMatOffset + aNumFillCols * k + j] * (flip ? flip[col] : 1.); 171195a0b26dSToby Isaac } 171295a0b26dSToby Isaac pointWork[r * aNumFillCols + j] = inVal; 171395a0b26dSToby Isaac } 171495a0b26dSToby Isaac } 171595a0b26dSToby Isaac /* assume that the columns are sorted, spend less time searching */ 171695a0b26dSToby Isaac for (j = 0, k = 0; j < aNumFillCols; j++) { 171795a0b26dSToby Isaac PetscInt col = ja[aMatOffset + j]; 171895a0b26dSToby Isaac for (; k < numFillCols; k++) { 1719ad540459SPierre Jolivet if (ja[matOffset + k] == col) break; 172095a0b26dSToby Isaac } 172163a3b9bcSJacob Faibussowitsch PetscCheck(k != numFillCols, PETSC_COMM_SELF, PETSC_ERR_PLIB, "No nonzero space for (%" PetscInt_FMT ", %" PetscInt_FMT ")", cOff, col); 1722ad540459SPierre Jolivet for (r = 0; r < cDof; r++) vals[matOffset + numFillCols * r + k] = pointWork[r * aNumFillCols + j]; 172395a0b26dSToby Isaac } 17249371c9d4SSatish Balay } else { 172595a0b26dSToby Isaac /* find where to put this portion of pointMat into the matrix */ 172695a0b26dSToby Isaac for (k = 0; k < numFillCols; k++) { 1727ad540459SPierre Jolivet if (ja[matOffset + k] == aOff) break; 172895a0b26dSToby Isaac } 172963a3b9bcSJacob Faibussowitsch PetscCheck(k != numFillCols, PETSC_COMM_SELF, PETSC_ERR_PLIB, "No nonzero space for (%" PetscInt_FMT ", %" PetscInt_FMT ")", cOff, aOff); 173095a0b26dSToby Isaac for (r = 0; r < cDof; r++) { 1731085f0adfSToby Isaac for (j = 0; j < aDof; j++) { 1732085f0adfSToby Isaac PetscInt col = perm ? perm[j] : j; 1733085f0adfSToby Isaac 1734085f0adfSToby Isaac vals[matOffset + numFillCols * r + k + col] += pointMat[r * numCols + offset + j] * (flip ? flip[col] : 1.); 173595a0b26dSToby Isaac } 173695a0b26dSToby Isaac } 173795a0b26dSToby Isaac } 173895a0b26dSToby Isaac offset += aDof; 173995a0b26dSToby Isaac } 1740085f0adfSToby Isaac if (numFields) { 17419566063dSJacob Faibussowitsch PetscCall(PetscSectionRestoreFieldPointSyms(section, f, closureSize, closure, &perms, &flips)); 1742085f0adfSToby Isaac } else { 17439566063dSJacob Faibussowitsch PetscCall(PetscSectionRestorePointSyms(section, closureSize, closure, &perms, &flips)); 1744085f0adfSToby Isaac } 174595a0b26dSToby Isaac } 17469566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, parent, PETSC_TRUE, &closureSize, &closure)); 174795a0b26dSToby Isaac } 174848a46eb9SPierre Jolivet for (row = 0; row < nRows; row++) PetscCall(MatSetValues(cMat, 1, &row, ia[row + 1] - ia[row], &ja[ia[row]], &vals[ia[row]], INSERT_VALUES)); 17499566063dSJacob Faibussowitsch PetscCall(MatRestoreRowIJ(cMat, 0, PETSC_FALSE, PETSC_FALSE, &nRows, &ia, &ja, &done)); 175028b400f6SJacob Faibussowitsch PetscCheck(done, PetscObjectComm((PetscObject)cMat), PETSC_ERR_PLIB, "Could not restore RowIJ of constraint matrix"); 17519566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(cMat, MAT_FINAL_ASSEMBLY)); 17529566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(cMat, MAT_FINAL_ASSEMBLY)); 17539566063dSJacob Faibussowitsch PetscCall(PetscFree(vals)); 175495a0b26dSToby Isaac } 175595a0b26dSToby Isaac 175695a0b26dSToby Isaac /* clean up */ 17579566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(anIS, &anchors)); 17589566063dSJacob Faibussowitsch PetscCall(PetscFree2(perm, iperm)); 17599566063dSJacob Faibussowitsch PetscCall(PetscFree(pointWork)); 17609566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 17613ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 176295a0b26dSToby Isaac } 176395a0b26dSToby Isaac 17646f5f1567SToby Isaac /* refine a single cell on rank 0: this is not intended to provide good local refinement, only to create an example of 17656f5f1567SToby Isaac * a non-conforming mesh. Local refinement comes later */ 1766d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexTreeRefineCell(DM dm, PetscInt cell, DM *ncdm) 1767d71ae5a4SJacob Faibussowitsch { 17686f5f1567SToby Isaac DM K; 1769420f55faSMatthew G. Knepley PetscMPIInt rank; 17706f5f1567SToby Isaac PetscInt dim, *pNewStart, *pNewEnd, *pNewCount, *pOldStart, *pOldEnd, offset, d, pStart, pEnd; 17716f5f1567SToby Isaac PetscInt numNewCones, *newConeSizes, *newCones, *newOrientations; 17726f5f1567SToby Isaac PetscInt *Kembedding; 17736f5f1567SToby Isaac PetscInt *cellClosure = NULL, nc; 17746f5f1567SToby Isaac PetscScalar *newVertexCoords; 17756f5f1567SToby Isaac PetscInt numPointsWithParents, *parents, *childIDs, *perm, *iperm, *preOrient, pOffset; 17766f5f1567SToby Isaac PetscSection parentSection; 17776f5f1567SToby Isaac 17786f5f1567SToby Isaac PetscFunctionBegin; 17799566063dSJacob Faibussowitsch PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank)); 17809566063dSJacob Faibussowitsch PetscCall(DMGetDimension(dm, &dim)); 17819566063dSJacob Faibussowitsch PetscCall(DMPlexCreate(PetscObjectComm((PetscObject)dm), ncdm)); 17829566063dSJacob Faibussowitsch PetscCall(DMSetDimension(*ncdm, dim)); 17836f5f1567SToby Isaac 17849566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); 17859566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &parentSection)); 17869566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(dm, &K)); 17876858538eSMatthew G. Knepley PetscCall(DMGetCoordinatesLocalSetUp(dm)); 1788dd400576SPatrick Sanan if (rank == 0) { 17896f5f1567SToby Isaac /* compute the new charts */ 17909566063dSJacob Faibussowitsch PetscCall(PetscMalloc5(dim + 1, &pNewCount, dim + 1, &pNewStart, dim + 1, &pNewEnd, dim + 1, &pOldStart, dim + 1, &pOldEnd)); 17916f5f1567SToby Isaac offset = 0; 17926f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 17936f5f1567SToby Isaac PetscInt pOldCount, kStart, kEnd, k; 17946f5f1567SToby Isaac 17956f5f1567SToby Isaac pNewStart[d] = offset; 17969566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(dm, d, &pOldStart[d], &pOldEnd[d])); 17979566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &kStart, &kEnd)); 17986f5f1567SToby Isaac pOldCount = pOldEnd[d] - pOldStart[d]; 17996f5f1567SToby Isaac /* adding the new points */ 18006f5f1567SToby Isaac pNewCount[d] = pOldCount + kEnd - kStart; 18016f5f1567SToby Isaac if (!d) { 18026f5f1567SToby Isaac /* removing the cell */ 18036f5f1567SToby Isaac pNewCount[d]--; 18046f5f1567SToby Isaac } 18056f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 18066f5f1567SToby Isaac PetscInt parent; 18079566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(K, k, &parent, NULL)); 18086f5f1567SToby Isaac if (parent == k) { 18096f5f1567SToby Isaac /* avoid double counting points that won't actually be new */ 18106f5f1567SToby Isaac pNewCount[d]--; 18116f5f1567SToby Isaac } 18126f5f1567SToby Isaac } 18136f5f1567SToby Isaac pNewEnd[d] = pNewStart[d] + pNewCount[d]; 18146f5f1567SToby Isaac offset = pNewEnd[d]; 18156f5f1567SToby Isaac } 18161dca8a05SBarry Smith PetscCheck(cell >= pOldStart[0] && cell < pOldEnd[0], PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "%" PetscInt_FMT " not in cell range [%" PetscInt_FMT ", %" PetscInt_FMT ")", cell, pOldStart[0], pOldEnd[0]); 18176f5f1567SToby Isaac /* get the current closure of the cell that we are removing */ 18189566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &nc, &cellClosure)); 18196f5f1567SToby Isaac 18209566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pNewEnd[dim], &newConeSizes)); 18216f5f1567SToby Isaac { 1822b5a892a1SMatthew G. Knepley DMPolytopeType pct, qct; 18236f5f1567SToby Isaac PetscInt kStart, kEnd, k, closureSizeK, *closureK = NULL, j; 18246f5f1567SToby Isaac 18259566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(K, &kStart, &kEnd)); 18269566063dSJacob Faibussowitsch PetscCall(PetscMalloc4(kEnd - kStart, &Kembedding, kEnd - kStart, &perm, kEnd - kStart, &iperm, kEnd - kStart, &preOrient)); 18276f5f1567SToby Isaac 18286f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 18296f5f1567SToby Isaac perm[k - kStart] = k; 18306f5f1567SToby Isaac iperm[k - kStart] = k - kStart; 18316f5f1567SToby Isaac preOrient[k - kStart] = 0; 18326f5f1567SToby Isaac } 18336f5f1567SToby Isaac 18349566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(K, 0, PETSC_TRUE, &closureSizeK, &closureK)); 18356f5f1567SToby Isaac for (j = 1; j < closureSizeK; j++) { 18366f5f1567SToby Isaac PetscInt parentOrientA = closureK[2 * j + 1]; 18376f5f1567SToby Isaac PetscInt parentOrientB = cellClosure[2 * j + 1]; 18386f5f1567SToby Isaac PetscInt p, q; 18396f5f1567SToby Isaac 18406f5f1567SToby Isaac p = closureK[2 * j]; 18416f5f1567SToby Isaac q = cellClosure[2 * j]; 18429566063dSJacob Faibussowitsch PetscCall(DMPlexGetCellType(K, p, &pct)); 18439566063dSJacob Faibussowitsch PetscCall(DMPlexGetCellType(dm, q, &qct)); 18446f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 1845ad540459SPierre Jolivet if (q >= pOldStart[d] && q < pOldEnd[d]) Kembedding[p] = (q - pOldStart[d]) + pNewStart[d]; 18466f5f1567SToby Isaac } 1847b5a892a1SMatthew G. Knepley parentOrientA = DMPolytopeConvertNewOrientation_Internal(pct, parentOrientA); 1848b5a892a1SMatthew G. Knepley parentOrientB = DMPolytopeConvertNewOrientation_Internal(qct, parentOrientB); 18496f5f1567SToby Isaac if (parentOrientA != parentOrientB) { 18506f5f1567SToby Isaac PetscInt numChildren, i; 18516f5f1567SToby Isaac const PetscInt *children; 18526f5f1567SToby Isaac 18539566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(K, p, &numChildren, &children)); 18546f5f1567SToby Isaac for (i = 0; i < numChildren; i++) { 18556f5f1567SToby Isaac PetscInt kPerm, oPerm; 18566f5f1567SToby Isaac 18576f5f1567SToby Isaac k = children[i]; 18589566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildSymmetry(K, p, parentOrientA, 0, k, parentOrientB, &oPerm, &kPerm)); 18596f5f1567SToby Isaac /* perm = what refTree position I'm in */ 18606f5f1567SToby Isaac perm[kPerm - kStart] = k; 18616f5f1567SToby Isaac /* iperm = who is at this position */ 18626f5f1567SToby Isaac iperm[k - kStart] = kPerm - kStart; 18636f5f1567SToby Isaac preOrient[kPerm - kStart] = oPerm; 18646f5f1567SToby Isaac } 18656f5f1567SToby Isaac } 18666f5f1567SToby Isaac } 18679566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(K, 0, PETSC_TRUE, &closureSizeK, &closureK)); 18686f5f1567SToby Isaac } 18699566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(parentSection, 0, pNewEnd[dim])); 18706f5f1567SToby Isaac offset = 0; 18716f5f1567SToby Isaac numNewCones = 0; 18726f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 18736f5f1567SToby Isaac PetscInt kStart, kEnd, k; 18746f5f1567SToby Isaac PetscInt p; 18756f5f1567SToby Isaac PetscInt size; 18766f5f1567SToby Isaac 18776f5f1567SToby Isaac for (p = pOldStart[d]; p < pOldEnd[d]; p++) { 18786f5f1567SToby Isaac /* skip cell 0 */ 18796f5f1567SToby Isaac if (p == cell) continue; 18806f5f1567SToby Isaac /* old cones to new cones */ 18819566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, p, &size)); 18826f5f1567SToby Isaac newConeSizes[offset++] = size; 18836f5f1567SToby Isaac numNewCones += size; 18846f5f1567SToby Isaac } 18856f5f1567SToby Isaac 18869566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &kStart, &kEnd)); 18876f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 18886f5f1567SToby Isaac PetscInt kParent; 18896f5f1567SToby Isaac 18909566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(K, k, &kParent, NULL)); 18916f5f1567SToby Isaac if (kParent != k) { 18926f5f1567SToby Isaac Kembedding[k] = offset; 18939566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, k, &size)); 18946f5f1567SToby Isaac newConeSizes[offset++] = size; 18956f5f1567SToby Isaac numNewCones += size; 189648a46eb9SPierre Jolivet if (kParent != 0) PetscCall(PetscSectionSetDof(parentSection, Kembedding[k], 1)); 18976f5f1567SToby Isaac } 18986f5f1567SToby Isaac } 18996f5f1567SToby Isaac } 19006f5f1567SToby Isaac 19019566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(parentSection)); 19029566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(parentSection, &numPointsWithParents)); 19039566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numNewCones, &newCones, numNewCones, &newOrientations)); 19049566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numPointsWithParents, &parents, numPointsWithParents, &childIDs)); 19056f5f1567SToby Isaac 19066f5f1567SToby Isaac /* fill new cones */ 19076f5f1567SToby Isaac offset = 0; 19086f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 19096f5f1567SToby Isaac PetscInt kStart, kEnd, k, l; 19106f5f1567SToby Isaac PetscInt p; 19116f5f1567SToby Isaac PetscInt size; 19126f5f1567SToby Isaac const PetscInt *cone, *orientation; 19136f5f1567SToby Isaac 19146f5f1567SToby Isaac for (p = pOldStart[d]; p < pOldEnd[d]; p++) { 19156f5f1567SToby Isaac /* skip cell 0 */ 19166f5f1567SToby Isaac if (p == cell) continue; 19176f5f1567SToby Isaac /* old cones to new cones */ 19189566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(dm, p, &size)); 19199566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(dm, p, &cone)); 19209566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(dm, p, &orientation)); 19216f5f1567SToby Isaac for (l = 0; l < size; l++) { 19226f5f1567SToby Isaac newCones[offset] = (cone[l] - pOldStart[d + 1]) + pNewStart[d + 1]; 19236f5f1567SToby Isaac newOrientations[offset++] = orientation[l]; 19246f5f1567SToby Isaac } 19256f5f1567SToby Isaac } 19266f5f1567SToby Isaac 19279566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, d, &kStart, &kEnd)); 19286f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 19296f5f1567SToby Isaac PetscInt kPerm = perm[k], kParent; 19306f5f1567SToby Isaac PetscInt preO = preOrient[k]; 19316f5f1567SToby Isaac 19329566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(K, k, &kParent, NULL)); 19336f5f1567SToby Isaac if (kParent != k) { 19346f5f1567SToby Isaac /* embed new cones */ 19359566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, k, &size)); 19369566063dSJacob Faibussowitsch PetscCall(DMPlexGetCone(K, kPerm, &cone)); 19379566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientation(K, kPerm, &orientation)); 19386f5f1567SToby Isaac for (l = 0; l < size; l++) { 19396f5f1567SToby Isaac PetscInt q, m = (preO >= 0) ? ((preO + l) % size) : ((size - (preO + 1) - l) % size); 19406f5f1567SToby Isaac PetscInt newO, lSize, oTrue; 1941b5a892a1SMatthew G. Knepley DMPolytopeType ct = DM_NUM_POLYTOPES; 19426f5f1567SToby Isaac 19436f5f1567SToby Isaac q = iperm[cone[m]]; 19446f5f1567SToby Isaac newCones[offset] = Kembedding[q]; 19459566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeSize(K, q, &lSize)); 1946b5a892a1SMatthew G. Knepley if (lSize == 2) ct = DM_POLYTOPE_SEGMENT; 1947b5a892a1SMatthew G. Knepley else if (lSize == 4) ct = DM_POLYTOPE_QUADRILATERAL; 1948b5a892a1SMatthew G. Knepley oTrue = DMPolytopeConvertNewOrientation_Internal(ct, orientation[m]); 19496f5f1567SToby Isaac oTrue = ((!lSize) || (preOrient[k] >= 0)) ? oTrue : -(oTrue + 2); 19506f5f1567SToby Isaac newO = DihedralCompose(lSize, oTrue, preOrient[q]); 1951b5a892a1SMatthew G. Knepley newOrientations[offset++] = DMPolytopeConvertOldOrientation_Internal(ct, newO); 19526f5f1567SToby Isaac } 19536f5f1567SToby Isaac if (kParent != 0) { 19546f5f1567SToby Isaac PetscInt newPoint = Kembedding[kParent]; 19559566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(parentSection, Kembedding[k], &pOffset)); 19566f5f1567SToby Isaac parents[pOffset] = newPoint; 19576f5f1567SToby Isaac childIDs[pOffset] = k; 19586f5f1567SToby Isaac } 19596f5f1567SToby Isaac } 19606f5f1567SToby Isaac } 19616f5f1567SToby Isaac } 19626f5f1567SToby Isaac 19639566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(dim * (pNewEnd[dim] - pNewStart[dim]), &newVertexCoords)); 19646f5f1567SToby Isaac 19656f5f1567SToby Isaac /* fill coordinates */ 19666f5f1567SToby Isaac offset = 0; 19676f5f1567SToby Isaac { 1968d90620a3SMatthew G. Knepley PetscInt kStart, kEnd, l; 19696f5f1567SToby Isaac PetscSection vSection; 19706f5f1567SToby Isaac PetscInt v; 19716f5f1567SToby Isaac Vec coords; 19726f5f1567SToby Isaac PetscScalar *coordvals; 19736f5f1567SToby Isaac PetscInt dof, off; 1974c111c6b7SMatthew G. Knepley PetscReal v0[3], J[9], detJ; 19756f5f1567SToby Isaac 197676bd3646SJed Brown if (PetscDefined(USE_DEBUG)) { 1977d90620a3SMatthew G. Knepley PetscInt k; 19789566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(K, 0, &kStart, &kEnd)); 19796f5f1567SToby Isaac for (k = kStart; k < kEnd; k++) { 19809566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(K, k, NULL, v0, J, NULL, &detJ)); 198163a3b9bcSJacob Faibussowitsch PetscCheck(detJ > 0., PETSC_COMM_SELF, PETSC_ERR_PLIB, "reference tree cell %" PetscInt_FMT " has bad determinant", k); 19826f5f1567SToby Isaac } 1983d90620a3SMatthew G. Knepley } 19849566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, NULL, v0, J, NULL, &detJ)); 19859566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateSection(dm, &vSection)); 19869566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(dm, &coords)); 19879566063dSJacob Faibussowitsch PetscCall(VecGetArray(coords, &coordvals)); 19886f5f1567SToby Isaac for (v = pOldStart[dim]; v < pOldEnd[dim]; v++) { 19899566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(vSection, v, &dof)); 19909566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(vSection, v, &off)); 1991ad540459SPierre Jolivet for (l = 0; l < dof; l++) newVertexCoords[offset++] = coordvals[off + l]; 19926f5f1567SToby Isaac } 19939566063dSJacob Faibussowitsch PetscCall(VecRestoreArray(coords, &coordvals)); 19946f5f1567SToby Isaac 19959566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateSection(K, &vSection)); 19969566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(K, &coords)); 19979566063dSJacob Faibussowitsch PetscCall(VecGetArray(coords, &coordvals)); 19989566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(K, 0, &kStart, &kEnd)); 19996f5f1567SToby Isaac for (v = kStart; v < kEnd; v++) { 20009bc368c7SMatthew G. Knepley PetscReal coord[3], newCoord[3]; 20016f5f1567SToby Isaac PetscInt vPerm = perm[v]; 20026f5f1567SToby Isaac PetscInt kParent; 2003c330f8ffSToby Isaac const PetscReal xi0[3] = {-1., -1., -1.}; 20046f5f1567SToby Isaac 20059566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(K, v, &kParent, NULL)); 20066f5f1567SToby Isaac if (kParent != v) { 20076f5f1567SToby Isaac /* this is a new vertex */ 20089566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(vSection, vPerm, &off)); 20099bc368c7SMatthew G. Knepley for (l = 0; l < dim; ++l) coord[l] = PetscRealPart(coordvals[off + l]); 2010367003a6SStefano Zampini CoordinatesRefToReal(dim, dim, xi0, v0, J, coord, newCoord); 20119bc368c7SMatthew G. Knepley for (l = 0; l < dim; ++l) newVertexCoords[offset + l] = newCoord[l]; 20126f5f1567SToby Isaac offset += dim; 20136f5f1567SToby Isaac } 20146f5f1567SToby Isaac } 20159566063dSJacob Faibussowitsch PetscCall(VecRestoreArray(coords, &coordvals)); 20166f5f1567SToby Isaac } 20176f5f1567SToby Isaac 20186f5f1567SToby Isaac /* need to reverse the order of pNewCount: vertices first, cells last */ 20196f5f1567SToby Isaac for (d = 0; d < (dim + 1) / 2; d++) { 20206f5f1567SToby Isaac PetscInt tmp; 20216f5f1567SToby Isaac 20226f5f1567SToby Isaac tmp = pNewCount[d]; 20236f5f1567SToby Isaac pNewCount[d] = pNewCount[dim - d]; 20246f5f1567SToby Isaac pNewCount[dim - d] = tmp; 20256f5f1567SToby Isaac } 20266f5f1567SToby Isaac 20279566063dSJacob Faibussowitsch PetscCall(DMPlexCreateFromDAG(*ncdm, dim, pNewCount, newConeSizes, newCones, newOrientations, newVertexCoords)); 20289566063dSJacob Faibussowitsch PetscCall(DMPlexSetReferenceTree(*ncdm, K)); 20299566063dSJacob Faibussowitsch PetscCall(DMPlexSetTree(*ncdm, parentSection, parents, childIDs)); 20306f5f1567SToby Isaac 20316f5f1567SToby Isaac /* clean up */ 20329566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &nc, &cellClosure)); 20339566063dSJacob Faibussowitsch PetscCall(PetscFree5(pNewCount, pNewStart, pNewEnd, pOldStart, pOldEnd)); 20349566063dSJacob Faibussowitsch PetscCall(PetscFree(newConeSizes)); 20359566063dSJacob Faibussowitsch PetscCall(PetscFree2(newCones, newOrientations)); 20369566063dSJacob Faibussowitsch PetscCall(PetscFree(newVertexCoords)); 20379566063dSJacob Faibussowitsch PetscCall(PetscFree2(parents, childIDs)); 20389566063dSJacob Faibussowitsch PetscCall(PetscFree4(Kembedding, perm, iperm, preOrient)); 20399371c9d4SSatish Balay } else { 20406f5f1567SToby Isaac PetscInt p, counts[4]; 20416f5f1567SToby Isaac PetscInt *coneSizes, *cones, *orientations; 20426f5f1567SToby Isaac Vec coordVec; 20436f5f1567SToby Isaac PetscScalar *coords; 20446f5f1567SToby Isaac 20456f5f1567SToby Isaac for (d = 0; d <= dim; d++) { 20466f5f1567SToby Isaac PetscInt dStart, dEnd; 20476f5f1567SToby Isaac 20489566063dSJacob Faibussowitsch PetscCall(DMPlexGetDepthStratum(dm, d, &dStart, &dEnd)); 20496f5f1567SToby Isaac counts[d] = dEnd - dStart; 20506f5f1567SToby Isaac } 20519566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEnd - pStart, &coneSizes)); 205248a46eb9SPierre Jolivet for (p = pStart; p < pEnd; p++) PetscCall(DMPlexGetConeSize(dm, p, &coneSizes[p - pStart])); 20539566063dSJacob Faibussowitsch PetscCall(DMPlexGetCones(dm, &cones)); 20549566063dSJacob Faibussowitsch PetscCall(DMPlexGetConeOrientations(dm, &orientations)); 20559566063dSJacob Faibussowitsch PetscCall(DMGetCoordinatesLocal(dm, &coordVec)); 20569566063dSJacob Faibussowitsch PetscCall(VecGetArray(coordVec, &coords)); 20576f5f1567SToby Isaac 20589566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(parentSection, pStart, pEnd)); 20599566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(parentSection)); 20609566063dSJacob Faibussowitsch PetscCall(DMPlexCreateFromDAG(*ncdm, dim, counts, coneSizes, cones, orientations, NULL)); 20619566063dSJacob Faibussowitsch PetscCall(DMPlexSetReferenceTree(*ncdm, K)); 20629566063dSJacob Faibussowitsch PetscCall(DMPlexSetTree(*ncdm, parentSection, NULL, NULL)); 20639566063dSJacob Faibussowitsch PetscCall(VecRestoreArray(coordVec, &coords)); 20646f5f1567SToby Isaac } 20659566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&parentSection)); 20666f5f1567SToby Isaac 20673ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 20686f5f1567SToby Isaac } 20696ecaa68aSToby Isaac 2070d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeInterpolatorTree(DM coarse, DM fine, PetscSF coarseToFine, PetscInt *childIds, Mat mat) 2071d71ae5a4SJacob Faibussowitsch { 20726ecaa68aSToby Isaac PetscSF coarseToFineEmbedded; 20736ecaa68aSToby Isaac PetscSection globalCoarse, globalFine; 20746ecaa68aSToby Isaac PetscSection localCoarse, localFine; 20756ecaa68aSToby Isaac PetscSection aSec, cSec; 20766ecaa68aSToby Isaac PetscSection rootIndicesSec, rootMatricesSec; 207746bdb399SToby Isaac PetscSection leafIndicesSec, leafMatricesSec; 207846bdb399SToby Isaac PetscInt *rootIndices, *leafIndices; 207946bdb399SToby Isaac PetscScalar *rootMatrices, *leafMatrices; 20806ecaa68aSToby Isaac IS aIS; 20816ecaa68aSToby Isaac const PetscInt *anchors; 20826ecaa68aSToby Isaac Mat cMat; 20834acb8e1eSToby Isaac PetscInt numFields, maxFields; 20846ecaa68aSToby Isaac PetscInt pStartC, pEndC, pStartF, pEndF, p; 20856ecaa68aSToby Isaac PetscInt aStart, aEnd, cStart, cEnd; 20861c58ffc4SToby Isaac PetscInt *maxChildIds; 2087e44e4e7fSToby Isaac PetscInt *offsets, *newOffsets, *offsetsCopy, *newOffsetsCopy, *rowOffsets, *numD, *numO; 20884acb8e1eSToby Isaac const PetscInt ***perms; 20894acb8e1eSToby Isaac const PetscScalar ***flips; 20906ecaa68aSToby Isaac 20916ecaa68aSToby Isaac PetscFunctionBegin; 20929566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 20939566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 20949566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 20956ecaa68aSToby Isaac { /* winnow fine points that don't have global dofs out of the sf */ 209689698031SToby Isaac PetscInt dof, cdof, numPointsWithDofs, offset, *pointsWithDofs, nleaves, l; 209789698031SToby Isaac const PetscInt *leaves; 20986ecaa68aSToby Isaac 20999566063dSJacob Faibussowitsch PetscCall(PetscSFGetGraph(coarseToFine, NULL, &nleaves, &leaves, NULL)); 210089698031SToby Isaac for (l = 0, numPointsWithDofs = 0; l < nleaves; l++) { 210189698031SToby Isaac p = leaves ? leaves[l] : l; 21029566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 21039566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 2104ad540459SPierre Jolivet if ((dof - cdof) > 0) numPointsWithDofs++; 21056ecaa68aSToby Isaac } 21069566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numPointsWithDofs, &pointsWithDofs)); 21077cc7abc7SToby Isaac for (l = 0, offset = 0; l < nleaves; l++) { 210889698031SToby Isaac p = leaves ? leaves[l] : l; 21099566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 21109566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 2111ad540459SPierre Jolivet if ((dof - cdof) > 0) pointsWithDofs[offset++] = l; 21126ecaa68aSToby Isaac } 21139566063dSJacob Faibussowitsch PetscCall(PetscSFCreateEmbeddedLeafSF(coarseToFine, numPointsWithDofs, pointsWithDofs, &coarseToFineEmbedded)); 21149566063dSJacob Faibussowitsch PetscCall(PetscFree(pointsWithDofs)); 21156ecaa68aSToby Isaac } 21166ecaa68aSToby Isaac /* communicate back to the coarse mesh which coarse points have children (that may require interpolation) */ 21179566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEndC - pStartC, &maxChildIds)); 2118ad540459SPierre Jolivet for (p = pStartC; p < pEndC; p++) maxChildIds[p - pStartC] = -2; 211957168dbeSPierre Jolivet PetscCall(PetscSFReduceBegin(coarseToFineEmbedded, MPIU_INT, childIds, maxChildIds, MPI_MAX)); 212057168dbeSPierre Jolivet PetscCall(PetscSFReduceEnd(coarseToFineEmbedded, MPIU_INT, childIds, maxChildIds, MPI_MAX)); 212146bdb399SToby Isaac 21229566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 21239566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 212446bdb399SToby Isaac 21259566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(coarse, &aSec, &aIS)); 21269566063dSJacob Faibussowitsch PetscCall(ISGetIndices(aIS, &anchors)); 21279566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(aSec, &aStart, &aEnd)); 212846bdb399SToby Isaac 21299566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(coarse, &cSec, &cMat, NULL)); 21309566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSec, &cStart, &cEnd)); 213146bdb399SToby Isaac 213246bdb399SToby Isaac /* create sections that will send to children the indices and matrices they will need to construct the interpolator */ 21339566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &rootIndicesSec)); 21349566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &rootMatricesSec)); 21359566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(rootIndicesSec, pStartC, pEndC)); 21369566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(rootMatricesSec, pStartC, pEndC)); 21379566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(localCoarse, &numFields)); 2138713c1c5dSToby Isaac maxFields = PetscMax(1, numFields); 21399566063dSJacob Faibussowitsch PetscCall(PetscMalloc7(maxFields + 1, &offsets, maxFields + 1, &offsetsCopy, maxFields + 1, &newOffsets, maxFields + 1, &newOffsetsCopy, maxFields + 1, &rowOffsets, maxFields + 1, &numD, maxFields + 1, &numO)); 21409566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(maxFields + 1, (PetscInt ****)&perms, maxFields + 1, (PetscScalar ****)&flips)); 21419566063dSJacob Faibussowitsch PetscCall(PetscMemzero((void *)perms, (maxFields + 1) * sizeof(const PetscInt **))); 21429566063dSJacob Faibussowitsch PetscCall(PetscMemzero((void *)flips, (maxFields + 1) * sizeof(const PetscScalar **))); 214346bdb399SToby Isaac 214446bdb399SToby Isaac for (p = pStartC; p < pEndC; p++) { /* count the sizes of the indices and matrices */ 21458d2f55e7SToby Isaac PetscInt dof, matSize = 0; 21466ecaa68aSToby Isaac PetscInt aDof = 0; 21476ecaa68aSToby Isaac PetscInt cDof = 0; 21486ecaa68aSToby Isaac PetscInt maxChildId = maxChildIds[p - pStartC]; 21496ecaa68aSToby Isaac PetscInt numRowIndices = 0; 21506ecaa68aSToby Isaac PetscInt numColIndices = 0; 2151f13f9184SToby Isaac PetscInt f; 21526ecaa68aSToby Isaac 21539566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 2154ad540459SPierre Jolivet if (dof < 0) dof = -(dof + 1); 215548a46eb9SPierre Jolivet if (p >= aStart && p < aEnd) PetscCall(PetscSectionGetDof(aSec, p, &aDof)); 215648a46eb9SPierre Jolivet if (p >= cStart && p < cEnd) PetscCall(PetscSectionGetDof(cSec, p, &cDof)); 2157f13f9184SToby Isaac for (f = 0; f <= numFields; f++) offsets[f] = 0; 2158f13f9184SToby Isaac for (f = 0; f <= numFields; f++) newOffsets[f] = 0; 21596ecaa68aSToby Isaac if (maxChildId >= 0) { /* this point has children (with dofs) that will need to be interpolated from the closure of p */ 2160f13f9184SToby Isaac PetscInt *closure = NULL, closureSize, cl; 21616ecaa68aSToby Isaac 21629566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 216346bdb399SToby Isaac for (cl = 0; cl < closureSize; cl++) { /* get the closure */ 21646ecaa68aSToby Isaac PetscInt c = closure[2 * cl], clDof; 21656ecaa68aSToby Isaac 21669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, c, &clDof)); 21676ecaa68aSToby Isaac numRowIndices += clDof; 21686ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 21699566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, c, f, &clDof)); 21706ecaa68aSToby Isaac offsets[f + 1] += clDof; 21716ecaa68aSToby Isaac } 21726ecaa68aSToby Isaac } 21736ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 21746ecaa68aSToby Isaac offsets[f + 1] += offsets[f]; 21756ecaa68aSToby Isaac newOffsets[f + 1] = offsets[f + 1]; 21766ecaa68aSToby Isaac } 217746bdb399SToby Isaac /* get the number of indices needed and their field offsets */ 21789566063dSJacob Faibussowitsch PetscCall(DMPlexAnchorsModifyMat(coarse, localCoarse, closureSize, numRowIndices, closure, NULL, NULL, NULL, &numColIndices, NULL, NULL, newOffsets, PETSC_FALSE)); 21799566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 21806ecaa68aSToby Isaac if (!numColIndices) { /* there are no hanging constraint modifications, so the matrix is just the identity: do not send it */ 21816ecaa68aSToby Isaac numColIndices = numRowIndices; 21826ecaa68aSToby Isaac matSize = 0; 21839371c9d4SSatish Balay } else if (numFields) { /* we send one submat for each field: sum their sizes */ 21846ecaa68aSToby Isaac matSize = 0; 21856ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 21866ecaa68aSToby Isaac PetscInt numRow, numCol; 21876ecaa68aSToby Isaac 21886ecaa68aSToby Isaac numRow = offsets[f + 1] - offsets[f]; 2189f13f9184SToby Isaac numCol = newOffsets[f + 1] - newOffsets[f]; 21906ecaa68aSToby Isaac matSize += numRow * numCol; 21916ecaa68aSToby Isaac } 21929371c9d4SSatish Balay } else { 21936ecaa68aSToby Isaac matSize = numRowIndices * numColIndices; 21946ecaa68aSToby Isaac } 2195f13f9184SToby Isaac } else if (maxChildId == -1) { 21968d2f55e7SToby Isaac if (cDof > 0) { /* this point's dofs are interpolated via cMat: get the submatrix of cMat */ 2197f13f9184SToby Isaac PetscInt aOff, a; 21986ecaa68aSToby Isaac 21999566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(aSec, p, &aOff)); 22006ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 22016ecaa68aSToby Isaac PetscInt fDof; 22026ecaa68aSToby Isaac 22039566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 220421968bf8SToby Isaac offsets[f + 1] = fDof; 22056ecaa68aSToby Isaac } 22066ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 22076ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff], aLocalDof; 22086ecaa68aSToby Isaac 22099566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, anchor, &aLocalDof)); 22106ecaa68aSToby Isaac numColIndices += aLocalDof; 22116ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 22126ecaa68aSToby Isaac PetscInt fDof; 22136ecaa68aSToby Isaac 22149566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, anchor, f, &fDof)); 221521968bf8SToby Isaac newOffsets[f + 1] += fDof; 22166ecaa68aSToby Isaac } 22176ecaa68aSToby Isaac } 22186ecaa68aSToby Isaac if (numFields) { 22196ecaa68aSToby Isaac matSize = 0; 2220ad540459SPierre Jolivet for (f = 0; f < numFields; f++) matSize += offsets[f + 1] * newOffsets[f + 1]; 22219371c9d4SSatish Balay } else { 22226ecaa68aSToby Isaac matSize = numColIndices * dof; 22236ecaa68aSToby Isaac } 22249371c9d4SSatish Balay } else { /* no children, and no constraints on dofs: just get the global indices */ 22256ecaa68aSToby Isaac numColIndices = dof; 22266ecaa68aSToby Isaac matSize = 0; 22276ecaa68aSToby Isaac } 22288d2f55e7SToby Isaac } 222946bdb399SToby Isaac /* we will pack the column indices with the field offsets */ 22309566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(rootIndicesSec, p, numColIndices ? numColIndices + 2 * numFields : 0)); 22319566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(rootMatricesSec, p, matSize)); 22326ecaa68aSToby Isaac } 22339566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(rootIndicesSec)); 22349566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(rootMatricesSec)); 22356ecaa68aSToby Isaac { 22366ecaa68aSToby Isaac PetscInt numRootIndices, numRootMatrices; 22376ecaa68aSToby Isaac 22389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(rootIndicesSec, &numRootIndices)); 22399566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(rootMatricesSec, &numRootMatrices)); 22409566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numRootIndices, &rootIndices, numRootMatrices, &rootMatrices)); 22416ecaa68aSToby Isaac for (p = pStartC; p < pEndC; p++) { 22426ecaa68aSToby Isaac PetscInt numRowIndices, numColIndices, matSize, dof; 2243f13f9184SToby Isaac PetscInt pIndOff, pMatOff, f; 22446ecaa68aSToby Isaac PetscInt *pInd; 22456ecaa68aSToby Isaac PetscInt maxChildId = maxChildIds[p - pStartC]; 22466ecaa68aSToby Isaac PetscScalar *pMat = NULL; 22476ecaa68aSToby Isaac 22489566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootIndicesSec, p, &numColIndices)); 2249ad540459SPierre Jolivet if (!numColIndices) continue; 2250f13f9184SToby Isaac for (f = 0; f <= numFields; f++) { 2251f13f9184SToby Isaac offsets[f] = 0; 2252f13f9184SToby Isaac newOffsets[f] = 0; 2253f13f9184SToby Isaac offsetsCopy[f] = 0; 2254f13f9184SToby Isaac newOffsetsCopy[f] = 0; 2255f13f9184SToby Isaac } 22566ecaa68aSToby Isaac numColIndices -= 2 * numFields; 22579566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootIndicesSec, p, &pIndOff)); 22586ecaa68aSToby Isaac pInd = &(rootIndices[pIndOff]); 22599566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootMatricesSec, p, &matSize)); 22606ecaa68aSToby Isaac if (matSize) { 22619566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootMatricesSec, p, &pMatOff)); 22626ecaa68aSToby Isaac pMat = &rootMatrices[pMatOff]; 22636ecaa68aSToby Isaac } 22649566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 2265ad540459SPierre Jolivet if (dof < 0) dof = -(dof + 1); 22666ecaa68aSToby Isaac if (maxChildId >= 0) { /* build an identity matrix, apply matrix constraints on the right */ 22676ecaa68aSToby Isaac PetscInt i, j; 22686ecaa68aSToby Isaac PetscInt numRowIndices = matSize / numColIndices; 22696ecaa68aSToby Isaac 22706ecaa68aSToby Isaac if (!numRowIndices) { /* don't need to calculate the mat, just the indices */ 22716ecaa68aSToby Isaac PetscInt numIndices, *indices; 22729566063dSJacob Faibussowitsch PetscCall(DMPlexGetClosureIndices(coarse, localCoarse, globalCoarse, p, PETSC_TRUE, &numIndices, &indices, offsets, NULL)); 227308401ef6SPierre Jolivet PetscCheck(numIndices == numColIndices, PETSC_COMM_SELF, PETSC_ERR_PLIB, "mismatching constraint indices calculations"); 2274ad540459SPierre Jolivet for (i = 0; i < numColIndices; i++) pInd[i] = indices[i]; 22756ecaa68aSToby Isaac for (i = 0; i < numFields; i++) { 227646bdb399SToby Isaac pInd[numColIndices + i] = offsets[i + 1]; 227746bdb399SToby Isaac pInd[numColIndices + numFields + i] = offsets[i + 1]; 22786ecaa68aSToby Isaac } 22799566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreClosureIndices(coarse, localCoarse, globalCoarse, p, PETSC_TRUE, &numIndices, &indices, offsets, NULL)); 22809371c9d4SSatish Balay } else { 22816ecaa68aSToby Isaac PetscInt closureSize, *closure = NULL, cl; 22826ecaa68aSToby Isaac PetscScalar *pMatIn, *pMatModified; 22836ecaa68aSToby Isaac PetscInt numPoints, *points; 22846ecaa68aSToby Isaac 22859566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(coarse, numRowIndices * numRowIndices, MPIU_SCALAR, &pMatIn)); 22866ecaa68aSToby Isaac for (i = 0; i < numRowIndices; i++) { /* initialize to the identity */ 2287ad540459SPierre Jolivet for (j = 0; j < numRowIndices; j++) pMatIn[i * numRowIndices + j] = (i == j) ? 1. : 0.; 22886ecaa68aSToby Isaac } 22899566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 22904acb8e1eSToby Isaac for (f = 0; f < maxFields; f++) { 22919566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionGetFieldPointSyms(localCoarse, f, closureSize, closure, &perms[f], &flips[f])); 22929566063dSJacob Faibussowitsch else PetscCall(PetscSectionGetPointSyms(localCoarse, closureSize, closure, &perms[f], &flips[f])); 22934acb8e1eSToby Isaac } 22946ecaa68aSToby Isaac if (numFields) { 22956ecaa68aSToby Isaac for (cl = 0; cl < closureSize; cl++) { 22966ecaa68aSToby Isaac PetscInt c = closure[2 * cl]; 22976ecaa68aSToby Isaac 22986ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 22996ecaa68aSToby Isaac PetscInt fDof; 23006ecaa68aSToby Isaac 23019566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, c, f, &fDof)); 23026ecaa68aSToby Isaac offsets[f + 1] += fDof; 23036ecaa68aSToby Isaac } 23046ecaa68aSToby Isaac } 23056ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 23066ecaa68aSToby Isaac offsets[f + 1] += offsets[f]; 23076ecaa68aSToby Isaac newOffsets[f + 1] = offsets[f + 1]; 23086ecaa68aSToby Isaac } 23096ecaa68aSToby Isaac } 23104acb8e1eSToby Isaac /* TODO : flips here ? */ 23116ecaa68aSToby Isaac /* apply hanging node constraints on the right, get the new points and the new offsets */ 23129566063dSJacob Faibussowitsch PetscCall(DMPlexAnchorsModifyMat(coarse, localCoarse, closureSize, numRowIndices, closure, perms, pMatIn, &numPoints, NULL, &points, &pMatModified, newOffsets, PETSC_FALSE)); 23134acb8e1eSToby Isaac for (f = 0; f < maxFields; f++) { 23149566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionRestoreFieldPointSyms(localCoarse, f, closureSize, closure, &perms[f], &flips[f])); 23159566063dSJacob Faibussowitsch else PetscCall(PetscSectionRestorePointSyms(localCoarse, closureSize, closure, &perms[f], &flips[f])); 23164acb8e1eSToby Isaac } 23174acb8e1eSToby Isaac for (f = 0; f < maxFields; f++) { 23189566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionGetFieldPointSyms(localCoarse, f, numPoints, points, &perms[f], &flips[f])); 23199566063dSJacob Faibussowitsch else PetscCall(PetscSectionGetPointSyms(localCoarse, numPoints, points, &perms[f], &flips[f])); 23204acb8e1eSToby Isaac } 23216ecaa68aSToby Isaac if (!numFields) { 2322ad540459SPierre Jolivet for (i = 0; i < numRowIndices * numColIndices; i++) pMat[i] = pMatModified[i]; 23239371c9d4SSatish Balay } else { 2324f13f9184SToby Isaac PetscInt i, j, count; 23256ecaa68aSToby Isaac for (f = 0, count = 0; f < numFields; f++) { 23266ecaa68aSToby Isaac for (i = offsets[f]; i < offsets[f + 1]; i++) { 2327ad540459SPierre Jolivet for (j = newOffsets[f]; j < newOffsets[f + 1]; j++, count++) pMat[count] = pMatModified[i * numColIndices + j]; 23286ecaa68aSToby Isaac } 23296ecaa68aSToby Isaac } 23306ecaa68aSToby Isaac } 23319566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numRowIndices * numColIndices, MPIU_SCALAR, &pMatModified)); 23329566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 23339566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numRowIndices * numColIndices, MPIU_SCALAR, &pMatIn)); 23346ecaa68aSToby Isaac if (numFields) { 233546bdb399SToby Isaac for (f = 0; f < numFields; f++) { 233646bdb399SToby Isaac pInd[numColIndices + f] = offsets[f + 1]; 233746bdb399SToby Isaac pInd[numColIndices + numFields + f] = newOffsets[f + 1]; 23386ecaa68aSToby Isaac } 23394acb8e1eSToby Isaac for (cl = 0; cl < numPoints; cl++) { 23404acb8e1eSToby Isaac PetscInt globalOff, c = points[2 * cl]; 23419566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, c, &globalOff)); 23429566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, c, globalOff < 0 ? -(globalOff + 1) : globalOff, newOffsets, PETSC_FALSE, perms, cl, NULL, pInd)); 23436ecaa68aSToby Isaac } 23446ecaa68aSToby Isaac } else { 23454acb8e1eSToby Isaac for (cl = 0; cl < numPoints; cl++) { 23464acb8e1eSToby Isaac PetscInt c = points[2 * cl], globalOff; 23474acb8e1eSToby Isaac const PetscInt *perm = perms[0] ? perms[0][cl] : NULL; 23484acb8e1eSToby Isaac 23499566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, c, &globalOff)); 23509566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, c, globalOff < 0 ? -(globalOff + 1) : globalOff, newOffsets, PETSC_FALSE, perm, NULL, pInd)); 23516ecaa68aSToby Isaac } 23526ecaa68aSToby Isaac } 23534acb8e1eSToby Isaac for (f = 0; f < maxFields; f++) { 23549566063dSJacob Faibussowitsch if (numFields) PetscCall(PetscSectionRestoreFieldPointSyms(localCoarse, f, numPoints, points, &perms[f], &flips[f])); 23559566063dSJacob Faibussowitsch else PetscCall(PetscSectionRestorePointSyms(localCoarse, numPoints, points, &perms[f], &flips[f])); 23564acb8e1eSToby Isaac } 23579566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numPoints, MPIU_SCALAR, &points)); 23586ecaa68aSToby Isaac } 23599371c9d4SSatish Balay } else if (matSize) { 23606ecaa68aSToby Isaac PetscInt cOff; 23616ecaa68aSToby Isaac PetscInt *rowIndices, *colIndices, a, aDof, aOff; 23626ecaa68aSToby Isaac 23636ecaa68aSToby Isaac numRowIndices = matSize / numColIndices; 236408401ef6SPierre Jolivet PetscCheck(numRowIndices == dof, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Miscounted dofs"); 23659566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(coarse, numRowIndices, MPIU_INT, &rowIndices)); 23669566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(coarse, numColIndices, MPIU_INT, &colIndices)); 23679566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(cSec, p, &cOff)); 23689566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(aSec, p, &aDof)); 23699566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(aSec, p, &aOff)); 23706ecaa68aSToby Isaac if (numFields) { 23716ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 23726ecaa68aSToby Isaac PetscInt fDof; 2373f13f9184SToby Isaac 23749566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSec, p, f, &fDof)); 23756ecaa68aSToby Isaac offsets[f + 1] = fDof; 23766ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 23776ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff]; 23789566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, anchor, f, &fDof)); 23796ecaa68aSToby Isaac newOffsets[f + 1] += fDof; 23806ecaa68aSToby Isaac } 23816ecaa68aSToby Isaac } 23826ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 23836ecaa68aSToby Isaac offsets[f + 1] += offsets[f]; 23846ecaa68aSToby Isaac offsetsCopy[f + 1] = offsets[f + 1]; 23856ecaa68aSToby Isaac newOffsets[f + 1] += newOffsets[f]; 23866ecaa68aSToby Isaac newOffsetsCopy[f + 1] = newOffsets[f + 1]; 23876ecaa68aSToby Isaac } 23889566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, p, cOff, offsetsCopy, PETSC_TRUE, NULL, -1, NULL, rowIndices)); 23896ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 23906ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff], lOff; 23919566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(localCoarse, anchor, &lOff)); 23929566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_TRUE, anchor, lOff, newOffsetsCopy, PETSC_TRUE, NULL, -1, NULL, colIndices)); 23936ecaa68aSToby Isaac } 23949371c9d4SSatish Balay } else { 23959566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, p, cOff, offsetsCopy, PETSC_TRUE, NULL, NULL, rowIndices)); 23966ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 23976ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff], lOff; 23989566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(localCoarse, anchor, &lOff)); 23999566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_TRUE, anchor, lOff, newOffsetsCopy, PETSC_TRUE, NULL, NULL, colIndices)); 24006ecaa68aSToby Isaac } 24016ecaa68aSToby Isaac } 24026ecaa68aSToby Isaac if (numFields) { 2403f13f9184SToby Isaac PetscInt count, a; 2404f13f9184SToby Isaac 24056ecaa68aSToby Isaac for (f = 0, count = 0; f < numFields; f++) { 24066ecaa68aSToby Isaac PetscInt iSize = offsets[f + 1] - offsets[f]; 24076ecaa68aSToby Isaac PetscInt jSize = newOffsets[f + 1] - newOffsets[f]; 24089566063dSJacob Faibussowitsch PetscCall(MatGetValues(cMat, iSize, &rowIndices[offsets[f]], jSize, &colIndices[newOffsets[f]], &pMat[count])); 24096ecaa68aSToby Isaac count += iSize * jSize; 241046bdb399SToby Isaac pInd[numColIndices + f] = offsets[f + 1]; 241146bdb399SToby Isaac pInd[numColIndices + numFields + f] = newOffsets[f + 1]; 24126ecaa68aSToby Isaac } 24136ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 24146ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff]; 24156ecaa68aSToby Isaac PetscInt gOff; 24169566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, anchor, &gOff)); 24179566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, anchor, gOff < 0 ? -(gOff + 1) : gOff, newOffsets, PETSC_FALSE, NULL, -1, NULL, pInd)); 24186ecaa68aSToby Isaac } 24199371c9d4SSatish Balay } else { 24206ecaa68aSToby Isaac PetscInt a; 24219566063dSJacob Faibussowitsch PetscCall(MatGetValues(cMat, numRowIndices, rowIndices, numColIndices, colIndices, pMat)); 24226ecaa68aSToby Isaac for (a = 0; a < aDof; a++) { 24236ecaa68aSToby Isaac PetscInt anchor = anchors[a + aOff]; 24246ecaa68aSToby Isaac PetscInt gOff; 24259566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, anchor, &gOff)); 24269566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, anchor, gOff < 0 ? -(gOff + 1) : gOff, newOffsets, PETSC_FALSE, NULL, NULL, pInd)); 24276ecaa68aSToby Isaac } 24286ecaa68aSToby Isaac } 24299566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numColIndices, MPIU_INT, &colIndices)); 24309566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(coarse, numRowIndices, MPIU_INT, &rowIndices)); 24319371c9d4SSatish Balay } else { 24326ecaa68aSToby Isaac PetscInt gOff; 24336ecaa68aSToby Isaac 24349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, p, &gOff)); 24356ecaa68aSToby Isaac if (numFields) { 24366ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 24376ecaa68aSToby Isaac PetscInt fDof; 24389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 24396ecaa68aSToby Isaac offsets[f + 1] = fDof + offsets[f]; 24406ecaa68aSToby Isaac } 24416ecaa68aSToby Isaac for (f = 0; f < numFields; f++) { 244246bdb399SToby Isaac pInd[numColIndices + f] = offsets[f + 1]; 244346bdb399SToby Isaac pInd[numColIndices + numFields + f] = offsets[f + 1]; 24446ecaa68aSToby Isaac } 24459566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsets, PETSC_FALSE, NULL, -1, NULL, pInd)); 2446367003a6SStefano Zampini } else { 24479566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsets, PETSC_FALSE, NULL, NULL, pInd)); 24486ecaa68aSToby Isaac } 24496ecaa68aSToby Isaac } 24506ecaa68aSToby Isaac } 24519566063dSJacob Faibussowitsch PetscCall(PetscFree(maxChildIds)); 24526ecaa68aSToby Isaac } 245346bdb399SToby Isaac { 245446bdb399SToby Isaac PetscSF indicesSF, matricesSF; 245546bdb399SToby Isaac PetscInt *remoteOffsetsIndices, *remoteOffsetsMatrices, numLeafIndices, numLeafMatrices; 245646bdb399SToby Isaac 24579566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)fine), &leafIndicesSec)); 24589566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)fine), &leafMatricesSec)); 24599566063dSJacob Faibussowitsch PetscCall(PetscSFDistributeSection(coarseToFineEmbedded, rootIndicesSec, &remoteOffsetsIndices, leafIndicesSec)); 24609566063dSJacob Faibussowitsch PetscCall(PetscSFDistributeSection(coarseToFineEmbedded, rootMatricesSec, &remoteOffsetsMatrices, leafMatricesSec)); 24619566063dSJacob Faibussowitsch PetscCall(PetscSFCreateSectionSF(coarseToFineEmbedded, rootIndicesSec, remoteOffsetsIndices, leafIndicesSec, &indicesSF)); 24629566063dSJacob Faibussowitsch PetscCall(PetscSFCreateSectionSF(coarseToFineEmbedded, rootMatricesSec, remoteOffsetsMatrices, leafMatricesSec, &matricesSF)); 24639566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 24649566063dSJacob Faibussowitsch PetscCall(PetscFree(remoteOffsetsIndices)); 24659566063dSJacob Faibussowitsch PetscCall(PetscFree(remoteOffsetsMatrices)); 24669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(leafIndicesSec, &numLeafIndices)); 24679566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(leafMatricesSec, &numLeafMatrices)); 24689566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(numLeafIndices, &leafIndices, numLeafMatrices, &leafMatrices)); 24699566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(indicesSF, MPIU_INT, rootIndices, leafIndices, MPI_REPLACE)); 24709566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(matricesSF, MPIU_SCALAR, rootMatrices, leafMatrices, MPI_REPLACE)); 24719566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(indicesSF, MPIU_INT, rootIndices, leafIndices, MPI_REPLACE)); 24729566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(matricesSF, MPIU_SCALAR, rootMatrices, leafMatrices, MPI_REPLACE)); 24739566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&matricesSF)); 24749566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&indicesSF)); 24759566063dSJacob Faibussowitsch PetscCall(PetscFree2(rootIndices, rootMatrices)); 24769566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootIndicesSec)); 24779566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootMatricesSec)); 247846bdb399SToby Isaac } 247946bdb399SToby Isaac /* count to preallocate */ 24809566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 248146bdb399SToby Isaac { 248246bdb399SToby Isaac PetscInt nGlobal; 248346bdb399SToby Isaac PetscInt *dnnz, *onnz; 2484b9a5774bSToby Isaac PetscLayout rowMap, colMap; 2485b9a5774bSToby Isaac PetscInt rowStart, rowEnd, colStart, colEnd; 24861c58ffc4SToby Isaac PetscInt maxDof; 24871c58ffc4SToby Isaac PetscInt *rowIndices; 24881c58ffc4SToby Isaac DM refTree; 24891c58ffc4SToby Isaac PetscInt **refPointFieldN; 24901c58ffc4SToby Isaac PetscScalar ***refPointFieldMats; 24911c58ffc4SToby Isaac PetscSection refConSec, refAnSec; 24920eb7e1eaSToby Isaac PetscInt pRefStart, pRefEnd, maxConDof, maxColumns, leafStart, leafEnd; 24931c58ffc4SToby Isaac PetscScalar *pointWork; 249446bdb399SToby Isaac 24959566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstrainedStorageSize(globalFine, &nGlobal)); 24969566063dSJacob Faibussowitsch PetscCall(PetscCalloc2(nGlobal, &dnnz, nGlobal, &onnz)); 24979566063dSJacob Faibussowitsch PetscCall(MatGetLayouts(mat, &rowMap, &colMap)); 24989566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(rowMap)); 24999566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(colMap)); 25009566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(rowMap, &rowStart, &rowEnd)); 25019566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(colMap, &colStart, &colEnd)); 25029566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localFine, &maxDof)); 25039566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(leafIndicesSec, &leafStart, &leafEnd)); 25049566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(fine, maxDof, MPIU_INT, &rowIndices)); 25050eb7e1eaSToby Isaac for (p = leafStart; p < leafEnd; p++) { 250646bdb399SToby Isaac PetscInt gDof, gcDof, gOff; 250746bdb399SToby Isaac PetscInt numColIndices, pIndOff, *pInd; 250846bdb399SToby Isaac PetscInt matSize; 250921968bf8SToby Isaac PetscInt i; 251046bdb399SToby Isaac 25119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &gDof)); 25129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &gcDof)); 2513ad540459SPierre Jolivet if ((gDof - gcDof) <= 0) continue; 25149566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalFine, p, &gOff)); 251508401ef6SPierre Jolivet PetscCheck(gOff >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "I though having global dofs meant a non-negative offset"); 25161dca8a05SBarry Smith PetscCheck(gOff >= rowStart && (gOff + gDof - gcDof) <= rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "I thought the row map would constrain the global dofs"); 25179566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafIndicesSec, p, &numColIndices)); 25189566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafIndicesSec, p, &pIndOff)); 251946bdb399SToby Isaac numColIndices -= 2 * numFields; 252008401ef6SPierre Jolivet PetscCheck(numColIndices > 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "global fine dof with no dofs to interpolate from"); 252146bdb399SToby Isaac pInd = &leafIndices[pIndOff]; 252221968bf8SToby Isaac offsets[0] = 0; 252321968bf8SToby Isaac offsetsCopy[0] = 0; 252421968bf8SToby Isaac newOffsets[0] = 0; 252521968bf8SToby Isaac newOffsetsCopy[0] = 0; 252646bdb399SToby Isaac if (numFields) { 252721968bf8SToby Isaac PetscInt f; 252846bdb399SToby Isaac for (f = 0; f < numFields; f++) { 252946bdb399SToby Isaac PetscInt rowDof; 253046bdb399SToby Isaac 25319566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localFine, p, f, &rowDof)); 253221968bf8SToby Isaac offsets[f + 1] = offsets[f] + rowDof; 253321968bf8SToby Isaac offsetsCopy[f + 1] = offsets[f + 1]; 253421968bf8SToby Isaac newOffsets[f + 1] = pInd[numColIndices + numFields + f]; 253521968bf8SToby Isaac numD[f] = 0; 253621968bf8SToby Isaac numO[f] = 0; 253746bdb399SToby Isaac } 25389566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, rowIndices)); 253946bdb399SToby Isaac for (f = 0; f < numFields; f++) { 254021968bf8SToby Isaac PetscInt colOffset = newOffsets[f]; 254121968bf8SToby Isaac PetscInt numFieldCols = newOffsets[f + 1] - newOffsets[f]; 254246bdb399SToby Isaac 254346bdb399SToby Isaac for (i = 0; i < numFieldCols; i++) { 254446bdb399SToby Isaac PetscInt gInd = pInd[i + colOffset]; 254546bdb399SToby Isaac 254646bdb399SToby Isaac if (gInd >= colStart && gInd < colEnd) { 254721968bf8SToby Isaac numD[f]++; 25489371c9d4SSatish Balay } else if (gInd >= 0) { /* negative means non-entry */ 254921968bf8SToby Isaac numO[f]++; 255046bdb399SToby Isaac } 255146bdb399SToby Isaac } 255246bdb399SToby Isaac } 25539371c9d4SSatish Balay } else { 25549566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, rowIndices)); 255521968bf8SToby Isaac numD[0] = 0; 255621968bf8SToby Isaac numO[0] = 0; 255746bdb399SToby Isaac for (i = 0; i < numColIndices; i++) { 255846bdb399SToby Isaac PetscInt gInd = pInd[i]; 255946bdb399SToby Isaac 256046bdb399SToby Isaac if (gInd >= colStart && gInd < colEnd) { 256121968bf8SToby Isaac numD[0]++; 25629371c9d4SSatish Balay } else if (gInd >= 0) { /* negative means non-entry */ 256321968bf8SToby Isaac numO[0]++; 256446bdb399SToby Isaac } 256546bdb399SToby Isaac } 256646bdb399SToby Isaac } 25679566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafMatricesSec, p, &matSize)); 256846bdb399SToby Isaac if (!matSize) { /* incoming matrix is identity */ 256946bdb399SToby Isaac PetscInt childId; 257046bdb399SToby Isaac 257146bdb399SToby Isaac childId = childIds[p - pStartF]; 257221968bf8SToby Isaac if (childId < 0) { /* no child interpolation: one nnz per */ 257346bdb399SToby Isaac if (numFields) { 2574b9a5774bSToby Isaac PetscInt f; 2575b9a5774bSToby Isaac for (f = 0; f < numFields; f++) { 257621968bf8SToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f], row; 257746bdb399SToby Isaac for (row = 0; row < numRows; row++) { 257821968bf8SToby Isaac PetscInt gIndCoarse = pInd[newOffsets[f] + row]; 257921968bf8SToby Isaac PetscInt gIndFine = rowIndices[offsets[f] + row]; 258046bdb399SToby Isaac if (gIndCoarse >= colStart && gIndCoarse < colEnd) { /* local */ 25811dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2582b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = 1; 25839371c9d4SSatish Balay } else if (gIndCoarse >= 0) { /* remote */ 25841dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2585b9a5774bSToby Isaac onnz[gIndFine - rowStart] = 1; 25869371c9d4SSatish Balay } else { /* constrained */ 258708401ef6SPierre Jolivet PetscCheck(gIndFine < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 258846bdb399SToby Isaac } 258946bdb399SToby Isaac } 259046bdb399SToby Isaac } 25919371c9d4SSatish Balay } else { 2592b9a5774bSToby Isaac PetscInt i; 2593b9a5774bSToby Isaac for (i = 0; i < gDof; i++) { 259446bdb399SToby Isaac PetscInt gIndCoarse = pInd[i]; 259546bdb399SToby Isaac PetscInt gIndFine = rowIndices[i]; 259646bdb399SToby Isaac if (gIndCoarse >= colStart && gIndCoarse < colEnd) { /* local */ 25971dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2598b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = 1; 25999371c9d4SSatish Balay } else if (gIndCoarse >= 0) { /* remote */ 26001dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2601b9a5774bSToby Isaac onnz[gIndFine - rowStart] = 1; 26029371c9d4SSatish Balay } else { /* constrained */ 260308401ef6SPierre Jolivet PetscCheck(gIndFine < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 260446bdb399SToby Isaac } 260546bdb399SToby Isaac } 260646bdb399SToby Isaac } 26079371c9d4SSatish Balay } else { /* interpolate from all */ 260846bdb399SToby Isaac if (numFields) { 2609b9a5774bSToby Isaac PetscInt f; 2610b9a5774bSToby Isaac for (f = 0; f < numFields; f++) { 261121968bf8SToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f], row; 261246bdb399SToby Isaac for (row = 0; row < numRows; row++) { 261321968bf8SToby Isaac PetscInt gIndFine = rowIndices[offsets[f] + row]; 261446bdb399SToby Isaac if (gIndFine >= 0) { 26151dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2616b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = numD[f]; 2617b9a5774bSToby Isaac onnz[gIndFine - rowStart] = numO[f]; 261846bdb399SToby Isaac } 261946bdb399SToby Isaac } 262046bdb399SToby Isaac } 26219371c9d4SSatish Balay } else { 2622b9a5774bSToby Isaac PetscInt i; 2623b9a5774bSToby Isaac for (i = 0; i < gDof; i++) { 262446bdb399SToby Isaac PetscInt gIndFine = rowIndices[i]; 262546bdb399SToby Isaac if (gIndFine >= 0) { 26261dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2627b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = numD[0]; 2628b9a5774bSToby Isaac onnz[gIndFine - rowStart] = numO[0]; 262946bdb399SToby Isaac } 263046bdb399SToby Isaac } 263146bdb399SToby Isaac } 263246bdb399SToby Isaac } 26339371c9d4SSatish Balay } else { /* interpolate from all */ 263446bdb399SToby Isaac if (numFields) { 2635b9a5774bSToby Isaac PetscInt f; 2636b9a5774bSToby Isaac for (f = 0; f < numFields; f++) { 263721968bf8SToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f], row; 263846bdb399SToby Isaac for (row = 0; row < numRows; row++) { 263921968bf8SToby Isaac PetscInt gIndFine = rowIndices[offsets[f] + row]; 264046bdb399SToby Isaac if (gIndFine >= 0) { 26411dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2642b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = numD[f]; 2643b9a5774bSToby Isaac onnz[gIndFine - rowStart] = numO[f]; 264446bdb399SToby Isaac } 264546bdb399SToby Isaac } 264646bdb399SToby Isaac } 26479371c9d4SSatish Balay } else { /* every dof get a full row */ 2648b9a5774bSToby Isaac PetscInt i; 2649b9a5774bSToby Isaac for (i = 0; i < gDof; i++) { 265046bdb399SToby Isaac PetscInt gIndFine = rowIndices[i]; 265146bdb399SToby Isaac if (gIndFine >= 0) { 26521dca8a05SBarry Smith PetscCheck(gIndFine >= rowStart && gIndFine < rowEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched number of constrained dofs"); 2653b9a5774bSToby Isaac dnnz[gIndFine - rowStart] = numD[0]; 2654b9a5774bSToby Isaac onnz[gIndFine - rowStart] = numO[0]; 265546bdb399SToby Isaac } 265646bdb399SToby Isaac } 265746bdb399SToby Isaac } 265846bdb399SToby Isaac } 265946bdb399SToby Isaac } 26609566063dSJacob Faibussowitsch PetscCall(MatXAIJSetPreallocation(mat, 1, dnnz, onnz, NULL, NULL)); 26619566063dSJacob Faibussowitsch PetscCall(PetscFree2(dnnz, onnz)); 266221968bf8SToby Isaac 26639566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(fine, &refTree)); 2664*d3a532e9SStefano Zampini PetscCall(DMCopyDisc(fine, refTree)); 2665*d3a532e9SStefano Zampini PetscCall(DMSetLocalSection(refTree, NULL)); 2666*d3a532e9SStefano Zampini PetscCall(DMSetDefaultConstraints(refTree, NULL, NULL, NULL)); 26679566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 26689566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 26699566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(refTree, &refAnSec, NULL)); 26709566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 26719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refConSec, &maxConDof)); 26729566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(leafIndicesSec, &maxColumns)); 26739566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxConDof * maxColumns, &pointWork)); 26740eb7e1eaSToby Isaac for (p = leafStart; p < leafEnd; p++) { 2675e44e4e7fSToby Isaac PetscInt gDof, gcDof, gOff; 2676e44e4e7fSToby Isaac PetscInt numColIndices, pIndOff, *pInd; 2677e44e4e7fSToby Isaac PetscInt matSize; 2678e44e4e7fSToby Isaac PetscInt childId; 2679e44e4e7fSToby Isaac 26809566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &gDof)); 26819566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &gcDof)); 2682ad540459SPierre Jolivet if ((gDof - gcDof) <= 0) continue; 2683e44e4e7fSToby Isaac childId = childIds[p - pStartF]; 26849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalFine, p, &gOff)); 26859566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafIndicesSec, p, &numColIndices)); 26869566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafIndicesSec, p, &pIndOff)); 2687e44e4e7fSToby Isaac numColIndices -= 2 * numFields; 2688e44e4e7fSToby Isaac pInd = &leafIndices[pIndOff]; 2689e44e4e7fSToby Isaac offsets[0] = 0; 2690e44e4e7fSToby Isaac offsetsCopy[0] = 0; 2691e44e4e7fSToby Isaac newOffsets[0] = 0; 2692e44e4e7fSToby Isaac newOffsetsCopy[0] = 0; 2693e44e4e7fSToby Isaac rowOffsets[0] = 0; 2694e44e4e7fSToby Isaac if (numFields) { 2695e44e4e7fSToby Isaac PetscInt f; 2696e44e4e7fSToby Isaac for (f = 0; f < numFields; f++) { 2697e44e4e7fSToby Isaac PetscInt rowDof; 2698e44e4e7fSToby Isaac 26999566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localFine, p, f, &rowDof)); 2700e44e4e7fSToby Isaac offsets[f + 1] = offsets[f] + rowDof; 2701e44e4e7fSToby Isaac offsetsCopy[f + 1] = offsets[f + 1]; 2702e44e4e7fSToby Isaac rowOffsets[f + 1] = pInd[numColIndices + f]; 2703e44e4e7fSToby Isaac newOffsets[f + 1] = pInd[numColIndices + numFields + f]; 2704e44e4e7fSToby Isaac } 27059566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, rowIndices)); 27069371c9d4SSatish Balay } else { 27079566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, rowIndices)); 27081c58ffc4SToby Isaac } 27099566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafMatricesSec, p, &matSize)); 2710e44e4e7fSToby Isaac if (!matSize) { /* incoming matrix is identity */ 2711e44e4e7fSToby Isaac if (childId < 0) { /* no child interpolation: scatter */ 2712e44e4e7fSToby Isaac if (numFields) { 2713e44e4e7fSToby Isaac PetscInt f; 2714e44e4e7fSToby Isaac for (f = 0; f < numFields; f++) { 2715e44e4e7fSToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f], row; 271648a46eb9SPierre Jolivet for (row = 0; row < numRows; row++) PetscCall(MatSetValue(mat, rowIndices[offsets[f] + row], pInd[newOffsets[f] + row], 1., INSERT_VALUES)); 271721968bf8SToby Isaac } 27189371c9d4SSatish Balay } else { 2719e44e4e7fSToby Isaac PetscInt numRows = gDof, row; 272048a46eb9SPierre Jolivet for (row = 0; row < numRows; row++) PetscCall(MatSetValue(mat, rowIndices[row], pInd[row], 1., INSERT_VALUES)); 2721e44e4e7fSToby Isaac } 27229371c9d4SSatish Balay } else { /* interpolate from all */ 2723e44e4e7fSToby Isaac if (numFields) { 2724e44e4e7fSToby Isaac PetscInt f; 2725e44e4e7fSToby Isaac for (f = 0; f < numFields; f++) { 2726e44e4e7fSToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f]; 2727e44e4e7fSToby Isaac PetscInt numCols = newOffsets[f + 1] - newOffsets[f]; 27289566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numRows, &rowIndices[offsets[f]], numCols, &pInd[newOffsets[f]], refPointFieldMats[childId - pRefStart][f], INSERT_VALUES)); 2729e44e4e7fSToby Isaac } 27309371c9d4SSatish Balay } else { 27319566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, gDof, rowIndices, numColIndices, pInd, refPointFieldMats[childId - pRefStart][0], INSERT_VALUES)); 2732e44e4e7fSToby Isaac } 2733e44e4e7fSToby Isaac } 27349371c9d4SSatish Balay } else { /* interpolate from all */ 2735e44e4e7fSToby Isaac PetscInt pMatOff; 2736e44e4e7fSToby Isaac PetscScalar *pMat; 2737e44e4e7fSToby Isaac 27389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafMatricesSec, p, &pMatOff)); 2739e44e4e7fSToby Isaac pMat = &leafMatrices[pMatOff]; 2740e44e4e7fSToby Isaac if (childId < 0) { /* copy the incoming matrix */ 2741e44e4e7fSToby Isaac if (numFields) { 2742e44e4e7fSToby Isaac PetscInt f, count; 2743e44e4e7fSToby Isaac for (f = 0, count = 0; f < numFields; f++) { 2744e44e4e7fSToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f]; 2745e44e4e7fSToby Isaac PetscInt numCols = newOffsets[f + 1] - newOffsets[f]; 2746e44e4e7fSToby Isaac PetscInt numInRows = rowOffsets[f + 1] - rowOffsets[f]; 2747e44e4e7fSToby Isaac PetscScalar *inMat = &pMat[count]; 2748e44e4e7fSToby Isaac 27499566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numRows, &rowIndices[offsets[f]], numCols, &pInd[newOffsets[f]], inMat, INSERT_VALUES)); 2750e44e4e7fSToby Isaac count += numCols * numInRows; 2751e44e4e7fSToby Isaac } 27529371c9d4SSatish Balay } else { 27539566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, gDof, rowIndices, numColIndices, pInd, pMat, INSERT_VALUES)); 2754e44e4e7fSToby Isaac } 27559371c9d4SSatish Balay } else { /* multiply the incoming matrix by the child interpolation */ 2756e44e4e7fSToby Isaac if (numFields) { 2757e44e4e7fSToby Isaac PetscInt f, count; 2758e44e4e7fSToby Isaac for (f = 0, count = 0; f < numFields; f++) { 2759e44e4e7fSToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f]; 2760e44e4e7fSToby Isaac PetscInt numCols = newOffsets[f + 1] - newOffsets[f]; 2761e44e4e7fSToby Isaac PetscInt numInRows = rowOffsets[f + 1] - rowOffsets[f]; 2762e44e4e7fSToby Isaac PetscScalar *inMat = &pMat[count]; 2763e44e4e7fSToby Isaac PetscInt i, j, k; 276408401ef6SPierre Jolivet PetscCheck(refPointFieldN[childId - pRefStart][f] == numInRows, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point constraint matrix multiply dimension mismatch"); 2765e44e4e7fSToby Isaac for (i = 0; i < numRows; i++) { 2766e44e4e7fSToby Isaac for (j = 0; j < numCols; j++) { 2767e44e4e7fSToby Isaac PetscScalar val = 0.; 2768ad540459SPierre Jolivet for (k = 0; k < numInRows; k++) val += refPointFieldMats[childId - pRefStart][f][i * numInRows + k] * inMat[k * numCols + j]; 2769e44e4e7fSToby Isaac pointWork[i * numCols + j] = val; 2770e44e4e7fSToby Isaac } 2771e44e4e7fSToby Isaac } 27729566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numRows, &rowIndices[offsets[f]], numCols, &pInd[newOffsets[f]], pointWork, INSERT_VALUES)); 2773e44e4e7fSToby Isaac count += numCols * numInRows; 2774e44e4e7fSToby Isaac } 27759371c9d4SSatish Balay } else { /* every dof gets a full row */ 2776e44e4e7fSToby Isaac PetscInt numRows = gDof; 2777e44e4e7fSToby Isaac PetscInt numCols = numColIndices; 2778e44e4e7fSToby Isaac PetscInt numInRows = matSize / numColIndices; 2779e44e4e7fSToby Isaac PetscInt i, j, k; 278008401ef6SPierre Jolivet PetscCheck(refPointFieldN[childId - pRefStart][0] == numInRows, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point constraint matrix multiply dimension mismatch"); 2781e44e4e7fSToby Isaac for (i = 0; i < numRows; i++) { 2782e44e4e7fSToby Isaac for (j = 0; j < numCols; j++) { 2783e44e4e7fSToby Isaac PetscScalar val = 0.; 2784ad540459SPierre Jolivet for (k = 0; k < numInRows; k++) val += refPointFieldMats[childId - pRefStart][0][i * numInRows + k] * pMat[k * numCols + j]; 2785e44e4e7fSToby Isaac pointWork[i * numCols + j] = val; 2786e44e4e7fSToby Isaac } 2787e44e4e7fSToby Isaac } 27889566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numRows, rowIndices, numCols, pInd, pointWork, INSERT_VALUES)); 2789e44e4e7fSToby Isaac } 2790e44e4e7fSToby Isaac } 2791e44e4e7fSToby Isaac } 2792e44e4e7fSToby Isaac } 27939566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 27949566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(fine, maxDof, MPIU_INT, &rowIndices)); 27959566063dSJacob Faibussowitsch PetscCall(PetscFree(pointWork)); 2796e44e4e7fSToby Isaac } 27979566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(mat, MAT_FINAL_ASSEMBLY)); 27989566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(mat, MAT_FINAL_ASSEMBLY)); 27999566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&leafIndicesSec)); 28009566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&leafMatricesSec)); 28019566063dSJacob Faibussowitsch PetscCall(PetscFree2(leafIndices, leafMatrices)); 28029566063dSJacob Faibussowitsch PetscCall(PetscFree2(*(PetscInt ****)&perms, *(PetscScalar ****)&flips)); 28039566063dSJacob Faibussowitsch PetscCall(PetscFree7(offsets, offsetsCopy, newOffsets, newOffsetsCopy, rowOffsets, numD, numO)); 28049566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(aIS, &anchors)); 28053ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 28066ecaa68aSToby Isaac } 2807154bca37SToby Isaac 28088d2f55e7SToby Isaac /* 28098d2f55e7SToby Isaac * Assuming a nodal basis (w.r.t. the dual basis) basis: 28108d2f55e7SToby Isaac * 28118d2f55e7SToby Isaac * for each coarse dof \phi^c_i: 28128d2f55e7SToby Isaac * for each quadrature point (w_l,x_l) in the dual basis definition of \phi^c_i: 28138d2f55e7SToby Isaac * for each fine dof \phi^f_j; 28148d2f55e7SToby Isaac * a_{i,j} = 0; 28158d2f55e7SToby Isaac * for each fine dof \phi^f_k: 28168d2f55e7SToby Isaac * a_{i,j} += interp_{i,k} * \phi^f_k(x_l) * \phi^f_j(x_l) * w_l 28178d2f55e7SToby Isaac * [^^^ this is = \phi^c_i ^^^] 28188d2f55e7SToby Isaac */ 2819d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeInjectorReferenceTree(DM refTree, Mat *inj) 2820d71ae5a4SJacob Faibussowitsch { 28218d2f55e7SToby Isaac PetscDS ds; 28228d2f55e7SToby Isaac PetscSection section, cSection; 28238d2f55e7SToby Isaac DMLabel canonical, depth; 28248d2f55e7SToby Isaac Mat cMat, mat; 28258d2f55e7SToby Isaac PetscInt *nnz; 28268d2f55e7SToby Isaac PetscInt f, dim, numFields, numSecFields, p, pStart, pEnd, cStart, cEnd; 28278d2f55e7SToby Isaac PetscInt m, n; 28288d2f55e7SToby Isaac PetscScalar *pointScalar; 28298d2f55e7SToby Isaac PetscReal *v0, *v0parent, *vtmp, *J, *Jparent, *invJ, *pointRef, detJ, detJparent; 28308d2f55e7SToby Isaac 28318d2f55e7SToby Isaac PetscFunctionBegin; 28329566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(refTree, §ion)); 28339566063dSJacob Faibussowitsch PetscCall(DMGetDimension(refTree, &dim)); 28349566063dSJacob Faibussowitsch PetscCall(PetscMalloc6(dim, &v0, dim, &v0parent, dim, &vtmp, dim * dim, &J, dim * dim, &Jparent, dim * dim, &invJ)); 28359566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(dim, &pointScalar, dim, &pointRef)); 28369566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 28379566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 28389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(section, &numSecFields)); 28399566063dSJacob Faibussowitsch PetscCall(DMGetLabel(refTree, "canonical", &canonical)); 28409566063dSJacob Faibussowitsch PetscCall(DMGetLabel(refTree, "depth", &depth)); 28419566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &cSection, &cMat, NULL)); 28429566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(refTree, &pStart, &pEnd)); 28439566063dSJacob Faibussowitsch PetscCall(DMPlexGetHeightStratum(refTree, 0, &cStart, &cEnd)); 28449566063dSJacob Faibussowitsch PetscCall(MatGetSize(cMat, &n, &m)); /* the injector has transpose sizes from the constraint matrix */ 28458d2f55e7SToby Isaac /* Step 1: compute non-zero pattern. A proper subset of constraint matrix non-zero */ 28469566063dSJacob Faibussowitsch PetscCall(PetscCalloc1(m, &nnz)); 28478d2f55e7SToby Isaac for (p = pStart; p < pEnd; p++) { /* a point will have non-zeros if it is canonical, it has dofs, and its children have dofs */ 28488d2f55e7SToby Isaac const PetscInt *children; 28498d2f55e7SToby Isaac PetscInt numChildren; 28508d2f55e7SToby Isaac PetscInt i, numChildDof, numSelfDof; 28518d2f55e7SToby Isaac 28528d2f55e7SToby Isaac if (canonical) { 28538d2f55e7SToby Isaac PetscInt pCanonical; 28549566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonical, p, &pCanonical)); 28558d2f55e7SToby Isaac if (p != pCanonical) continue; 28568d2f55e7SToby Isaac } 28579566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(refTree, p, &numChildren, &children)); 28588d2f55e7SToby Isaac if (!numChildren) continue; 28598d2f55e7SToby Isaac for (i = 0, numChildDof = 0; i < numChildren; i++) { 28608d2f55e7SToby Isaac PetscInt child = children[i]; 28618d2f55e7SToby Isaac PetscInt dof; 28628d2f55e7SToby Isaac 28639566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, child, &dof)); 28648d2f55e7SToby Isaac numChildDof += dof; 28658d2f55e7SToby Isaac } 28669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &numSelfDof)); 28678d2f55e7SToby Isaac if (!numChildDof || !numSelfDof) continue; 28688d2f55e7SToby Isaac for (f = 0; f < numFields; f++) { 28698d2f55e7SToby Isaac PetscInt selfOff; 28708d2f55e7SToby Isaac 28718d2f55e7SToby Isaac if (numSecFields) { /* count the dofs for just this field */ 28728d2f55e7SToby Isaac for (i = 0, numChildDof = 0; i < numChildren; i++) { 28738d2f55e7SToby Isaac PetscInt child = children[i]; 28748d2f55e7SToby Isaac PetscInt dof; 28758d2f55e7SToby Isaac 28769566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, child, f, &dof)); 28778d2f55e7SToby Isaac numChildDof += dof; 28788d2f55e7SToby Isaac } 28799566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, p, f, &numSelfDof)); 28809566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(section, p, f, &selfOff)); 28819371c9d4SSatish Balay } else { 28829566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, p, &selfOff)); 28838d2f55e7SToby Isaac } 2884ad540459SPierre Jolivet for (i = 0; i < numSelfDof; i++) nnz[selfOff + i] = numChildDof; 28858d2f55e7SToby Isaac } 28868d2f55e7SToby Isaac } 28879566063dSJacob Faibussowitsch PetscCall(MatCreateAIJ(PETSC_COMM_SELF, m, n, m, n, -1, nnz, -1, NULL, &mat)); 28889566063dSJacob Faibussowitsch PetscCall(PetscFree(nnz)); 28898d2f55e7SToby Isaac /* Setp 2: compute entries */ 28908d2f55e7SToby Isaac for (p = pStart; p < pEnd; p++) { 28918d2f55e7SToby Isaac const PetscInt *children; 28928d2f55e7SToby Isaac PetscInt numChildren; 28938d2f55e7SToby Isaac PetscInt i, numChildDof, numSelfDof; 28948d2f55e7SToby Isaac 28958d2f55e7SToby Isaac /* same conditions about when entries occur */ 28968d2f55e7SToby Isaac if (canonical) { 28978d2f55e7SToby Isaac PetscInt pCanonical; 28989566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(canonical, p, &pCanonical)); 28998d2f55e7SToby Isaac if (p != pCanonical) continue; 29008d2f55e7SToby Isaac } 29019566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeChildren(refTree, p, &numChildren, &children)); 29028d2f55e7SToby Isaac if (!numChildren) continue; 29038d2f55e7SToby Isaac for (i = 0, numChildDof = 0; i < numChildren; i++) { 29048d2f55e7SToby Isaac PetscInt child = children[i]; 29058d2f55e7SToby Isaac PetscInt dof; 29068d2f55e7SToby Isaac 29079566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, child, &dof)); 29088d2f55e7SToby Isaac numChildDof += dof; 29098d2f55e7SToby Isaac } 29109566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(section, p, &numSelfDof)); 29118d2f55e7SToby Isaac if (!numChildDof || !numSelfDof) continue; 29128d2f55e7SToby Isaac 29138d2f55e7SToby Isaac for (f = 0; f < numFields; f++) { 291459fc6756SToby Isaac PetscInt pI = -1, cI = -1; 291552a3aeb4SToby Isaac PetscInt selfOff, Nc, parentCell; 29168d2f55e7SToby Isaac PetscInt cellShapeOff; 29178d2f55e7SToby Isaac PetscObject disc; 29188d2f55e7SToby Isaac PetscDualSpace dsp; 29198d2f55e7SToby Isaac PetscClassId classId; 29208d2f55e7SToby Isaac PetscScalar *pointMat; 29213b1c2a6aSToby Isaac PetscInt *matRows, *matCols; 29228d2f55e7SToby Isaac PetscInt pO = PETSC_MIN_INT; 29238d2f55e7SToby Isaac const PetscInt *depthNumDof; 29248d2f55e7SToby Isaac 29258d2f55e7SToby Isaac if (numSecFields) { 29268d2f55e7SToby Isaac for (i = 0, numChildDof = 0; i < numChildren; i++) { 29278d2f55e7SToby Isaac PetscInt child = children[i]; 29288d2f55e7SToby Isaac PetscInt dof; 29298d2f55e7SToby Isaac 29309566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, child, f, &dof)); 29318d2f55e7SToby Isaac numChildDof += dof; 29328d2f55e7SToby Isaac } 29339566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(section, p, f, &numSelfDof)); 29349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(section, p, f, &selfOff)); 29359371c9d4SSatish Balay } else { 29369566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(section, p, &selfOff)); 29378d2f55e7SToby Isaac } 29388d2f55e7SToby Isaac 29393b1c2a6aSToby Isaac /* find a cell whose closure contains p */ 29408d2f55e7SToby Isaac if (p >= cStart && p < cEnd) { 29418d2f55e7SToby Isaac parentCell = p; 29429371c9d4SSatish Balay } else { 29438d2f55e7SToby Isaac PetscInt *star = NULL; 29448d2f55e7SToby Isaac PetscInt numStar; 29458d2f55e7SToby Isaac 29468d2f55e7SToby Isaac parentCell = -1; 29479566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, p, PETSC_FALSE, &numStar, &star)); 29488d2f55e7SToby Isaac for (i = numStar - 1; i >= 0; i--) { 29498d2f55e7SToby Isaac PetscInt c = star[2 * i]; 29508d2f55e7SToby Isaac 29518d2f55e7SToby Isaac if (c >= cStart && c < cEnd) { 29528d2f55e7SToby Isaac parentCell = c; 29538d2f55e7SToby Isaac break; 29548d2f55e7SToby Isaac } 29558d2f55e7SToby Isaac } 29569566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, p, PETSC_FALSE, &numStar, &star)); 29578d2f55e7SToby Isaac } 2958a5b23f4aSJose E. Roman /* determine the offset of p's shape functions within parentCell's shape functions */ 29599566063dSJacob Faibussowitsch PetscCall(PetscDSGetDiscretization(ds, f, &disc)); 29609566063dSJacob Faibussowitsch PetscCall(PetscObjectGetClassId(disc, &classId)); 2961c5356c36SToby Isaac if (classId == PETSCFE_CLASSID) { 29629566063dSJacob Faibussowitsch PetscCall(PetscFEGetDualSpace((PetscFE)disc, &dsp)); 29639371c9d4SSatish Balay } else if (classId == PETSCFV_CLASSID) { 29649566063dSJacob Faibussowitsch PetscCall(PetscFVGetDualSpace((PetscFV)disc, &dsp)); 29659371c9d4SSatish Balay } else { 29669b90b7cdSMatthew G. Knepley SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Unsupported discretization object"); 2967c5356c36SToby Isaac } 29689566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetNumDof(dsp, &depthNumDof)); 29699566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetNumComponents(dsp, &Nc)); 29708d2f55e7SToby Isaac { 29718d2f55e7SToby Isaac PetscInt *closure = NULL; 29728d2f55e7SToby Isaac PetscInt numClosure; 29738d2f55e7SToby Isaac 29749566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, parentCell, PETSC_TRUE, &numClosure, &closure)); 297559fc6756SToby Isaac for (i = 0, pI = -1, cellShapeOff = 0; i < numClosure; i++) { 29768d2f55e7SToby Isaac PetscInt point = closure[2 * i], pointDepth; 29778d2f55e7SToby Isaac 29788d2f55e7SToby Isaac pO = closure[2 * i + 1]; 297959fc6756SToby Isaac if (point == p) { 298059fc6756SToby Isaac pI = i; 298159fc6756SToby Isaac break; 298259fc6756SToby Isaac } 29839566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(depth, point, &pointDepth)); 29848d2f55e7SToby Isaac cellShapeOff += depthNumDof[pointDepth]; 29858d2f55e7SToby Isaac } 29869566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, parentCell, PETSC_TRUE, &numClosure, &closure)); 29878d2f55e7SToby Isaac } 29888d2f55e7SToby Isaac 29899566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(refTree, numSelfDof * numChildDof, MPIU_SCALAR, &pointMat)); 29909566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(refTree, numSelfDof + numChildDof, MPIU_INT, &matRows)); 299152a3aeb4SToby Isaac matCols = matRows + numSelfDof; 2992ad540459SPierre Jolivet for (i = 0; i < numSelfDof; i++) matRows[i] = selfOff + i; 299352a3aeb4SToby Isaac for (i = 0; i < numSelfDof * numChildDof; i++) pointMat[i] = 0.; 29943b1c2a6aSToby Isaac { 29953b1c2a6aSToby Isaac PetscInt colOff = 0; 29963b1c2a6aSToby Isaac 29973b1c2a6aSToby Isaac for (i = 0; i < numChildren; i++) { 29983b1c2a6aSToby Isaac PetscInt child = children[i]; 29993b1c2a6aSToby Isaac PetscInt dof, off, j; 30003b1c2a6aSToby Isaac 30013b1c2a6aSToby Isaac if (numSecFields) { 30029566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSection, child, f, &dof)); 30039566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(cSection, child, f, &off)); 30049371c9d4SSatish Balay } else { 30059566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSection, child, &dof)); 30069566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(cSection, child, &off)); 30073b1c2a6aSToby Isaac } 30083b1c2a6aSToby Isaac 3009ad540459SPierre Jolivet for (j = 0; j < dof; j++) matCols[colOff++] = off + j; 30103b1c2a6aSToby Isaac } 30113b1c2a6aSToby Isaac } 30128d2f55e7SToby Isaac if (classId == PETSCFE_CLASSID) { 30138d2f55e7SToby Isaac PetscFE fe = (PetscFE)disc; 30148d2f55e7SToby Isaac PetscInt fSize; 301559fc6756SToby Isaac const PetscInt ***perms; 301659fc6756SToby Isaac const PetscScalar ***flips; 301759fc6756SToby Isaac const PetscInt *pperms; 301859fc6756SToby Isaac 30199566063dSJacob Faibussowitsch PetscCall(PetscFEGetDualSpace(fe, &dsp)); 30209566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetDimension(dsp, &fSize)); 30219566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetSymmetries(dsp, &perms, &flips)); 302259fc6756SToby Isaac pperms = perms ? perms[pI] ? perms[pI][pO] : NULL : NULL; 302352a3aeb4SToby Isaac for (i = 0; i < numSelfDof; i++) { /* for every shape function */ 30248d2f55e7SToby Isaac PetscQuadrature q; 302552a3aeb4SToby Isaac PetscInt dim, thisNc, numPoints, j, k; 30268d2f55e7SToby Isaac const PetscReal *points; 30278d2f55e7SToby Isaac const PetscReal *weights; 30288d2f55e7SToby Isaac PetscInt *closure = NULL; 30298d2f55e7SToby Isaac PetscInt numClosure; 303059fc6756SToby Isaac PetscInt iCell = pperms ? pperms[i] : i; 303159fc6756SToby Isaac PetscInt parentCellShapeDof = cellShapeOff + iCell; 3032ef0bb6c7SMatthew G. Knepley PetscTabulation Tparent; 30338d2f55e7SToby Isaac 30349566063dSJacob Faibussowitsch PetscCall(PetscDualSpaceGetFunctional(dsp, parentCellShapeDof, &q)); 30359566063dSJacob Faibussowitsch PetscCall(PetscQuadratureGetData(q, &dim, &thisNc, &numPoints, &points, &weights)); 303663a3b9bcSJacob Faibussowitsch PetscCheck(thisNc == Nc, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Functional dim %" PetscInt_FMT " does not much basis dim %" PetscInt_FMT, thisNc, Nc); 30379566063dSJacob Faibussowitsch PetscCall(PetscFECreateTabulation(fe, 1, numPoints, points, 0, &Tparent)); /* I'm expecting a nodal basis: weights[:]' * Bparent[:,cellShapeDof] = 1. */ 30383b1c2a6aSToby Isaac for (j = 0; j < numPoints; j++) { 30398d2f55e7SToby Isaac PetscInt childCell = -1; 304052a3aeb4SToby Isaac PetscReal *parentValAtPoint; 3041c330f8ffSToby Isaac const PetscReal xi0[3] = {-1., -1., -1.}; 30428d2f55e7SToby Isaac const PetscReal *pointReal = &points[dim * j]; 30438d2f55e7SToby Isaac const PetscScalar *point; 3044ef0bb6c7SMatthew G. Knepley PetscTabulation Tchild; 30458d2f55e7SToby Isaac PetscInt childCellShapeOff, pointMatOff; 30468d2f55e7SToby Isaac #if defined(PETSC_USE_COMPLEX) 30478d2f55e7SToby Isaac PetscInt d; 30488d2f55e7SToby Isaac 3049ad540459SPierre Jolivet for (d = 0; d < dim; d++) pointScalar[d] = points[dim * j + d]; 30508d2f55e7SToby Isaac point = pointScalar; 30518d2f55e7SToby Isaac #else 30528d2f55e7SToby Isaac point = pointReal; 30538d2f55e7SToby Isaac #endif 30548d2f55e7SToby Isaac 3055ef0bb6c7SMatthew G. Knepley parentValAtPoint = &Tparent->T[0][(fSize * j + parentCellShapeDof) * Nc]; 30563b1c2a6aSToby Isaac 30573b1c2a6aSToby Isaac for (k = 0; k < numChildren; k++) { /* locate the point in a child's star cell*/ 30588d2f55e7SToby Isaac PetscInt child = children[k]; 30598d2f55e7SToby Isaac PetscInt *star = NULL; 30608d2f55e7SToby Isaac PetscInt numStar, s; 30618d2f55e7SToby Isaac 30629566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, child, PETSC_FALSE, &numStar, &star)); 30638d2f55e7SToby Isaac for (s = numStar - 1; s >= 0; s--) { 30648d2f55e7SToby Isaac PetscInt c = star[2 * s]; 30658d2f55e7SToby Isaac 30668d2f55e7SToby Isaac if (c < cStart || c >= cEnd) continue; 30679566063dSJacob Faibussowitsch PetscCall(DMPlexLocatePoint_Internal(refTree, dim, point, c, &childCell)); 30688d2f55e7SToby Isaac if (childCell >= 0) break; 30698d2f55e7SToby Isaac } 30709566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, child, PETSC_FALSE, &numStar, &star)); 30718d2f55e7SToby Isaac if (childCell >= 0) break; 30728d2f55e7SToby Isaac } 307308401ef6SPierre Jolivet PetscCheck(childCell >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not locate quadrature point"); 30749566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(refTree, childCell, NULL, v0, J, invJ, &detJ)); 30759566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFEM(refTree, parentCell, NULL, v0parent, Jparent, NULL, &detJparent)); 3076c330f8ffSToby Isaac CoordinatesRefToReal(dim, dim, xi0, v0parent, Jparent, pointReal, vtmp); 3077c330f8ffSToby Isaac CoordinatesRealToRef(dim, dim, xi0, v0, invJ, vtmp, pointRef); 30788d2f55e7SToby Isaac 30799566063dSJacob Faibussowitsch PetscCall(PetscFECreateTabulation(fe, 1, 1, pointRef, 0, &Tchild)); 30809566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(refTree, childCell, PETSC_TRUE, &numClosure, &closure)); 30813b1c2a6aSToby Isaac for (k = 0, pointMatOff = 0; k < numChildren; k++) { /* point is located in cell => child dofs support at point are in closure of cell */ 3082c5356c36SToby Isaac PetscInt child = children[k], childDepth, childDof, childO = PETSC_MIN_INT; 30838d2f55e7SToby Isaac PetscInt l; 308459fc6756SToby Isaac const PetscInt *cperms; 30858d2f55e7SToby Isaac 30869566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(depth, child, &childDepth)); 30878d2f55e7SToby Isaac childDof = depthNumDof[childDepth]; 308859fc6756SToby Isaac for (l = 0, cI = -1, childCellShapeOff = 0; l < numClosure; l++) { 30898d2f55e7SToby Isaac PetscInt point = closure[2 * l]; 30908d2f55e7SToby Isaac PetscInt pointDepth; 30918d2f55e7SToby Isaac 30928d2f55e7SToby Isaac childO = closure[2 * l + 1]; 309359fc6756SToby Isaac if (point == child) { 309459fc6756SToby Isaac cI = l; 309559fc6756SToby Isaac break; 309659fc6756SToby Isaac } 30979566063dSJacob Faibussowitsch PetscCall(DMLabelGetValue(depth, point, &pointDepth)); 30988d2f55e7SToby Isaac childCellShapeOff += depthNumDof[pointDepth]; 30998d2f55e7SToby Isaac } 31008d2f55e7SToby Isaac if (l == numClosure) { 31018d2f55e7SToby Isaac pointMatOff += childDof; 31028d2f55e7SToby Isaac continue; /* child is not in the closure of the cell: has nothing to contribute to this point */ 31038d2f55e7SToby Isaac } 310459fc6756SToby Isaac cperms = perms ? perms[cI] ? perms[cI][childO] : NULL : NULL; 31058d2f55e7SToby Isaac for (l = 0; l < childDof; l++) { 310659fc6756SToby Isaac PetscInt lCell = cperms ? cperms[l] : l; 310759fc6756SToby Isaac PetscInt childCellDof = childCellShapeOff + lCell; 310852a3aeb4SToby Isaac PetscReal *childValAtPoint; 310952a3aeb4SToby Isaac PetscReal val = 0.; 31108d2f55e7SToby Isaac 3111ef0bb6c7SMatthew G. Knepley childValAtPoint = &Tchild->T[0][childCellDof * Nc]; 3112ad540459SPierre Jolivet for (m = 0; m < Nc; m++) val += weights[j * Nc + m] * parentValAtPoint[m] * childValAtPoint[m]; 311352a3aeb4SToby Isaac 311452a3aeb4SToby Isaac pointMat[i * numChildDof + pointMatOff + l] += val; 31158d2f55e7SToby Isaac } 31168d2f55e7SToby Isaac pointMatOff += childDof; 31178d2f55e7SToby Isaac } 31189566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(refTree, childCell, PETSC_TRUE, &numClosure, &closure)); 31199566063dSJacob Faibussowitsch PetscCall(PetscTabulationDestroy(&Tchild)); 31208d2f55e7SToby Isaac } 31219566063dSJacob Faibussowitsch PetscCall(PetscTabulationDestroy(&Tparent)); 31228d2f55e7SToby Isaac } 31239371c9d4SSatish Balay } else { /* just the volume-weighted averages of the children */ 31243b1c2a6aSToby Isaac PetscReal parentVol; 3125bfaa5bdcSToby Isaac PetscInt childCell; 31263b1c2a6aSToby Isaac 31279566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFVM(refTree, p, &parentVol, NULL, NULL)); 3128bfaa5bdcSToby Isaac for (i = 0, childCell = 0; i < numChildren; i++) { 312952a3aeb4SToby Isaac PetscInt child = children[i], j; 31303b1c2a6aSToby Isaac PetscReal childVol; 31313b1c2a6aSToby Isaac 31323b1c2a6aSToby Isaac if (child < cStart || child >= cEnd) continue; 31339566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFVM(refTree, child, &childVol, NULL, NULL)); 3134ad540459SPierre Jolivet for (j = 0; j < Nc; j++) pointMat[j * numChildDof + Nc * childCell + j] = childVol / parentVol; 3135bfaa5bdcSToby Isaac childCell++; 31363b1c2a6aSToby Isaac } 31378d2f55e7SToby Isaac } 31383b1c2a6aSToby Isaac /* Insert pointMat into mat */ 31399566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, numSelfDof, matRows, numChildDof, matCols, pointMat, INSERT_VALUES)); 31409566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(refTree, numSelfDof + numChildDof, MPIU_INT, &matRows)); 31419566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(refTree, numSelfDof * numChildDof, MPIU_SCALAR, &pointMat)); 31428d2f55e7SToby Isaac } 31438d2f55e7SToby Isaac } 31449566063dSJacob Faibussowitsch PetscCall(PetscFree6(v0, v0parent, vtmp, J, Jparent, invJ)); 31459566063dSJacob Faibussowitsch PetscCall(PetscFree2(pointScalar, pointRef)); 31469566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(mat, MAT_FINAL_ASSEMBLY)); 31479566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(mat, MAT_FINAL_ASSEMBLY)); 31488d2f55e7SToby Isaac *inj = mat; 31493ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 31508d2f55e7SToby Isaac } 31518d2f55e7SToby Isaac 3152d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexReferenceTreeGetChildrenMatrices_Injection(DM refTree, Mat inj, PetscScalar ****childrenMats) 3153d71ae5a4SJacob Faibussowitsch { 3154f30e825dSToby Isaac PetscDS ds; 3155f30e825dSToby Isaac PetscInt numFields, f, pRefStart, pRefEnd, p, *rows, *cols, maxDof; 3156f30e825dSToby Isaac PetscScalar ***refPointFieldMats; 3157f30e825dSToby Isaac PetscSection refConSec, refSection; 3158f30e825dSToby Isaac 3159f30e825dSToby Isaac PetscFunctionBegin; 31609566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 31619566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 31629566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 31639566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(refTree, &refSection)); 31649566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 31659566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pRefEnd - pRefStart, &refPointFieldMats)); 31669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(refConSec, &maxDof)); 31679566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof, &rows)); 31689566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof * maxDof, &cols)); 3169f30e825dSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 3170f30e825dSToby Isaac PetscInt parent, pDof, parentDof; 3171f30e825dSToby Isaac 31729566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, p, &parent, NULL)); 31739566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &pDof)); 31749566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, parent, &parentDof)); 3175f30e825dSToby Isaac if (!pDof || !parentDof || parent == p) continue; 3176f30e825dSToby Isaac 31779566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numFields, &refPointFieldMats[p - pRefStart])); 3178f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 317952a3aeb4SToby Isaac PetscInt cDof, cOff, numCols, r; 3180f30e825dSToby Isaac 3181f30e825dSToby Isaac if (numFields > 1) { 31829566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refConSec, p, f, &cDof)); 31839566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refConSec, p, f, &cOff)); 31849371c9d4SSatish Balay } else { 31859566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &cDof)); 31869566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refConSec, p, &cOff)); 3187f30e825dSToby Isaac } 3188f30e825dSToby Isaac 3189ad540459SPierre Jolivet for (r = 0; r < cDof; r++) rows[r] = cOff + r; 3190f30e825dSToby Isaac numCols = 0; 3191f30e825dSToby Isaac { 3192f30e825dSToby Isaac PetscInt aDof, aOff, j; 3193f30e825dSToby Isaac 3194f30e825dSToby Isaac if (numFields > 1) { 31959566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refSection, parent, f, &aDof)); 31969566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldOffset(refSection, parent, f, &aOff)); 31979371c9d4SSatish Balay } else { 31989566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, parent, &aDof)); 31999566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(refSection, parent, &aOff)); 3200f30e825dSToby Isaac } 3201f30e825dSToby Isaac 3202ad540459SPierre Jolivet for (j = 0; j < aDof; j++) cols[numCols++] = aOff + j; 3203f30e825dSToby Isaac } 32049566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(cDof * numCols, &refPointFieldMats[p - pRefStart][f])); 3205f30e825dSToby Isaac /* transpose of constraint matrix */ 32069566063dSJacob Faibussowitsch PetscCall(MatGetValues(inj, numCols, cols, cDof, rows, refPointFieldMats[p - pRefStart][f])); 3207f30e825dSToby Isaac } 3208f30e825dSToby Isaac } 3209f30e825dSToby Isaac *childrenMats = refPointFieldMats; 32109566063dSJacob Faibussowitsch PetscCall(PetscFree(rows)); 32119566063dSJacob Faibussowitsch PetscCall(PetscFree(cols)); 32123ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 3213f30e825dSToby Isaac } 3214f30e825dSToby Isaac 3215d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexReferenceTreeRestoreChildrenMatrices_Injection(DM refTree, Mat inj, PetscScalar ****childrenMats) 3216d71ae5a4SJacob Faibussowitsch { 3217f30e825dSToby Isaac PetscDS ds; 3218f30e825dSToby Isaac PetscScalar ***refPointFieldMats; 3219f30e825dSToby Isaac PetscInt numFields, pRefStart, pRefEnd, p, f; 3220c6154584SToby Isaac PetscSection refConSec, refSection; 3221f30e825dSToby Isaac 3222f30e825dSToby Isaac PetscFunctionBegin; 3223f30e825dSToby Isaac refPointFieldMats = *childrenMats; 3224f30e825dSToby Isaac *childrenMats = NULL; 32259566063dSJacob Faibussowitsch PetscCall(DMGetDS(refTree, &ds)); 32269566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(refTree, &refSection)); 32279566063dSJacob Faibussowitsch PetscCall(PetscDSGetNumFields(ds, &numFields)); 32289566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 32299566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 3230f30e825dSToby Isaac for (p = pRefStart; p < pRefEnd; p++) { 3231f30e825dSToby Isaac PetscInt parent, pDof, parentDof; 3232f30e825dSToby Isaac 32339566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, p, &parent, NULL)); 32349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &pDof)); 32359566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refSection, parent, &parentDof)); 3236f30e825dSToby Isaac if (!pDof || !parentDof || parent == p) continue; 3237f30e825dSToby Isaac 3238f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 3239f30e825dSToby Isaac PetscInt cDof; 3240f30e825dSToby Isaac 3241f30e825dSToby Isaac if (numFields > 1) { 32429566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(refConSec, p, f, &cDof)); 32439371c9d4SSatish Balay } else { 32449566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(refConSec, p, &cDof)); 3245f30e825dSToby Isaac } 3246f30e825dSToby Isaac 32479566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats[p - pRefStart][f])); 3248f30e825dSToby Isaac } 32499566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats[p - pRefStart])); 3250f30e825dSToby Isaac } 32519566063dSJacob Faibussowitsch PetscCall(PetscFree(refPointFieldMats)); 32523ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 3253f30e825dSToby Isaac } 3254f30e825dSToby Isaac 3255d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexReferenceTreeGetInjector(DM refTree, Mat *injRef) 3256d71ae5a4SJacob Faibussowitsch { 3257ebf164c7SToby Isaac Mat cMatRef; 32586148253fSToby Isaac PetscObject injRefObj; 32598d2f55e7SToby Isaac 3260154bca37SToby Isaac PetscFunctionBegin; 32619566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, NULL, &cMatRef, NULL)); 32629566063dSJacob Faibussowitsch PetscCall(PetscObjectQuery((PetscObject)cMatRef, "DMPlexComputeInjectorTree_refTree", &injRefObj)); 3263ebf164c7SToby Isaac *injRef = (Mat)injRefObj; 3264ebf164c7SToby Isaac if (!*injRef) { 32659566063dSJacob Faibussowitsch PetscCall(DMPlexComputeInjectorReferenceTree(refTree, injRef)); 32669566063dSJacob Faibussowitsch PetscCall(PetscObjectCompose((PetscObject)cMatRef, "DMPlexComputeInjectorTree_refTree", (PetscObject)*injRef)); 3267ec92bd66SToby Isaac /* there is now a reference in cMatRef, which should be the only one for symmetry with the above case */ 32689566063dSJacob Faibussowitsch PetscCall(PetscObjectDereference((PetscObject)*injRef)); 3269ebf164c7SToby Isaac } 32703ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 32716148253fSToby Isaac } 3272f30e825dSToby Isaac 3273d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexTransferInjectorTree(DM coarse, DM fine, PetscSF coarseToFine, const PetscInt *childIds, Vec fineVec, PetscInt numFields, PetscInt *offsets, PetscSection *rootMultiSec, PetscSection *multiLeafSec, PetscInt **gatheredIndices, PetscScalar **gatheredValues) 3274d71ae5a4SJacob Faibussowitsch { 3275c921d74cSToby Isaac PetscInt pStartF, pEndF, pStartC, pEndC, p, maxDof, numMulti; 3276ebf164c7SToby Isaac PetscSection globalCoarse, globalFine; 3277ebf164c7SToby Isaac PetscSection localCoarse, localFine, leafIndicesSec; 3278c921d74cSToby Isaac PetscSection multiRootSec, rootIndicesSec; 3279c921d74cSToby Isaac PetscInt *leafInds, *rootInds = NULL; 3280c921d74cSToby Isaac const PetscInt *rootDegrees; 3281c921d74cSToby Isaac PetscScalar *leafVals = NULL, *rootVals = NULL; 3282ebf164c7SToby Isaac PetscSF coarseToFineEmbedded; 3283ebf164c7SToby Isaac 3284ebf164c7SToby Isaac PetscFunctionBegin; 32859566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 32869566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 32879566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 32889566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 32899566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)fine), &leafIndicesSec)); 32909566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(leafIndicesSec, pStartF, pEndF)); 32919566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localFine, &maxDof)); 32928d2f55e7SToby Isaac { /* winnow fine points that don't have global dofs out of the sf */ 32937e96bdafSToby Isaac PetscInt l, nleaves, dof, cdof, numPointsWithDofs, offset, *pointsWithDofs, numIndices; 32947e96bdafSToby Isaac const PetscInt *leaves; 32958d2f55e7SToby Isaac 32969566063dSJacob Faibussowitsch PetscCall(PetscSFGetGraph(coarseToFine, NULL, &nleaves, &leaves, NULL)); 32977e96bdafSToby Isaac for (l = 0, numPointsWithDofs = 0; l < nleaves; l++) { 32987e96bdafSToby Isaac p = leaves ? leaves[l] : l; 32999566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 33009566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 33018d2f55e7SToby Isaac if ((dof - cdof) > 0) { 33028d2f55e7SToby Isaac numPointsWithDofs++; 3303f30e825dSToby Isaac 33049566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localFine, p, &dof)); 33059566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(leafIndicesSec, p, dof + 1)); 33068d2f55e7SToby Isaac } 33078d2f55e7SToby Isaac } 33089566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numPointsWithDofs, &pointsWithDofs)); 33099566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(leafIndicesSec)); 33109566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(leafIndicesSec, &numIndices)); 33119566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(gatheredIndices ? numIndices : (maxDof + 1), &leafInds)); 33129566063dSJacob Faibussowitsch if (gatheredValues) PetscCall(PetscMalloc1(numIndices, &leafVals)); 33137e96bdafSToby Isaac for (l = 0, offset = 0; l < nleaves; l++) { 33147e96bdafSToby Isaac p = leaves ? leaves[l] : l; 33159566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 33169566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 33178d2f55e7SToby Isaac if ((dof - cdof) > 0) { 3318f30e825dSToby Isaac PetscInt off, gOff; 3319f30e825dSToby Isaac PetscInt *pInd; 3320c921d74cSToby Isaac PetscScalar *pVal = NULL; 3321f30e825dSToby Isaac 33227e96bdafSToby Isaac pointsWithDofs[offset++] = l; 3323f30e825dSToby Isaac 33249566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafIndicesSec, p, &off)); 3325f30e825dSToby Isaac 3326c921d74cSToby Isaac pInd = gatheredIndices ? (&leafInds[off + 1]) : leafInds; 3327c921d74cSToby Isaac if (gatheredValues) { 3328c921d74cSToby Isaac PetscInt i; 3329c921d74cSToby Isaac 3330c921d74cSToby Isaac pVal = &leafVals[off + 1]; 3331c921d74cSToby Isaac for (i = 0; i < dof; i++) pVal[i] = 0.; 3332c921d74cSToby Isaac } 33339566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalFine, p, &gOff)); 3334f30e825dSToby Isaac 3335f30e825dSToby Isaac offsets[0] = 0; 3336f30e825dSToby Isaac if (numFields) { 3337f30e825dSToby Isaac PetscInt f; 3338f30e825dSToby Isaac 3339f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 3340f30e825dSToby Isaac PetscInt fDof; 33419566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localFine, p, f, &fDof)); 3342f30e825dSToby Isaac offsets[f + 1] = fDof + offsets[f]; 3343f30e825dSToby Isaac } 33449566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localFine, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsets, PETSC_FALSE, NULL, -1, NULL, pInd)); 3345367003a6SStefano Zampini } else { 33469566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localFine, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsets, PETSC_FALSE, NULL, NULL, pInd)); 3347f30e825dSToby Isaac } 33489566063dSJacob Faibussowitsch if (gatheredValues) PetscCall(VecGetValues(fineVec, dof, pInd, pVal)); 33498d2f55e7SToby Isaac } 33508d2f55e7SToby Isaac } 33519566063dSJacob Faibussowitsch PetscCall(PetscSFCreateEmbeddedLeafSF(coarseToFine, numPointsWithDofs, pointsWithDofs, &coarseToFineEmbedded)); 33529566063dSJacob Faibussowitsch PetscCall(PetscFree(pointsWithDofs)); 33538d2f55e7SToby Isaac } 3354f30e825dSToby Isaac 33559566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 33569566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 33579566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 3358f30e825dSToby Isaac 33596148253fSToby Isaac { /* there may be the case where an sf root has a parent: broadcast parents back to children */ 33606148253fSToby Isaac MPI_Datatype threeInt; 33616148253fSToby Isaac PetscMPIInt rank; 33626148253fSToby Isaac PetscInt(*parentNodeAndIdCoarse)[3]; 33636148253fSToby Isaac PetscInt(*parentNodeAndIdFine)[3]; 33646148253fSToby Isaac PetscInt p, nleaves, nleavesToParents; 33656148253fSToby Isaac PetscSF pointSF, sfToParents; 33666148253fSToby Isaac const PetscInt *ilocal; 33676148253fSToby Isaac const PetscSFNode *iremote; 33686148253fSToby Isaac PetscSFNode *iremoteToParents; 33696148253fSToby Isaac PetscInt *ilocalToParents; 33706148253fSToby Isaac 33719566063dSJacob Faibussowitsch PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)coarse), &rank)); 33729566063dSJacob Faibussowitsch PetscCallMPI(MPI_Type_contiguous(3, MPIU_INT, &threeInt)); 33739566063dSJacob Faibussowitsch PetscCallMPI(MPI_Type_commit(&threeInt)); 33749566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(pEndC - pStartC, &parentNodeAndIdCoarse, pEndF - pStartF, &parentNodeAndIdFine)); 33759566063dSJacob Faibussowitsch PetscCall(DMGetPointSF(coarse, &pointSF)); 33769566063dSJacob Faibussowitsch PetscCall(PetscSFGetGraph(pointSF, NULL, &nleaves, &ilocal, &iremote)); 33776148253fSToby Isaac for (p = pStartC; p < pEndC; p++) { 33786148253fSToby Isaac PetscInt parent, childId; 33799566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(coarse, p, &parent, &childId)); 33806148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][0] = rank; 33816148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][1] = parent - pStartC; 33826148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][2] = (p == parent) ? -1 : childId; 33836148253fSToby Isaac if (nleaves > 0) { 33846148253fSToby Isaac PetscInt leaf = -1; 33856148253fSToby Isaac 33866148253fSToby Isaac if (ilocal) { 33879566063dSJacob Faibussowitsch PetscCall(PetscFindInt(parent, nleaves, ilocal, &leaf)); 33889371c9d4SSatish Balay } else { 33896148253fSToby Isaac leaf = p - pStartC; 33906148253fSToby Isaac } 33916148253fSToby Isaac if (leaf >= 0) { 33926148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][0] = iremote[leaf].rank; 33936148253fSToby Isaac parentNodeAndIdCoarse[p - pStartC][1] = iremote[leaf].index; 33946148253fSToby Isaac } 33956148253fSToby Isaac } 33966148253fSToby Isaac } 33976148253fSToby Isaac for (p = pStartF; p < pEndF; p++) { 33986148253fSToby Isaac parentNodeAndIdFine[p - pStartF][0] = -1; 33996148253fSToby Isaac parentNodeAndIdFine[p - pStartF][1] = -1; 34006148253fSToby Isaac parentNodeAndIdFine[p - pStartF][2] = -1; 34016148253fSToby Isaac } 34029566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(coarseToFineEmbedded, threeInt, parentNodeAndIdCoarse, parentNodeAndIdFine, MPI_REPLACE)); 34039566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(coarseToFineEmbedded, threeInt, parentNodeAndIdCoarse, parentNodeAndIdFine, MPI_REPLACE)); 34046148253fSToby Isaac for (p = pStartF, nleavesToParents = 0; p < pEndF; p++) { 3405f30e825dSToby Isaac PetscInt dof; 3406f30e825dSToby Isaac 34079566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafIndicesSec, p, &dof)); 3408f30e825dSToby Isaac if (dof) { 3409f30e825dSToby Isaac PetscInt off; 3410f30e825dSToby Isaac 34119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafIndicesSec, p, &off)); 3412c921d74cSToby Isaac if (gatheredIndices) { 3413c921d74cSToby Isaac leafInds[off] = PetscMax(childIds[p - pStartF], parentNodeAndIdFine[p - pStartF][2]); 3414c921d74cSToby Isaac } else if (gatheredValues) { 3415c921d74cSToby Isaac leafVals[off] = (PetscScalar)PetscMax(childIds[p - pStartF], parentNodeAndIdFine[p - pStartF][2]); 3416c921d74cSToby Isaac } 3417f30e825dSToby Isaac } 3418ad540459SPierre Jolivet if (parentNodeAndIdFine[p - pStartF][0] >= 0) nleavesToParents++; 34196148253fSToby Isaac } 34209566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(nleavesToParents, &ilocalToParents)); 34219566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(nleavesToParents, &iremoteToParents)); 34226148253fSToby Isaac for (p = pStartF, nleavesToParents = 0; p < pEndF; p++) { 34236148253fSToby Isaac if (parentNodeAndIdFine[p - pStartF][0] >= 0) { 34246148253fSToby Isaac ilocalToParents[nleavesToParents] = p - pStartF; 34256148253fSToby Isaac iremoteToParents[nleavesToParents].rank = parentNodeAndIdFine[p - pStartF][0]; 34266148253fSToby Isaac iremoteToParents[nleavesToParents].index = parentNodeAndIdFine[p - pStartF][1]; 34276148253fSToby Isaac nleavesToParents++; 34286148253fSToby Isaac } 34296148253fSToby Isaac } 34309566063dSJacob Faibussowitsch PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)coarse), &sfToParents)); 34319566063dSJacob Faibussowitsch PetscCall(PetscSFSetGraph(sfToParents, pEndC - pStartC, nleavesToParents, ilocalToParents, PETSC_OWN_POINTER, iremoteToParents, PETSC_OWN_POINTER)); 34329566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 34336148253fSToby Isaac 34346148253fSToby Isaac coarseToFineEmbedded = sfToParents; 34356148253fSToby Isaac 34369566063dSJacob Faibussowitsch PetscCall(PetscFree2(parentNodeAndIdCoarse, parentNodeAndIdFine)); 34379566063dSJacob Faibussowitsch PetscCallMPI(MPI_Type_free(&threeInt)); 34386148253fSToby Isaac } 3439f30e825dSToby Isaac 34406148253fSToby Isaac { /* winnow out coarse points that don't have dofs */ 34416148253fSToby Isaac PetscInt dof, cdof, numPointsWithDofs, offset, *pointsWithDofs; 34426148253fSToby Isaac PetscSF sfDofsOnly; 34436148253fSToby Isaac 34446148253fSToby Isaac for (p = pStartC, numPointsWithDofs = 0; p < pEndC; p++) { 34459566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 34469566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 3447ad540459SPierre Jolivet if ((dof - cdof) > 0) numPointsWithDofs++; 34486148253fSToby Isaac } 34499566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numPointsWithDofs, &pointsWithDofs)); 34506148253fSToby Isaac for (p = pStartC, offset = 0; p < pEndC; p++) { 34519566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 34529566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 3453ad540459SPierre Jolivet if ((dof - cdof) > 0) pointsWithDofs[offset++] = p - pStartC; 34546148253fSToby Isaac } 34559566063dSJacob Faibussowitsch PetscCall(PetscSFCreateEmbeddedRootSF(coarseToFineEmbedded, numPointsWithDofs, pointsWithDofs, &sfDofsOnly)); 34569566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 34579566063dSJacob Faibussowitsch PetscCall(PetscFree(pointsWithDofs)); 34586148253fSToby Isaac coarseToFineEmbedded = sfDofsOnly; 34596148253fSToby Isaac } 3460f30e825dSToby Isaac 34616148253fSToby Isaac /* communicate back to the coarse mesh which coarse points have children (that may require injection) */ 34629566063dSJacob Faibussowitsch PetscCall(PetscSFComputeDegreeBegin(coarseToFineEmbedded, &rootDegrees)); 34639566063dSJacob Faibussowitsch PetscCall(PetscSFComputeDegreeEnd(coarseToFineEmbedded, &rootDegrees)); 34649566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &multiRootSec)); 34659566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(multiRootSec, pStartC, pEndC)); 346648a46eb9SPierre Jolivet for (p = pStartC; p < pEndC; p++) PetscCall(PetscSectionSetDof(multiRootSec, p, rootDegrees[p - pStartC])); 34679566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(multiRootSec)); 34689566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(multiRootSec, &numMulti)); 34699566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &rootIndicesSec)); 3470f30e825dSToby Isaac { /* distribute the leaf section */ 3471f30e825dSToby Isaac PetscSF multi, multiInv, indicesSF; 3472f30e825dSToby Isaac PetscInt *remoteOffsets, numRootIndices; 34738d2f55e7SToby Isaac 34749566063dSJacob Faibussowitsch PetscCall(PetscSFGetMultiSF(coarseToFineEmbedded, &multi)); 34759566063dSJacob Faibussowitsch PetscCall(PetscSFCreateInverseSF(multi, &multiInv)); 34769566063dSJacob Faibussowitsch PetscCall(PetscSFDistributeSection(multiInv, leafIndicesSec, &remoteOffsets, rootIndicesSec)); 34779566063dSJacob Faibussowitsch PetscCall(PetscSFCreateSectionSF(multiInv, leafIndicesSec, remoteOffsets, rootIndicesSec, &indicesSF)); 34789566063dSJacob Faibussowitsch PetscCall(PetscFree(remoteOffsets)); 34799566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&multiInv)); 34809566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(rootIndicesSec, &numRootIndices)); 3481c921d74cSToby Isaac if (gatheredIndices) { 34829566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numRootIndices, &rootInds)); 34839566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(indicesSF, MPIU_INT, leafInds, rootInds, MPI_REPLACE)); 34849566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(indicesSF, MPIU_INT, leafInds, rootInds, MPI_REPLACE)); 3485c921d74cSToby Isaac } 3486c921d74cSToby Isaac if (gatheredValues) { 34879566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numRootIndices, &rootVals)); 34889566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(indicesSF, MPIU_SCALAR, leafVals, rootVals, MPI_REPLACE)); 34899566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(indicesSF, MPIU_SCALAR, leafVals, rootVals, MPI_REPLACE)); 3490c921d74cSToby Isaac } 34919566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&indicesSF)); 34928d2f55e7SToby Isaac } 34939566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&leafIndicesSec)); 34949566063dSJacob Faibussowitsch PetscCall(PetscFree(leafInds)); 34959566063dSJacob Faibussowitsch PetscCall(PetscFree(leafVals)); 34969566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 3497c921d74cSToby Isaac *rootMultiSec = multiRootSec; 3498c921d74cSToby Isaac *multiLeafSec = rootIndicesSec; 3499c921d74cSToby Isaac if (gatheredIndices) *gatheredIndices = rootInds; 3500c921d74cSToby Isaac if (gatheredValues) *gatheredValues = rootVals; 35013ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 3502ebf164c7SToby Isaac } 3503ebf164c7SToby Isaac 3504d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeInjectorTree(DM coarse, DM fine, PetscSF coarseToFine, PetscInt *childIds, Mat mat) 3505d71ae5a4SJacob Faibussowitsch { 3506ebf164c7SToby Isaac DM refTree; 3507c921d74cSToby Isaac PetscSection multiRootSec, rootIndicesSec; 3508ebf164c7SToby Isaac PetscSection globalCoarse, globalFine; 3509ebf164c7SToby Isaac PetscSection localCoarse, localFine; 3510ebf164c7SToby Isaac PetscSection cSecRef; 3511277f51e8SBarry Smith PetscInt *rootIndices = NULL, *parentIndices, pRefStart, pRefEnd; 3512ebf164c7SToby Isaac Mat injRef; 3513c921d74cSToby Isaac PetscInt numFields, maxDof; 3514ebf164c7SToby Isaac PetscInt pStartC, pEndC, pStartF, pEndF, p; 3515ebf164c7SToby Isaac PetscInt *offsets, *offsetsCopy, *rowOffsets; 3516ebf164c7SToby Isaac PetscLayout rowMap, colMap; 3517ebf164c7SToby Isaac PetscInt rowStart, rowEnd, colStart, colEnd, *nnzD, *nnzO; 3518ebf164c7SToby Isaac PetscScalar ***childrenMats = NULL; /* gcc -O gives 'may be used uninitialized' warning'. Initializing to suppress this warning */ 3519ebf164c7SToby Isaac 3520ebf164c7SToby Isaac PetscFunctionBegin; 3521ebf164c7SToby Isaac 3522ebf164c7SToby Isaac /* get the templates for the fine-to-coarse injection from the reference tree */ 35239566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(coarse, &refTree)); 3524*d3a532e9SStefano Zampini PetscCall(DMCopyDisc(coarse, refTree)); 3525*d3a532e9SStefano Zampini PetscCall(DMSetLocalSection(refTree, NULL)); 3526*d3a532e9SStefano Zampini PetscCall(DMSetDefaultConstraints(refTree, NULL, NULL, NULL)); 35279566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &cSecRef, NULL, NULL)); 35289566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSecRef, &pRefStart, &pRefEnd)); 35299566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetInjector(refTree, &injRef)); 3530ebf164c7SToby Isaac 35319566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 35329566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 35339566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 35349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(localFine, &numFields)); 35359566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 35369566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 35379566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 35389566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localCoarse, &maxDof)); 3539ebf164c7SToby Isaac { 3540ebf164c7SToby Isaac PetscInt maxFields = PetscMax(1, numFields) + 1; 35419566063dSJacob Faibussowitsch PetscCall(PetscMalloc3(maxFields, &offsets, maxFields, &offsetsCopy, maxFields, &rowOffsets)); 3542ebf164c7SToby Isaac } 3543ebf164c7SToby Isaac 35449566063dSJacob Faibussowitsch PetscCall(DMPlexTransferInjectorTree(coarse, fine, coarseToFine, childIds, NULL, numFields, offsets, &multiRootSec, &rootIndicesSec, &rootIndices, NULL)); 35458d2f55e7SToby Isaac 35469566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(maxDof, &parentIndices)); 3547f30e825dSToby Isaac 3548f30e825dSToby Isaac /* count indices */ 35499566063dSJacob Faibussowitsch PetscCall(MatGetLayouts(mat, &rowMap, &colMap)); 35509566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(rowMap)); 35519566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(colMap)); 35529566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(rowMap, &rowStart, &rowEnd)); 35539566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(colMap, &colStart, &colEnd)); 35549566063dSJacob Faibussowitsch PetscCall(PetscCalloc2(rowEnd - rowStart, &nnzD, rowEnd - rowStart, &nnzO)); 3555f30e825dSToby Isaac for (p = pStartC; p < pEndC; p++) { 3556f30e825dSToby Isaac PetscInt numLeaves, leafStart, leafEnd, l, dof, cdof, gOff; 35578d2f55e7SToby Isaac 35589566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 35599566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 3560f30e825dSToby Isaac if ((dof - cdof) <= 0) continue; 35619566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, p, &gOff)); 35628d2f55e7SToby Isaac 35638d2f55e7SToby Isaac rowOffsets[0] = 0; 3564f30e825dSToby Isaac offsetsCopy[0] = 0; 35658d2f55e7SToby Isaac if (numFields) { 35668d2f55e7SToby Isaac PetscInt f; 35678d2f55e7SToby Isaac 3568f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 3569f30e825dSToby Isaac PetscInt fDof; 35709566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 3571f30e825dSToby Isaac rowOffsets[f + 1] = offsetsCopy[f + 1] = fDof + rowOffsets[f]; 35728d2f55e7SToby Isaac } 35739566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, parentIndices)); 3574367003a6SStefano Zampini } else { 35759566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, parentIndices)); 3576f30e825dSToby Isaac rowOffsets[1] = offsetsCopy[0]; 35778d2f55e7SToby Isaac } 3578f30e825dSToby Isaac 35799566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(multiRootSec, p, &numLeaves)); 35809566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(multiRootSec, p, &leafStart)); 3581f30e825dSToby Isaac leafEnd = leafStart + numLeaves; 3582f30e825dSToby Isaac for (l = leafStart; l < leafEnd; l++) { 3583f30e825dSToby Isaac PetscInt numIndices, childId, offset; 3584f30e825dSToby Isaac const PetscInt *childIndices; 3585f30e825dSToby Isaac 35869566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootIndicesSec, l, &numIndices)); 35879566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootIndicesSec, l, &offset)); 3588f30e825dSToby Isaac childId = rootIndices[offset++]; 3589f30e825dSToby Isaac childIndices = &rootIndices[offset]; 3590f30e825dSToby Isaac numIndices--; 3591f30e825dSToby Isaac 3592f30e825dSToby Isaac if (childId == -1) { /* equivalent points: scatter */ 3593f30e825dSToby Isaac PetscInt i; 3594f30e825dSToby Isaac 3595f30e825dSToby Isaac for (i = 0; i < numIndices; i++) { 3596f30e825dSToby Isaac PetscInt colIndex = childIndices[i]; 3597f30e825dSToby Isaac PetscInt rowIndex = parentIndices[i]; 3598f30e825dSToby Isaac if (rowIndex < 0) continue; 359908401ef6SPierre Jolivet PetscCheck(colIndex >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unconstrained fine and constrained coarse"); 3600a47f92cbSToby Isaac if (colIndex >= colStart && colIndex < colEnd) { 3601f30e825dSToby Isaac nnzD[rowIndex - rowStart] = 1; 36029371c9d4SSatish Balay } else { 3603f30e825dSToby Isaac nnzO[rowIndex - rowStart] = 1; 3604f30e825dSToby Isaac } 3605f30e825dSToby Isaac } 36069371c9d4SSatish Balay } else { 3607f30e825dSToby Isaac PetscInt parentId, f, lim; 3608f30e825dSToby Isaac 36099566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, childId, &parentId, NULL)); 3610f30e825dSToby Isaac 3611f30e825dSToby Isaac lim = PetscMax(1, numFields); 3612f30e825dSToby Isaac offsets[0] = 0; 36138d2f55e7SToby Isaac if (numFields) { 36148d2f55e7SToby Isaac PetscInt f; 3615f30e825dSToby Isaac 36168d2f55e7SToby Isaac for (f = 0; f < numFields; f++) { 3617f30e825dSToby Isaac PetscInt fDof; 36189566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSecRef, childId, f, &fDof)); 3619f30e825dSToby Isaac 3620f30e825dSToby Isaac offsets[f + 1] = fDof + offsets[f]; 36218d2f55e7SToby Isaac } 36229371c9d4SSatish Balay } else { 3623f30e825dSToby Isaac PetscInt cDof; 3624f30e825dSToby Isaac 36259566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSecRef, childId, &cDof)); 3626f30e825dSToby Isaac offsets[1] = cDof; 3627f30e825dSToby Isaac } 3628f30e825dSToby Isaac for (f = 0; f < lim; f++) { 3629f30e825dSToby Isaac PetscInt parentStart = rowOffsets[f], parentEnd = rowOffsets[f + 1]; 3630f30e825dSToby Isaac PetscInt childStart = offsets[f], childEnd = offsets[f + 1]; 3631f30e825dSToby Isaac PetscInt i, numD = 0, numO = 0; 3632f30e825dSToby Isaac 3633f30e825dSToby Isaac for (i = childStart; i < childEnd; i++) { 3634f30e825dSToby Isaac PetscInt colIndex = childIndices[i]; 3635f30e825dSToby Isaac 3636f30e825dSToby Isaac if (colIndex < 0) continue; 3637f30e825dSToby Isaac if (colIndex >= colStart && colIndex < colEnd) { 3638f30e825dSToby Isaac numD++; 36399371c9d4SSatish Balay } else { 3640f30e825dSToby Isaac numO++; 3641f30e825dSToby Isaac } 3642f30e825dSToby Isaac } 3643f30e825dSToby Isaac for (i = parentStart; i < parentEnd; i++) { 3644f30e825dSToby Isaac PetscInt rowIndex = parentIndices[i]; 3645f30e825dSToby Isaac 3646f30e825dSToby Isaac if (rowIndex < 0) continue; 3647f30e825dSToby Isaac nnzD[rowIndex - rowStart] += numD; 3648f30e825dSToby Isaac nnzO[rowIndex - rowStart] += numO; 36498d2f55e7SToby Isaac } 36508d2f55e7SToby Isaac } 36518d2f55e7SToby Isaac } 3652f30e825dSToby Isaac } 3653f30e825dSToby Isaac } 3654f30e825dSToby Isaac /* preallocate */ 36559566063dSJacob Faibussowitsch PetscCall(MatXAIJSetPreallocation(mat, 1, nnzD, nnzO, NULL, NULL)); 36569566063dSJacob Faibussowitsch PetscCall(PetscFree2(nnzD, nnzO)); 3657f30e825dSToby Isaac /* insert values */ 36589566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices_Injection(refTree, injRef, &childrenMats)); 3659f30e825dSToby Isaac for (p = pStartC; p < pEndC; p++) { 3660f30e825dSToby Isaac PetscInt numLeaves, leafStart, leafEnd, l, dof, cdof, gOff; 3661f30e825dSToby Isaac 36629566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 36639566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 3664f30e825dSToby Isaac if ((dof - cdof) <= 0) continue; 36659566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, p, &gOff)); 3666f30e825dSToby Isaac 3667f30e825dSToby Isaac rowOffsets[0] = 0; 3668f30e825dSToby Isaac offsetsCopy[0] = 0; 36698d2f55e7SToby Isaac if (numFields) { 36708d2f55e7SToby Isaac PetscInt f; 3671f30e825dSToby Isaac 36728d2f55e7SToby Isaac for (f = 0; f < numFields; f++) { 3673f30e825dSToby Isaac PetscInt fDof; 36749566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 3675f30e825dSToby Isaac rowOffsets[f + 1] = offsetsCopy[f + 1] = fDof + rowOffsets[f]; 3676f30e825dSToby Isaac } 36779566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, parentIndices)); 3678367003a6SStefano Zampini } else { 36799566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, parentIndices)); 3680f30e825dSToby Isaac rowOffsets[1] = offsetsCopy[0]; 3681f30e825dSToby Isaac } 3682f30e825dSToby Isaac 36839566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(multiRootSec, p, &numLeaves)); 36849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(multiRootSec, p, &leafStart)); 3685f30e825dSToby Isaac leafEnd = leafStart + numLeaves; 3686f30e825dSToby Isaac for (l = leafStart; l < leafEnd; l++) { 3687f30e825dSToby Isaac PetscInt numIndices, childId, offset; 3688f30e825dSToby Isaac const PetscInt *childIndices; 3689f30e825dSToby Isaac 36909566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootIndicesSec, l, &numIndices)); 36919566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootIndicesSec, l, &offset)); 3692f30e825dSToby Isaac childId = rootIndices[offset++]; 3693f30e825dSToby Isaac childIndices = &rootIndices[offset]; 3694f30e825dSToby Isaac numIndices--; 3695f30e825dSToby Isaac 3696f30e825dSToby Isaac if (childId == -1) { /* equivalent points: scatter */ 3697f30e825dSToby Isaac PetscInt i; 3698f30e825dSToby Isaac 369948a46eb9SPierre Jolivet for (i = 0; i < numIndices; i++) PetscCall(MatSetValue(mat, parentIndices[i], childIndices[i], 1., INSERT_VALUES)); 37009371c9d4SSatish Balay } else { 3701f30e825dSToby Isaac PetscInt parentId, f, lim; 37028d2f55e7SToby Isaac 37039566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, childId, &parentId, NULL)); 3704f30e825dSToby Isaac 3705f30e825dSToby Isaac lim = PetscMax(1, numFields); 3706f30e825dSToby Isaac offsets[0] = 0; 37078d2f55e7SToby Isaac if (numFields) { 3708f30e825dSToby Isaac PetscInt f; 37098d2f55e7SToby Isaac 3710f30e825dSToby Isaac for (f = 0; f < numFields; f++) { 3711f30e825dSToby Isaac PetscInt fDof; 37129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSecRef, childId, f, &fDof)); 3713f30e825dSToby Isaac 3714f30e825dSToby Isaac offsets[f + 1] = fDof + offsets[f]; 37158d2f55e7SToby Isaac } 37169371c9d4SSatish Balay } else { 3717f30e825dSToby Isaac PetscInt cDof; 3718f30e825dSToby Isaac 37199566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSecRef, childId, &cDof)); 3720f30e825dSToby Isaac offsets[1] = cDof; 37218d2f55e7SToby Isaac } 3722f30e825dSToby Isaac for (f = 0; f < lim; f++) { 3723f30e825dSToby Isaac PetscScalar *childMat = &childrenMats[childId - pRefStart][f][0]; 3724f30e825dSToby Isaac PetscInt *rowIndices = &parentIndices[rowOffsets[f]]; 3725f30e825dSToby Isaac const PetscInt *colIndices = &childIndices[offsets[f]]; 3726f30e825dSToby Isaac 37279566063dSJacob Faibussowitsch PetscCall(MatSetValues(mat, rowOffsets[f + 1] - rowOffsets[f], rowIndices, offsets[f + 1] - offsets[f], colIndices, childMat, INSERT_VALUES)); 37288d2f55e7SToby Isaac } 37298d2f55e7SToby Isaac } 37308d2f55e7SToby Isaac } 37318d2f55e7SToby Isaac } 37329566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&multiRootSec)); 37339566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootIndicesSec)); 37349566063dSJacob Faibussowitsch PetscCall(PetscFree(parentIndices)); 37359566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices_Injection(refTree, injRef, &childrenMats)); 37369566063dSJacob Faibussowitsch PetscCall(PetscFree(rootIndices)); 37379566063dSJacob Faibussowitsch PetscCall(PetscFree3(offsets, offsetsCopy, rowOffsets)); 3738f30e825dSToby Isaac 37399566063dSJacob Faibussowitsch PetscCall(MatAssemblyBegin(mat, MAT_FINAL_ASSEMBLY)); 37409566063dSJacob Faibussowitsch PetscCall(MatAssemblyEnd(mat, MAT_FINAL_ASSEMBLY)); 37413ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 3742154bca37SToby Isaac } 374338fc2455SToby Isaac 3744d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexTransferVecTree_Interpolate(DM coarse, Vec vecCoarseLocal, DM fine, Vec vecFine, PetscSF coarseToFine, PetscInt *cids, Vec grad, Vec cellGeom) 3745d71ae5a4SJacob Faibussowitsch { 374662095d54SToby Isaac PetscSF coarseToFineEmbedded; 374762095d54SToby Isaac PetscSection globalCoarse, globalFine; 374862095d54SToby Isaac PetscSection localCoarse, localFine; 374962095d54SToby Isaac PetscSection aSec, cSec; 375062095d54SToby Isaac PetscSection rootValuesSec; 375162095d54SToby Isaac PetscSection leafValuesSec; 375262095d54SToby Isaac PetscScalar *rootValues, *leafValues; 375362095d54SToby Isaac IS aIS; 375462095d54SToby Isaac const PetscInt *anchors; 375562095d54SToby Isaac Mat cMat; 375662095d54SToby Isaac PetscInt numFields; 3757412e9a14SMatthew G. Knepley PetscInt pStartC, pEndC, pStartF, pEndF, p, cellStart, cellEnd; 375862095d54SToby Isaac PetscInt aStart, aEnd, cStart, cEnd; 375962095d54SToby Isaac PetscInt *maxChildIds; 376062095d54SToby Isaac PetscInt *offsets, *newOffsets, *offsetsCopy, *newOffsetsCopy, *rowOffsets, *numD, *numO; 37610eb7e1eaSToby Isaac PetscFV fv = NULL; 37620eb7e1eaSToby Isaac PetscInt dim, numFVcomps = -1, fvField = -1; 37630eb7e1eaSToby Isaac DM cellDM = NULL, gradDM = NULL; 37640eb7e1eaSToby Isaac const PetscScalar *cellGeomArray = NULL; 37650eb7e1eaSToby Isaac const PetscScalar *gradArray = NULL; 376662095d54SToby Isaac 3767ebf164c7SToby Isaac PetscFunctionBegin; 37689566063dSJacob Faibussowitsch PetscCall(VecSetOption(vecFine, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE)); 37699566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 37709566063dSJacob Faibussowitsch PetscCall(DMPlexGetSimplexOrBoxCells(coarse, 0, &cellStart, &cellEnd)); 37719566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 37729566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 37739566063dSJacob Faibussowitsch PetscCall(DMGetCoordinateDim(coarse, &dim)); 377462095d54SToby Isaac { /* winnow fine points that don't have global dofs out of the sf */ 3775e4a60869SToby Isaac PetscInt nleaves, l; 3776e4a60869SToby Isaac const PetscInt *leaves; 377762095d54SToby Isaac PetscInt dof, cdof, numPointsWithDofs, offset, *pointsWithDofs; 377862095d54SToby Isaac 37799566063dSJacob Faibussowitsch PetscCall(PetscSFGetGraph(coarseToFine, NULL, &nleaves, &leaves, NULL)); 3780e4a60869SToby Isaac 3781e4a60869SToby Isaac for (l = 0, numPointsWithDofs = 0; l < nleaves; l++) { 3782e4a60869SToby Isaac PetscInt p = leaves ? leaves[l] : l; 3783e4a60869SToby Isaac 37849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 37859566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 3786ad540459SPierre Jolivet if ((dof - cdof) > 0) numPointsWithDofs++; 378762095d54SToby Isaac } 37889566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numPointsWithDofs, &pointsWithDofs)); 37894833aeb0SToby Isaac for (l = 0, offset = 0; l < nleaves; l++) { 3790e4a60869SToby Isaac PetscInt p = leaves ? leaves[l] : l; 3791e4a60869SToby Isaac 37929566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &dof)); 37939566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &cdof)); 3794ad540459SPierre Jolivet if ((dof - cdof) > 0) pointsWithDofs[offset++] = l; 379562095d54SToby Isaac } 37969566063dSJacob Faibussowitsch PetscCall(PetscSFCreateEmbeddedLeafSF(coarseToFine, numPointsWithDofs, pointsWithDofs, &coarseToFineEmbedded)); 37979566063dSJacob Faibussowitsch PetscCall(PetscFree(pointsWithDofs)); 379862095d54SToby Isaac } 379962095d54SToby Isaac /* communicate back to the coarse mesh which coarse points have children (that may require interpolation) */ 38009566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(pEndC - pStartC, &maxChildIds)); 3801ad540459SPierre Jolivet for (p = pStartC; p < pEndC; p++) maxChildIds[p - pStartC] = -2; 38029566063dSJacob Faibussowitsch PetscCall(PetscSFReduceBegin(coarseToFineEmbedded, MPIU_INT, cids, maxChildIds, MPIU_MAX)); 38039566063dSJacob Faibussowitsch PetscCall(PetscSFReduceEnd(coarseToFineEmbedded, MPIU_INT, cids, maxChildIds, MPIU_MAX)); 380462095d54SToby Isaac 38059566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 38069566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 380762095d54SToby Isaac 38089566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(coarse, &aSec, &aIS)); 38099566063dSJacob Faibussowitsch PetscCall(ISGetIndices(aIS, &anchors)); 38109566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(aSec, &aStart, &aEnd)); 381162095d54SToby Isaac 38129566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(coarse, &cSec, &cMat, NULL)); 38139566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSec, &cStart, &cEnd)); 381462095d54SToby Isaac 381562095d54SToby Isaac /* create sections that will send to children the indices and matrices they will need to construct the interpolator */ 38169566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)coarse), &rootValuesSec)); 38179566063dSJacob Faibussowitsch PetscCall(PetscSectionSetChart(rootValuesSec, pStartC, pEndC)); 38189566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(localCoarse, &numFields)); 381962095d54SToby Isaac { 382062095d54SToby Isaac PetscInt maxFields = PetscMax(1, numFields) + 1; 38219566063dSJacob Faibussowitsch PetscCall(PetscMalloc7(maxFields, &offsets, maxFields, &offsetsCopy, maxFields, &newOffsets, maxFields, &newOffsetsCopy, maxFields, &rowOffsets, maxFields, &numD, maxFields, &numO)); 382262095d54SToby Isaac } 38230eb7e1eaSToby Isaac if (grad) { 38240eb7e1eaSToby Isaac PetscInt i; 38250eb7e1eaSToby Isaac 38269566063dSJacob Faibussowitsch PetscCall(VecGetDM(cellGeom, &cellDM)); 38279566063dSJacob Faibussowitsch PetscCall(VecGetArrayRead(cellGeom, &cellGeomArray)); 38289566063dSJacob Faibussowitsch PetscCall(VecGetDM(grad, &gradDM)); 38299566063dSJacob Faibussowitsch PetscCall(VecGetArrayRead(grad, &gradArray)); 38300eb7e1eaSToby Isaac for (i = 0; i < PetscMax(1, numFields); i++) { 38310eb7e1eaSToby Isaac PetscObject obj; 38320eb7e1eaSToby Isaac PetscClassId id; 38330eb7e1eaSToby Isaac 38349566063dSJacob Faibussowitsch PetscCall(DMGetField(coarse, i, NULL, &obj)); 38359566063dSJacob Faibussowitsch PetscCall(PetscObjectGetClassId(obj, &id)); 38360eb7e1eaSToby Isaac if (id == PETSCFV_CLASSID) { 38370eb7e1eaSToby Isaac fv = (PetscFV)obj; 38389566063dSJacob Faibussowitsch PetscCall(PetscFVGetNumComponents(fv, &numFVcomps)); 38390eb7e1eaSToby Isaac fvField = i; 38400eb7e1eaSToby Isaac break; 38410eb7e1eaSToby Isaac } 38420eb7e1eaSToby Isaac } 38430eb7e1eaSToby Isaac } 384462095d54SToby Isaac 384562095d54SToby Isaac for (p = pStartC; p < pEndC; p++) { /* count the sizes of the indices and matrices */ 384662095d54SToby Isaac PetscInt dof; 384762095d54SToby Isaac PetscInt maxChildId = maxChildIds[p - pStartC]; 384862095d54SToby Isaac PetscInt numValues = 0; 384962095d54SToby Isaac 38509566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 3851ad540459SPierre Jolivet if (dof < 0) dof = -(dof + 1); 385262095d54SToby Isaac offsets[0] = 0; 385362095d54SToby Isaac newOffsets[0] = 0; 385462095d54SToby Isaac if (maxChildId >= 0) { /* this point has children (with dofs) that will need to be interpolated from the closure of p */ 385562095d54SToby Isaac PetscInt *closure = NULL, closureSize, cl; 385662095d54SToby Isaac 38579566063dSJacob Faibussowitsch PetscCall(DMPlexGetTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 385862095d54SToby Isaac for (cl = 0; cl < closureSize; cl++) { /* get the closure */ 385962095d54SToby Isaac PetscInt c = closure[2 * cl], clDof; 386062095d54SToby Isaac 38619566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, c, &clDof)); 386262095d54SToby Isaac numValues += clDof; 386362095d54SToby Isaac } 38649566063dSJacob Faibussowitsch PetscCall(DMPlexRestoreTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure)); 38659371c9d4SSatish Balay } else if (maxChildId == -1) { 38669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, p, &numValues)); 386762095d54SToby Isaac } 386862095d54SToby Isaac /* we will pack the column indices with the field offsets */ 386978b7adb5SToby Isaac if (maxChildId >= 0 && grad && p >= cellStart && p < cellEnd) { 38700eb7e1eaSToby Isaac /* also send the centroid, and the gradient */ 38710eb7e1eaSToby Isaac numValues += dim * (1 + numFVcomps); 38720eb7e1eaSToby Isaac } 38739566063dSJacob Faibussowitsch PetscCall(PetscSectionSetDof(rootValuesSec, p, numValues)); 387462095d54SToby Isaac } 38759566063dSJacob Faibussowitsch PetscCall(PetscSectionSetUp(rootValuesSec)); 387662095d54SToby Isaac { 387762095d54SToby Isaac PetscInt numRootValues; 387862095d54SToby Isaac const PetscScalar *coarseArray; 387962095d54SToby Isaac 38809566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(rootValuesSec, &numRootValues)); 38819566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numRootValues, &rootValues)); 38829566063dSJacob Faibussowitsch PetscCall(VecGetArrayRead(vecCoarseLocal, &coarseArray)); 388362095d54SToby Isaac for (p = pStartC; p < pEndC; p++) { 388462095d54SToby Isaac PetscInt numValues; 388562095d54SToby Isaac PetscInt pValOff; 388662095d54SToby Isaac PetscScalar *pVal; 388762095d54SToby Isaac PetscInt maxChildId = maxChildIds[p - pStartC]; 388862095d54SToby Isaac 38899566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootValuesSec, p, &numValues)); 3890ad540459SPierre Jolivet if (!numValues) continue; 38919566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootValuesSec, p, &pValOff)); 389262095d54SToby Isaac pVal = &(rootValues[pValOff]); 389362095d54SToby Isaac if (maxChildId >= 0) { /* build an identity matrix, apply matrix constraints on the right */ 38940eb7e1eaSToby Isaac PetscInt closureSize = numValues; 38959566063dSJacob Faibussowitsch PetscCall(DMPlexVecGetClosure(coarse, NULL, vecCoarseLocal, p, &closureSize, &pVal)); 38960eb7e1eaSToby Isaac if (grad && p >= cellStart && p < cellEnd) { 3897193eb951SToby Isaac PetscFVCellGeom *cg; 38986dd00756SToby Isaac PetscScalar *gradVals = NULL; 38990eb7e1eaSToby Isaac PetscInt i; 39000eb7e1eaSToby Isaac 39010eb7e1eaSToby Isaac pVal += (numValues - dim * (1 + numFVcomps)); 39020eb7e1eaSToby Isaac 39039566063dSJacob Faibussowitsch PetscCall(DMPlexPointLocalRead(cellDM, p, cellGeomArray, (void *)&cg)); 39040eb7e1eaSToby Isaac for (i = 0; i < dim; i++) pVal[i] = cg->centroid[i]; 39050eb7e1eaSToby Isaac pVal += dim; 39069566063dSJacob Faibussowitsch PetscCall(DMPlexPointGlobalRead(gradDM, p, gradArray, (void *)&gradVals)); 39070eb7e1eaSToby Isaac for (i = 0; i < dim * numFVcomps; i++) pVal[i] = gradVals[i]; 39080eb7e1eaSToby Isaac } 39099371c9d4SSatish Balay } else if (maxChildId == -1) { 391078b7adb5SToby Isaac PetscInt lDof, lOff, i; 391178b7adb5SToby Isaac 39129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, p, &lDof)); 39139566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(localCoarse, p, &lOff)); 391478b7adb5SToby Isaac for (i = 0; i < lDof; i++) pVal[i] = coarseArray[lOff + i]; 391578b7adb5SToby Isaac } 391678b7adb5SToby Isaac } 39179566063dSJacob Faibussowitsch PetscCall(VecRestoreArrayRead(vecCoarseLocal, &coarseArray)); 39189566063dSJacob Faibussowitsch PetscCall(PetscFree(maxChildIds)); 391962095d54SToby Isaac } 392062095d54SToby Isaac { 392162095d54SToby Isaac PetscSF valuesSF; 392262095d54SToby Isaac PetscInt *remoteOffsetsValues, numLeafValues; 392362095d54SToby Isaac 39249566063dSJacob Faibussowitsch PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)fine), &leafValuesSec)); 39259566063dSJacob Faibussowitsch PetscCall(PetscSFDistributeSection(coarseToFineEmbedded, rootValuesSec, &remoteOffsetsValues, leafValuesSec)); 39269566063dSJacob Faibussowitsch PetscCall(PetscSFCreateSectionSF(coarseToFineEmbedded, rootValuesSec, remoteOffsetsValues, leafValuesSec, &valuesSF)); 39279566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&coarseToFineEmbedded)); 39289566063dSJacob Faibussowitsch PetscCall(PetscFree(remoteOffsetsValues)); 39299566063dSJacob Faibussowitsch PetscCall(PetscSectionGetStorageSize(leafValuesSec, &numLeafValues)); 39309566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(numLeafValues, &leafValues)); 39319566063dSJacob Faibussowitsch PetscCall(PetscSFBcastBegin(valuesSF, MPIU_SCALAR, rootValues, leafValues, MPI_REPLACE)); 39329566063dSJacob Faibussowitsch PetscCall(PetscSFBcastEnd(valuesSF, MPIU_SCALAR, rootValues, leafValues, MPI_REPLACE)); 39339566063dSJacob Faibussowitsch PetscCall(PetscSFDestroy(&valuesSF)); 39349566063dSJacob Faibussowitsch PetscCall(PetscFree(rootValues)); 39359566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootValuesSec)); 393662095d54SToby Isaac } 39379566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 393862095d54SToby Isaac { 393962095d54SToby Isaac PetscInt maxDof; 394062095d54SToby Isaac PetscInt *rowIndices; 394162095d54SToby Isaac DM refTree; 394262095d54SToby Isaac PetscInt **refPointFieldN; 394362095d54SToby Isaac PetscScalar ***refPointFieldMats; 394462095d54SToby Isaac PetscSection refConSec, refAnSec; 39450eb7e1eaSToby Isaac PetscInt pRefStart, pRefEnd, leafStart, leafEnd; 394662095d54SToby Isaac PetscScalar *pointWork; 394762095d54SToby Isaac 39489566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localFine, &maxDof)); 39499566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(fine, maxDof, MPIU_INT, &rowIndices)); 39509566063dSJacob Faibussowitsch PetscCall(DMGetWorkArray(fine, maxDof, MPIU_SCALAR, &pointWork)); 39519566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(fine, &refTree)); 39529566063dSJacob Faibussowitsch PetscCall(DMCopyDisc(fine, refTree)); 39539566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 39549566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &refConSec, NULL, NULL)); 39559566063dSJacob Faibussowitsch PetscCall(DMPlexGetAnchors(refTree, &refAnSec, NULL)); 39569566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(refConSec, &pRefStart, &pRefEnd)); 39579566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(leafValuesSec, &leafStart, &leafEnd)); 39589566063dSJacob Faibussowitsch PetscCall(DMPlexGetSimplexOrBoxCells(fine, 0, &cellStart, &cellEnd)); 39590eb7e1eaSToby Isaac for (p = leafStart; p < leafEnd; p++) { 396062095d54SToby Isaac PetscInt gDof, gcDof, gOff, lDof; 396162095d54SToby Isaac PetscInt numValues, pValOff; 396262095d54SToby Isaac PetscInt childId; 396362095d54SToby Isaac const PetscScalar *pVal; 39640eb7e1eaSToby Isaac const PetscScalar *fvGradData = NULL; 396562095d54SToby Isaac 39669566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalFine, p, &gDof)); 39679566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localFine, p, &lDof)); 39689566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalFine, p, &gcDof)); 3969ad540459SPierre Jolivet if ((gDof - gcDof) <= 0) continue; 39709566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalFine, p, &gOff)); 39719566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(leafValuesSec, p, &numValues)); 397262095d54SToby Isaac if (!numValues) continue; 39739566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(leafValuesSec, p, &pValOff)); 397462095d54SToby Isaac pVal = &leafValues[pValOff]; 397562095d54SToby Isaac offsets[0] = 0; 397662095d54SToby Isaac offsetsCopy[0] = 0; 397762095d54SToby Isaac newOffsets[0] = 0; 397862095d54SToby Isaac newOffsetsCopy[0] = 0; 39794833aeb0SToby Isaac childId = cids[p - pStartF]; 398062095d54SToby Isaac if (numFields) { 398162095d54SToby Isaac PetscInt f; 398262095d54SToby Isaac for (f = 0; f < numFields; f++) { 398362095d54SToby Isaac PetscInt rowDof; 398462095d54SToby Isaac 39859566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localFine, p, f, &rowDof)); 398662095d54SToby Isaac offsets[f + 1] = offsets[f] + rowDof; 398762095d54SToby Isaac offsetsCopy[f + 1] = offsets[f + 1]; 398862095d54SToby Isaac /* TODO: closure indices */ 39899f4e70e1SToby Isaac newOffsets[f + 1] = newOffsets[f] + ((childId == -1) ? rowDof : refPointFieldN[childId - pRefStart][f]); 399062095d54SToby Isaac } 39919566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, rowIndices)); 39929371c9d4SSatish Balay } else { 39934833aeb0SToby Isaac offsets[0] = 0; 39944833aeb0SToby Isaac offsets[1] = lDof; 39954833aeb0SToby Isaac newOffsets[0] = 0; 39964833aeb0SToby Isaac newOffsets[1] = (childId == -1) ? lDof : refPointFieldN[childId - pRefStart][0]; 39979566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localFine, PETSC_FALSE, p, gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, rowIndices)); 399862095d54SToby Isaac } 399962095d54SToby Isaac if (childId == -1) { /* no child interpolation: one nnz per */ 40009566063dSJacob Faibussowitsch PetscCall(VecSetValues(vecFine, numValues, rowIndices, pVal, INSERT_VALUES)); 400162095d54SToby Isaac } else { 400262095d54SToby Isaac PetscInt f; 400362095d54SToby Isaac 400478b7adb5SToby Isaac if (grad && p >= cellStart && p < cellEnd) { 400578b7adb5SToby Isaac numValues -= (dim * (1 + numFVcomps)); 400678b7adb5SToby Isaac fvGradData = &pVal[numValues]; 400778b7adb5SToby Isaac } 400862095d54SToby Isaac for (f = 0; f < PetscMax(1, numFields); f++) { 400962095d54SToby Isaac const PetscScalar *childMat = refPointFieldMats[childId - pRefStart][f]; 401062095d54SToby Isaac PetscInt numRows = offsets[f + 1] - offsets[f]; 401162095d54SToby Isaac PetscInt numCols = newOffsets[f + 1] - newOffsets[f]; 401262095d54SToby Isaac const PetscScalar *cVal = &pVal[newOffsets[f]]; 401362095d54SToby Isaac PetscScalar *rVal = &pointWork[offsets[f]]; 401462095d54SToby Isaac PetscInt i, j; 401562095d54SToby Isaac 4016708c7f19SToby Isaac #if 0 401763a3b9bcSJacob Faibussowitsch PetscCall(PetscInfo(coarse,"childId %" PetscInt_FMT ", numRows %" PetscInt_FMT ", numCols %" PetscInt_FMT ", refPointFieldN %" PetscInt_FMT " maxDof %" PetscInt_FMT "\n",childId,numRows,numCols,refPointFieldN[childId - pRefStart][f], maxDof)); 4018708c7f19SToby Isaac #endif 401962095d54SToby Isaac for (i = 0; i < numRows; i++) { 402062095d54SToby Isaac PetscScalar val = 0.; 4021ad540459SPierre Jolivet for (j = 0; j < numCols; j++) val += childMat[i * numCols + j] * cVal[j]; 402262095d54SToby Isaac rVal[i] = val; 402362095d54SToby Isaac } 40240eb7e1eaSToby Isaac if (f == fvField && p >= cellStart && p < cellEnd) { 40250eb7e1eaSToby Isaac PetscReal centroid[3]; 40260eb7e1eaSToby Isaac PetscScalar diff[3]; 40270eb7e1eaSToby Isaac const PetscScalar *parentCentroid = &fvGradData[0]; 40280eb7e1eaSToby Isaac const PetscScalar *gradient = &fvGradData[dim]; 40290eb7e1eaSToby Isaac 40309566063dSJacob Faibussowitsch PetscCall(DMPlexComputeCellGeometryFVM(fine, p, NULL, centroid, NULL)); 4031ad540459SPierre Jolivet for (i = 0; i < dim; i++) diff[i] = centroid[i] - parentCentroid[i]; 40320eb7e1eaSToby Isaac for (i = 0; i < numFVcomps; i++) { 40330eb7e1eaSToby Isaac PetscScalar val = 0.; 40340eb7e1eaSToby Isaac 4035ad540459SPierre Jolivet for (j = 0; j < dim; j++) val += gradient[dim * i + j] * diff[j]; 40360eb7e1eaSToby Isaac rVal[i] += val; 40370eb7e1eaSToby Isaac } 40380eb7e1eaSToby Isaac } 40399566063dSJacob Faibussowitsch PetscCall(VecSetValues(vecFine, numRows, &rowIndices[offsets[f]], rVal, INSERT_VALUES)); 404062095d54SToby Isaac } 404162095d54SToby Isaac } 404262095d54SToby Isaac } 40439566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices(refTree, &refPointFieldMats, &refPointFieldN)); 40449566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(fine, maxDof, MPIU_SCALAR, &pointWork)); 40459566063dSJacob Faibussowitsch PetscCall(DMRestoreWorkArray(fine, maxDof, MPIU_INT, &rowIndices)); 404662095d54SToby Isaac } 40479566063dSJacob Faibussowitsch PetscCall(PetscFree(leafValues)); 40489566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&leafValuesSec)); 40499566063dSJacob Faibussowitsch PetscCall(PetscFree7(offsets, offsetsCopy, newOffsets, newOffsetsCopy, rowOffsets, numD, numO)); 40509566063dSJacob Faibussowitsch PetscCall(ISRestoreIndices(aIS, &anchors)); 40513ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 4052ebf164c7SToby Isaac } 4053ebf164c7SToby Isaac 4054d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexTransferVecTree_Inject(DM fine, Vec vecFine, DM coarse, Vec vecCoarse, PetscSF coarseToFine, PetscInt *cids) 4055d71ae5a4SJacob Faibussowitsch { 4056c921d74cSToby Isaac DM refTree; 4057c921d74cSToby Isaac PetscSection multiRootSec, rootIndicesSec; 4058c921d74cSToby Isaac PetscSection globalCoarse, globalFine; 4059c921d74cSToby Isaac PetscSection localCoarse, localFine; 4060c921d74cSToby Isaac PetscSection cSecRef; 4061c921d74cSToby Isaac PetscInt *parentIndices, pRefStart, pRefEnd; 4062d3bc4906SToby Isaac PetscScalar *rootValues, *parentValues; 4063c921d74cSToby Isaac Mat injRef; 4064c921d74cSToby Isaac PetscInt numFields, maxDof; 4065c921d74cSToby Isaac PetscInt pStartC, pEndC, pStartF, pEndF, p; 4066c921d74cSToby Isaac PetscInt *offsets, *offsetsCopy, *rowOffsets; 4067c921d74cSToby Isaac PetscLayout rowMap, colMap; 4068c921d74cSToby Isaac PetscInt rowStart, rowEnd, colStart, colEnd; 4069c921d74cSToby Isaac PetscScalar ***childrenMats = NULL; /* gcc -O gives 'may be used uninitialized' warning'. Initializing to suppress this warning */ 4070c921d74cSToby Isaac 4071ebf164c7SToby Isaac PetscFunctionBegin; 4072c921d74cSToby Isaac /* get the templates for the fine-to-coarse injection from the reference tree */ 40739566063dSJacob Faibussowitsch PetscCall(VecSetOption(vecFine, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE)); 40749566063dSJacob Faibussowitsch PetscCall(VecSetOption(vecCoarse, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE)); 40759566063dSJacob Faibussowitsch PetscCall(DMPlexGetReferenceTree(coarse, &refTree)); 40769566063dSJacob Faibussowitsch PetscCall(DMCopyDisc(coarse, refTree)); 40779566063dSJacob Faibussowitsch PetscCall(DMGetDefaultConstraints(refTree, &cSecRef, NULL, NULL)); 40789566063dSJacob Faibussowitsch PetscCall(PetscSectionGetChart(cSecRef, &pRefStart, &pRefEnd)); 40799566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetInjector(refTree, &injRef)); 4080c921d74cSToby Isaac 40819566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(fine, &pStartF, &pEndF)); 40829566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(fine, &localFine)); 40839566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(fine, &globalFine)); 40849566063dSJacob Faibussowitsch PetscCall(PetscSectionGetNumFields(localFine, &numFields)); 40859566063dSJacob Faibussowitsch PetscCall(DMPlexGetChart(coarse, &pStartC, &pEndC)); 40869566063dSJacob Faibussowitsch PetscCall(DMGetLocalSection(coarse, &localCoarse)); 40879566063dSJacob Faibussowitsch PetscCall(DMGetGlobalSection(coarse, &globalCoarse)); 40889566063dSJacob Faibussowitsch PetscCall(PetscSectionGetMaxDof(localCoarse, &maxDof)); 4089c921d74cSToby Isaac { 4090c921d74cSToby Isaac PetscInt maxFields = PetscMax(1, numFields) + 1; 40919566063dSJacob Faibussowitsch PetscCall(PetscMalloc3(maxFields, &offsets, maxFields, &offsetsCopy, maxFields, &rowOffsets)); 4092c921d74cSToby Isaac } 4093c921d74cSToby Isaac 40949566063dSJacob Faibussowitsch PetscCall(DMPlexTransferInjectorTree(coarse, fine, coarseToFine, cids, vecFine, numFields, offsets, &multiRootSec, &rootIndicesSec, NULL, &rootValues)); 4095c921d74cSToby Isaac 40969566063dSJacob Faibussowitsch PetscCall(PetscMalloc2(maxDof, &parentIndices, maxDof, &parentValues)); 4097c921d74cSToby Isaac 4098c921d74cSToby Isaac /* count indices */ 40999566063dSJacob Faibussowitsch PetscCall(VecGetLayout(vecFine, &colMap)); 41009566063dSJacob Faibussowitsch PetscCall(VecGetLayout(vecCoarse, &rowMap)); 41019566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(rowMap)); 41029566063dSJacob Faibussowitsch PetscCall(PetscLayoutSetUp(colMap)); 41039566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(rowMap, &rowStart, &rowEnd)); 41049566063dSJacob Faibussowitsch PetscCall(PetscLayoutGetRange(colMap, &colStart, &colEnd)); 4105c921d74cSToby Isaac /* insert values */ 41069566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeGetChildrenMatrices_Injection(refTree, injRef, &childrenMats)); 4107c921d74cSToby Isaac for (p = pStartC; p < pEndC; p++) { 4108c921d74cSToby Isaac PetscInt numLeaves, leafStart, leafEnd, l, dof, cdof, gOff; 410978b7adb5SToby Isaac PetscBool contribute = PETSC_FALSE; 4110c921d74cSToby Isaac 41119566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(globalCoarse, p, &dof)); 41129566063dSJacob Faibussowitsch PetscCall(PetscSectionGetConstraintDof(globalCoarse, p, &cdof)); 4113c921d74cSToby Isaac if ((dof - cdof) <= 0) continue; 41149566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(localCoarse, p, &dof)); 41159566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(globalCoarse, p, &gOff)); 4116c921d74cSToby Isaac 4117c921d74cSToby Isaac rowOffsets[0] = 0; 4118c921d74cSToby Isaac offsetsCopy[0] = 0; 4119c921d74cSToby Isaac if (numFields) { 4120c921d74cSToby Isaac PetscInt f; 4121c921d74cSToby Isaac 4122c921d74cSToby Isaac for (f = 0; f < numFields; f++) { 4123c921d74cSToby Isaac PetscInt fDof; 41249566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(localCoarse, p, f, &fDof)); 4125c921d74cSToby Isaac rowOffsets[f + 1] = offsetsCopy[f + 1] = fDof + rowOffsets[f]; 4126c921d74cSToby Isaac } 41279566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPointFields_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, -1, NULL, parentIndices)); 4128367003a6SStefano Zampini } else { 41299566063dSJacob Faibussowitsch PetscCall(DMPlexGetIndicesPoint_Internal(localCoarse, PETSC_FALSE, p, gOff < 0 ? -(gOff + 1) : gOff, offsetsCopy, PETSC_FALSE, NULL, NULL, parentIndices)); 4130c921d74cSToby Isaac rowOffsets[1] = offsetsCopy[0]; 4131c921d74cSToby Isaac } 4132c921d74cSToby Isaac 41339566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(multiRootSec, p, &numLeaves)); 41349566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(multiRootSec, p, &leafStart)); 4135c921d74cSToby Isaac leafEnd = leafStart + numLeaves; 41362f65e181SToby Isaac for (l = 0; l < dof; l++) parentValues[l] = 0.; 4137c921d74cSToby Isaac for (l = leafStart; l < leafEnd; l++) { 4138c921d74cSToby Isaac PetscInt numIndices, childId, offset; 4139c921d74cSToby Isaac const PetscScalar *childValues; 4140c921d74cSToby Isaac 41419566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(rootIndicesSec, l, &numIndices)); 41429566063dSJacob Faibussowitsch PetscCall(PetscSectionGetOffset(rootIndicesSec, l, &offset)); 4143c921d74cSToby Isaac childId = (PetscInt)PetscRealPart(rootValues[offset++]); 4144c921d74cSToby Isaac childValues = &rootValues[offset]; 4145c921d74cSToby Isaac numIndices--; 4146c921d74cSToby Isaac 4147c921d74cSToby Isaac if (childId == -2) { /* skip */ 4148c921d74cSToby Isaac continue; 4149c921d74cSToby Isaac } else if (childId == -1) { /* equivalent points: scatter */ 41502f65e181SToby Isaac PetscInt m; 41512f65e181SToby Isaac 415278b7adb5SToby Isaac contribute = PETSC_TRUE; 41532f65e181SToby Isaac for (m = 0; m < numIndices; m++) parentValues[m] = childValues[m]; 4154beedf8abSToby Isaac } else { /* contributions from children: sum with injectors from reference tree */ 4155d3bc4906SToby Isaac PetscInt parentId, f, lim; 4156d3bc4906SToby Isaac 415778b7adb5SToby Isaac contribute = PETSC_TRUE; 41589566063dSJacob Faibussowitsch PetscCall(DMPlexGetTreeParent(refTree, childId, &parentId, NULL)); 4159d3bc4906SToby Isaac 4160d3bc4906SToby Isaac lim = PetscMax(1, numFields); 4161d3bc4906SToby Isaac offsets[0] = 0; 4162d3bc4906SToby Isaac if (numFields) { 4163d3bc4906SToby Isaac PetscInt f; 4164d3bc4906SToby Isaac 4165d3bc4906SToby Isaac for (f = 0; f < numFields; f++) { 4166d3bc4906SToby Isaac PetscInt fDof; 41679566063dSJacob Faibussowitsch PetscCall(PetscSectionGetFieldDof(cSecRef, childId, f, &fDof)); 4168d3bc4906SToby Isaac 4169d3bc4906SToby Isaac offsets[f + 1] = fDof + offsets[f]; 4170d3bc4906SToby Isaac } 41719371c9d4SSatish Balay } else { 4172d3bc4906SToby Isaac PetscInt cDof; 4173d3bc4906SToby Isaac 41749566063dSJacob Faibussowitsch PetscCall(PetscSectionGetDof(cSecRef, childId, &cDof)); 4175d3bc4906SToby Isaac offsets[1] = cDof; 4176d3bc4906SToby Isaac } 4177d3bc4906SToby Isaac for (f = 0; f < lim; f++) { 4178d3bc4906SToby Isaac PetscScalar *childMat = &childrenMats[childId - pRefStart][f][0]; 4179d3bc4906SToby Isaac PetscInt n = offsets[f + 1] - offsets[f]; 4180e328ff09SToby Isaac PetscInt m = rowOffsets[f + 1] - rowOffsets[f]; 4181d3bc4906SToby Isaac PetscInt i, j; 4182d3bc4906SToby Isaac const PetscScalar *colValues = &childValues[offsets[f]]; 4183d3bc4906SToby Isaac 4184e328ff09SToby Isaac for (i = 0; i < m; i++) { 4185d3bc4906SToby Isaac PetscScalar val = 0.; 4186ad540459SPierre Jolivet for (j = 0; j < n; j++) val += childMat[n * i + j] * colValues[j]; 4187e328ff09SToby Isaac parentValues[rowOffsets[f] + i] += val; 4188d3bc4906SToby Isaac } 4189d3bc4906SToby Isaac } 4190c921d74cSToby Isaac } 4191c921d74cSToby Isaac } 41929566063dSJacob Faibussowitsch if (contribute) PetscCall(VecSetValues(vecCoarse, dof, parentIndices, parentValues, INSERT_VALUES)); 4193c921d74cSToby Isaac } 41949566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&multiRootSec)); 41959566063dSJacob Faibussowitsch PetscCall(PetscSectionDestroy(&rootIndicesSec)); 41969566063dSJacob Faibussowitsch PetscCall(PetscFree2(parentIndices, parentValues)); 41979566063dSJacob Faibussowitsch PetscCall(DMPlexReferenceTreeRestoreChildrenMatrices_Injection(refTree, injRef, &childrenMats)); 41989566063dSJacob Faibussowitsch PetscCall(PetscFree(rootValues)); 41999566063dSJacob Faibussowitsch PetscCall(PetscFree3(offsets, offsetsCopy, rowOffsets)); 42003ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 4201ebf164c7SToby Isaac } 4202ebf164c7SToby Isaac 4203ff1f73f7SToby Isaac /*@ 4204ff1f73f7SToby Isaac DMPlexTransferVecTree - transfer a vector between two meshes that differ from each other by refinement/coarsening 4205ff1f73f7SToby Isaac that can be represented by a common reference tree used by both. This routine can be used for a combination of 4206ff1f73f7SToby Isaac coarsening and refinement at the same time. 4207ff1f73f7SToby Isaac 420820f4b53cSBarry Smith Collective 4209ff1f73f7SToby Isaac 4210ff1f73f7SToby Isaac Input Parameters: 4211a1cb98faSBarry Smith + dmIn - The `DMPLEX` mesh for the input vector 421220f4b53cSBarry Smith . dmOut - The second `DMPLEX` mesh 4213ff1f73f7SToby Isaac . vecIn - The input vector 421420f4b53cSBarry Smith . sfRefine - A star forest indicating points in the mesh `dmIn` (roots in the star forest) that are parents to points in 421520f4b53cSBarry Smith the mesh `dmOut` (leaves in the star forest), i.e. where `dmOut` is more refined than `dmIn` 421620f4b53cSBarry Smith . sfCoarsen - A star forest indicating points in the mesh `dmOut` (roots in the star forest) that are parents to points in 421720f4b53cSBarry Smith the mesh `dmIn` (leaves in the star forest), i.e. where `dmOut` is more coarsened than `dmIn` 421820f4b53cSBarry Smith . cidsRefine - The childIds of the points in `dmOut`. These childIds relate back to the reference tree: childid[j] = k implies 421920f4b53cSBarry Smith that mesh point j of `dmOut` was refined from a point in `dmIn` just as the mesh point k in the reference 422020f4b53cSBarry Smith tree was refined from its parent. childid[j] = -1 indicates that the point j in `dmOut` is exactly 422120f4b53cSBarry Smith equivalent to its root in `dmIn`, so no interpolation is necessary. childid[j] = -2 indicates that this 422220f4b53cSBarry Smith point j in `dmOut` is not a leaf of `sfRefine`. 422320f4b53cSBarry Smith . cidsCoarsen - The childIds of the points in `dmIn`. These childIds relate back to the reference tree: childid[j] = k implies 422420f4b53cSBarry Smith that mesh point j of dmIn coarsens to a point in `dmOut` just as the mesh point k in the reference 422520f4b53cSBarry Smith tree coarsens to its parent. childid[j] = -2 indicates that point j in `dmOut` is not a leaf in `sfCoarsen`. 422620f4b53cSBarry Smith . useBCs - `PETSC_TRUE` indicates that boundary values should be inserted into `vecIn` before transfer. 4227ff1f73f7SToby Isaac - time - Used if boundary values are time dependent. 4228ff1f73f7SToby Isaac 42292fe279fdSBarry Smith Output Parameter: 42308966356dSPierre Jolivet . vecOut - Using interpolation and injection operators calculated on the reference tree, the transferred 423120f4b53cSBarry Smith projection of `vecIn` from `dmIn` to `dmOut`. Note that any field discretized with a `PetscFV` finite volume 4232ff1f73f7SToby Isaac method that uses gradient reconstruction will use reconstructed gradients when interpolating from 4233ff1f73f7SToby Isaac coarse points to fine points. 4234ff1f73f7SToby Isaac 4235ff1f73f7SToby Isaac Level: developer 4236ff1f73f7SToby Isaac 42371cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscSF`, `Vec`, `PetscFV`, `DMPlexSetReferenceTree()`, `DMPlexGetReferenceTree()`, `PetscFVGetComputeGradients()` 4238ff1f73f7SToby Isaac @*/ 4239d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexTransferVecTree(DM dmIn, Vec vecIn, DM dmOut, Vec vecOut, PetscSF sfRefine, PetscSF sfCoarsen, PetscInt *cidsRefine, PetscInt *cidsCoarsen, PetscBool useBCs, PetscReal time) 4240d71ae5a4SJacob Faibussowitsch { 424138fc2455SToby Isaac PetscFunctionBegin; 42429566063dSJacob Faibussowitsch PetscCall(VecSet(vecOut, 0.0)); 4243ff1f73f7SToby Isaac if (sfRefine) { 4244fbfa57b9SToby Isaac Vec vecInLocal; 42450eb7e1eaSToby Isaac DM dmGrad = NULL; 42460eb7e1eaSToby Isaac Vec faceGeom = NULL, cellGeom = NULL, grad = NULL; 4247fbfa57b9SToby Isaac 42489566063dSJacob Faibussowitsch PetscCall(DMGetLocalVector(dmIn, &vecInLocal)); 42499566063dSJacob Faibussowitsch PetscCall(VecSet(vecInLocal, 0.0)); 42500eb7e1eaSToby Isaac { 42510eb7e1eaSToby Isaac PetscInt numFields, i; 42520eb7e1eaSToby Isaac 42539566063dSJacob Faibussowitsch PetscCall(DMGetNumFields(dmIn, &numFields)); 42540eb7e1eaSToby Isaac for (i = 0; i < numFields; i++) { 42550eb7e1eaSToby Isaac PetscObject obj; 42560eb7e1eaSToby Isaac PetscClassId classid; 42570eb7e1eaSToby Isaac 42589566063dSJacob Faibussowitsch PetscCall(DMGetField(dmIn, i, NULL, &obj)); 42599566063dSJacob Faibussowitsch PetscCall(PetscObjectGetClassId(obj, &classid)); 42600eb7e1eaSToby Isaac if (classid == PETSCFV_CLASSID) { 42619566063dSJacob Faibussowitsch PetscCall(DMPlexGetDataFVM(dmIn, (PetscFV)obj, &cellGeom, &faceGeom, &dmGrad)); 42620eb7e1eaSToby Isaac break; 42630eb7e1eaSToby Isaac } 42640eb7e1eaSToby Isaac } 42650eb7e1eaSToby Isaac } 42661baa6e33SBarry Smith if (useBCs) PetscCall(DMPlexInsertBoundaryValues(dmIn, PETSC_TRUE, vecInLocal, time, faceGeom, cellGeom, NULL)); 42679566063dSJacob Faibussowitsch PetscCall(DMGlobalToLocalBegin(dmIn, vecIn, INSERT_VALUES, vecInLocal)); 42689566063dSJacob Faibussowitsch PetscCall(DMGlobalToLocalEnd(dmIn, vecIn, INSERT_VALUES, vecInLocal)); 42690eb7e1eaSToby Isaac if (dmGrad) { 42709566063dSJacob Faibussowitsch PetscCall(DMGetGlobalVector(dmGrad, &grad)); 42719566063dSJacob Faibussowitsch PetscCall(DMPlexReconstructGradientsFVM(dmIn, vecInLocal, grad)); 42720eb7e1eaSToby Isaac } 42739566063dSJacob Faibussowitsch PetscCall(DMPlexTransferVecTree_Interpolate(dmIn, vecInLocal, dmOut, vecOut, sfRefine, cidsRefine, grad, cellGeom)); 42749566063dSJacob Faibussowitsch PetscCall(DMRestoreLocalVector(dmIn, &vecInLocal)); 427548a46eb9SPierre Jolivet if (dmGrad) PetscCall(DMRestoreGlobalVector(dmGrad, &grad)); 4276ebf164c7SToby Isaac } 42771baa6e33SBarry Smith if (sfCoarsen) PetscCall(DMPlexTransferVecTree_Inject(dmIn, vecIn, dmOut, vecOut, sfCoarsen, cidsCoarsen)); 42789566063dSJacob Faibussowitsch PetscCall(VecAssemblyBegin(vecOut)); 42799566063dSJacob Faibussowitsch PetscCall(VecAssemblyEnd(vecOut)); 42803ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 428138fc2455SToby Isaac } 4282