xref: /petsc/src/dm/impls/plex/plextree.c (revision f489ac74240ff84cac916603ef355e2f8f5405a9)
1af0996ceSBarry Smith #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2af0996ceSBarry Smith #include <petsc/private/isimpl.h>
3af0996ceSBarry Smith #include <petsc/private/petscfeimpl.h>
4d6a7ad0dSToby Isaac #include <petscsf.h>
50c37af3bSToby Isaac #include <petscds.h>
6d6a7ad0dSToby Isaac 
7d6a7ad0dSToby Isaac /** hierarchy routines */
8d6a7ad0dSToby Isaac 
9d6a7ad0dSToby Isaac /*@
10d6a7ad0dSToby Isaac   DMPlexSetReferenceTree - set the reference tree for hierarchically non-conforming meshes.
11d6a7ad0dSToby Isaac 
12d6a7ad0dSToby Isaac   Not collective
13d6a7ad0dSToby Isaac 
14d6a7ad0dSToby Isaac   Input Parameters:
15d6a7ad0dSToby Isaac + dm - The DMPlex object
16d6a7ad0dSToby Isaac - ref - The reference tree DMPlex object
17d6a7ad0dSToby Isaac 
180b7167a0SToby Isaac   Level: intermediate
19d6a7ad0dSToby Isaac 
20da43764aSToby Isaac .seealso: DMPlexGetReferenceTree(), DMPlexCreateDefaultReferenceTree()
21d6a7ad0dSToby Isaac @*/
22d6a7ad0dSToby Isaac PetscErrorCode DMPlexSetReferenceTree(DM dm, DM ref)
23d6a7ad0dSToby Isaac {
24d6a7ad0dSToby Isaac   DM_Plex        *mesh = (DM_Plex *)dm->data;
25d6a7ad0dSToby Isaac   PetscErrorCode  ierr;
26d6a7ad0dSToby Isaac 
27d6a7ad0dSToby Isaac   PetscFunctionBegin;
28d6a7ad0dSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2947a1df27SMatthew G. Knepley   if (ref) {PetscValidHeaderSpecific(ref, DM_CLASSID, 2);}
30d6a7ad0dSToby Isaac   ierr = PetscObjectReference((PetscObject)ref);CHKERRQ(ierr);
31d6a7ad0dSToby Isaac   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
32d6a7ad0dSToby Isaac   mesh->referenceTree = ref;
33d6a7ad0dSToby Isaac   PetscFunctionReturn(0);
34d6a7ad0dSToby Isaac }
35d6a7ad0dSToby Isaac 
36d6a7ad0dSToby Isaac /*@
37d6a7ad0dSToby Isaac   DMPlexGetReferenceTree - get the reference tree for hierarchically non-conforming meshes.
38d6a7ad0dSToby Isaac 
39d6a7ad0dSToby Isaac   Not collective
40d6a7ad0dSToby Isaac 
41d6a7ad0dSToby Isaac   Input Parameters:
42d6a7ad0dSToby Isaac . dm - The DMPlex object
43d6a7ad0dSToby Isaac 
44d6a7ad0dSToby Isaac   Output Parameters
45d6a7ad0dSToby Isaac . ref - The reference tree DMPlex object
46d6a7ad0dSToby Isaac 
470b7167a0SToby Isaac   Level: intermediate
48d6a7ad0dSToby Isaac 
49da43764aSToby Isaac .seealso: DMPlexSetReferenceTree(), DMPlexCreateDefaultReferenceTree()
50d6a7ad0dSToby Isaac @*/
51d6a7ad0dSToby Isaac PetscErrorCode DMPlexGetReferenceTree(DM dm, DM *ref)
52d6a7ad0dSToby Isaac {
53d6a7ad0dSToby Isaac   DM_Plex        *mesh = (DM_Plex *)dm->data;
54d6a7ad0dSToby Isaac 
55d6a7ad0dSToby Isaac   PetscFunctionBegin;
56d6a7ad0dSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
57d6a7ad0dSToby Isaac   PetscValidPointer(ref,2);
58d6a7ad0dSToby Isaac   *ref = mesh->referenceTree;
59d6a7ad0dSToby Isaac   PetscFunctionReturn(0);
60d6a7ad0dSToby Isaac }
61d6a7ad0dSToby Isaac 
62dcbd3bf7SToby Isaac static PetscErrorCode DMPlexReferenceTreeGetChildSymmetry_Default(DM dm, PetscInt parent, PetscInt parentOrientA, PetscInt childOrientA, PetscInt childA, PetscInt parentOrientB, PetscInt *childOrientB, PetscInt *childB)
63dcbd3bf7SToby Isaac {
64dcbd3bf7SToby Isaac   PetscInt       coneSize, dStart, dEnd, dim, ABswap, oAvert, oBvert, ABswapVert;
65dcbd3bf7SToby Isaac   PetscErrorCode ierr;
66dcbd3bf7SToby Isaac 
67dcbd3bf7SToby Isaac   PetscFunctionBegin;
68dcbd3bf7SToby Isaac   if (parentOrientA == parentOrientB) {
69dcbd3bf7SToby Isaac     if (childOrientB) *childOrientB = childOrientA;
70dcbd3bf7SToby Isaac     if (childB) *childB = childA;
71dcbd3bf7SToby Isaac     PetscFunctionReturn(0);
72dcbd3bf7SToby Isaac   }
73dcbd3bf7SToby Isaac   for (dim = 0; dim < 3; dim++) {
74dcbd3bf7SToby Isaac     ierr = DMPlexGetDepthStratum(dm,dim,&dStart,&dEnd);CHKERRQ(ierr);
75dcbd3bf7SToby Isaac     if (parent >= dStart && parent <= dEnd) {
76dcbd3bf7SToby Isaac       break;
77dcbd3bf7SToby Isaac     }
78dcbd3bf7SToby Isaac   }
79dcbd3bf7SToby Isaac   if (dim > 2) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot perform child symmetry for %d-cells",dim);
80dcbd3bf7SToby Isaac   if (!dim) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"A vertex has no children");
81dcbd3bf7SToby Isaac   if (childA < dStart || childA >= dEnd) {
82dcbd3bf7SToby Isaac     /* this is a lower-dimensional child: bootstrap */
83dcbd3bf7SToby Isaac     PetscInt size, i, sA = -1, sB, sOrientB, sConeSize;
84dcbd3bf7SToby Isaac     const PetscInt *supp, *coneA, *coneB, *oA, *oB;
85dcbd3bf7SToby Isaac 
86dcbd3bf7SToby Isaac     ierr = DMPlexGetSupportSize(dm,childA,&size);CHKERRQ(ierr);
87dcbd3bf7SToby Isaac     ierr = DMPlexGetSupport(dm,childA,&supp);CHKERRQ(ierr);
88dcbd3bf7SToby Isaac 
89dcbd3bf7SToby Isaac     /* find a point sA in supp(childA) that has the same parent */
90dcbd3bf7SToby Isaac     for (i = 0; i < size; i++) {
91dcbd3bf7SToby Isaac       PetscInt sParent;
92dcbd3bf7SToby Isaac 
93dcbd3bf7SToby Isaac       sA   = supp[i];
94dcbd3bf7SToby Isaac       if (sA == parent) continue;
95dcbd3bf7SToby Isaac       ierr = DMPlexGetTreeParent(dm,sA,&sParent,NULL);CHKERRQ(ierr);
96dcbd3bf7SToby Isaac       if (sParent == parent) {
97dcbd3bf7SToby Isaac         break;
98dcbd3bf7SToby Isaac       }
99dcbd3bf7SToby Isaac     }
100dcbd3bf7SToby Isaac     if (i == size) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"could not find support in children");
101dcbd3bf7SToby Isaac     /* find out which point sB is in an equivalent position to sA under
102dcbd3bf7SToby Isaac      * parentOrientB */
103dcbd3bf7SToby Isaac     ierr = DMPlexReferenceTreeGetChildSymmetry_Default(dm,parent,parentOrientA,0,sA,parentOrientB,&sOrientB,&sB);CHKERRQ(ierr);
104dcbd3bf7SToby Isaac     ierr = DMPlexGetConeSize(dm,sA,&sConeSize);CHKERRQ(ierr);
105dcbd3bf7SToby Isaac     ierr = DMPlexGetCone(dm,sA,&coneA);CHKERRQ(ierr);
106dcbd3bf7SToby Isaac     ierr = DMPlexGetCone(dm,sB,&coneB);CHKERRQ(ierr);
107dcbd3bf7SToby Isaac     ierr = DMPlexGetConeOrientation(dm,sA,&oA);CHKERRQ(ierr);
108dcbd3bf7SToby Isaac     ierr = DMPlexGetConeOrientation(dm,sB,&oB);CHKERRQ(ierr);
109dcbd3bf7SToby Isaac     /* step through the cone of sA in natural order */
110dcbd3bf7SToby Isaac     for (i = 0; i < sConeSize; i++) {
111dcbd3bf7SToby Isaac       if (coneA[i] == childA) {
112dcbd3bf7SToby Isaac         /* if childA is at position i in coneA,
113dcbd3bf7SToby Isaac          * then we want the point that is at sOrientB*i in coneB */
114dcbd3bf7SToby Isaac         PetscInt j = (sOrientB >= 0) ? ((sOrientB + i) % sConeSize) : ((sConeSize -(sOrientB+1) - i) % sConeSize);
115dcbd3bf7SToby Isaac         if (childB) *childB = coneB[j];
116dcbd3bf7SToby Isaac         if (childOrientB) {
117dcbd3bf7SToby Isaac           PetscInt oBtrue;
118dcbd3bf7SToby Isaac 
119dcbd3bf7SToby Isaac           ierr          = DMPlexGetConeSize(dm,childA,&coneSize);CHKERRQ(ierr);
120dcbd3bf7SToby Isaac           /* compose sOrientB and oB[j] */
121dcbd3bf7SToby Isaac           if (coneSize != 0 && coneSize != 2) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected a vertex or an edge");
122dcbd3bf7SToby Isaac           /* we may have to flip an edge */
123dcbd3bf7SToby Isaac           oBtrue        = coneSize ? ((sOrientB >= 0) ? oB[j] : -(oB[j] + 2)) : 0;
124dcbd3bf7SToby Isaac           ABswap        = DihedralSwap(coneSize,oA[i],oBtrue);CHKERRQ(ierr);
125dcbd3bf7SToby Isaac           *childOrientB = DihedralCompose(coneSize,childOrientA,ABswap);
126dcbd3bf7SToby Isaac         }
127dcbd3bf7SToby Isaac         break;
128dcbd3bf7SToby Isaac       }
129dcbd3bf7SToby Isaac     }
130dcbd3bf7SToby Isaac     if (i == sConeSize) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"support cone mismatch");
131dcbd3bf7SToby Isaac     PetscFunctionReturn(0);
132dcbd3bf7SToby Isaac   }
133dcbd3bf7SToby Isaac   /* get the cone size and symmetry swap */
134dcbd3bf7SToby Isaac   ierr   = DMPlexGetConeSize(dm,parent,&coneSize);CHKERRQ(ierr);
135dcbd3bf7SToby Isaac   ABswap = DihedralSwap(coneSize, parentOrientA, parentOrientB);
136dcbd3bf7SToby Isaac   if (dim == 2) {
137dcbd3bf7SToby Isaac     /* orientations refer to cones: we want them to refer to vertices:
138dcbd3bf7SToby Isaac      * if it's a rotation, they are the same, but if the order is reversed, a
139dcbd3bf7SToby Isaac      * permutation that puts side i first does *not* put vertex i first */
140dcbd3bf7SToby Isaac     oAvert     = (parentOrientA >= 0) ? parentOrientA : -((-parentOrientA % coneSize) + 1);
141dcbd3bf7SToby Isaac     oBvert     = (parentOrientB >= 0) ? parentOrientB : -((-parentOrientB % coneSize) + 1);
142dcbd3bf7SToby Isaac     ABswapVert = DihedralSwap(coneSize, oAvert, oBvert);
143947b95d8SBarry Smith   } else {
144dcbd3bf7SToby Isaac     ABswapVert = ABswap;
145dcbd3bf7SToby Isaac   }
146dcbd3bf7SToby Isaac   if (childB) {
147dcbd3bf7SToby Isaac     /* assume that each child corresponds to a vertex, in the same order */
148dcbd3bf7SToby Isaac     PetscInt p, posA = -1, numChildren, i;
149dcbd3bf7SToby Isaac     const PetscInt *children;
150dcbd3bf7SToby Isaac 
151dcbd3bf7SToby Isaac     /* count which position the child is in */
152dcbd3bf7SToby Isaac     ierr = DMPlexGetTreeChildren(dm,parent,&numChildren,&children);CHKERRQ(ierr);
153dcbd3bf7SToby Isaac     for (i = 0; i < numChildren; i++) {
154dcbd3bf7SToby Isaac       p = children[i];
155dcbd3bf7SToby Isaac       if (p == childA) {
156dcbd3bf7SToby Isaac         posA = i;
157dcbd3bf7SToby Isaac         break;
158dcbd3bf7SToby Isaac       }
159dcbd3bf7SToby Isaac     }
160dcbd3bf7SToby Isaac     if (posA >= coneSize) {
161dcbd3bf7SToby Isaac       /* this is the triangle in the middle of a uniformly refined triangle: it
162dcbd3bf7SToby Isaac        * is invariant */
163dcbd3bf7SToby Isaac       if (dim != 2 || posA != 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Expected a middle triangle, got something else");
164dcbd3bf7SToby Isaac       *childB = childA;
165dcbd3bf7SToby Isaac     }
166dcbd3bf7SToby Isaac     else {
167dcbd3bf7SToby Isaac       /* figure out position B by applying ABswapVert */
168dcbd3bf7SToby Isaac       PetscInt posB;
169dcbd3bf7SToby Isaac 
170dcbd3bf7SToby Isaac       posB = (ABswapVert >= 0) ? ((ABswapVert + posA) % coneSize) : ((coneSize -(ABswapVert + 1) - posA) % coneSize);
171dcbd3bf7SToby Isaac       if (childB) *childB = children[posB];
172dcbd3bf7SToby Isaac     }
173dcbd3bf7SToby Isaac   }
174dcbd3bf7SToby Isaac   if (childOrientB) *childOrientB = DihedralCompose(coneSize,childOrientA,ABswap);
175dcbd3bf7SToby Isaac   PetscFunctionReturn(0);
176dcbd3bf7SToby Isaac }
177dcbd3bf7SToby Isaac 
178dcbd3bf7SToby Isaac /*@
179dcbd3bf7SToby Isaac   DMPlexReferenceTreeGetChildSymmetry - Given a reference tree, transform a childid and orientation from one parent frame to another
180dcbd3bf7SToby Isaac 
181dcbd3bf7SToby Isaac   Input Parameters:
182dcbd3bf7SToby Isaac + dm - the reference tree DMPlex object
183dcbd3bf7SToby Isaac . parent - the parent point
184dcbd3bf7SToby Isaac . parentOrientA - the reference orientation for describing the parent
185dcbd3bf7SToby Isaac . childOrientA - the reference orientation for describing the child
186dcbd3bf7SToby Isaac . childA - the reference childID for describing the child
187dcbd3bf7SToby Isaac - parentOrientB - the new orientation for describing the parent
188dcbd3bf7SToby Isaac 
189dcbd3bf7SToby Isaac   Output Parameters:
190dcbd3bf7SToby Isaac + childOrientB - if not NULL, set to the new oreintation for describing the child
191ff1f73f7SToby Isaac - childB - if not NULL, the new childID for describing the child
192dcbd3bf7SToby Isaac 
193dcbd3bf7SToby Isaac   Level: developer
194dcbd3bf7SToby Isaac 
195dcbd3bf7SToby Isaac .seealso: DMPlexGetReferenceTree(), DMPlexSetReferenceTree(), DMPlexSetTree()
196dcbd3bf7SToby Isaac @*/
197dcbd3bf7SToby Isaac PetscErrorCode DMPlexReferenceTreeGetChildSymmetry(DM dm, PetscInt parent, PetscInt parentOrientA, PetscInt childOrientA, PetscInt childA, PetscInt parentOrientB, PetscInt *childOrientB, PetscInt *childB)
198dcbd3bf7SToby Isaac {
199dcbd3bf7SToby Isaac   DM_Plex        *mesh = (DM_Plex *)dm->data;
200dcbd3bf7SToby Isaac   PetscErrorCode ierr;
201dcbd3bf7SToby Isaac 
202dcbd3bf7SToby Isaac   PetscFunctionBegin;
203dcbd3bf7SToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
204dcbd3bf7SToby Isaac   if (!mesh->getchildsymmetry) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"DMPlexReferenceTreeGetChildSymmetry not implemented");
205dcbd3bf7SToby Isaac   ierr = mesh->getchildsymmetry(dm,parent,parentOrientA,childOrientA,childA,parentOrientB,childOrientB,childB);CHKERRQ(ierr);
206dcbd3bf7SToby Isaac   PetscFunctionReturn(0);
207dcbd3bf7SToby Isaac }
208dcbd3bf7SToby Isaac 
209776742edSToby Isaac static PetscErrorCode DMPlexSetTree_Internal(DM,PetscSection,PetscInt*,PetscInt*,PetscBool,PetscBool);
210f9f063d4SToby Isaac 
2110e2cc29aSToby Isaac PetscErrorCode DMPlexCreateReferenceTree_Union(DM K, DM Kref, const char *labelName, DM *ref)
212da43764aSToby Isaac {
2130e2cc29aSToby Isaac   MPI_Comm       comm;
2140e2cc29aSToby Isaac   PetscInt       dim, p, pStart, pEnd, pRefStart, pRefEnd, d, offset, parentSize, *parents, *childIDs;
215da43764aSToby Isaac   PetscInt      *permvals, *unionCones, *coneSizes, *unionOrientations, numUnionPoints, *numDimPoints, numCones, numVerts;
216da43764aSToby Isaac   DMLabel        identity, identityRef;
21710f7e118SToby Isaac   PetscSection   unionSection, unionConeSection, parentSection;
218da43764aSToby Isaac   PetscScalar   *unionCoords;
219da43764aSToby Isaac   IS             perm;
220da43764aSToby Isaac   PetscErrorCode ierr;
221da43764aSToby Isaac 
222da43764aSToby Isaac   PetscFunctionBegin;
2230e2cc29aSToby Isaac   comm = PetscObjectComm((PetscObject)K);
2240e2cc29aSToby Isaac   ierr = DMGetDimension(K, &dim);CHKERRQ(ierr);
225da43764aSToby Isaac   ierr = DMPlexGetChart(K, &pStart, &pEnd);CHKERRQ(ierr);
2260e2cc29aSToby Isaac   ierr = DMGetLabel(K, labelName, &identity);CHKERRQ(ierr);
2270e2cc29aSToby Isaac   ierr = DMGetLabel(Kref, labelName, &identityRef);CHKERRQ(ierr);
228da43764aSToby Isaac   ierr = DMPlexGetChart(Kref, &pRefStart, &pRefEnd);CHKERRQ(ierr);
229da43764aSToby Isaac   ierr = PetscSectionCreate(comm, &unionSection);CHKERRQ(ierr);
230da43764aSToby Isaac   ierr = PetscSectionSetChart(unionSection, 0, (pEnd - pStart) + (pRefEnd - pRefStart));CHKERRQ(ierr);
231da43764aSToby Isaac   /* count points that will go in the union */
232da43764aSToby Isaac   for (p = pStart; p < pEnd; p++) {
233da43764aSToby Isaac     ierr = PetscSectionSetDof(unionSection, p - pStart, 1);CHKERRQ(ierr);
234da43764aSToby Isaac   }
235da43764aSToby Isaac   for (p = pRefStart; p < pRefEnd; p++) {
236da43764aSToby Isaac     PetscInt q, qSize;
237da43764aSToby Isaac     ierr = DMLabelGetValue(identityRef, p, &q);CHKERRQ(ierr);
238da43764aSToby Isaac     ierr = DMLabelGetStratumSize(identityRef, q, &qSize);CHKERRQ(ierr);
239da43764aSToby Isaac     if (qSize > 1) {
240da43764aSToby Isaac       ierr = PetscSectionSetDof(unionSection, p - pRefStart + (pEnd - pStart), 1);CHKERRQ(ierr);
241da43764aSToby Isaac     }
242da43764aSToby Isaac   }
243854ce69bSBarry Smith   ierr = PetscMalloc1(pEnd - pStart + pRefEnd - pRefStart,&permvals);CHKERRQ(ierr);
244da43764aSToby Isaac   offset = 0;
245da43764aSToby Isaac   /* stratify points in the union by topological dimension */
246da43764aSToby Isaac   for (d = 0; d <= dim; d++) {
247da43764aSToby Isaac     PetscInt cStart, cEnd, c;
248da43764aSToby Isaac 
249da43764aSToby Isaac     ierr = DMPlexGetHeightStratum(K, d, &cStart, &cEnd);CHKERRQ(ierr);
250da43764aSToby Isaac     for (c = cStart; c < cEnd; c++) {
251da43764aSToby Isaac       permvals[offset++] = c;
252da43764aSToby Isaac     }
253da43764aSToby Isaac 
254da43764aSToby Isaac     ierr = DMPlexGetHeightStratum(Kref, d, &cStart, &cEnd);CHKERRQ(ierr);
255da43764aSToby Isaac     for (c = cStart; c < cEnd; c++) {
256da43764aSToby Isaac       permvals[offset++] = c + (pEnd - pStart);
257da43764aSToby Isaac     }
258da43764aSToby Isaac   }
259da43764aSToby Isaac   ierr = ISCreateGeneral(comm, (pEnd - pStart) + (pRefEnd - pRefStart), permvals, PETSC_OWN_POINTER, &perm);CHKERRQ(ierr);
260da43764aSToby Isaac   ierr = PetscSectionSetPermutation(unionSection,perm);CHKERRQ(ierr);
261da43764aSToby Isaac   ierr = PetscSectionSetUp(unionSection);CHKERRQ(ierr);
262da43764aSToby Isaac   ierr = PetscSectionGetStorageSize(unionSection,&numUnionPoints);CHKERRQ(ierr);
263da43764aSToby Isaac   ierr = PetscMalloc2(numUnionPoints,&coneSizes,dim+1,&numDimPoints);CHKERRQ(ierr);
264da43764aSToby Isaac   /* count dimension points */
265da43764aSToby Isaac   for (d = 0; d <= dim; d++) {
266da43764aSToby Isaac     PetscInt cStart, cOff, cOff2;
267da43764aSToby Isaac     ierr = DMPlexGetHeightStratum(K,d,&cStart,NULL);CHKERRQ(ierr);
268da43764aSToby Isaac     ierr = PetscSectionGetOffset(unionSection,cStart-pStart,&cOff);CHKERRQ(ierr);
269da43764aSToby Isaac     if (d < dim) {
270da43764aSToby Isaac       ierr = DMPlexGetHeightStratum(K,d+1,&cStart,NULL);CHKERRQ(ierr);
271da43764aSToby Isaac       ierr = PetscSectionGetOffset(unionSection,cStart-pStart,&cOff2);CHKERRQ(ierr);
272da43764aSToby Isaac     }
273da43764aSToby Isaac     else {
274da43764aSToby Isaac       cOff2 = numUnionPoints;
275da43764aSToby Isaac     }
276da43764aSToby Isaac     numDimPoints[dim - d] = cOff2 - cOff;
277da43764aSToby Isaac   }
278da43764aSToby Isaac   ierr = PetscSectionCreate(comm, &unionConeSection);CHKERRQ(ierr);
279da43764aSToby Isaac   ierr = PetscSectionSetChart(unionConeSection, 0, numUnionPoints);CHKERRQ(ierr);
280da43764aSToby Isaac   /* count the cones in the union */
281da43764aSToby Isaac   for (p = pStart; p < pEnd; p++) {
282da43764aSToby Isaac     PetscInt dof, uOff;
283da43764aSToby Isaac 
284da43764aSToby Isaac     ierr = DMPlexGetConeSize(K, p, &dof);CHKERRQ(ierr);
285da43764aSToby Isaac     ierr = PetscSectionGetOffset(unionSection, p - pStart,&uOff);CHKERRQ(ierr);
286da43764aSToby Isaac     ierr = PetscSectionSetDof(unionConeSection, uOff, dof);CHKERRQ(ierr);
287da43764aSToby Isaac     coneSizes[uOff] = dof;
288da43764aSToby Isaac   }
289da43764aSToby Isaac   for (p = pRefStart; p < pRefEnd; p++) {
290da43764aSToby Isaac     PetscInt dof, uDof, uOff;
291da43764aSToby Isaac 
292da43764aSToby Isaac     ierr = DMPlexGetConeSize(Kref, p, &dof);CHKERRQ(ierr);
293da43764aSToby Isaac     ierr = PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart),&uDof);CHKERRQ(ierr);
294da43764aSToby Isaac     ierr = PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart),&uOff);CHKERRQ(ierr);
295da43764aSToby Isaac     if (uDof) {
296da43764aSToby Isaac       ierr = PetscSectionSetDof(unionConeSection, uOff, dof);CHKERRQ(ierr);
297da43764aSToby Isaac       coneSizes[uOff] = dof;
298da43764aSToby Isaac     }
299da43764aSToby Isaac   }
300da43764aSToby Isaac   ierr = PetscSectionSetUp(unionConeSection);CHKERRQ(ierr);
301da43764aSToby Isaac   ierr = PetscSectionGetStorageSize(unionConeSection,&numCones);CHKERRQ(ierr);
302da43764aSToby Isaac   ierr = PetscMalloc2(numCones,&unionCones,numCones,&unionOrientations);CHKERRQ(ierr);
303da43764aSToby Isaac   /* write the cones in the union */
304da43764aSToby Isaac   for (p = pStart; p < pEnd; p++) {
305da43764aSToby Isaac     PetscInt dof, uOff, c, cOff;
306da43764aSToby Isaac     const PetscInt *cone, *orientation;
307da43764aSToby Isaac 
308da43764aSToby Isaac     ierr = DMPlexGetConeSize(K, p, &dof);CHKERRQ(ierr);
309da43764aSToby Isaac     ierr = DMPlexGetCone(K, p, &cone);CHKERRQ(ierr);
310da43764aSToby Isaac     ierr = DMPlexGetConeOrientation(K, p, &orientation);CHKERRQ(ierr);
311da43764aSToby Isaac     ierr = PetscSectionGetOffset(unionSection, p - pStart,&uOff);CHKERRQ(ierr);
312da43764aSToby Isaac     ierr = PetscSectionGetOffset(unionConeSection,uOff,&cOff);CHKERRQ(ierr);
313da43764aSToby Isaac     for (c = 0; c < dof; c++) {
314da43764aSToby Isaac       PetscInt e, eOff;
315da43764aSToby Isaac       e                           = cone[c];
316da43764aSToby Isaac       ierr                        = PetscSectionGetOffset(unionSection, e - pStart, &eOff);CHKERRQ(ierr);
317da43764aSToby Isaac       unionCones[cOff + c]        = eOff;
318da43764aSToby Isaac       unionOrientations[cOff + c] = orientation[c];
319da43764aSToby Isaac     }
320da43764aSToby Isaac   }
321da43764aSToby Isaac   for (p = pRefStart; p < pRefEnd; p++) {
322da43764aSToby Isaac     PetscInt dof, uDof, uOff, c, cOff;
323da43764aSToby Isaac     const PetscInt *cone, *orientation;
324da43764aSToby Isaac 
325da43764aSToby Isaac     ierr = DMPlexGetConeSize(Kref, p, &dof);CHKERRQ(ierr);
326da43764aSToby Isaac     ierr = DMPlexGetCone(Kref, p, &cone);CHKERRQ(ierr);
327da43764aSToby Isaac     ierr = DMPlexGetConeOrientation(Kref, p, &orientation);CHKERRQ(ierr);
328da43764aSToby Isaac     ierr = PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart),&uDof);CHKERRQ(ierr);
329da43764aSToby Isaac     ierr = PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart),&uOff);CHKERRQ(ierr);
330da43764aSToby Isaac     if (uDof) {
331da43764aSToby Isaac       ierr = PetscSectionGetOffset(unionConeSection,uOff,&cOff);CHKERRQ(ierr);
332da43764aSToby Isaac       for (c = 0; c < dof; c++) {
333da43764aSToby Isaac         PetscInt e, eOff, eDof;
334da43764aSToby Isaac 
335da43764aSToby Isaac         e    = cone[c];
336da43764aSToby Isaac         ierr = PetscSectionGetDof(unionSection, e - pRefStart + (pEnd - pStart),&eDof);CHKERRQ(ierr);
337da43764aSToby Isaac         if (eDof) {
338da43764aSToby Isaac           ierr = PetscSectionGetOffset(unionSection, e - pRefStart + (pEnd - pStart), &eOff);CHKERRQ(ierr);
339da43764aSToby Isaac         }
340da43764aSToby Isaac         else {
341da43764aSToby Isaac           ierr = DMLabelGetValue(identityRef, e, &e);CHKERRQ(ierr);
342da43764aSToby Isaac           ierr = PetscSectionGetOffset(unionSection, e - pStart, &eOff);CHKERRQ(ierr);
343da43764aSToby Isaac         }
344da43764aSToby Isaac         unionCones[cOff + c]        = eOff;
345da43764aSToby Isaac         unionOrientations[cOff + c] = orientation[c];
346da43764aSToby Isaac       }
347da43764aSToby Isaac     }
348da43764aSToby Isaac   }
349da43764aSToby Isaac   /* get the coordinates */
350da43764aSToby Isaac   {
351da43764aSToby Isaac     PetscInt vStart, vEnd, vRefStart, vRefEnd, v, vDof, vOff;
352da43764aSToby Isaac     PetscSection KcoordsSec, KrefCoordsSec;
353da43764aSToby Isaac     Vec      KcoordsVec, KrefCoordsVec;
354da43764aSToby Isaac     PetscScalar *Kcoords;
355da43764aSToby Isaac 
356da43764aSToby Isaac     DMGetCoordinateSection(K, &KcoordsSec);CHKERRQ(ierr);
357da43764aSToby Isaac     DMGetCoordinatesLocal(K, &KcoordsVec);CHKERRQ(ierr);
358da43764aSToby Isaac     DMGetCoordinateSection(Kref, &KrefCoordsSec);CHKERRQ(ierr);
359da43764aSToby Isaac     DMGetCoordinatesLocal(Kref, &KrefCoordsVec);CHKERRQ(ierr);
360da43764aSToby Isaac 
361da43764aSToby Isaac     numVerts = numDimPoints[0];
362da43764aSToby Isaac     ierr     = PetscMalloc1(numVerts * dim,&unionCoords);CHKERRQ(ierr);
363da43764aSToby Isaac     ierr     = DMPlexGetDepthStratum(K,0,&vStart,&vEnd);CHKERRQ(ierr);
364da43764aSToby Isaac 
365da43764aSToby Isaac     offset = 0;
366da43764aSToby Isaac     for (v = vStart; v < vEnd; v++) {
367da43764aSToby Isaac       ierr = PetscSectionGetOffset(unionSection,v - pStart,&vOff);CHKERRQ(ierr);
368da43764aSToby Isaac       ierr = VecGetValuesSection(KcoordsVec, KcoordsSec, v, &Kcoords);CHKERRQ(ierr);
369da43764aSToby Isaac       for (d = 0; d < dim; d++) {
370da43764aSToby Isaac         unionCoords[offset * dim + d] = Kcoords[d];
371da43764aSToby Isaac       }
372da43764aSToby Isaac       offset++;
373da43764aSToby Isaac     }
374da43764aSToby Isaac     ierr = DMPlexGetDepthStratum(Kref,0,&vRefStart,&vRefEnd);CHKERRQ(ierr);
375da43764aSToby Isaac     for (v = vRefStart; v < vRefEnd; v++) {
376da43764aSToby Isaac       ierr = PetscSectionGetDof(unionSection,v - pRefStart + (pEnd - pStart),&vDof);CHKERRQ(ierr);
377da43764aSToby Isaac       ierr = PetscSectionGetOffset(unionSection,v - pRefStart + (pEnd - pStart),&vOff);CHKERRQ(ierr);
378da43764aSToby Isaac       ierr = VecGetValuesSection(KrefCoordsVec, KrefCoordsSec, v, &Kcoords);CHKERRQ(ierr);
379da43764aSToby Isaac       if (vDof) {
380da43764aSToby Isaac         for (d = 0; d < dim; d++) {
381da43764aSToby Isaac           unionCoords[offset * dim + d] = Kcoords[d];
382da43764aSToby Isaac         }
383da43764aSToby Isaac         offset++;
384da43764aSToby Isaac       }
385da43764aSToby Isaac     }
386da43764aSToby Isaac   }
387da43764aSToby Isaac   ierr = DMCreate(comm,ref);CHKERRQ(ierr);
388da43764aSToby Isaac   ierr = DMSetType(*ref,DMPLEX);CHKERRQ(ierr);
38928f4b327SMatthew G. Knepley   ierr = DMSetDimension(*ref,dim);CHKERRQ(ierr);
390da43764aSToby Isaac   ierr = DMPlexCreateFromDAG(*ref,dim,numDimPoints,coneSizes,unionCones,unionOrientations,unionCoords);CHKERRQ(ierr);
39110f7e118SToby Isaac   /* set the tree */
39210f7e118SToby Isaac   ierr = PetscSectionCreate(comm,&parentSection);CHKERRQ(ierr);
39310f7e118SToby Isaac   ierr = PetscSectionSetChart(parentSection,0,numUnionPoints);CHKERRQ(ierr);
39410f7e118SToby Isaac   for (p = pRefStart; p < pRefEnd; p++) {
39510f7e118SToby Isaac     PetscInt uDof, uOff;
39610f7e118SToby Isaac 
39710f7e118SToby Isaac     ierr = PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart),&uDof);CHKERRQ(ierr);
39810f7e118SToby Isaac     ierr = PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart),&uOff);CHKERRQ(ierr);
39910f7e118SToby Isaac     if (uDof) {
40010f7e118SToby Isaac       PetscSectionSetDof(parentSection,uOff,1);CHKERRQ(ierr);
40110f7e118SToby Isaac     }
40210f7e118SToby Isaac   }
40310f7e118SToby Isaac   ierr = PetscSectionSetUp(parentSection);CHKERRQ(ierr);
40410f7e118SToby Isaac   ierr = PetscSectionGetStorageSize(parentSection,&parentSize);CHKERRQ(ierr);
40510f7e118SToby Isaac   ierr = PetscMalloc2(parentSize,&parents,parentSize,&childIDs);CHKERRQ(ierr);
40610f7e118SToby Isaac   for (p = pRefStart; p < pRefEnd; p++) {
40710f7e118SToby Isaac     PetscInt uDof, uOff;
40810f7e118SToby Isaac 
40910f7e118SToby Isaac     ierr = PetscSectionGetDof(unionSection, p - pRefStart + (pEnd - pStart),&uDof);CHKERRQ(ierr);
41010f7e118SToby Isaac     ierr = PetscSectionGetOffset(unionSection, p - pRefStart + (pEnd - pStart),&uOff);CHKERRQ(ierr);
41110f7e118SToby Isaac     if (uDof) {
41210f7e118SToby Isaac       PetscInt pOff, parent, parentU;
41310f7e118SToby Isaac       PetscSectionGetOffset(parentSection,uOff,&pOff);CHKERRQ(ierr);
41410f7e118SToby Isaac       DMLabelGetValue(identityRef,p,&parent);CHKERRQ(ierr);
41510f7e118SToby Isaac       ierr = PetscSectionGetOffset(unionSection, parent - pStart,&parentU);CHKERRQ(ierr);
41610f7e118SToby Isaac       parents[pOff] = parentU;
41710f7e118SToby Isaac       childIDs[pOff] = uOff;
41810f7e118SToby Isaac     }
41910f7e118SToby Isaac   }
420776742edSToby Isaac   ierr = DMPlexSetTree_Internal(*ref,parentSection,parents,childIDs,PETSC_TRUE,PETSC_FALSE);CHKERRQ(ierr);
42110f7e118SToby Isaac   ierr = PetscSectionDestroy(&parentSection);CHKERRQ(ierr);
42210f7e118SToby Isaac   ierr = PetscFree2(parents,childIDs);CHKERRQ(ierr);
42310f7e118SToby Isaac 
424da43764aSToby Isaac   /* clean up */
425da43764aSToby Isaac   ierr = PetscSectionDestroy(&unionSection);CHKERRQ(ierr);
426da43764aSToby Isaac   ierr = PetscSectionDestroy(&unionConeSection);CHKERRQ(ierr);
427da43764aSToby Isaac   ierr = ISDestroy(&perm);CHKERRQ(ierr);
428da43764aSToby Isaac   ierr = PetscFree(unionCoords);CHKERRQ(ierr);
429da43764aSToby Isaac   ierr = PetscFree2(unionCones,unionOrientations);CHKERRQ(ierr);
430da43764aSToby Isaac   ierr = PetscFree2(coneSizes,numDimPoints);CHKERRQ(ierr);
4310e2cc29aSToby Isaac   PetscFunctionReturn(0);
4320e2cc29aSToby Isaac }
4330e2cc29aSToby Isaac 
4340e2cc29aSToby Isaac /*@
4350e2cc29aSToby Isaac   DMPlexCreateDefaultReferenceTree - create a reference tree for isotropic hierarchical mesh refinement.
4360e2cc29aSToby Isaac 
4370e2cc29aSToby Isaac   Collective on comm
4380e2cc29aSToby Isaac 
4390e2cc29aSToby Isaac   Input Parameters:
4400e2cc29aSToby Isaac + comm    - the MPI communicator
4410e2cc29aSToby Isaac . dim     - the spatial dimension
4420e2cc29aSToby Isaac - simplex - Flag for simplex, otherwise use a tensor-product cell
4430e2cc29aSToby Isaac 
4440e2cc29aSToby Isaac   Output Parameters:
4450e2cc29aSToby Isaac . ref     - the reference tree DMPlex object
4460e2cc29aSToby Isaac 
4470e2cc29aSToby Isaac   Level: intermediate
4480e2cc29aSToby Isaac 
4490e2cc29aSToby Isaac .keywords: reference cell
4500e2cc29aSToby Isaac .seealso: DMPlexSetReferenceTree(), DMPlexGetReferenceTree()
4510e2cc29aSToby Isaac @*/
4520e2cc29aSToby Isaac PetscErrorCode DMPlexCreateDefaultReferenceTree(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *ref)
4530e2cc29aSToby Isaac {
4540e2cc29aSToby Isaac   DM_Plex       *mesh;
4550e2cc29aSToby Isaac   DM             K, Kref;
4560e2cc29aSToby Isaac   PetscInt       p, pStart, pEnd;
4570e2cc29aSToby Isaac   DMLabel        identity;
4580e2cc29aSToby Isaac   PetscErrorCode ierr;
4590e2cc29aSToby Isaac 
4600e2cc29aSToby Isaac   PetscFunctionBegin;
4610e2cc29aSToby Isaac #if 1
4620e2cc29aSToby Isaac   comm = PETSC_COMM_SELF;
4630e2cc29aSToby Isaac #endif
4640e2cc29aSToby Isaac   /* create a reference element */
4650e2cc29aSToby Isaac   ierr = DMPlexCreateReferenceCell(comm, dim, simplex, &K);CHKERRQ(ierr);
4660e2cc29aSToby Isaac   ierr = DMCreateLabel(K, "identity");CHKERRQ(ierr);
4670e2cc29aSToby Isaac   ierr = DMGetLabel(K, "identity", &identity);CHKERRQ(ierr);
4680e2cc29aSToby Isaac   ierr = DMPlexGetChart(K, &pStart, &pEnd);CHKERRQ(ierr);
4690e2cc29aSToby Isaac   for (p = pStart; p < pEnd; p++) {
4700e2cc29aSToby Isaac     ierr = DMLabelSetValue(identity, p, p);CHKERRQ(ierr);
4710e2cc29aSToby Isaac   }
4720e2cc29aSToby Isaac   /* refine it */
4730e2cc29aSToby Isaac   ierr = DMRefine(K,comm,&Kref);CHKERRQ(ierr);
4740e2cc29aSToby Isaac 
4750e2cc29aSToby Isaac   /* the reference tree is the union of these two, without duplicating
4760e2cc29aSToby Isaac    * points that appear in both */
4770e2cc29aSToby Isaac   ierr = DMPlexCreateReferenceTree_Union(K, Kref, "identity", ref);CHKERRQ(ierr);
4780e2cc29aSToby Isaac   mesh = (DM_Plex *) (*ref)->data;
4790e2cc29aSToby Isaac   mesh->getchildsymmetry = DMPlexReferenceTreeGetChildSymmetry_Default;
480da43764aSToby Isaac   ierr = DMDestroy(&K);CHKERRQ(ierr);
481da43764aSToby Isaac   ierr = DMDestroy(&Kref);CHKERRQ(ierr);
482da43764aSToby Isaac   PetscFunctionReturn(0);
483da43764aSToby Isaac }
484da43764aSToby Isaac 
485878b19aaSToby Isaac static PetscErrorCode DMPlexTreeSymmetrize(DM dm)
486878b19aaSToby Isaac {
487878b19aaSToby Isaac   DM_Plex        *mesh = (DM_Plex *)dm->data;
488878b19aaSToby Isaac   PetscSection   childSec, pSec;
489878b19aaSToby Isaac   PetscInt       p, pSize, cSize, parMax = PETSC_MIN_INT, parMin = PETSC_MAX_INT;
490878b19aaSToby Isaac   PetscInt       *offsets, *children, pStart, pEnd;
491878b19aaSToby Isaac   PetscErrorCode ierr;
492878b19aaSToby Isaac 
493878b19aaSToby Isaac   PetscFunctionBegin;
494878b19aaSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
495878b19aaSToby Isaac   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
496878b19aaSToby Isaac   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
497878b19aaSToby Isaac   pSec = mesh->parentSection;
498878b19aaSToby Isaac   if (!pSec) PetscFunctionReturn(0);
499878b19aaSToby Isaac   ierr = PetscSectionGetStorageSize(pSec,&pSize);CHKERRQ(ierr);
500878b19aaSToby Isaac   for (p = 0; p < pSize; p++) {
501878b19aaSToby Isaac     PetscInt par = mesh->parents[p];
502878b19aaSToby Isaac 
503878b19aaSToby Isaac     parMax = PetscMax(parMax,par+1);
504878b19aaSToby Isaac     parMin = PetscMin(parMin,par);
505878b19aaSToby Isaac   }
506878b19aaSToby Isaac   if (parMin > parMax) {
507878b19aaSToby Isaac     parMin = -1;
508878b19aaSToby Isaac     parMax = -1;
509878b19aaSToby Isaac   }
510878b19aaSToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)pSec),&childSec);CHKERRQ(ierr);
511878b19aaSToby Isaac   ierr = PetscSectionSetChart(childSec,parMin,parMax);CHKERRQ(ierr);
512878b19aaSToby Isaac   for (p = 0; p < pSize; p++) {
513878b19aaSToby Isaac     PetscInt par = mesh->parents[p];
514878b19aaSToby Isaac 
515878b19aaSToby Isaac     ierr = PetscSectionAddDof(childSec,par,1);CHKERRQ(ierr);
516878b19aaSToby Isaac   }
517878b19aaSToby Isaac   ierr = PetscSectionSetUp(childSec);CHKERRQ(ierr);
518878b19aaSToby Isaac   ierr = PetscSectionGetStorageSize(childSec,&cSize);CHKERRQ(ierr);
519878b19aaSToby Isaac   ierr = PetscMalloc1(cSize,&children);CHKERRQ(ierr);
520878b19aaSToby Isaac   ierr = PetscCalloc1(parMax-parMin,&offsets);CHKERRQ(ierr);
521878b19aaSToby Isaac   ierr = PetscSectionGetChart(pSec,&pStart,&pEnd);CHKERRQ(ierr);
522878b19aaSToby Isaac   for (p = pStart; p < pEnd; p++) {
523878b19aaSToby Isaac     PetscInt dof, off, i;
524878b19aaSToby Isaac 
525878b19aaSToby Isaac     ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
526878b19aaSToby Isaac     ierr = PetscSectionGetOffset(pSec,p,&off);CHKERRQ(ierr);
527878b19aaSToby Isaac     for (i = 0; i < dof; i++) {
528878b19aaSToby Isaac       PetscInt par = mesh->parents[off + i], cOff;
529878b19aaSToby Isaac 
530878b19aaSToby Isaac       ierr = PetscSectionGetOffset(childSec,par,&cOff);CHKERRQ(ierr);
531878b19aaSToby Isaac       children[cOff + offsets[par-parMin]++] = p;
532878b19aaSToby Isaac     }
533878b19aaSToby Isaac   }
534878b19aaSToby Isaac   mesh->childSection = childSec;
535878b19aaSToby Isaac   mesh->children = children;
536878b19aaSToby Isaac   ierr = PetscFree(offsets);CHKERRQ(ierr);
537878b19aaSToby Isaac   PetscFunctionReturn(0);
538878b19aaSToby Isaac }
539878b19aaSToby Isaac 
5406dd5a8c8SToby Isaac static PetscErrorCode AnchorsFlatten (PetscSection section, IS is, PetscSection *sectionNew, IS *isNew)
5416dd5a8c8SToby Isaac {
5426dd5a8c8SToby Isaac   PetscInt       pStart, pEnd, size, sizeNew, i, p, *valsNew = NULL;
5436dd5a8c8SToby Isaac   const PetscInt *vals;
5446dd5a8c8SToby Isaac   PetscSection   secNew;
5456dd5a8c8SToby Isaac   PetscBool      anyNew, globalAnyNew;
5466dd5a8c8SToby Isaac   PetscBool      compress;
5476dd5a8c8SToby Isaac   PetscErrorCode ierr;
5486dd5a8c8SToby Isaac 
5496dd5a8c8SToby Isaac   PetscFunctionBegin;
5506dd5a8c8SToby Isaac   ierr = PetscSectionGetChart(section,&pStart,&pEnd);CHKERRQ(ierr);
5516dd5a8c8SToby Isaac   ierr = ISGetLocalSize(is,&size);CHKERRQ(ierr);
5526dd5a8c8SToby Isaac   ierr = ISGetIndices(is,&vals);CHKERRQ(ierr);
5536dd5a8c8SToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)section),&secNew);CHKERRQ(ierr);
5546dd5a8c8SToby Isaac   ierr = PetscSectionSetChart(secNew,pStart,pEnd);CHKERRQ(ierr);
5556dd5a8c8SToby Isaac   for (i = 0; i < size; i++) {
5566dd5a8c8SToby Isaac     PetscInt dof;
5576dd5a8c8SToby Isaac 
5586dd5a8c8SToby Isaac     p = vals[i];
5596dd5a8c8SToby Isaac     if (p < pStart || p >= pEnd) continue;
5606dd5a8c8SToby Isaac     ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
5616dd5a8c8SToby Isaac     if (dof) break;
5626dd5a8c8SToby Isaac   }
5636dd5a8c8SToby Isaac   if (i == size) {
5646dd5a8c8SToby Isaac     ierr     = PetscSectionSetUp(secNew);CHKERRQ(ierr);
5656dd5a8c8SToby Isaac     anyNew   = PETSC_FALSE;
5666dd5a8c8SToby Isaac     compress = PETSC_FALSE;
5676dd5a8c8SToby Isaac     sizeNew  = 0;
5686dd5a8c8SToby Isaac   }
5696dd5a8c8SToby Isaac   else {
5706dd5a8c8SToby Isaac     anyNew = PETSC_TRUE;
5716dd5a8c8SToby Isaac     for (p = pStart; p < pEnd; p++) {
5726dd5a8c8SToby Isaac       PetscInt dof, off;
5736dd5a8c8SToby Isaac 
5746dd5a8c8SToby Isaac       ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
5756dd5a8c8SToby Isaac       ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
5766dd5a8c8SToby Isaac       for (i = 0; i < dof; i++) {
5776dd5a8c8SToby Isaac         PetscInt q = vals[off + i], qDof = 0;
5786dd5a8c8SToby Isaac 
5796dd5a8c8SToby Isaac         if (q >= pStart && q < pEnd) {
5806dd5a8c8SToby Isaac           ierr = PetscSectionGetDof(section, q, &qDof);CHKERRQ(ierr);
5816dd5a8c8SToby Isaac         }
5826dd5a8c8SToby Isaac         if (qDof) {
5836dd5a8c8SToby Isaac           ierr = PetscSectionAddDof(secNew, p, qDof);CHKERRQ(ierr);
5846dd5a8c8SToby Isaac         }
5856dd5a8c8SToby Isaac         else {
5866dd5a8c8SToby Isaac           ierr = PetscSectionAddDof(secNew, p, 1);CHKERRQ(ierr);
5876dd5a8c8SToby Isaac         }
5886dd5a8c8SToby Isaac       }
5896dd5a8c8SToby Isaac     }
5906dd5a8c8SToby Isaac     ierr = PetscSectionSetUp(secNew);CHKERRQ(ierr);
5916dd5a8c8SToby Isaac     ierr = PetscSectionGetStorageSize(secNew,&sizeNew);CHKERRQ(ierr);
5926dd5a8c8SToby Isaac     ierr = PetscMalloc1(sizeNew,&valsNew);CHKERRQ(ierr);
5936dd5a8c8SToby Isaac     compress = PETSC_FALSE;
5946dd5a8c8SToby Isaac     for (p = pStart; p < pEnd; p++) {
5956dd5a8c8SToby Isaac       PetscInt dof, off, count, offNew, dofNew;
5966dd5a8c8SToby Isaac 
5976dd5a8c8SToby Isaac       ierr  = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
5986dd5a8c8SToby Isaac       ierr  = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
5996dd5a8c8SToby Isaac       ierr  = PetscSectionGetDof(secNew, p, &dofNew);CHKERRQ(ierr);
6006dd5a8c8SToby Isaac       ierr  = PetscSectionGetOffset(secNew, p, &offNew);CHKERRQ(ierr);
6016dd5a8c8SToby Isaac       count = 0;
6026dd5a8c8SToby Isaac       for (i = 0; i < dof; i++) {
6036dd5a8c8SToby Isaac         PetscInt q = vals[off + i], qDof = 0, qOff = 0, j;
6046dd5a8c8SToby Isaac 
6056dd5a8c8SToby Isaac         if (q >= pStart && q < pEnd) {
6066dd5a8c8SToby Isaac           ierr = PetscSectionGetDof(section, q, &qDof);CHKERRQ(ierr);
6076dd5a8c8SToby Isaac           ierr = PetscSectionGetOffset(section, q, &qOff);CHKERRQ(ierr);
6086dd5a8c8SToby Isaac         }
6096dd5a8c8SToby Isaac         if (qDof) {
6106dd5a8c8SToby Isaac           PetscInt oldCount = count;
6116dd5a8c8SToby Isaac 
6126dd5a8c8SToby Isaac           for (j = 0; j < qDof; j++) {
6136dd5a8c8SToby Isaac             PetscInt k, r = vals[qOff + j];
6146dd5a8c8SToby Isaac 
6156dd5a8c8SToby Isaac             for (k = 0; k < oldCount; k++) {
6166dd5a8c8SToby Isaac               if (valsNew[offNew + k] == r) {
6176dd5a8c8SToby Isaac                 break;
6186dd5a8c8SToby Isaac               }
6196dd5a8c8SToby Isaac             }
6206dd5a8c8SToby Isaac             if (k == oldCount) {
6216dd5a8c8SToby Isaac               valsNew[offNew + count++] = r;
6226dd5a8c8SToby Isaac             }
6236dd5a8c8SToby Isaac           }
6246dd5a8c8SToby Isaac         }
6256dd5a8c8SToby Isaac         else {
6266dd5a8c8SToby Isaac           PetscInt k, oldCount = count;
6276dd5a8c8SToby Isaac 
6286dd5a8c8SToby Isaac           for (k = 0; k < oldCount; k++) {
6296dd5a8c8SToby Isaac             if (valsNew[offNew + k] == q) {
6306dd5a8c8SToby Isaac               break;
6316dd5a8c8SToby Isaac             }
6326dd5a8c8SToby Isaac           }
6336dd5a8c8SToby Isaac           if (k == oldCount) {
6346dd5a8c8SToby Isaac             valsNew[offNew + count++] = q;
6356dd5a8c8SToby Isaac           }
6366dd5a8c8SToby Isaac         }
6376dd5a8c8SToby Isaac       }
6386dd5a8c8SToby Isaac       if (count < dofNew) {
6396dd5a8c8SToby Isaac         ierr = PetscSectionSetDof(secNew, p, count);CHKERRQ(ierr);
6406dd5a8c8SToby Isaac         compress = PETSC_TRUE;
6416dd5a8c8SToby Isaac       }
6426dd5a8c8SToby Isaac     }
6436dd5a8c8SToby Isaac   }
6446dd5a8c8SToby Isaac   ierr = ISRestoreIndices(is,&vals);CHKERRQ(ierr);
645b2566f29SBarry Smith   ierr = MPIU_Allreduce(&anyNew,&globalAnyNew,1,MPIU_BOOL,MPI_LOR,PetscObjectComm((PetscObject)secNew));CHKERRQ(ierr);
6466dd5a8c8SToby Isaac   if (!globalAnyNew) {
6476dd5a8c8SToby Isaac     ierr = PetscSectionDestroy(&secNew);CHKERRQ(ierr);
6486dd5a8c8SToby Isaac     *sectionNew = NULL;
6496dd5a8c8SToby Isaac     *isNew = NULL;
6506dd5a8c8SToby Isaac   }
6516dd5a8c8SToby Isaac   else {
6526dd5a8c8SToby Isaac     PetscBool globalCompress;
6536dd5a8c8SToby Isaac 
654b2566f29SBarry Smith     ierr = MPIU_Allreduce(&compress,&globalCompress,1,MPIU_BOOL,MPI_LOR,PetscObjectComm((PetscObject)secNew));CHKERRQ(ierr);
6556dd5a8c8SToby Isaac     if (compress) {
6566dd5a8c8SToby Isaac       PetscSection secComp;
6576dd5a8c8SToby Isaac       PetscInt *valsComp = NULL;
6586dd5a8c8SToby Isaac 
6596dd5a8c8SToby Isaac       ierr = PetscSectionCreate(PetscObjectComm((PetscObject)section),&secComp);CHKERRQ(ierr);
6606dd5a8c8SToby Isaac       ierr = PetscSectionSetChart(secComp,pStart,pEnd);CHKERRQ(ierr);
6616dd5a8c8SToby Isaac       for (p = pStart; p < pEnd; p++) {
6626dd5a8c8SToby Isaac         PetscInt dof;
6636dd5a8c8SToby Isaac 
6646dd5a8c8SToby Isaac         ierr = PetscSectionGetDof(secNew, p, &dof);CHKERRQ(ierr);
6656dd5a8c8SToby Isaac         ierr = PetscSectionSetDof(secComp, p, dof);CHKERRQ(ierr);
6666dd5a8c8SToby Isaac       }
6676dd5a8c8SToby Isaac       ierr = PetscSectionSetUp(secComp);CHKERRQ(ierr);
6686dd5a8c8SToby Isaac       ierr = PetscSectionGetStorageSize(secComp,&sizeNew);CHKERRQ(ierr);
6696dd5a8c8SToby Isaac       ierr = PetscMalloc1(sizeNew,&valsComp);CHKERRQ(ierr);
6706dd5a8c8SToby Isaac       for (p = pStart; p < pEnd; p++) {
6716dd5a8c8SToby Isaac         PetscInt dof, off, offNew, j;
6726dd5a8c8SToby Isaac 
6736dd5a8c8SToby Isaac         ierr = PetscSectionGetDof(secNew, p, &dof);CHKERRQ(ierr);
6746dd5a8c8SToby Isaac         ierr = PetscSectionGetOffset(secNew, p, &off);CHKERRQ(ierr);
6756dd5a8c8SToby Isaac         ierr = PetscSectionGetOffset(secComp, p, &offNew);CHKERRQ(ierr);
6766dd5a8c8SToby Isaac         for (j = 0; j < dof; j++) {
6776dd5a8c8SToby Isaac           valsComp[offNew + j] = valsNew[off + j];
6786dd5a8c8SToby Isaac         }
6796dd5a8c8SToby Isaac       }
6806dd5a8c8SToby Isaac       ierr    = PetscSectionDestroy(&secNew);CHKERRQ(ierr);
6816dd5a8c8SToby Isaac       secNew  = secComp;
6826dd5a8c8SToby Isaac       ierr    = PetscFree(valsNew);CHKERRQ(ierr);
6836dd5a8c8SToby Isaac       valsNew = valsComp;
6846dd5a8c8SToby Isaac     }
6856dd5a8c8SToby Isaac     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)is),sizeNew,valsNew,PETSC_OWN_POINTER,isNew);CHKERRQ(ierr);
6866dd5a8c8SToby Isaac   }
6876dd5a8c8SToby Isaac   PetscFunctionReturn(0);
6886dd5a8c8SToby Isaac }
6896dd5a8c8SToby Isaac 
690f7c74593SToby Isaac static PetscErrorCode DMPlexCreateAnchors_Tree(DM dm)
69166af876cSToby Isaac {
69266af876cSToby Isaac   PetscInt       p, pStart, pEnd, *anchors, size;
69366af876cSToby Isaac   PetscInt       aMin = PETSC_MAX_INT, aMax = PETSC_MIN_INT;
69466af876cSToby Isaac   PetscSection   aSec;
695f9f063d4SToby Isaac   DMLabel        canonLabel;
69666af876cSToby Isaac   IS             aIS;
69766af876cSToby Isaac   PetscErrorCode ierr;
69866af876cSToby Isaac 
69966af876cSToby Isaac   PetscFunctionBegin;
70066af876cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
70166af876cSToby Isaac   ierr = DMPlexGetChart(dm,&pStart,&pEnd);CHKERRQ(ierr);
702c58f1c22SToby Isaac   ierr = DMGetLabel(dm,"canonical",&canonLabel);CHKERRQ(ierr);
70366af876cSToby Isaac   for (p = pStart; p < pEnd; p++) {
70466af876cSToby Isaac     PetscInt parent;
70566af876cSToby Isaac 
706f9f063d4SToby Isaac     if (canonLabel) {
707f9f063d4SToby Isaac       PetscInt canon;
708f9f063d4SToby Isaac 
709f9f063d4SToby Isaac       ierr = DMLabelGetValue(canonLabel,p,&canon);CHKERRQ(ierr);
710f9f063d4SToby Isaac       if (p != canon) continue;
711f9f063d4SToby Isaac     }
71266af876cSToby Isaac     ierr = DMPlexGetTreeParent(dm,p,&parent,NULL);CHKERRQ(ierr);
71366af876cSToby Isaac     if (parent != p) {
71466af876cSToby Isaac       aMin = PetscMin(aMin,p);
71566af876cSToby Isaac       aMax = PetscMax(aMax,p+1);
71666af876cSToby Isaac     }
71766af876cSToby Isaac   }
71866af876cSToby Isaac   if (aMin > aMax) {
71966af876cSToby Isaac     aMin = -1;
72066af876cSToby Isaac     aMax = -1;
72166af876cSToby Isaac   }
722e228b242SToby Isaac   ierr = PetscSectionCreate(PETSC_COMM_SELF,&aSec);CHKERRQ(ierr);
72366af876cSToby Isaac   ierr = PetscSectionSetChart(aSec,aMin,aMax);CHKERRQ(ierr);
72466af876cSToby Isaac   for (p = aMin; p < aMax; p++) {
72566af876cSToby Isaac     PetscInt parent, ancestor = p;
72666af876cSToby Isaac 
727f9f063d4SToby Isaac     if (canonLabel) {
728f9f063d4SToby Isaac       PetscInt canon;
729f9f063d4SToby Isaac 
730f9f063d4SToby Isaac       ierr = DMLabelGetValue(canonLabel,p,&canon);CHKERRQ(ierr);
731f9f063d4SToby Isaac       if (p != canon) continue;
732f9f063d4SToby Isaac     }
73366af876cSToby Isaac     ierr = DMPlexGetTreeParent(dm,p,&parent,NULL);CHKERRQ(ierr);
73466af876cSToby Isaac     while (parent != ancestor) {
73566af876cSToby Isaac       ancestor = parent;
73666af876cSToby Isaac       ierr     = DMPlexGetTreeParent(dm,ancestor,&parent,NULL);CHKERRQ(ierr);
73766af876cSToby Isaac     }
73866af876cSToby Isaac     if (ancestor != p) {
73966af876cSToby Isaac       PetscInt closureSize, *closure = NULL;
74066af876cSToby Isaac 
74166af876cSToby Isaac       ierr = DMPlexGetTransitiveClosure(dm,ancestor,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
74266af876cSToby Isaac       ierr = PetscSectionSetDof(aSec,p,closureSize);CHKERRQ(ierr);
74366af876cSToby Isaac       ierr = DMPlexRestoreTransitiveClosure(dm,ancestor,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
74466af876cSToby Isaac     }
74566af876cSToby Isaac   }
74666af876cSToby Isaac   ierr = PetscSectionSetUp(aSec);CHKERRQ(ierr);
74766af876cSToby Isaac   ierr = PetscSectionGetStorageSize(aSec,&size);CHKERRQ(ierr);
74866af876cSToby Isaac   ierr = PetscMalloc1(size,&anchors);CHKERRQ(ierr);
74966af876cSToby Isaac   for (p = aMin; p < aMax; p++) {
75066af876cSToby Isaac     PetscInt parent, ancestor = p;
75166af876cSToby Isaac 
752f9f063d4SToby Isaac     if (canonLabel) {
753f9f063d4SToby Isaac       PetscInt canon;
754f9f063d4SToby Isaac 
755f9f063d4SToby Isaac       ierr = DMLabelGetValue(canonLabel,p,&canon);CHKERRQ(ierr);
756f9f063d4SToby Isaac       if (p != canon) continue;
757f9f063d4SToby Isaac     }
75866af876cSToby Isaac     ierr = DMPlexGetTreeParent(dm,p,&parent,NULL);CHKERRQ(ierr);
75966af876cSToby Isaac     while (parent != ancestor) {
76066af876cSToby Isaac       ancestor = parent;
76166af876cSToby Isaac       ierr     = DMPlexGetTreeParent(dm,ancestor,&parent,NULL);CHKERRQ(ierr);
76266af876cSToby Isaac     }
76366af876cSToby Isaac     if (ancestor != p) {
76466af876cSToby Isaac       PetscInt j, closureSize, *closure = NULL, aOff;
76566af876cSToby Isaac 
76666af876cSToby Isaac       ierr = PetscSectionGetOffset(aSec,p,&aOff);CHKERRQ(ierr);
76766af876cSToby Isaac 
76866af876cSToby Isaac       ierr = DMPlexGetTransitiveClosure(dm,ancestor,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
76966af876cSToby Isaac       for (j = 0; j < closureSize; j++) {
77066af876cSToby Isaac         anchors[aOff + j] = closure[2*j];
77166af876cSToby Isaac       }
77266af876cSToby Isaac       ierr = DMPlexRestoreTransitiveClosure(dm,ancestor,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
77366af876cSToby Isaac     }
77466af876cSToby Isaac   }
775e228b242SToby Isaac   ierr = ISCreateGeneral(PETSC_COMM_SELF,size,anchors,PETSC_OWN_POINTER,&aIS);CHKERRQ(ierr);
7766dd5a8c8SToby Isaac   {
7776dd5a8c8SToby Isaac     PetscSection aSecNew = aSec;
7786dd5a8c8SToby Isaac     IS           aISNew  = aIS;
7796dd5a8c8SToby Isaac 
7806dd5a8c8SToby Isaac     ierr = PetscObjectReference((PetscObject)aSec);CHKERRQ(ierr);
7816dd5a8c8SToby Isaac     ierr = PetscObjectReference((PetscObject)aIS);CHKERRQ(ierr);
7826dd5a8c8SToby Isaac     while (aSecNew) {
7836dd5a8c8SToby Isaac       ierr    = PetscSectionDestroy(&aSec);CHKERRQ(ierr);
7846dd5a8c8SToby Isaac       ierr    = ISDestroy(&aIS);CHKERRQ(ierr);
7856dd5a8c8SToby Isaac       aSec    = aSecNew;
7866dd5a8c8SToby Isaac       aIS     = aISNew;
7876dd5a8c8SToby Isaac       aSecNew = NULL;
7886dd5a8c8SToby Isaac       aISNew  = NULL;
7896dd5a8c8SToby Isaac       ierr    = AnchorsFlatten(aSec,aIS,&aSecNew,&aISNew);CHKERRQ(ierr);
7906dd5a8c8SToby Isaac     }
7916dd5a8c8SToby Isaac   }
792a17985deSToby Isaac   ierr = DMPlexSetAnchors(dm,aSec,aIS);CHKERRQ(ierr);
79366af876cSToby Isaac   ierr = PetscSectionDestroy(&aSec);CHKERRQ(ierr);
79466af876cSToby Isaac   ierr = ISDestroy(&aIS);CHKERRQ(ierr);
79566af876cSToby Isaac   PetscFunctionReturn(0);
79666af876cSToby Isaac }
79766af876cSToby Isaac 
7986461c1adSToby Isaac static PetscErrorCode DMPlexGetTrueSupportSize(DM dm,PetscInt p,PetscInt *dof,PetscInt *numTrueSupp)
7996461c1adSToby Isaac {
8006461c1adSToby Isaac   PetscErrorCode ierr;
8016461c1adSToby Isaac 
8026461c1adSToby Isaac   PetscFunctionBegin;
8036461c1adSToby Isaac   if (numTrueSupp[p] == -1) {
8046461c1adSToby Isaac     PetscInt i, alldof;
8056461c1adSToby Isaac     const PetscInt *supp;
8066461c1adSToby Isaac     PetscInt count = 0;
8076461c1adSToby Isaac 
8086461c1adSToby Isaac     ierr = DMPlexGetSupportSize(dm,p,&alldof);CHKERRQ(ierr);
8096461c1adSToby Isaac     ierr = DMPlexGetSupport(dm,p,&supp);CHKERRQ(ierr);
8106461c1adSToby Isaac     for (i = 0; i < alldof; i++) {
8116461c1adSToby Isaac       PetscInt q = supp[i], numCones, j;
8126461c1adSToby Isaac       const PetscInt *cone;
8136461c1adSToby Isaac 
8146461c1adSToby Isaac       ierr = DMPlexGetConeSize(dm,q,&numCones);CHKERRQ(ierr);
8156461c1adSToby Isaac       ierr = DMPlexGetCone(dm,q,&cone);CHKERRQ(ierr);
8166461c1adSToby Isaac       for (j = 0; j < numCones; j++) {
8176461c1adSToby Isaac         if (cone[j] == p) break;
8186461c1adSToby Isaac       }
8196461c1adSToby Isaac       if (j < numCones) count++;
8206461c1adSToby Isaac     }
8216461c1adSToby Isaac     numTrueSupp[p] = count;
8226461c1adSToby Isaac   }
8236461c1adSToby Isaac   *dof = numTrueSupp[p];
8246461c1adSToby Isaac   PetscFunctionReturn(0);
8256461c1adSToby Isaac }
8266461c1adSToby Isaac 
827776742edSToby Isaac static PetscErrorCode DMPlexTreeExchangeSupports(DM dm)
828776742edSToby Isaac {
829776742edSToby Isaac   DM_Plex *mesh = (DM_Plex *)dm->data;
830776742edSToby Isaac   PetscSection newSupportSection;
831776742edSToby Isaac   PetscInt newSize, *newSupports, pStart, pEnd, p, d, depth;
8326461c1adSToby Isaac   PetscInt *numTrueSupp;
833776742edSToby Isaac   PetscInt *offsets;
834776742edSToby Isaac   PetscErrorCode ierr;
835776742edSToby Isaac 
836776742edSToby Isaac   PetscFunctionBegin;
837776742edSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
838776742edSToby Isaac   /* symmetrize the hierarchy */
839776742edSToby Isaac   ierr = DMPlexGetDepth(dm,&depth);CHKERRQ(ierr);
840e228b242SToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)(mesh->supportSection)),&newSupportSection);CHKERRQ(ierr);
841776742edSToby Isaac   ierr = DMPlexGetChart(dm,&pStart,&pEnd);CHKERRQ(ierr);
842776742edSToby Isaac   ierr = PetscSectionSetChart(newSupportSection,pStart,pEnd);CHKERRQ(ierr);
843776742edSToby Isaac   ierr = PetscCalloc1(pEnd,&offsets);CHKERRQ(ierr);
8446461c1adSToby Isaac   ierr = PetscMalloc1(pEnd,&numTrueSupp);CHKERRQ(ierr);
8456461c1adSToby Isaac   for (p = 0; p < pEnd; p++) numTrueSupp[p] = -1;
8466461c1adSToby Isaac   /* if a point is in the (true) support of q, it should be in the support of
847776742edSToby Isaac    * parent(q) */
848776742edSToby Isaac   for (d = 0; d <= depth; d++) {
849776742edSToby Isaac     ierr = DMPlexGetHeightStratum(dm,d,&pStart,&pEnd);CHKERRQ(ierr);
850776742edSToby Isaac     for (p = pStart; p < pEnd; ++p) {
851776742edSToby Isaac       PetscInt dof, q, qdof, parent;
852776742edSToby Isaac 
8536461c1adSToby Isaac       ierr = DMPlexGetTrueSupportSize(dm,p,&dof,numTrueSupp);CHKERRQ(ierr);
854776742edSToby Isaac       ierr = PetscSectionAddDof(newSupportSection, p, dof);CHKERRQ(ierr);
855776742edSToby Isaac       q    = p;
856776742edSToby Isaac       ierr = DMPlexGetTreeParent(dm,q,&parent,NULL);CHKERRQ(ierr);
857776742edSToby Isaac       while (parent != q && parent >= pStart && parent < pEnd) {
858776742edSToby Isaac         q = parent;
859776742edSToby Isaac 
8606461c1adSToby Isaac         ierr = DMPlexGetTrueSupportSize(dm,q,&qdof,numTrueSupp);CHKERRQ(ierr);
861776742edSToby Isaac         ierr = PetscSectionAddDof(newSupportSection,p,qdof);CHKERRQ(ierr);
862776742edSToby Isaac         ierr = PetscSectionAddDof(newSupportSection,q,dof);CHKERRQ(ierr);
863776742edSToby Isaac         ierr = DMPlexGetTreeParent(dm,q,&parent,NULL);CHKERRQ(ierr);
864776742edSToby Isaac       }
865776742edSToby Isaac     }
866776742edSToby Isaac   }
867776742edSToby Isaac   ierr = PetscSectionSetUp(newSupportSection);CHKERRQ(ierr);
868776742edSToby Isaac   ierr = PetscSectionGetStorageSize(newSupportSection,&newSize);CHKERRQ(ierr);
869776742edSToby Isaac   ierr = PetscMalloc1(newSize,&newSupports);CHKERRQ(ierr);
870776742edSToby Isaac   for (d = 0; d <= depth; d++) {
871776742edSToby Isaac     ierr = DMPlexGetHeightStratum(dm,d,&pStart,&pEnd);CHKERRQ(ierr);
872776742edSToby Isaac     for (p = pStart; p < pEnd; p++) {
873776742edSToby Isaac       PetscInt dof, off, q, qdof, qoff, newDof, newOff, newqOff, i, parent;
874776742edSToby Isaac 
875776742edSToby Isaac       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
876776742edSToby Isaac       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
877776742edSToby Isaac       ierr = PetscSectionGetDof(newSupportSection, p, &newDof);CHKERRQ(ierr);
878776742edSToby Isaac       ierr = PetscSectionGetOffset(newSupportSection, p, &newOff);CHKERRQ(ierr);
879776742edSToby Isaac       for (i = 0; i < dof; i++) {
8806461c1adSToby Isaac         PetscInt numCones, j;
8816461c1adSToby Isaac         const PetscInt *cone;
8826461c1adSToby Isaac         PetscInt q = mesh->supports[off + i];
8836461c1adSToby Isaac 
8846461c1adSToby Isaac         ierr = DMPlexGetConeSize(dm,q,&numCones);CHKERRQ(ierr);
8856461c1adSToby Isaac         ierr = DMPlexGetCone(dm,q,&cone);CHKERRQ(ierr);
8866461c1adSToby Isaac         for (j = 0; j < numCones; j++) {
8876461c1adSToby Isaac           if (cone[j] == p) break;
8886461c1adSToby Isaac         }
8896461c1adSToby Isaac         if (j < numCones) newSupports[newOff+offsets[p]++] = q;
890776742edSToby Isaac       }
891776742edSToby Isaac       mesh->maxSupportSize = PetscMax(mesh->maxSupportSize,newDof);
892776742edSToby Isaac 
893776742edSToby Isaac       q    = p;
894776742edSToby Isaac       ierr = DMPlexGetTreeParent(dm,q,&parent,NULL);CHKERRQ(ierr);
895776742edSToby Isaac       while (parent != q && parent >= pStart && parent < pEnd) {
896776742edSToby Isaac         q = parent;
897776742edSToby Isaac         ierr = PetscSectionGetDof(mesh->supportSection, q, &qdof);CHKERRQ(ierr);
898776742edSToby Isaac         ierr = PetscSectionGetOffset(mesh->supportSection, q, &qoff);CHKERRQ(ierr);
899776742edSToby Isaac         ierr = PetscSectionGetOffset(newSupportSection, q, &newqOff);CHKERRQ(ierr);
900776742edSToby Isaac         for (i = 0; i < qdof; i++) {
9016461c1adSToby Isaac           PetscInt numCones, j;
9026461c1adSToby Isaac           const PetscInt *cone;
9036461c1adSToby Isaac           PetscInt r = mesh->supports[qoff + i];
9046461c1adSToby Isaac 
9056461c1adSToby Isaac           ierr = DMPlexGetConeSize(dm,r,&numCones);CHKERRQ(ierr);
9066461c1adSToby Isaac           ierr = DMPlexGetCone(dm,r,&cone);CHKERRQ(ierr);
9076461c1adSToby Isaac           for (j = 0; j < numCones; j++) {
9086461c1adSToby Isaac             if (cone[j] == q) break;
9096461c1adSToby Isaac           }
9106461c1adSToby Isaac           if (j < numCones) newSupports[newOff+offsets[p]++] = r;
911776742edSToby Isaac         }
912776742edSToby Isaac         for (i = 0; i < dof; i++) {
9136461c1adSToby Isaac           PetscInt numCones, j;
9146461c1adSToby Isaac           const PetscInt *cone;
9156461c1adSToby Isaac           PetscInt r = mesh->supports[off + i];
9166461c1adSToby Isaac 
9176461c1adSToby Isaac           ierr = DMPlexGetConeSize(dm,r,&numCones);CHKERRQ(ierr);
9186461c1adSToby Isaac           ierr = DMPlexGetCone(dm,r,&cone);CHKERRQ(ierr);
9196461c1adSToby Isaac           for (j = 0; j < numCones; j++) {
9206461c1adSToby Isaac             if (cone[j] == p) break;
9216461c1adSToby Isaac           }
9226461c1adSToby Isaac           if (j < numCones) newSupports[newqOff+offsets[q]++] = r;
923776742edSToby Isaac         }
924776742edSToby Isaac         ierr = DMPlexGetTreeParent(dm,q,&parent,NULL);CHKERRQ(ierr);
925776742edSToby Isaac       }
926776742edSToby Isaac     }
927776742edSToby Isaac   }
928776742edSToby Isaac   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
929776742edSToby Isaac   mesh->supportSection = newSupportSection;
930776742edSToby Isaac   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
931776742edSToby Isaac   mesh->supports = newSupports;
932776742edSToby Isaac   ierr = PetscFree(offsets);CHKERRQ(ierr);
9336461c1adSToby Isaac   ierr = PetscFree(numTrueSupp);CHKERRQ(ierr);
934776742edSToby Isaac 
935776742edSToby Isaac   PetscFunctionReturn(0);
936776742edSToby Isaac }
937776742edSToby Isaac 
938f7c74593SToby Isaac static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_Direct(DM,PetscSection,PetscSection,Mat);
939f7c74593SToby Isaac static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_FromReference(DM,PetscSection,PetscSection,Mat);
940f7c74593SToby Isaac 
941776742edSToby Isaac static PetscErrorCode DMPlexSetTree_Internal(DM dm, PetscSection parentSection, PetscInt *parents, PetscInt *childIDs, PetscBool computeCanonical, PetscBool exchangeSupports)
942f9f063d4SToby Isaac {
943f9f063d4SToby Isaac   DM_Plex       *mesh = (DM_Plex *)dm->data;
944f9f063d4SToby Isaac   DM             refTree;
945f9f063d4SToby Isaac   PetscInt       size;
946f9f063d4SToby Isaac   PetscErrorCode ierr;
947f9f063d4SToby Isaac 
948f9f063d4SToby Isaac   PetscFunctionBegin;
949f9f063d4SToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
950f9f063d4SToby Isaac   PetscValidHeaderSpecific(parentSection, PETSC_SECTION_CLASSID, 2);
951f9f063d4SToby Isaac   ierr = PetscObjectReference((PetscObject)parentSection);CHKERRQ(ierr);
952f9f063d4SToby Isaac   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
953f9f063d4SToby Isaac   mesh->parentSection = parentSection;
954f9f063d4SToby Isaac   ierr = PetscSectionGetStorageSize(parentSection,&size);CHKERRQ(ierr);
955f9f063d4SToby Isaac   if (parents != mesh->parents) {
956f9f063d4SToby Isaac     ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
957f9f063d4SToby Isaac     ierr = PetscMalloc1(size,&mesh->parents);CHKERRQ(ierr);
958f9f063d4SToby Isaac     ierr = PetscMemcpy(mesh->parents, parents, size * sizeof(*parents));CHKERRQ(ierr);
959f9f063d4SToby Isaac   }
960f9f063d4SToby Isaac   if (childIDs != mesh->childIDs) {
961f9f063d4SToby Isaac     ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
962f9f063d4SToby Isaac     ierr = PetscMalloc1(size,&mesh->childIDs);CHKERRQ(ierr);
963f9f063d4SToby Isaac     ierr = PetscMemcpy(mesh->childIDs, childIDs, size * sizeof(*childIDs));CHKERRQ(ierr);
964f9f063d4SToby Isaac   }
965f9f063d4SToby Isaac   ierr = DMPlexGetReferenceTree(dm,&refTree);CHKERRQ(ierr);
966f9f063d4SToby Isaac   if (refTree) {
967f9f063d4SToby Isaac     DMLabel canonLabel;
968f9f063d4SToby Isaac 
969c58f1c22SToby Isaac     ierr = DMGetLabel(refTree,"canonical",&canonLabel);CHKERRQ(ierr);
970f9f063d4SToby Isaac     if (canonLabel) {
971f9f063d4SToby Isaac       PetscInt i;
972f9f063d4SToby Isaac 
973f9f063d4SToby Isaac       for (i = 0; i < size; i++) {
974f9f063d4SToby Isaac         PetscInt canon;
975f9f063d4SToby Isaac         ierr = DMLabelGetValue(canonLabel, mesh->childIDs[i], &canon);CHKERRQ(ierr);
976f9f063d4SToby Isaac         if (canon >= 0) {
977f9f063d4SToby Isaac           mesh->childIDs[i] = canon;
978f9f063d4SToby Isaac         }
979f9f063d4SToby Isaac       }
980f9f063d4SToby Isaac     }
981f7c74593SToby Isaac     mesh->computeanchormatrix = DMPlexComputeAnchorMatrix_Tree_FromReference;
982f7c74593SToby Isaac   }
983f7c74593SToby Isaac   else {
984f7c74593SToby Isaac     mesh->computeanchormatrix = DMPlexComputeAnchorMatrix_Tree_Direct;
985f9f063d4SToby Isaac   }
986f9f063d4SToby Isaac   ierr = DMPlexTreeSymmetrize(dm);CHKERRQ(ierr);
987f9f063d4SToby Isaac   if (computeCanonical) {
988f9f063d4SToby Isaac     PetscInt d, dim;
989f9f063d4SToby Isaac 
990f9f063d4SToby Isaac     /* add the canonical label */
99128f4b327SMatthew G. Knepley     ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
992c58f1c22SToby Isaac     ierr = DMCreateLabel(dm,"canonical");CHKERRQ(ierr);
993f9f063d4SToby Isaac     for (d = 0; d <= dim; d++) {
994f9f063d4SToby Isaac       PetscInt p, dStart, dEnd, canon = -1, cNumChildren;
995f9f063d4SToby Isaac       const PetscInt *cChildren;
996f9f063d4SToby Isaac 
997f9f063d4SToby Isaac       ierr = DMPlexGetDepthStratum(dm,d,&dStart,&dEnd);CHKERRQ(ierr);
998f9f063d4SToby Isaac       for (p = dStart; p < dEnd; p++) {
999f9f063d4SToby Isaac         ierr = DMPlexGetTreeChildren(dm,p,&cNumChildren,&cChildren);CHKERRQ(ierr);
1000f9f063d4SToby Isaac         if (cNumChildren) {
1001f9f063d4SToby Isaac           canon = p;
1002f9f063d4SToby Isaac           break;
1003f9f063d4SToby Isaac         }
1004f9f063d4SToby Isaac       }
1005f9f063d4SToby Isaac       if (canon == -1) continue;
1006f9f063d4SToby Isaac       for (p = dStart; p < dEnd; p++) {
1007f9f063d4SToby Isaac         PetscInt numChildren, i;
1008f9f063d4SToby Isaac         const PetscInt *children;
1009f9f063d4SToby Isaac 
1010f9f063d4SToby Isaac         ierr = DMPlexGetTreeChildren(dm,p,&numChildren,&children);CHKERRQ(ierr);
1011f9f063d4SToby Isaac         if (numChildren) {
1012f9f063d4SToby Isaac           if (numChildren != cNumChildren) SETERRQ2(PetscObjectComm((PetscObject)dm),PETSC_ERR_PLIB,"All parent points in a stratum should have the same number of children: %d != %d", numChildren, cNumChildren);
1013c58f1c22SToby Isaac           ierr = DMSetLabelValue(dm,"canonical",p,canon);CHKERRQ(ierr);
1014f9f063d4SToby Isaac           for (i = 0; i < numChildren; i++) {
1015c58f1c22SToby Isaac             ierr = DMSetLabelValue(dm,"canonical",children[i],cChildren[i]);CHKERRQ(ierr);
1016f9f063d4SToby Isaac           }
1017f9f063d4SToby Isaac         }
1018f9f063d4SToby Isaac       }
1019f9f063d4SToby Isaac     }
1020f9f063d4SToby Isaac   }
1021776742edSToby Isaac   if (exchangeSupports) {
1022776742edSToby Isaac     ierr = DMPlexTreeExchangeSupports(dm);CHKERRQ(ierr);
1023776742edSToby Isaac   }
1024f7c74593SToby Isaac   mesh->createanchors = DMPlexCreateAnchors_Tree;
1025f7c74593SToby Isaac   /* reset anchors */
1026f7c74593SToby Isaac   ierr = DMPlexSetAnchors(dm,NULL,NULL);CHKERRQ(ierr);
1027f9f063d4SToby Isaac   PetscFunctionReturn(0);
1028f9f063d4SToby Isaac }
1029f9f063d4SToby Isaac 
10300b7167a0SToby Isaac /*@
10310b7167a0SToby Isaac   DMPlexSetTree - set the tree that describes the hierarchy of non-conforming mesh points.  This routine also creates
10320b7167a0SToby Isaac   the point-to-point constraints determined by the tree: a point is constained to the points in the closure of its
10330b7167a0SToby Isaac   tree root.
10340b7167a0SToby Isaac 
10350b7167a0SToby Isaac   Collective on dm
10360b7167a0SToby Isaac 
10370b7167a0SToby Isaac   Input Parameters:
10380b7167a0SToby Isaac + dm - the DMPlex object
10390b7167a0SToby Isaac . parentSection - a section describing the tree: a point has a parent if it has 1 dof in the section; the section
10400b7167a0SToby Isaac                   offset indexes the parent and childID list; the reference count of parentSection is incremented
10410b7167a0SToby Isaac . parents - a list of the point parents; copied, can be destroyed
10420b7167a0SToby Isaac - childIDs - identifies the relationship of the child point to the parent point; if there is a reference tree, then
10430b7167a0SToby Isaac              the child corresponds to the point in the reference tree with index childIDs; copied, can be destroyed
10440b7167a0SToby Isaac 
10450b7167a0SToby Isaac   Level: intermediate
10460b7167a0SToby Isaac 
1047a17985deSToby Isaac .seealso: DMPlexGetTree(), DMPlexSetReferenceTree(), DMPlexSetAnchors(), DMPlexGetTreeParent(), DMPlexGetTreeChildren()
10480b7167a0SToby Isaac @*/
1049b2f41788SToby Isaac PetscErrorCode DMPlexSetTree(DM dm, PetscSection parentSection, PetscInt parents[], PetscInt childIDs[])
10500b7167a0SToby Isaac {
10510b7167a0SToby Isaac   PetscErrorCode ierr;
10520b7167a0SToby Isaac 
10530b7167a0SToby Isaac   PetscFunctionBegin;
1054776742edSToby Isaac   ierr = DMPlexSetTree_Internal(dm,parentSection,parents,childIDs,PETSC_FALSE,PETSC_TRUE);CHKERRQ(ierr);
10550b7167a0SToby Isaac   PetscFunctionReturn(0);
10560b7167a0SToby Isaac }
10570b7167a0SToby Isaac 
1058b2f41788SToby Isaac /*@
1059b2f41788SToby Isaac   DMPlexGetTree - get the tree that describes the hierarchy of non-conforming mesh points.
1060b2f41788SToby Isaac   Collective on dm
1061b2f41788SToby Isaac 
1062b2f41788SToby Isaac   Input Parameters:
1063b2f41788SToby Isaac . dm - the DMPlex object
1064b2f41788SToby Isaac 
1065b2f41788SToby Isaac   Output Parameters:
1066b2f41788SToby Isaac + parentSection - a section describing the tree: a point has a parent if it has 1 dof in the section; the section
1067b2f41788SToby Isaac                   offset indexes the parent and childID list
1068b2f41788SToby Isaac . parents - a list of the point parents
1069b2f41788SToby Isaac . childIDs - identifies the relationship of the child point to the parent point; if there is a reference tree, then
1070b2f41788SToby Isaac              the child corresponds to the point in the reference tree with index childID
1071b2f41788SToby Isaac . childSection - the inverse of the parent section
1072b2f41788SToby Isaac - children - a list of the point children
1073b2f41788SToby Isaac 
1074b2f41788SToby Isaac   Level: intermediate
1075b2f41788SToby Isaac 
1076a17985deSToby Isaac .seealso: DMPlexSetTree(), DMPlexSetReferenceTree(), DMPlexSetAnchors(), DMPlexGetTreeParent(), DMPlexGetTreeChildren()
1077b2f41788SToby Isaac @*/
1078b2f41788SToby Isaac PetscErrorCode DMPlexGetTree(DM dm, PetscSection *parentSection, PetscInt *parents[], PetscInt *childIDs[], PetscSection *childSection, PetscInt *children[])
1079b2f41788SToby Isaac {
1080b2f41788SToby Isaac   DM_Plex        *mesh = (DM_Plex *)dm->data;
1081b2f41788SToby Isaac 
1082b2f41788SToby Isaac   PetscFunctionBegin;
1083b2f41788SToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1084b2f41788SToby Isaac   if (parentSection) *parentSection = mesh->parentSection;
1085b2f41788SToby Isaac   if (parents)       *parents       = mesh->parents;
1086b2f41788SToby Isaac   if (childIDs)      *childIDs      = mesh->childIDs;
1087b2f41788SToby Isaac   if (childSection)  *childSection  = mesh->childSection;
1088b2f41788SToby Isaac   if (children)      *children      = mesh->children;
1089b2f41788SToby Isaac   PetscFunctionReturn(0);
1090b2f41788SToby Isaac }
1091b2f41788SToby Isaac 
1092d961a43aSToby Isaac /*@
1093d961a43aSToby Isaac   DMPlexGetTreeParent - get the parent of a point in the tree describing the point hierarchy (not the Sieve DAG)
1094d961a43aSToby Isaac 
1095d961a43aSToby Isaac   Input Parameters:
1096d961a43aSToby Isaac + dm - the DMPlex object
1097d961a43aSToby Isaac - point - the query point
1098d961a43aSToby Isaac 
1099d961a43aSToby Isaac   Output Parameters:
1100d961a43aSToby Isaac + parent - if not NULL, set to the parent of the point, or the point itself if the point does not have a parent
1101d961a43aSToby Isaac - childID - if not NULL, set to the child ID of the point with respect to its parent, or 0 if the point
1102d961a43aSToby Isaac             does not have a parent
1103d961a43aSToby Isaac 
1104d961a43aSToby Isaac   Level: intermediate
1105d961a43aSToby Isaac 
1106d961a43aSToby Isaac .seealso: DMPlexSetTree(), DMPlexGetTree(), DMPlexGetTreeChildren()
1107d961a43aSToby Isaac @*/
1108d961a43aSToby Isaac PetscErrorCode DMPlexGetTreeParent(DM dm, PetscInt point, PetscInt *parent, PetscInt *childID)
1109d961a43aSToby Isaac {
1110d961a43aSToby Isaac   DM_Plex       *mesh = (DM_Plex *)dm->data;
1111d961a43aSToby Isaac   PetscSection   pSec;
1112d961a43aSToby Isaac   PetscErrorCode ierr;
1113d961a43aSToby Isaac 
1114d961a43aSToby Isaac   PetscFunctionBegin;
1115d961a43aSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1116d961a43aSToby Isaac   pSec = mesh->parentSection;
1117d961a43aSToby Isaac   if (pSec && point >= pSec->pStart && point < pSec->pEnd) {
1118d961a43aSToby Isaac     PetscInt dof;
1119d961a43aSToby Isaac 
1120d961a43aSToby Isaac     ierr = PetscSectionGetDof (pSec, point, &dof);CHKERRQ(ierr);
1121d961a43aSToby Isaac     if (dof) {
1122d961a43aSToby Isaac       PetscInt off;
1123d961a43aSToby Isaac 
1124d961a43aSToby Isaac       ierr = PetscSectionGetOffset (pSec, point, &off);CHKERRQ(ierr);
1125d961a43aSToby Isaac       if (parent)  *parent = mesh->parents[off];
1126d961a43aSToby Isaac       if (childID) *childID = mesh->childIDs[off];
1127d961a43aSToby Isaac       PetscFunctionReturn(0);
1128d961a43aSToby Isaac     }
1129d961a43aSToby Isaac   }
1130d961a43aSToby Isaac   if (parent) {
1131d961a43aSToby Isaac     *parent = point;
1132d961a43aSToby Isaac   }
1133d961a43aSToby Isaac   if (childID) {
1134d961a43aSToby Isaac     *childID = 0;
1135d961a43aSToby Isaac   }
1136d961a43aSToby Isaac   PetscFunctionReturn(0);
1137d961a43aSToby Isaac }
1138d961a43aSToby Isaac 
1139d961a43aSToby Isaac /*@C
1140d961a43aSToby Isaac   DMPlexGetTreeChildren - get the children of a point in the tree describing the point hierarchy (not the Sieve DAG)
1141d961a43aSToby Isaac 
1142d961a43aSToby Isaac   Input Parameters:
1143d961a43aSToby Isaac + dm - the DMPlex object
1144d961a43aSToby Isaac - point - the query point
1145d961a43aSToby Isaac 
1146d961a43aSToby Isaac   Output Parameters:
1147d961a43aSToby Isaac + numChildren - if not NULL, set to the number of children
1148d961a43aSToby Isaac - children - if not NULL, set to a list children, or set to NULL if the point has no children
1149d961a43aSToby Isaac 
1150d961a43aSToby Isaac   Level: intermediate
1151d961a43aSToby Isaac 
1152d961a43aSToby Isaac   Fortran Notes:
1153d961a43aSToby Isaac   Since it returns an array, this routine is only available in Fortran 90, and you must
1154d961a43aSToby Isaac   include petsc.h90 in your code.
1155d961a43aSToby Isaac 
1156d961a43aSToby Isaac .seealso: DMPlexSetTree(), DMPlexGetTree(), DMPlexGetTreeParent()
1157d961a43aSToby Isaac @*/
1158d961a43aSToby Isaac PetscErrorCode DMPlexGetTreeChildren(DM dm, PetscInt point, PetscInt *numChildren, const PetscInt *children[])
1159d961a43aSToby Isaac {
1160d961a43aSToby Isaac   DM_Plex       *mesh = (DM_Plex *)dm->data;
1161d961a43aSToby Isaac   PetscSection   childSec;
1162d961a43aSToby Isaac   PetscInt       dof = 0;
1163d961a43aSToby Isaac   PetscErrorCode ierr;
1164d961a43aSToby Isaac 
1165d961a43aSToby Isaac   PetscFunctionBegin;
1166d961a43aSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1167d961a43aSToby Isaac   childSec = mesh->childSection;
1168d961a43aSToby Isaac   if (childSec && point >= childSec->pStart && point < childSec->pEnd) {
1169d961a43aSToby Isaac     ierr = PetscSectionGetDof (childSec, point, &dof);CHKERRQ(ierr);
1170d961a43aSToby Isaac   }
1171d961a43aSToby Isaac   if (numChildren) *numChildren = dof;
1172d961a43aSToby Isaac   if (children) {
1173d961a43aSToby Isaac     if (dof) {
1174d961a43aSToby Isaac       PetscInt off;
1175d961a43aSToby Isaac 
1176d961a43aSToby Isaac       ierr = PetscSectionGetOffset (childSec, point, &off);CHKERRQ(ierr);
1177d961a43aSToby Isaac       *children = &mesh->children[off];
1178d961a43aSToby Isaac     }
1179d961a43aSToby Isaac     else {
1180d961a43aSToby Isaac       *children = NULL;
1181d961a43aSToby Isaac     }
1182d961a43aSToby Isaac   }
1183d961a43aSToby Isaac   PetscFunctionReturn(0);
1184d961a43aSToby Isaac }
11850c37af3bSToby Isaac 
118652a3aeb4SToby Isaac static PetscErrorCode EvaluateBasis(PetscSpace space, PetscInt nBasis, PetscInt nFunctionals, PetscInt nComps, PetscInt nPoints, const PetscInt *pointsPerFn, const PetscReal *points, const PetscReal *weights, PetscReal *work, Mat basisAtPoints)
1187b3a4bf2aSToby Isaac {
118852a3aeb4SToby Isaac   PetscInt       f, b, p, c, offset, qPoints;
1189b3a4bf2aSToby Isaac   PetscErrorCode ierr;
1190b3a4bf2aSToby Isaac 
1191b3a4bf2aSToby Isaac   PetscFunctionBegin;
1192b3a4bf2aSToby Isaac   ierr = PetscSpaceEvaluate(space,nPoints,points,work,NULL,NULL);CHKERRQ(ierr);
119352a3aeb4SToby Isaac   for (f = 0, offset = 0; f < nFunctionals; f++) {
119452a3aeb4SToby Isaac     qPoints = pointsPerFn[f];
119552a3aeb4SToby Isaac     for (b = 0; b < nBasis; b++) {
1196b3a4bf2aSToby Isaac       PetscScalar val = 0.;
1197b3a4bf2aSToby Isaac 
119852a3aeb4SToby Isaac       for (p = 0; p < qPoints; p++) {
119952a3aeb4SToby Isaac         for (c = 0; c < nComps; c++) {
120052a3aeb4SToby Isaac           val += work[((offset + p) * nBasis + b) * nComps + c] * weights[(offset + p) * nComps + c];
1201b3a4bf2aSToby Isaac         }
120252a3aeb4SToby Isaac       }
120352a3aeb4SToby Isaac       ierr = MatSetValue(basisAtPoints,b,f,val,INSERT_VALUES);CHKERRQ(ierr);
1204b3a4bf2aSToby Isaac     }
1205b3a4bf2aSToby Isaac     offset += qPoints;
1206b3a4bf2aSToby Isaac   }
1207b3a4bf2aSToby Isaac   ierr = MatAssemblyBegin(basisAtPoints,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
1208b3a4bf2aSToby Isaac   ierr = MatAssemblyEnd(basisAtPoints,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
1209b3a4bf2aSToby Isaac   PetscFunctionReturn(0);
1210b3a4bf2aSToby Isaac }
1211b3a4bf2aSToby Isaac 
1212f7c74593SToby Isaac static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_Direct(DM dm, PetscSection section, PetscSection cSec, Mat cMat)
12130c37af3bSToby Isaac {
12140c37af3bSToby Isaac   PetscDS        ds;
12150c37af3bSToby Isaac   PetscInt       spdim;
12160c37af3bSToby Isaac   PetscInt       numFields, f, c, cStart, cEnd, pStart, pEnd, conStart, conEnd;
12170c37af3bSToby Isaac   const PetscInt *anchors;
1218f7c74593SToby Isaac   PetscSection   aSec;
12190c37af3bSToby Isaac   PetscReal      *v0, *v0parent, *vtmp, *J, *Jparent, *invJparent, detJ, detJparent;
12200c37af3bSToby Isaac   IS             aIS;
12210c37af3bSToby Isaac   PetscErrorCode ierr;
12220c37af3bSToby Isaac 
12230c37af3bSToby Isaac   PetscFunctionBegin;
12240c37af3bSToby Isaac   ierr = DMPlexGetChart(dm,&pStart,&pEnd);CHKERRQ(ierr);
12250c37af3bSToby Isaac   ierr = DMGetDS(dm,&ds);CHKERRQ(ierr);
12260c37af3bSToby Isaac   ierr = PetscDSGetNumFields(ds,&numFields);CHKERRQ(ierr);
12270c37af3bSToby Isaac   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
1228a17985deSToby Isaac   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
12290c37af3bSToby Isaac   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
12300c37af3bSToby Isaac   ierr = PetscSectionGetChart(cSec,&conStart,&conEnd);CHKERRQ(ierr);
123128f4b327SMatthew G. Knepley   ierr = DMGetDimension(dm,&spdim);CHKERRQ(ierr);
12320c37af3bSToby Isaac   ierr = PetscMalloc6(spdim,&v0,spdim,&v0parent,spdim,&vtmp,spdim*spdim,&J,spdim*spdim,&Jparent,spdim*spdim,&invJparent);CHKERRQ(ierr);
12330c37af3bSToby Isaac 
12340c37af3bSToby Isaac   for (f = 0; f < numFields; f++) {
12350dd1b1feSToby Isaac     PetscObject disc;
12360dd1b1feSToby Isaac     PetscClassId id;
1237b3a4bf2aSToby Isaac     PetscSpace     bspace;
1238b3a4bf2aSToby Isaac     PetscDualSpace dspace;
12399c3cf19fSMatthew G. Knepley     PetscInt i, j, k, nPoints, Nc, offset;
124052a3aeb4SToby Isaac     PetscInt fSize, maxDof;
1241b3a4bf2aSToby Isaac     PetscReal   *weights, *pointsRef, *pointsReal, *work;
12422c44ad04SToby Isaac     PetscScalar *scwork, *X;
12432c44ad04SToby Isaac     PetscInt  *sizes, *workIndRow, *workIndCol;
12440c37af3bSToby Isaac     Mat Amat, Bmat, Xmat;
12452c44ad04SToby Isaac     const PetscInt    *numDof  = NULL;
1246085f0adfSToby Isaac     const PetscInt    ***perms = NULL;
1247085f0adfSToby Isaac     const PetscScalar ***flips = NULL;
12480c37af3bSToby Isaac 
12490dd1b1feSToby Isaac     ierr = PetscDSGetDiscretization(ds,f,&disc);CHKERRQ(ierr);
12500dd1b1feSToby Isaac     ierr = PetscObjectGetClassId(disc,&id);CHKERRQ(ierr);
12510dd1b1feSToby Isaac     if (id == PETSCFE_CLASSID) {
1252b3a4bf2aSToby Isaac       PetscFE fe = (PetscFE) disc;
1253b3a4bf2aSToby Isaac 
1254b3a4bf2aSToby Isaac       ierr = PetscFEGetBasisSpace(fe,&bspace);CHKERRQ(ierr);
1255b3a4bf2aSToby Isaac       ierr = PetscFEGetDualSpace(fe,&dspace);CHKERRQ(ierr);
1256b3a4bf2aSToby Isaac       ierr = PetscDualSpaceGetDimension(dspace,&fSize);CHKERRQ(ierr);
125752a3aeb4SToby Isaac       ierr = PetscFEGetNumComponents(fe,&Nc);CHKERRQ(ierr);
12580dd1b1feSToby Isaac     }
12590dd1b1feSToby Isaac     else if (id == PETSCFV_CLASSID) {
1260b3a4bf2aSToby Isaac       PetscFV fv = (PetscFV) disc;
1261b3a4bf2aSToby Isaac 
126252a3aeb4SToby Isaac       ierr = PetscFVGetNumComponents(fv,&Nc);CHKERRQ(ierr);
1263b3a4bf2aSToby Isaac       ierr = PetscSpaceCreate(PetscObjectComm((PetscObject)fv),&bspace);CHKERRQ(ierr);
1264b3a4bf2aSToby Isaac       ierr = PetscSpaceSetType(bspace,PETSCSPACEPOLYNOMIAL);CHKERRQ(ierr);
1265b3a4bf2aSToby Isaac       ierr = PetscSpaceSetOrder(bspace,0);CHKERRQ(ierr);
126652a3aeb4SToby Isaac       ierr = PetscSpaceSetNumComponents(bspace,Nc);CHKERRQ(ierr);
1267b3a4bf2aSToby Isaac       ierr = PetscSpacePolynomialSetNumVariables(bspace,spdim);CHKERRQ(ierr);
1268b3a4bf2aSToby Isaac       ierr = PetscSpaceSetUp(bspace);CHKERRQ(ierr);
1269b3a4bf2aSToby Isaac       ierr = PetscFVGetDualSpace(fv,&dspace);CHKERRQ(ierr);
1270b3a4bf2aSToby Isaac       ierr = PetscDualSpaceGetDimension(dspace,&fSize);CHKERRQ(ierr);
12710dd1b1feSToby Isaac     }
12720dd1b1feSToby Isaac     else SETERRQ1(PetscObjectComm(disc),PETSC_ERR_ARG_UNKNOWN_TYPE, "PetscDS discretization id %d not recognized.", id);
12732c44ad04SToby Isaac     ierr = PetscDualSpaceGetNumDof(dspace,&numDof);CHKERRQ(ierr);
12742c44ad04SToby Isaac     for (i = 0, maxDof = 0; i <= spdim; i++) {maxDof = PetscMax(maxDof,numDof[i]);}
1275085f0adfSToby Isaac     ierr = PetscDualSpaceGetSymmetries(dspace,&perms,&flips);CHKERRQ(ierr);
12760dd1b1feSToby Isaac 
12770c37af3bSToby Isaac     ierr = MatCreate(PETSC_COMM_SELF,&Amat);CHKERRQ(ierr);
12780c37af3bSToby Isaac     ierr = MatSetSizes(Amat,fSize,fSize,fSize,fSize);CHKERRQ(ierr);
12790c37af3bSToby Isaac     ierr = MatSetType(Amat,MATSEQDENSE);CHKERRQ(ierr);
12800c37af3bSToby Isaac     ierr = MatSetUp(Amat);CHKERRQ(ierr);
12810c37af3bSToby Isaac     ierr = MatDuplicate(Amat,MAT_DO_NOT_COPY_VALUES,&Bmat);CHKERRQ(ierr);
12820c37af3bSToby Isaac     ierr = MatDuplicate(Amat,MAT_DO_NOT_COPY_VALUES,&Xmat);CHKERRQ(ierr);
12830c37af3bSToby Isaac     nPoints = 0;
12840c37af3bSToby Isaac     for (i = 0; i < fSize; i++) {
128552a3aeb4SToby Isaac       PetscInt        qPoints, thisNc;
12860c37af3bSToby Isaac       PetscQuadrature quad;
12870c37af3bSToby Isaac 
1288b3a4bf2aSToby Isaac       ierr = PetscDualSpaceGetFunctional(dspace,i,&quad);CHKERRQ(ierr);
128952a3aeb4SToby Isaac       ierr = PetscQuadratureGetData(quad,NULL,&thisNc,&qPoints,NULL,NULL);CHKERRQ(ierr);
129052a3aeb4SToby Isaac       if (thisNc != Nc) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Functional dim %D does not much basis dim %D\n",thisNc,Nc);
12910c37af3bSToby Isaac       nPoints += qPoints;
12920c37af3bSToby Isaac     }
129352a3aeb4SToby Isaac     ierr = PetscMalloc7(fSize,&sizes,nPoints*Nc,&weights,spdim*nPoints,&pointsRef,spdim*nPoints,&pointsReal,nPoints*fSize*Nc,&work,maxDof,&workIndRow,maxDof,&workIndCol);CHKERRQ(ierr);
12942c44ad04SToby Isaac     ierr = PetscMalloc1(maxDof * maxDof,&scwork);CHKERRQ(ierr);
12950c37af3bSToby Isaac     offset = 0;
12960c37af3bSToby Isaac     for (i = 0; i < fSize; i++) {
12970c37af3bSToby Isaac       PetscInt        qPoints;
12980c37af3bSToby Isaac       const PetscReal    *p, *w;
12990c37af3bSToby Isaac       PetscQuadrature quad;
13000c37af3bSToby Isaac 
1301b3a4bf2aSToby Isaac       ierr = PetscDualSpaceGetFunctional(dspace,i,&quad);CHKERRQ(ierr);
130252a3aeb4SToby Isaac       ierr = PetscQuadratureGetData(quad,NULL,NULL,&qPoints,&p,&w);CHKERRQ(ierr);
130352a3aeb4SToby Isaac       ierr = PetscMemcpy(weights+Nc*offset,w,Nc*qPoints*sizeof(*w));CHKERRQ(ierr);
13040c37af3bSToby Isaac       ierr = PetscMemcpy(pointsRef+spdim*offset,p,spdim*qPoints*sizeof(*p));CHKERRQ(ierr);
1305b3a4bf2aSToby Isaac       sizes[i] = qPoints;
13060c37af3bSToby Isaac       offset  += qPoints;
13070c37af3bSToby Isaac     }
130852a3aeb4SToby Isaac     ierr = EvaluateBasis(bspace,fSize,fSize,Nc,nPoints,sizes,pointsRef,weights,work,Amat);CHKERRQ(ierr);
13090c37af3bSToby Isaac     ierr = MatLUFactor(Amat,NULL,NULL,NULL);CHKERRQ(ierr);
13100c37af3bSToby Isaac     for (c = cStart; c < cEnd; c++) {
13110c37af3bSToby Isaac       PetscInt        parent;
13120c37af3bSToby Isaac       PetscInt        closureSize, closureSizeP, *closure = NULL, *closureP = NULL;
13130c37af3bSToby Isaac       PetscInt        *childOffsets, *parentOffsets;
13140c37af3bSToby Isaac 
13150c37af3bSToby Isaac       ierr = DMPlexGetTreeParent(dm,c,&parent,NULL);CHKERRQ(ierr);
13160c37af3bSToby Isaac       if (parent == c) continue;
13170c37af3bSToby Isaac       ierr = DMPlexGetTransitiveClosure(dm,c,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
13180c37af3bSToby Isaac       for (i = 0; i < closureSize; i++) {
13190c37af3bSToby Isaac         PetscInt p = closure[2*i];
13200c37af3bSToby Isaac         PetscInt conDof;
13210c37af3bSToby Isaac 
13220c37af3bSToby Isaac         if (p < conStart || p >= conEnd) continue;
1323085f0adfSToby Isaac         if (numFields) {
13240c37af3bSToby Isaac           ierr = PetscSectionGetFieldDof(cSec,p,f,&conDof);CHKERRQ(ierr);
13250c37af3bSToby Isaac         }
13260c37af3bSToby Isaac         else {
13270c37af3bSToby Isaac           ierr = PetscSectionGetDof(cSec,p,&conDof);CHKERRQ(ierr);
13280c37af3bSToby Isaac         }
13290c37af3bSToby Isaac         if (conDof) break;
13300c37af3bSToby Isaac       }
13310c37af3bSToby Isaac       if (i == closureSize) {
13320c37af3bSToby Isaac         ierr = DMPlexRestoreTransitiveClosure(dm,c,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
13330c37af3bSToby Isaac         continue;
13340c37af3bSToby Isaac       }
13350c37af3bSToby Isaac 
133673a7f2aaSMatthew G. Knepley       ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, v0, J, NULL, &detJ);CHKERRQ(ierr);
133773a7f2aaSMatthew G. Knepley       ierr = DMPlexComputeCellGeometryFEM(dm, parent, NULL, v0parent, Jparent, invJparent, &detJparent);CHKERRQ(ierr);
13380c37af3bSToby Isaac       for (i = 0; i < nPoints; i++) {
1339b3a4bf2aSToby Isaac         CoordinatesRefToReal(spdim, spdim, v0, J, &pointsRef[i*spdim],vtmp);
1340b3a4bf2aSToby Isaac         CoordinatesRealToRef(spdim, spdim, v0parent, invJparent, vtmp, &pointsReal[i*spdim]);
13410c37af3bSToby Isaac       }
134252a3aeb4SToby Isaac       ierr = EvaluateBasis(bspace,fSize,fSize,Nc,nPoints,sizes,pointsReal,weights,work,Bmat);CHKERRQ(ierr);
13430c37af3bSToby Isaac       ierr = MatMatSolve(Amat,Bmat,Xmat);CHKERRQ(ierr);
13442c44ad04SToby Isaac       ierr = MatDenseGetArray(Xmat,&X);CHKERRQ(ierr);
13450c37af3bSToby Isaac       ierr = DMPlexGetTransitiveClosure(dm,parent,PETSC_TRUE,&closureSizeP,&closureP);CHKERRQ(ierr);
13460c37af3bSToby Isaac       ierr = PetscMalloc2(closureSize+1,&childOffsets,closureSizeP+1,&parentOffsets);CHKERRQ(ierr);
13470c37af3bSToby Isaac       childOffsets[0] = 0;
13480c37af3bSToby Isaac       for (i = 0; i < closureSize; i++) {
13490c37af3bSToby Isaac         PetscInt p = closure[2*i];
13500c37af3bSToby Isaac         PetscInt dof;
13510c37af3bSToby Isaac 
1352085f0adfSToby Isaac         if (numFields) {
13530c37af3bSToby Isaac           ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
13540c37af3bSToby Isaac         }
13550c37af3bSToby Isaac         else {
13560c37af3bSToby Isaac           ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
13570c37af3bSToby Isaac         }
135852a3aeb4SToby Isaac         childOffsets[i+1]=childOffsets[i]+dof;
13590c37af3bSToby Isaac       }
13600c37af3bSToby Isaac       parentOffsets[0] = 0;
13610c37af3bSToby Isaac       for (i = 0; i < closureSizeP; i++) {
13620c37af3bSToby Isaac         PetscInt p = closureP[2*i];
13630c37af3bSToby Isaac         PetscInt dof;
13640c37af3bSToby Isaac 
1365085f0adfSToby Isaac         if (numFields) {
13660c37af3bSToby Isaac           ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
13670c37af3bSToby Isaac         }
13680c37af3bSToby Isaac         else {
13690c37af3bSToby Isaac           ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
13700c37af3bSToby Isaac         }
137152a3aeb4SToby Isaac         parentOffsets[i+1]=parentOffsets[i]+dof;
13720c37af3bSToby Isaac       }
13730c37af3bSToby Isaac       for (i = 0; i < closureSize; i++) {
13742c44ad04SToby Isaac         PetscInt conDof, conOff, aDof, aOff, nWork;
13750c37af3bSToby Isaac         PetscInt p = closure[2*i];
13760c37af3bSToby Isaac         PetscInt o = closure[2*i+1];
1377085f0adfSToby Isaac         const PetscInt    *perm;
1378085f0adfSToby Isaac         const PetscScalar *flip;
13790c37af3bSToby Isaac 
13800c37af3bSToby Isaac         if (p < conStart || p >= conEnd) continue;
1381085f0adfSToby Isaac         if (numFields) {
13820c37af3bSToby Isaac           ierr = PetscSectionGetFieldDof(cSec,p,f,&conDof);CHKERRQ(ierr);
13830c37af3bSToby Isaac           ierr = PetscSectionGetFieldOffset(cSec,p,f,&conOff);CHKERRQ(ierr);
13840c37af3bSToby Isaac         }
13850c37af3bSToby Isaac         else {
13860c37af3bSToby Isaac           ierr = PetscSectionGetDof(cSec,p,&conDof);CHKERRQ(ierr);
13870c37af3bSToby Isaac           ierr = PetscSectionGetOffset(cSec,p,&conOff);CHKERRQ(ierr);
13880c37af3bSToby Isaac         }
13890c37af3bSToby Isaac         if (!conDof) continue;
1390085f0adfSToby Isaac         perm  = (perms && perms[i]) ? perms[i][o] : NULL;
1391085f0adfSToby Isaac         flip  = (flips && flips[i]) ? flips[i][o] : NULL;
13920c37af3bSToby Isaac         ierr  = PetscSectionGetDof(aSec,p,&aDof);CHKERRQ(ierr);
13930c37af3bSToby Isaac         ierr  = PetscSectionGetOffset(aSec,p,&aOff);CHKERRQ(ierr);
13942c44ad04SToby Isaac         nWork = childOffsets[i+1]-childOffsets[i];
13950c37af3bSToby Isaac         for (k = 0; k < aDof; k++) {
13960c37af3bSToby Isaac           PetscInt a = anchors[aOff + k];
13970c37af3bSToby Isaac           PetscInt aSecDof, aSecOff;
13980c37af3bSToby Isaac 
1399085f0adfSToby Isaac           if (numFields) {
14000c37af3bSToby Isaac             ierr = PetscSectionGetFieldDof(section,a,f,&aSecDof);CHKERRQ(ierr);
14010c37af3bSToby Isaac             ierr = PetscSectionGetFieldOffset(section,a,f,&aSecOff);CHKERRQ(ierr);
14020c37af3bSToby Isaac           }
14030c37af3bSToby Isaac           else {
14040c37af3bSToby Isaac             ierr = PetscSectionGetDof(section,a,&aSecDof);CHKERRQ(ierr);
14050c37af3bSToby Isaac             ierr = PetscSectionGetOffset(section,a,&aSecOff);CHKERRQ(ierr);
14060c37af3bSToby Isaac           }
14070c37af3bSToby Isaac           if (!aSecDof) continue;
14080c37af3bSToby Isaac 
14090c37af3bSToby Isaac           for (j = 0; j < closureSizeP; j++) {
14100c37af3bSToby Isaac             PetscInt q = closureP[2*j];
14110c37af3bSToby Isaac             PetscInt oq = closureP[2*j+1];
14122c44ad04SToby Isaac 
14132c44ad04SToby Isaac             if (q == a) {
141452a3aeb4SToby Isaac               PetscInt           r, s, nWorkP;
1415085f0adfSToby Isaac               const PetscInt    *permP;
1416085f0adfSToby Isaac               const PetscScalar *flipP;
1417085f0adfSToby Isaac 
1418085f0adfSToby Isaac               permP  = (perms && perms[j]) ? perms[j][oq] : NULL;
1419085f0adfSToby Isaac               flipP  = (flips && flips[j]) ? flips[j][oq] : NULL;
14202c44ad04SToby Isaac               nWorkP = parentOffsets[j+1]-parentOffsets[j];
14212c44ad04SToby Isaac               /* get a copy of the child-to-anchor portion of the matrix, and transpose so that rows correspond to the
14222c44ad04SToby Isaac                * child and columns correspond to the anchor: BUT the maxrix returned by MatDenseGetArray is
14232c44ad04SToby Isaac                * column-major, so transpose-transpose = do nothing */
14242c44ad04SToby Isaac               for (r = 0; r < nWork; r++) {
14252c44ad04SToby Isaac                 for (s = 0; s < nWorkP; s++) {
14262c44ad04SToby Isaac                   scwork[r * nWorkP + s] = X[fSize * (r + childOffsets[i]) + (s + parentOffsets[j])];
14272c44ad04SToby Isaac                 }
14282c44ad04SToby Isaac               }
142952a3aeb4SToby Isaac               for (r = 0; r < nWork; r++)  {workIndRow[perm  ? perm[r]  : r] = conOff  + r;}
143052a3aeb4SToby Isaac               for (s = 0; s < nWorkP; s++) {workIndCol[permP ? permP[s] : s] = aSecOff + s;}
14312c44ad04SToby Isaac               if (flip) {
14322c44ad04SToby Isaac                 for (r = 0; r < nWork; r++) {
14332c44ad04SToby Isaac                   for (s = 0; s < nWorkP; s++) {
14342c44ad04SToby Isaac                     scwork[r * nWorkP + s] *= flip[r];
14352c44ad04SToby Isaac                   }
14362c44ad04SToby Isaac                 }
14372c44ad04SToby Isaac               }
14382c44ad04SToby Isaac               if (flipP) {
14392c44ad04SToby Isaac                 for (r = 0; r < nWork; r++) {
14402c44ad04SToby Isaac                   for (s = 0; s < nWorkP; s++) {
14412c44ad04SToby Isaac                     scwork[r * nWorkP + s] *= flipP[s];
14422c44ad04SToby Isaac                   }
14432c44ad04SToby Isaac                 }
14442c44ad04SToby Isaac               }
14452c44ad04SToby Isaac               ierr = MatSetValues(cMat,nWork,workIndRow,nWorkP,workIndCol,scwork,INSERT_VALUES);CHKERRQ(ierr);
14462c44ad04SToby Isaac               break;
14470c37af3bSToby Isaac             }
14480c37af3bSToby Isaac           }
14490c37af3bSToby Isaac         }
14500c37af3bSToby Isaac       }
14512c44ad04SToby Isaac       ierr = MatDenseRestoreArray(Xmat,&X);CHKERRQ(ierr);
14520c37af3bSToby Isaac       ierr = PetscFree2(childOffsets,parentOffsets);CHKERRQ(ierr);
14530c37af3bSToby Isaac       ierr = DMPlexRestoreTransitiveClosure(dm,c,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
14540c37af3bSToby Isaac       ierr = DMPlexRestoreTransitiveClosure(dm,parent,PETSC_TRUE,&closureSizeP,&closureP);CHKERRQ(ierr);
14550c37af3bSToby Isaac     }
14560c37af3bSToby Isaac     ierr = MatDestroy(&Amat);CHKERRQ(ierr);
14570c37af3bSToby Isaac     ierr = MatDestroy(&Bmat);CHKERRQ(ierr);
14580c37af3bSToby Isaac     ierr = MatDestroy(&Xmat);CHKERRQ(ierr);
14592c44ad04SToby Isaac     ierr = PetscFree(scwork);CHKERRQ(ierr);
14602c44ad04SToby Isaac     ierr = PetscFree7(sizes,weights,pointsRef,pointsReal,work,workIndRow,workIndCol);CHKERRQ(ierr);
1461b3a4bf2aSToby Isaac     if (id == PETSCFV_CLASSID) {
1462b3a4bf2aSToby Isaac       ierr = PetscSpaceDestroy(&bspace);CHKERRQ(ierr);
1463b3a4bf2aSToby Isaac     }
14640c37af3bSToby Isaac   }
14650c37af3bSToby Isaac   ierr = MatAssemblyBegin(cMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
14660c37af3bSToby Isaac   ierr = MatAssemblyEnd(cMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
14670c37af3bSToby Isaac   ierr = PetscFree6(v0,v0parent,vtmp,J,Jparent,invJparent);CHKERRQ(ierr);
14680c37af3bSToby Isaac   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
14690c37af3bSToby Isaac 
14700c37af3bSToby Isaac   PetscFunctionReturn(0);
14710c37af3bSToby Isaac }
147295a0b26dSToby Isaac 
147321968bf8SToby Isaac static PetscErrorCode DMPlexReferenceTreeGetChildrenMatrices(DM refTree, PetscScalar ****childrenMats, PetscInt ***childrenN)
147495a0b26dSToby Isaac {
1475f7c74593SToby Isaac   Mat               refCmat;
147621968bf8SToby Isaac   PetscDS           ds;
1477085f0adfSToby Isaac   PetscInt          numFields, maxFields, f, pRefStart, pRefEnd, p, *rows, *cols, maxDof, maxAnDof, **refPointFieldN;
147821968bf8SToby Isaac   PetscScalar       ***refPointFieldMats;
147921968bf8SToby Isaac   PetscSection      refConSec, refAnSec, refSection;
148021968bf8SToby Isaac   IS                refAnIS;
148121968bf8SToby Isaac   const PetscInt    *refAnchors;
1482085f0adfSToby Isaac   const PetscInt    **perms;
1483085f0adfSToby Isaac   const PetscScalar **flips;
148495a0b26dSToby Isaac   PetscErrorCode    ierr;
148595a0b26dSToby Isaac 
148695a0b26dSToby Isaac   PetscFunctionBegin;
148721968bf8SToby Isaac   ierr = DMGetDS(refTree,&ds);CHKERRQ(ierr);
148895a0b26dSToby Isaac   ierr = PetscDSGetNumFields(ds,&numFields);CHKERRQ(ierr);
1489085f0adfSToby Isaac   maxFields = PetscMax(1,numFields);
1490f7c74593SToby Isaac   ierr = DMGetDefaultConstraints(refTree,&refConSec,&refCmat);CHKERRQ(ierr);
1491a17985deSToby Isaac   ierr = DMPlexGetAnchors(refTree,&refAnSec,&refAnIS);CHKERRQ(ierr);
149295a0b26dSToby Isaac   ierr = ISGetIndices(refAnIS,&refAnchors);CHKERRQ(ierr);
149395a0b26dSToby Isaac   ierr = DMGetDefaultSection(refTree,&refSection);CHKERRQ(ierr);
149495a0b26dSToby Isaac   ierr = PetscSectionGetChart(refConSec,&pRefStart,&pRefEnd);CHKERRQ(ierr);
149595a0b26dSToby Isaac   ierr = PetscMalloc1(pRefEnd-pRefStart,&refPointFieldMats);CHKERRQ(ierr);
149695a0b26dSToby Isaac   ierr = PetscMalloc1(pRefEnd-pRefStart,&refPointFieldN);CHKERRQ(ierr);
149795a0b26dSToby Isaac   ierr = PetscSectionGetMaxDof(refConSec,&maxDof);CHKERRQ(ierr);
149895a0b26dSToby Isaac   ierr = PetscSectionGetMaxDof(refAnSec,&maxAnDof);CHKERRQ(ierr);
149995a0b26dSToby Isaac   ierr = PetscMalloc1(maxDof,&rows);CHKERRQ(ierr);
150095a0b26dSToby Isaac   ierr = PetscMalloc1(maxDof*maxAnDof,&cols);CHKERRQ(ierr);
150195a0b26dSToby Isaac   for (p = pRefStart; p < pRefEnd; p++) {
150295a0b26dSToby Isaac     PetscInt parent, closureSize, *closure = NULL, pDof;
150395a0b26dSToby Isaac 
150495a0b26dSToby Isaac     ierr = DMPlexGetTreeParent(refTree,p,&parent,NULL);CHKERRQ(ierr);
150595a0b26dSToby Isaac     ierr = PetscSectionGetDof(refConSec,p,&pDof);CHKERRQ(ierr);
150695a0b26dSToby Isaac     if (!pDof || parent == p) continue;
150795a0b26dSToby Isaac 
1508085f0adfSToby Isaac     ierr = PetscMalloc1(maxFields,&refPointFieldMats[p-pRefStart]);CHKERRQ(ierr);
1509085f0adfSToby Isaac     ierr = PetscCalloc1(maxFields,&refPointFieldN[p-pRefStart]);CHKERRQ(ierr);
151095a0b26dSToby Isaac     ierr = DMPlexGetTransitiveClosure(refTree,parent,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
1511085f0adfSToby Isaac     for (f = 0; f < maxFields; f++) {
1512085f0adfSToby Isaac       PetscInt cDof, cOff, numCols, r, i;
151395a0b26dSToby Isaac 
1514085f0adfSToby Isaac       if (f < numFields) {
151595a0b26dSToby Isaac         ierr = PetscSectionGetFieldDof(refConSec,p,f,&cDof);CHKERRQ(ierr);
151695a0b26dSToby Isaac         ierr = PetscSectionGetFieldOffset(refConSec,p,f,&cOff);CHKERRQ(ierr);
1517085f0adfSToby Isaac         ierr = PetscSectionGetFieldPointSyms(refSection,f,closureSize,closure,&perms,&flips);CHKERRQ(ierr);
1518085f0adfSToby Isaac       } else {
151995a0b26dSToby Isaac         ierr = PetscSectionGetDof(refConSec,p,&cDof);CHKERRQ(ierr);
152095a0b26dSToby Isaac         ierr = PetscSectionGetOffset(refConSec,p,&cOff);CHKERRQ(ierr);
1521085f0adfSToby Isaac         ierr = PetscSectionGetPointSyms(refSection,closureSize,closure,&perms,&flips);CHKERRQ(ierr);
152295a0b26dSToby Isaac       }
152395a0b26dSToby Isaac 
152495a0b26dSToby Isaac       for (r = 0; r < cDof; r++) {
152595a0b26dSToby Isaac         rows[r] = cOff + r;
152695a0b26dSToby Isaac       }
152795a0b26dSToby Isaac       numCols = 0;
152895a0b26dSToby Isaac       for (i = 0; i < closureSize; i++) {
152995a0b26dSToby Isaac         PetscInt          q = closure[2*i];
153095a0b26dSToby Isaac         PetscInt          aDof, aOff, j;
1531085f0adfSToby Isaac         const PetscInt    *perm = perms ? perms[i] : NULL;
153295a0b26dSToby Isaac 
1533085f0adfSToby Isaac         if (numFields) {
153495a0b26dSToby Isaac           ierr = PetscSectionGetFieldDof(refSection,q,f,&aDof);CHKERRQ(ierr);
153595a0b26dSToby Isaac           ierr = PetscSectionGetFieldOffset(refSection,q,f,&aOff);CHKERRQ(ierr);
153695a0b26dSToby Isaac         }
153795a0b26dSToby Isaac         else {
153895a0b26dSToby Isaac           ierr = PetscSectionGetDof(refSection,q,&aDof);CHKERRQ(ierr);
153995a0b26dSToby Isaac           ierr = PetscSectionGetOffset(refSection,q,&aOff);CHKERRQ(ierr);
154095a0b26dSToby Isaac         }
154195a0b26dSToby Isaac 
154295a0b26dSToby Isaac         for (j = 0; j < aDof; j++) {
1543085f0adfSToby Isaac           cols[numCols++] = aOff + (perm ? perm[j] : j);
154495a0b26dSToby Isaac         }
154595a0b26dSToby Isaac       }
154695a0b26dSToby Isaac       refPointFieldN[p-pRefStart][f] = numCols;
154795a0b26dSToby Isaac       ierr = PetscMalloc1(cDof*numCols,&refPointFieldMats[p-pRefStart][f]);CHKERRQ(ierr);
154895a0b26dSToby Isaac       ierr = MatGetValues(refCmat,cDof,rows,numCols,cols,refPointFieldMats[p-pRefStart][f]);CHKERRQ(ierr);
1549085f0adfSToby Isaac       if (flips) {
1550085f0adfSToby Isaac         PetscInt colOff = 0;
1551085f0adfSToby Isaac 
1552085f0adfSToby Isaac         for (i = 0; i < closureSize; i++) {
1553085f0adfSToby Isaac           PetscInt          q = closure[2*i];
1554085f0adfSToby Isaac           PetscInt          aDof, aOff, j;
1555085f0adfSToby Isaac           const PetscScalar *flip = flips ? flips[i] : NULL;
1556085f0adfSToby Isaac 
1557085f0adfSToby Isaac           if (numFields) {
1558085f0adfSToby Isaac             ierr = PetscSectionGetFieldDof(refSection,q,f,&aDof);CHKERRQ(ierr);
1559085f0adfSToby Isaac             ierr = PetscSectionGetFieldOffset(refSection,q,f,&aOff);CHKERRQ(ierr);
1560085f0adfSToby Isaac           }
1561085f0adfSToby Isaac           else {
1562085f0adfSToby Isaac             ierr = PetscSectionGetDof(refSection,q,&aDof);CHKERRQ(ierr);
1563085f0adfSToby Isaac             ierr = PetscSectionGetOffset(refSection,q,&aOff);CHKERRQ(ierr);
1564085f0adfSToby Isaac           }
1565085f0adfSToby Isaac           if (flip) {
1566085f0adfSToby Isaac             PetscInt k;
1567085f0adfSToby Isaac             for (k = 0; k < cDof; k++) {
1568085f0adfSToby Isaac               for (j = 0; j < aDof; j++) {
1569085f0adfSToby Isaac                 refPointFieldMats[p-pRefStart][f][k * numCols + colOff + j] *= flip[j];
1570085f0adfSToby Isaac               }
1571085f0adfSToby Isaac             }
1572085f0adfSToby Isaac           }
1573085f0adfSToby Isaac           colOff += aDof;
1574085f0adfSToby Isaac         }
1575085f0adfSToby Isaac       }
1576085f0adfSToby Isaac       if (numFields) {
1577085f0adfSToby Isaac         ierr = PetscSectionRestoreFieldPointSyms(refSection,f,closureSize,closure,&perms,&flips);CHKERRQ(ierr);
1578085f0adfSToby Isaac       } else {
1579085f0adfSToby Isaac         ierr = PetscSectionRestorePointSyms(refSection,closureSize,closure,&perms,&flips);CHKERRQ(ierr);
1580085f0adfSToby Isaac       }
158195a0b26dSToby Isaac     }
158295a0b26dSToby Isaac     ierr = DMPlexRestoreTransitiveClosure(refTree,parent,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
158395a0b26dSToby Isaac   }
158421968bf8SToby Isaac   *childrenMats = refPointFieldMats;
158521968bf8SToby Isaac   *childrenN = refPointFieldN;
158621968bf8SToby Isaac   ierr = ISRestoreIndices(refAnIS,&refAnchors);CHKERRQ(ierr);
158721968bf8SToby Isaac   ierr = PetscFree(rows);CHKERRQ(ierr);
158821968bf8SToby Isaac   ierr = PetscFree(cols);CHKERRQ(ierr);
158921968bf8SToby Isaac   PetscFunctionReturn(0);
159021968bf8SToby Isaac }
159121968bf8SToby Isaac 
159221968bf8SToby Isaac static PetscErrorCode DMPlexReferenceTreeRestoreChildrenMatrices(DM refTree, PetscScalar ****childrenMats, PetscInt ***childrenN)
159321968bf8SToby Isaac {
159421968bf8SToby Isaac   PetscDS        ds;
159521968bf8SToby Isaac   PetscInt       **refPointFieldN;
159621968bf8SToby Isaac   PetscScalar    ***refPointFieldMats;
1597085f0adfSToby Isaac   PetscInt       numFields, maxFields, pRefStart, pRefEnd, p, f;
159821968bf8SToby Isaac   PetscSection   refConSec;
159921968bf8SToby Isaac   PetscErrorCode ierr;
160021968bf8SToby Isaac 
160121968bf8SToby Isaac   PetscFunctionBegin;
160221968bf8SToby Isaac   refPointFieldN = *childrenN;
160321968bf8SToby Isaac   *childrenN = NULL;
160421968bf8SToby Isaac   refPointFieldMats = *childrenMats;
160521968bf8SToby Isaac   *childrenMats = NULL;
160621968bf8SToby Isaac   ierr = DMGetDS(refTree,&ds);CHKERRQ(ierr);
160721968bf8SToby Isaac   ierr = PetscDSGetNumFields(ds,&numFields);CHKERRQ(ierr);
1608085f0adfSToby Isaac   maxFields = PetscMax(1,numFields);CHKERRQ(ierr);
160921968bf8SToby Isaac   ierr = DMGetDefaultConstraints(refTree,&refConSec,NULL);CHKERRQ(ierr);
1610e44e4e7fSToby Isaac   ierr = PetscSectionGetChart(refConSec,&pRefStart,&pRefEnd);CHKERRQ(ierr);
161121968bf8SToby Isaac   for (p = pRefStart; p < pRefEnd; p++) {
161221968bf8SToby Isaac     PetscInt parent, pDof;
161321968bf8SToby Isaac 
161421968bf8SToby Isaac     ierr = DMPlexGetTreeParent(refTree,p,&parent,NULL);CHKERRQ(ierr);
161521968bf8SToby Isaac     ierr = PetscSectionGetDof(refConSec,p,&pDof);CHKERRQ(ierr);
161621968bf8SToby Isaac     if (!pDof || parent == p) continue;
161721968bf8SToby Isaac 
1618085f0adfSToby Isaac     for (f = 0; f < maxFields; f++) {
161921968bf8SToby Isaac       PetscInt cDof;
162021968bf8SToby Isaac 
1621085f0adfSToby Isaac       if (numFields) {
162221968bf8SToby Isaac         ierr = PetscSectionGetFieldDof(refConSec,p,f,&cDof);CHKERRQ(ierr);
162321968bf8SToby Isaac       }
162421968bf8SToby Isaac       else {
162521968bf8SToby Isaac         ierr = PetscSectionGetDof(refConSec,p,&cDof);CHKERRQ(ierr);
162621968bf8SToby Isaac       }
162721968bf8SToby Isaac 
162821968bf8SToby Isaac       ierr = PetscFree(refPointFieldMats[p - pRefStart][f]);CHKERRQ(ierr);
162921968bf8SToby Isaac     }
163021968bf8SToby Isaac     ierr = PetscFree(refPointFieldMats[p - pRefStart]);CHKERRQ(ierr);
163121968bf8SToby Isaac     ierr = PetscFree(refPointFieldN[p - pRefStart]);CHKERRQ(ierr);
163221968bf8SToby Isaac   }
163321968bf8SToby Isaac   ierr = PetscFree(refPointFieldMats);CHKERRQ(ierr);
163421968bf8SToby Isaac   ierr = PetscFree(refPointFieldN);CHKERRQ(ierr);
163521968bf8SToby Isaac   PetscFunctionReturn(0);
163621968bf8SToby Isaac }
163721968bf8SToby Isaac 
163821968bf8SToby Isaac static PetscErrorCode DMPlexComputeAnchorMatrix_Tree_FromReference(DM dm, PetscSection section, PetscSection conSec, Mat cMat)
163921968bf8SToby Isaac {
164021968bf8SToby Isaac   DM             refTree;
164121968bf8SToby Isaac   PetscDS        ds;
164221968bf8SToby Isaac   Mat            refCmat;
1643085f0adfSToby Isaac   PetscInt       numFields, maxFields, f, pRefStart, pRefEnd, p, maxDof, maxAnDof, *perm, *iperm, pStart, pEnd, conStart, conEnd, **refPointFieldN;
164421968bf8SToby Isaac   PetscScalar ***refPointFieldMats, *pointWork;
164521968bf8SToby Isaac   PetscSection   refConSec, refAnSec, anSec;
164621968bf8SToby Isaac   IS             refAnIS, anIS;
164721968bf8SToby Isaac   const PetscInt *anchors;
164821968bf8SToby Isaac   PetscErrorCode ierr;
164921968bf8SToby Isaac 
165021968bf8SToby Isaac   PetscFunctionBegin;
165121968bf8SToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
165221968bf8SToby Isaac   ierr = DMGetDS(dm,&ds);CHKERRQ(ierr);
165321968bf8SToby Isaac   ierr = PetscDSGetNumFields(ds,&numFields);CHKERRQ(ierr);
1654085f0adfSToby Isaac   maxFields = PetscMax(1,numFields);
165521968bf8SToby Isaac   ierr = DMPlexGetReferenceTree(dm,&refTree);CHKERRQ(ierr);
165621968bf8SToby Isaac   ierr = DMSetDS(refTree,ds);CHKERRQ(ierr);
165721968bf8SToby Isaac   ierr = DMGetDefaultConstraints(refTree,&refConSec,&refCmat);CHKERRQ(ierr);
165821968bf8SToby Isaac   ierr = DMPlexGetAnchors(refTree,&refAnSec,&refAnIS);CHKERRQ(ierr);
165921968bf8SToby Isaac   ierr = DMPlexGetAnchors(dm,&anSec,&anIS);CHKERRQ(ierr);
166021968bf8SToby Isaac   ierr = ISGetIndices(anIS,&anchors);CHKERRQ(ierr);
166121968bf8SToby Isaac   ierr = PetscSectionGetChart(refConSec,&pRefStart,&pRefEnd);CHKERRQ(ierr);
166221968bf8SToby Isaac   ierr = PetscSectionGetChart(conSec,&conStart,&conEnd);CHKERRQ(ierr);
166321968bf8SToby Isaac   ierr = PetscSectionGetMaxDof(refConSec,&maxDof);CHKERRQ(ierr);
166421968bf8SToby Isaac   ierr = PetscSectionGetMaxDof(refAnSec,&maxAnDof);CHKERRQ(ierr);
166521968bf8SToby Isaac   ierr = PetscMalloc1(maxDof*maxDof*maxAnDof,&pointWork);CHKERRQ(ierr);
166621968bf8SToby Isaac 
166721968bf8SToby Isaac   /* step 1: get submats for every constrained point in the reference tree */
166821968bf8SToby Isaac   ierr = DMPlexReferenceTreeGetChildrenMatrices(refTree,&refPointFieldMats,&refPointFieldN);CHKERRQ(ierr);
166995a0b26dSToby Isaac 
167095a0b26dSToby Isaac   /* step 2: compute the preorder */
167195a0b26dSToby Isaac   ierr = DMPlexGetChart(dm,&pStart,&pEnd);CHKERRQ(ierr);
167295a0b26dSToby Isaac   ierr = PetscMalloc2(pEnd-pStart,&perm,pEnd-pStart,&iperm);CHKERRQ(ierr);
167395a0b26dSToby Isaac   for (p = pStart; p < pEnd; p++) {
167495a0b26dSToby Isaac     perm[p - pStart] = p;
167595a0b26dSToby Isaac     iperm[p - pStart] = p-pStart;
167695a0b26dSToby Isaac   }
167795a0b26dSToby Isaac   for (p = 0; p < pEnd - pStart;) {
167895a0b26dSToby Isaac     PetscInt point = perm[p];
167995a0b26dSToby Isaac     PetscInt parent;
168095a0b26dSToby Isaac 
168195a0b26dSToby Isaac     ierr = DMPlexGetTreeParent(dm,point,&parent,NULL);CHKERRQ(ierr);
168295a0b26dSToby Isaac     if (parent == point) {
168395a0b26dSToby Isaac       p++;
168495a0b26dSToby Isaac     }
168595a0b26dSToby Isaac     else {
168695a0b26dSToby Isaac       PetscInt size, closureSize, *closure = NULL, i;
168795a0b26dSToby Isaac 
168895a0b26dSToby Isaac       ierr = DMPlexGetTransitiveClosure(dm,parent,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
168995a0b26dSToby Isaac       for (i = 0; i < closureSize; i++) {
169095a0b26dSToby Isaac         PetscInt q = closure[2*i];
169195a0b26dSToby Isaac         if (iperm[q-pStart] > iperm[point-pStart]) {
169295a0b26dSToby Isaac           /* swap */
169395a0b26dSToby Isaac           perm[p]               = q;
169495a0b26dSToby Isaac           perm[iperm[q-pStart]] = point;
169595a0b26dSToby Isaac           iperm[point-pStart]   = iperm[q-pStart];
169695a0b26dSToby Isaac           iperm[q-pStart]       = p;
169795a0b26dSToby Isaac           break;
169895a0b26dSToby Isaac         }
169995a0b26dSToby Isaac       }
170095a0b26dSToby Isaac       size = closureSize;
170195a0b26dSToby Isaac       ierr = DMPlexRestoreTransitiveClosure(dm,parent,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
170295a0b26dSToby Isaac       if (i == size) {
170395a0b26dSToby Isaac         p++;
170495a0b26dSToby Isaac       }
170595a0b26dSToby Isaac     }
170695a0b26dSToby Isaac   }
170795a0b26dSToby Isaac 
170895a0b26dSToby Isaac   /* step 3: fill the constraint matrix */
170995a0b26dSToby Isaac   /* we are going to use a preorder progressive fill strategy.  Mat doesn't
171095a0b26dSToby Isaac    * allow progressive fill without assembly, so we are going to set up the
171195a0b26dSToby Isaac    * values outside of the Mat first.
171295a0b26dSToby Isaac    */
171395a0b26dSToby Isaac   {
171495a0b26dSToby Isaac     PetscInt nRows, row, nnz;
171595a0b26dSToby Isaac     PetscBool done;
171695a0b26dSToby Isaac     const PetscInt *ia, *ja;
171795a0b26dSToby Isaac     PetscScalar *vals;
171895a0b26dSToby Isaac 
171995a0b26dSToby Isaac     ierr = MatGetRowIJ(cMat,0,PETSC_FALSE,PETSC_FALSE,&nRows,&ia,&ja,&done);CHKERRQ(ierr);
172095a0b26dSToby Isaac     if (!done) SETERRQ(PetscObjectComm((PetscObject)cMat),PETSC_ERR_PLIB,"Could not get RowIJ of constraint matrix");
172195a0b26dSToby Isaac     nnz  = ia[nRows];
172295a0b26dSToby Isaac     /* malloc and then zero rows right before we fill them: this way valgrind
172395a0b26dSToby Isaac      * can tell if we are doing progressive fill in the wrong order */
172495a0b26dSToby Isaac     ierr = PetscMalloc1(nnz,&vals);CHKERRQ(ierr);
172595a0b26dSToby Isaac     for (p = 0; p < pEnd - pStart; p++) {
172695a0b26dSToby Isaac       PetscInt        parent, childid, closureSize, *closure = NULL;
172795a0b26dSToby Isaac       PetscInt        point = perm[p], pointDof;
172895a0b26dSToby Isaac 
172995a0b26dSToby Isaac       ierr = DMPlexGetTreeParent(dm,point,&parent,&childid);CHKERRQ(ierr);
173095a0b26dSToby Isaac       if ((point < conStart) || (point >= conEnd) || (parent == point)) continue;
173195a0b26dSToby Isaac       ierr = PetscSectionGetDof(conSec,point,&pointDof);CHKERRQ(ierr);
173295a0b26dSToby Isaac       if (!pointDof) continue;
173395a0b26dSToby Isaac       ierr = DMPlexGetTransitiveClosure(dm,parent,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
1734085f0adfSToby Isaac       for (f = 0; f < maxFields; f++) {
1735085f0adfSToby Isaac         PetscInt cDof, cOff, numCols, numFillCols, i, r, matOffset, offset;
173695a0b26dSToby Isaac         PetscScalar *pointMat;
1737085f0adfSToby Isaac         const PetscInt    **perms;
1738085f0adfSToby Isaac         const PetscScalar **flips;
173995a0b26dSToby Isaac 
1740085f0adfSToby Isaac         if (numFields) {
174195a0b26dSToby Isaac           ierr = PetscSectionGetFieldDof(conSec,point,f,&cDof);CHKERRQ(ierr);
174295a0b26dSToby Isaac           ierr = PetscSectionGetFieldOffset(conSec,point,f,&cOff);CHKERRQ(ierr);
174395a0b26dSToby Isaac         }
174495a0b26dSToby Isaac         else {
174595a0b26dSToby Isaac           ierr = PetscSectionGetDof(conSec,point,&cDof);CHKERRQ(ierr);
174695a0b26dSToby Isaac           ierr = PetscSectionGetOffset(conSec,point,&cOff);CHKERRQ(ierr);
174795a0b26dSToby Isaac         }
174895a0b26dSToby Isaac         if (!cDof) continue;
1749085f0adfSToby Isaac         if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,closureSize,closure,&perms,&flips);CHKERRQ(ierr);}
1750085f0adfSToby Isaac         else           {ierr = PetscSectionGetPointSyms(section,closureSize,closure,&perms,&flips);CHKERRQ(ierr);}
175195a0b26dSToby Isaac 
175295a0b26dSToby Isaac         /* make sure that every row for this point is the same size */
175395a0b26dSToby Isaac #if defined(PETSC_USE_DEBUG)
175495a0b26dSToby Isaac         for (r = 0; r < cDof; r++) {
175595a0b26dSToby Isaac           if (cDof > 1 && r) {
175621968bf8SToby Isaac             if ((ia[cOff+r+1]-ia[cOff+r]) != (ia[cOff+r]-ia[cOff+r-1])) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Two point rows have different nnz: %D vs. %D", (ia[cOff+r+1]-ia[cOff+r]), (ia[cOff+r]-ia[cOff+r-1]));
175795a0b26dSToby Isaac           }
175895a0b26dSToby Isaac         }
175995a0b26dSToby Isaac #endif
176095a0b26dSToby Isaac         /* zero rows */
176195a0b26dSToby Isaac         for (i = ia[cOff] ; i< ia[cOff+cDof];i++) {
176295a0b26dSToby Isaac           vals[i] = 0.;
176395a0b26dSToby Isaac         }
176495a0b26dSToby Isaac         matOffset = ia[cOff];
176595a0b26dSToby Isaac         numFillCols = ia[cOff+1] - matOffset;
176695a0b26dSToby Isaac         pointMat = refPointFieldMats[childid-pRefStart][f];
176795a0b26dSToby Isaac         numCols = refPointFieldN[childid-pRefStart][f];
176895a0b26dSToby Isaac         offset = 0;
176995a0b26dSToby Isaac         for (i = 0; i < closureSize; i++) {
177095a0b26dSToby Isaac           PetscInt q = closure[2*i];
177195a0b26dSToby Isaac           PetscInt aDof, aOff, j, k, qConDof, qConOff;
1772085f0adfSToby Isaac           const PetscInt    *perm = perms ? perms[i] : NULL;
1773085f0adfSToby Isaac           const PetscScalar *flip = flips ? flips[i] : NULL;
177495a0b26dSToby Isaac 
177595a0b26dSToby Isaac           qConDof = qConOff = 0;
1776085f0adfSToby Isaac           if (numFields) {
177795a0b26dSToby Isaac             ierr = PetscSectionGetFieldDof(section,q,f,&aDof);CHKERRQ(ierr);
177895a0b26dSToby Isaac             ierr = PetscSectionGetFieldOffset(section,q,f,&aOff);CHKERRQ(ierr);
177995a0b26dSToby Isaac             if (q >= conStart && q < conEnd) {
178095a0b26dSToby Isaac               ierr = PetscSectionGetFieldDof(conSec,q,f,&qConDof);CHKERRQ(ierr);
178195a0b26dSToby Isaac               ierr = PetscSectionGetFieldOffset(conSec,q,f,&qConOff);CHKERRQ(ierr);
178295a0b26dSToby Isaac             }
178395a0b26dSToby Isaac           }
178495a0b26dSToby Isaac           else {
178595a0b26dSToby Isaac             ierr = PetscSectionGetDof(section,q,&aDof);CHKERRQ(ierr);
178695a0b26dSToby Isaac             ierr = PetscSectionGetOffset(section,q,&aOff);CHKERRQ(ierr);
178795a0b26dSToby Isaac             if (q >= conStart && q < conEnd) {
178895a0b26dSToby Isaac               ierr = PetscSectionGetDof(conSec,q,&qConDof);CHKERRQ(ierr);
178995a0b26dSToby Isaac               ierr = PetscSectionGetOffset(conSec,q,&qConOff);CHKERRQ(ierr);
179095a0b26dSToby Isaac             }
179195a0b26dSToby Isaac           }
179295a0b26dSToby Isaac           if (!aDof) continue;
179395a0b26dSToby Isaac           if (qConDof) {
179495a0b26dSToby Isaac             /* this point has anchors: its rows of the matrix should already
179595a0b26dSToby Isaac              * be filled, thanks to preordering */
179695a0b26dSToby Isaac             /* first multiply into pointWork, then set in matrix */
179795a0b26dSToby Isaac             PetscInt aMatOffset = ia[qConOff];
179895a0b26dSToby Isaac             PetscInt aNumFillCols = ia[qConOff + 1] - aMatOffset;
179995a0b26dSToby Isaac             for (r = 0; r < cDof; r++) {
180095a0b26dSToby Isaac               for (j = 0; j < aNumFillCols; j++) {
180195a0b26dSToby Isaac                 PetscScalar inVal = 0;
180295a0b26dSToby Isaac                 for (k = 0; k < aDof; k++) {
1803085f0adfSToby Isaac                   PetscInt col = perm ? perm[k] : k;
180495a0b26dSToby Isaac 
1805085f0adfSToby Isaac                   inVal += pointMat[r * numCols + offset + col] * vals[aMatOffset + aNumFillCols * k + j] * (flip ? flip[col] : 1.);
180695a0b26dSToby Isaac                 }
180795a0b26dSToby Isaac                 pointWork[r * aNumFillCols + j] = inVal;
180895a0b26dSToby Isaac               }
180995a0b26dSToby Isaac             }
181095a0b26dSToby Isaac             /* assume that the columns are sorted, spend less time searching */
181195a0b26dSToby Isaac             for (j = 0, k = 0; j < aNumFillCols; j++) {
181295a0b26dSToby Isaac               PetscInt col = ja[aMatOffset + j];
181395a0b26dSToby Isaac               for (;k < numFillCols; k++) {
181495a0b26dSToby Isaac                 if (ja[matOffset + k] == col) {
181595a0b26dSToby Isaac                   break;
181695a0b26dSToby Isaac                 }
181795a0b26dSToby Isaac               }
181895a0b26dSToby Isaac               if (k == numFillCols) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"No nonzero space for (%d, %d)", cOff, col);
181995a0b26dSToby Isaac               for (r = 0; r < cDof; r++) {
182095a0b26dSToby Isaac                 vals[matOffset + numFillCols * r + k] = pointWork[r * aNumFillCols + j];
182195a0b26dSToby Isaac               }
182295a0b26dSToby Isaac             }
182395a0b26dSToby Isaac           }
182495a0b26dSToby Isaac           else {
182595a0b26dSToby Isaac             /* find where to put this portion of pointMat into the matrix */
182695a0b26dSToby Isaac             for (k = 0; k < numFillCols; k++) {
182795a0b26dSToby Isaac               if (ja[matOffset + k] == aOff) {
182895a0b26dSToby Isaac                 break;
182995a0b26dSToby Isaac               }
183095a0b26dSToby Isaac             }
183195a0b26dSToby Isaac             if (k == numFillCols) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"No nonzero space for (%d, %d)", cOff, aOff);
183295a0b26dSToby Isaac             for (r = 0; r < cDof; r++) {
1833085f0adfSToby Isaac               for (j = 0; j < aDof; j++) {
1834085f0adfSToby Isaac                 PetscInt col = perm ? perm[j] : j;
1835085f0adfSToby Isaac 
1836085f0adfSToby Isaac                 vals[matOffset + numFillCols * r + k + col] += pointMat[r * numCols + offset + j] * (flip ? flip[col] : 1.);
183795a0b26dSToby Isaac               }
183895a0b26dSToby Isaac             }
183995a0b26dSToby Isaac           }
184095a0b26dSToby Isaac           offset += aDof;
184195a0b26dSToby Isaac         }
1842085f0adfSToby Isaac         if (numFields) {
1843085f0adfSToby Isaac           ierr = PetscSectionRestoreFieldPointSyms(section,f,closureSize,closure,&perms,&flips);CHKERRQ(ierr);
1844085f0adfSToby Isaac         } else {
1845085f0adfSToby Isaac           ierr = PetscSectionRestorePointSyms(section,closureSize,closure,&perms,&flips);CHKERRQ(ierr);
1846085f0adfSToby Isaac         }
184795a0b26dSToby Isaac       }
184895a0b26dSToby Isaac       ierr = DMPlexRestoreTransitiveClosure(dm,parent,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
184995a0b26dSToby Isaac     }
185095a0b26dSToby Isaac     for (row = 0; row < nRows; row++) {
185195a0b26dSToby Isaac       ierr = MatSetValues(cMat,1,&row,ia[row+1]-ia[row],&ja[ia[row]],&vals[ia[row]],INSERT_VALUES);CHKERRQ(ierr);
185295a0b26dSToby Isaac     }
185395a0b26dSToby Isaac     ierr = MatRestoreRowIJ(cMat,0,PETSC_FALSE,PETSC_FALSE,&nRows,&ia,&ja,&done);CHKERRQ(ierr);
185495a0b26dSToby Isaac     if (!done) SETERRQ(PetscObjectComm((PetscObject)cMat),PETSC_ERR_PLIB,"Could not restore RowIJ of constraint matrix");
185595a0b26dSToby Isaac     ierr = MatAssemblyBegin(cMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
185695a0b26dSToby Isaac     ierr = MatAssemblyEnd(cMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
185795a0b26dSToby Isaac     ierr = PetscFree(vals);CHKERRQ(ierr);
185895a0b26dSToby Isaac   }
185995a0b26dSToby Isaac 
186095a0b26dSToby Isaac   /* clean up */
186195a0b26dSToby Isaac   ierr = ISRestoreIndices(anIS,&anchors);CHKERRQ(ierr);
186295a0b26dSToby Isaac   ierr = PetscFree2(perm,iperm);CHKERRQ(ierr);
186395a0b26dSToby Isaac   ierr = PetscFree(pointWork);CHKERRQ(ierr);
186421968bf8SToby Isaac   ierr = DMPlexReferenceTreeRestoreChildrenMatrices(refTree,&refPointFieldMats,&refPointFieldN);CHKERRQ(ierr);
186595a0b26dSToby Isaac   PetscFunctionReturn(0);
186695a0b26dSToby Isaac }
186795a0b26dSToby Isaac 
18686f5f1567SToby Isaac /* refine a single cell on rank 0: this is not intended to provide good local refinement, only to create an example of
18696f5f1567SToby Isaac  * a non-conforming mesh.  Local refinement comes later */
18706f5f1567SToby Isaac PetscErrorCode DMPlexTreeRefineCell (DM dm, PetscInt cell, DM *ncdm)
18716f5f1567SToby Isaac {
18726f5f1567SToby Isaac   DM K;
1873420f55faSMatthew G. Knepley   PetscMPIInt rank;
18746f5f1567SToby Isaac   PetscInt dim, *pNewStart, *pNewEnd, *pNewCount, *pOldStart, *pOldEnd, offset, d, pStart, pEnd;
18756f5f1567SToby Isaac   PetscInt numNewCones, *newConeSizes, *newCones, *newOrientations;
18766f5f1567SToby Isaac   PetscInt *Kembedding;
18776f5f1567SToby Isaac   PetscInt *cellClosure=NULL, nc;
18786f5f1567SToby Isaac   PetscScalar *newVertexCoords;
18796f5f1567SToby Isaac   PetscInt numPointsWithParents, *parents, *childIDs, *perm, *iperm, *preOrient, pOffset;
18806f5f1567SToby Isaac   PetscSection parentSection;
18816f5f1567SToby Isaac   PetscErrorCode ierr;
18826f5f1567SToby Isaac 
18836f5f1567SToby Isaac   PetscFunctionBegin;
18846f5f1567SToby Isaac   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm),&rank);CHKERRQ(ierr);
188528f4b327SMatthew G. Knepley   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
18866f5f1567SToby Isaac   ierr = DMPlexCreate(PetscObjectComm((PetscObject)dm), ncdm);CHKERRQ(ierr);
188728f4b327SMatthew G. Knepley   ierr = DMSetDimension(*ncdm,dim);CHKERRQ(ierr);
18886f5f1567SToby Isaac 
18896f5f1567SToby Isaac   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
18906f5f1567SToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm),&parentSection);CHKERRQ(ierr);
18916f5f1567SToby Isaac   ierr = DMPlexGetReferenceTree(dm,&K);CHKERRQ(ierr);
18926f5f1567SToby Isaac   if (!rank) {
18936f5f1567SToby Isaac     /* compute the new charts */
18946f5f1567SToby Isaac     ierr = PetscMalloc5(dim+1,&pNewCount,dim+1,&pNewStart,dim+1,&pNewEnd,dim+1,&pOldStart,dim+1,&pOldEnd);CHKERRQ(ierr);
18956f5f1567SToby Isaac     offset = 0;
18966f5f1567SToby Isaac     for (d = 0; d <= dim; d++) {
18976f5f1567SToby Isaac       PetscInt pOldCount, kStart, kEnd, k;
18986f5f1567SToby Isaac 
18996f5f1567SToby Isaac       pNewStart[d] = offset;
19006f5f1567SToby Isaac       ierr = DMPlexGetHeightStratum(dm,d,&pOldStart[d],&pOldEnd[d]);CHKERRQ(ierr);
19016f5f1567SToby Isaac       ierr = DMPlexGetHeightStratum(K,d,&kStart,&kEnd);CHKERRQ(ierr);
19026f5f1567SToby Isaac       pOldCount = pOldEnd[d] - pOldStart[d];
19036f5f1567SToby Isaac       /* adding the new points */
19046f5f1567SToby Isaac       pNewCount[d] = pOldCount + kEnd - kStart;
19056f5f1567SToby Isaac       if (!d) {
19066f5f1567SToby Isaac         /* removing the cell */
19076f5f1567SToby Isaac         pNewCount[d]--;
19086f5f1567SToby Isaac       }
19096f5f1567SToby Isaac       for (k = kStart; k < kEnd; k++) {
19106f5f1567SToby Isaac         PetscInt parent;
19116f5f1567SToby Isaac         ierr = DMPlexGetTreeParent(K,k,&parent,NULL);CHKERRQ(ierr);
19126f5f1567SToby Isaac         if (parent == k) {
19136f5f1567SToby Isaac           /* avoid double counting points that won't actually be new */
19146f5f1567SToby Isaac           pNewCount[d]--;
19156f5f1567SToby Isaac         }
19166f5f1567SToby Isaac       }
19176f5f1567SToby Isaac       pNewEnd[d] = pNewStart[d] + pNewCount[d];
19186f5f1567SToby Isaac       offset = pNewEnd[d];
19196f5f1567SToby Isaac 
19206f5f1567SToby Isaac     }
19216f5f1567SToby Isaac     if (cell < pOldStart[0] || cell >= pOldEnd[0]) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"%d not in cell range [%d, %d)", cell, pOldStart[0], pOldEnd[0]);
19226f5f1567SToby Isaac     /* get the current closure of the cell that we are removing */
19236f5f1567SToby Isaac     ierr = DMPlexGetTransitiveClosure(dm,cell,PETSC_TRUE,&nc,&cellClosure);CHKERRQ(ierr);
19246f5f1567SToby Isaac 
19256f5f1567SToby Isaac     ierr = PetscMalloc1(pNewEnd[dim],&newConeSizes);CHKERRQ(ierr);
19266f5f1567SToby Isaac     {
19276f5f1567SToby Isaac       PetscInt kStart, kEnd, k, closureSizeK, *closureK = NULL, j;
19286f5f1567SToby Isaac 
19296f5f1567SToby Isaac       ierr = DMPlexGetChart(K,&kStart,&kEnd);CHKERRQ(ierr);
19306f5f1567SToby Isaac       ierr = PetscMalloc4(kEnd-kStart,&Kembedding,kEnd-kStart,&perm,kEnd-kStart,&iperm,kEnd-kStart,&preOrient);CHKERRQ(ierr);
19316f5f1567SToby Isaac 
19326f5f1567SToby Isaac       for (k = kStart; k < kEnd; k++) {
19336f5f1567SToby Isaac         perm[k - kStart] = k;
19346f5f1567SToby Isaac         iperm [k - kStart] = k - kStart;
19356f5f1567SToby Isaac         preOrient[k - kStart] = 0;
19366f5f1567SToby Isaac       }
19376f5f1567SToby Isaac 
19386f5f1567SToby Isaac       ierr = DMPlexGetTransitiveClosure(K,0,PETSC_TRUE,&closureSizeK,&closureK);CHKERRQ(ierr);
19396f5f1567SToby Isaac       for (j = 1; j < closureSizeK; j++) {
19406f5f1567SToby Isaac         PetscInt parentOrientA = closureK[2*j+1];
19416f5f1567SToby Isaac         PetscInt parentOrientB = cellClosure[2*j+1];
19426f5f1567SToby Isaac         PetscInt p, q;
19436f5f1567SToby Isaac 
19446f5f1567SToby Isaac         p = closureK[2*j];
19456f5f1567SToby Isaac         q = cellClosure[2*j];
19466f5f1567SToby Isaac         for (d = 0; d <= dim; d++) {
19476f5f1567SToby Isaac           if (q >= pOldStart[d] && q < pOldEnd[d]) {
19486f5f1567SToby Isaac             Kembedding[p] = (q - pOldStart[d]) + pNewStart[d];
19496f5f1567SToby Isaac           }
19506f5f1567SToby Isaac         }
19516f5f1567SToby Isaac         if (parentOrientA != parentOrientB) {
19526f5f1567SToby Isaac           PetscInt numChildren, i;
19536f5f1567SToby Isaac           const PetscInt *children;
19546f5f1567SToby Isaac 
19556f5f1567SToby Isaac           ierr = DMPlexGetTreeChildren(K,p,&numChildren,&children);CHKERRQ(ierr);
19566f5f1567SToby Isaac           for (i = 0; i < numChildren; i++) {
19576f5f1567SToby Isaac             PetscInt kPerm, oPerm;
19586f5f1567SToby Isaac 
19596f5f1567SToby Isaac             k    = children[i];
19606f5f1567SToby Isaac             ierr = DMPlexReferenceTreeGetChildSymmetry(K,p,parentOrientA,0,k,parentOrientB,&oPerm,&kPerm);CHKERRQ(ierr);
19616f5f1567SToby Isaac             /* perm = what refTree position I'm in */
19626f5f1567SToby Isaac             perm[kPerm-kStart]      = k;
19636f5f1567SToby Isaac             /* iperm = who is at this position */
19646f5f1567SToby Isaac             iperm[k-kStart]         = kPerm-kStart;
19656f5f1567SToby Isaac             preOrient[kPerm-kStart] = oPerm;
19666f5f1567SToby Isaac           }
19676f5f1567SToby Isaac         }
19686f5f1567SToby Isaac       }
19696f5f1567SToby Isaac       ierr = DMPlexRestoreTransitiveClosure(K,0,PETSC_TRUE,&closureSizeK,&closureK);CHKERRQ(ierr);
19706f5f1567SToby Isaac     }
19716f5f1567SToby Isaac     ierr = PetscSectionSetChart(parentSection,0,pNewEnd[dim]);CHKERRQ(ierr);
19726f5f1567SToby Isaac     offset = 0;
19736f5f1567SToby Isaac     numNewCones = 0;
19746f5f1567SToby Isaac     for (d = 0; d <= dim; d++) {
19756f5f1567SToby Isaac       PetscInt kStart, kEnd, k;
19766f5f1567SToby Isaac       PetscInt p;
19776f5f1567SToby Isaac       PetscInt size;
19786f5f1567SToby Isaac 
19796f5f1567SToby Isaac       for (p = pOldStart[d]; p < pOldEnd[d]; p++) {
19806f5f1567SToby Isaac         /* skip cell 0 */
19816f5f1567SToby Isaac         if (p == cell) continue;
19826f5f1567SToby Isaac         /* old cones to new cones */
19836f5f1567SToby Isaac         ierr = DMPlexGetConeSize(dm,p,&size);CHKERRQ(ierr);
19846f5f1567SToby Isaac         newConeSizes[offset++] = size;
19856f5f1567SToby Isaac         numNewCones += size;
19866f5f1567SToby Isaac       }
19876f5f1567SToby Isaac 
19886f5f1567SToby Isaac       ierr = DMPlexGetHeightStratum(K,d,&kStart,&kEnd);CHKERRQ(ierr);
19896f5f1567SToby Isaac       for (k = kStart; k < kEnd; k++) {
19906f5f1567SToby Isaac         PetscInt kParent;
19916f5f1567SToby Isaac 
19926f5f1567SToby Isaac         ierr = DMPlexGetTreeParent(K,k,&kParent,NULL);CHKERRQ(ierr);
19936f5f1567SToby Isaac         if (kParent != k) {
19946f5f1567SToby Isaac           Kembedding[k] = offset;
19956f5f1567SToby Isaac           ierr = DMPlexGetConeSize(K,k,&size);CHKERRQ(ierr);
19966f5f1567SToby Isaac           newConeSizes[offset++] = size;
19976f5f1567SToby Isaac           numNewCones += size;
19986f5f1567SToby Isaac           if (kParent != 0) {
19996f5f1567SToby Isaac             ierr = PetscSectionSetDof(parentSection,Kembedding[k],1);CHKERRQ(ierr);
20006f5f1567SToby Isaac           }
20016f5f1567SToby Isaac         }
20026f5f1567SToby Isaac       }
20036f5f1567SToby Isaac     }
20046f5f1567SToby Isaac 
20056f5f1567SToby Isaac     ierr = PetscSectionSetUp(parentSection);CHKERRQ(ierr);
20066f5f1567SToby Isaac     ierr = PetscSectionGetStorageSize(parentSection,&numPointsWithParents);CHKERRQ(ierr);
20076f5f1567SToby Isaac     ierr = PetscMalloc2(numNewCones,&newCones,numNewCones,&newOrientations);CHKERRQ(ierr);
20086f5f1567SToby Isaac     ierr = PetscMalloc2(numPointsWithParents,&parents,numPointsWithParents,&childIDs);CHKERRQ(ierr);
20096f5f1567SToby Isaac 
20106f5f1567SToby Isaac     /* fill new cones */
20116f5f1567SToby Isaac     offset = 0;
20126f5f1567SToby Isaac     for (d = 0; d <= dim; d++) {
20136f5f1567SToby Isaac       PetscInt kStart, kEnd, k, l;
20146f5f1567SToby Isaac       PetscInt p;
20156f5f1567SToby Isaac       PetscInt size;
20166f5f1567SToby Isaac       const PetscInt *cone, *orientation;
20176f5f1567SToby Isaac 
20186f5f1567SToby Isaac       for (p = pOldStart[d]; p < pOldEnd[d]; p++) {
20196f5f1567SToby Isaac         /* skip cell 0 */
20206f5f1567SToby Isaac         if (p == cell) continue;
20216f5f1567SToby Isaac         /* old cones to new cones */
20226f5f1567SToby Isaac         ierr = DMPlexGetConeSize(dm,p,&size);CHKERRQ(ierr);
20236f5f1567SToby Isaac         ierr = DMPlexGetCone(dm,p,&cone);CHKERRQ(ierr);
20246f5f1567SToby Isaac         ierr = DMPlexGetConeOrientation(dm,p,&orientation);CHKERRQ(ierr);
20256f5f1567SToby Isaac         for (l = 0; l < size; l++) {
20266f5f1567SToby Isaac           newCones[offset]          = (cone[l] - pOldStart[d + 1]) + pNewStart[d + 1];
20276f5f1567SToby Isaac           newOrientations[offset++] = orientation[l];
20286f5f1567SToby Isaac         }
20296f5f1567SToby Isaac       }
20306f5f1567SToby Isaac 
20316f5f1567SToby Isaac       ierr = DMPlexGetHeightStratum(K,d,&kStart,&kEnd);CHKERRQ(ierr);
20326f5f1567SToby Isaac       for (k = kStart; k < kEnd; k++) {
20336f5f1567SToby Isaac         PetscInt kPerm = perm[k], kParent;
20346f5f1567SToby Isaac         PetscInt preO  = preOrient[k];
20356f5f1567SToby Isaac 
20366f5f1567SToby Isaac         ierr = DMPlexGetTreeParent(K,k,&kParent,NULL);CHKERRQ(ierr);
20376f5f1567SToby Isaac         if (kParent != k) {
20386f5f1567SToby Isaac           /* embed new cones */
20396f5f1567SToby Isaac           ierr = DMPlexGetConeSize(K,k,&size);CHKERRQ(ierr);
20406f5f1567SToby Isaac           ierr = DMPlexGetCone(K,kPerm,&cone);CHKERRQ(ierr);
20416f5f1567SToby Isaac           ierr = DMPlexGetConeOrientation(K,kPerm,&orientation);CHKERRQ(ierr);
20426f5f1567SToby Isaac           for (l = 0; l < size; l++) {
20436f5f1567SToby Isaac             PetscInt q, m = (preO >= 0) ? ((preO + l) % size) : ((size -(preO + 1) - l) % size);
20446f5f1567SToby Isaac             PetscInt newO, lSize, oTrue;
20456f5f1567SToby Isaac 
20466f5f1567SToby Isaac             q                         = iperm[cone[m]];
20476f5f1567SToby Isaac             newCones[offset]          = Kembedding[q];
20486f5f1567SToby Isaac             ierr                      = DMPlexGetConeSize(K,q,&lSize);CHKERRQ(ierr);
20496f5f1567SToby Isaac             oTrue                     = orientation[m];
20506f5f1567SToby Isaac             oTrue                     = ((!lSize) || (preOrient[k] >= 0)) ? oTrue : -(oTrue + 2);
20516f5f1567SToby Isaac             newO                      = DihedralCompose(lSize,oTrue,preOrient[q]);
20526f5f1567SToby Isaac             newOrientations[offset++] = newO;
20536f5f1567SToby Isaac           }
20546f5f1567SToby Isaac           if (kParent != 0) {
20556f5f1567SToby Isaac             PetscInt newPoint = Kembedding[kParent];
20566f5f1567SToby Isaac             ierr              = PetscSectionGetOffset(parentSection,Kembedding[k],&pOffset);CHKERRQ(ierr);
20576f5f1567SToby Isaac             parents[pOffset]  = newPoint;
20586f5f1567SToby Isaac             childIDs[pOffset] = k;
20596f5f1567SToby Isaac           }
20606f5f1567SToby Isaac         }
20616f5f1567SToby Isaac       }
20626f5f1567SToby Isaac     }
20636f5f1567SToby Isaac 
20646f5f1567SToby Isaac     ierr = PetscMalloc1(dim*(pNewEnd[dim]-pNewStart[dim]),&newVertexCoords);CHKERRQ(ierr);
20656f5f1567SToby Isaac 
20666f5f1567SToby Isaac     /* fill coordinates */
20676f5f1567SToby Isaac     offset = 0;
20686f5f1567SToby Isaac     {
2069d90620a3SMatthew G. Knepley       PetscInt kStart, kEnd, l;
20706f5f1567SToby Isaac       PetscSection vSection;
20716f5f1567SToby Isaac       PetscInt v;
20726f5f1567SToby Isaac       Vec coords;
20736f5f1567SToby Isaac       PetscScalar *coordvals;
20746f5f1567SToby Isaac       PetscInt dof, off;
2075c111c6b7SMatthew G. Knepley       PetscReal v0[3], J[9], detJ;
20766f5f1567SToby Isaac 
20776f5f1567SToby Isaac #if defined(PETSC_USE_DEBUG)
2078d90620a3SMatthew G. Knepley       {
2079d90620a3SMatthew G. Knepley         PetscInt k;
20806f5f1567SToby Isaac         ierr = DMPlexGetHeightStratum(K,0,&kStart,&kEnd);CHKERRQ(ierr);
20816f5f1567SToby Isaac         for (k = kStart; k < kEnd; k++) {
208273a7f2aaSMatthew G. Knepley           ierr = DMPlexComputeCellGeometryFEM(K, k, NULL, v0, J, NULL, &detJ);CHKERRQ(ierr);
20836f5f1567SToby Isaac           if (detJ <= 0.) SETERRQ1 (PETSC_COMM_SELF,PETSC_ERR_PLIB,"reference tree cell %d has bad determinant",k);
20846f5f1567SToby Isaac         }
2085d90620a3SMatthew G. Knepley       }
20866f5f1567SToby Isaac #endif
208773a7f2aaSMatthew G. Knepley       ierr = DMPlexComputeCellGeometryFEM(dm, cell, NULL, v0, J, NULL, &detJ);CHKERRQ(ierr);
20886f5f1567SToby Isaac       ierr = DMGetCoordinateSection(dm,&vSection);CHKERRQ(ierr);
20896f5f1567SToby Isaac       ierr = DMGetCoordinatesLocal(dm,&coords);CHKERRQ(ierr);
20906f5f1567SToby Isaac       ierr = VecGetArray(coords,&coordvals);CHKERRQ(ierr);
20916f5f1567SToby Isaac       for (v = pOldStart[dim]; v < pOldEnd[dim]; v++) {
20926f5f1567SToby Isaac 
20936f5f1567SToby Isaac         ierr = PetscSectionGetDof(vSection,v,&dof);CHKERRQ(ierr);
20946f5f1567SToby Isaac         ierr = PetscSectionGetOffset(vSection,v,&off);CHKERRQ(ierr);
20956f5f1567SToby Isaac         for (l = 0; l < dof; l++) {
20966f5f1567SToby Isaac           newVertexCoords[offset++] = coordvals[off + l];
20976f5f1567SToby Isaac         }
20986f5f1567SToby Isaac       }
20996f5f1567SToby Isaac       ierr = VecRestoreArray(coords,&coordvals);CHKERRQ(ierr);
21006f5f1567SToby Isaac 
21016f5f1567SToby Isaac       ierr = DMGetCoordinateSection(K,&vSection);CHKERRQ(ierr);
21026f5f1567SToby Isaac       ierr = DMGetCoordinatesLocal(K,&coords);CHKERRQ(ierr);
21036f5f1567SToby Isaac       ierr = VecGetArray(coords,&coordvals);CHKERRQ(ierr);
21046f5f1567SToby Isaac       ierr = DMPlexGetDepthStratum(K,0,&kStart,&kEnd);CHKERRQ(ierr);
21056f5f1567SToby Isaac       for (v = kStart; v < kEnd; v++) {
21069bc368c7SMatthew G. Knepley         PetscReal coord[3], newCoord[3];
21076f5f1567SToby Isaac         PetscInt  vPerm = perm[v];
21086f5f1567SToby Isaac         PetscInt  kParent;
21096f5f1567SToby Isaac 
21106f5f1567SToby Isaac         ierr = DMPlexGetTreeParent(K,v,&kParent,NULL);CHKERRQ(ierr);
21116f5f1567SToby Isaac         if (kParent != v) {
21126f5f1567SToby Isaac           /* this is a new vertex */
21136f5f1567SToby Isaac           ierr = PetscSectionGetOffset(vSection,vPerm,&off);CHKERRQ(ierr);
21149bc368c7SMatthew G. Knepley           for (l = 0; l < dim; ++l) coord[l] = PetscRealPart(coordvals[off+l]);
21159bc368c7SMatthew G. Knepley           CoordinatesRefToReal(dim, dim, v0, J, coord, newCoord);CHKERRQ(ierr);
21169bc368c7SMatthew G. Knepley           for (l = 0; l < dim; ++l) newVertexCoords[offset+l] = newCoord[l];
21176f5f1567SToby Isaac           offset += dim;
21186f5f1567SToby Isaac         }
21196f5f1567SToby Isaac       }
21206f5f1567SToby Isaac       ierr = VecRestoreArray(coords,&coordvals);CHKERRQ(ierr);
21216f5f1567SToby Isaac     }
21226f5f1567SToby Isaac 
21236f5f1567SToby Isaac     /* need to reverse the order of pNewCount: vertices first, cells last */
21246f5f1567SToby Isaac     for (d = 0; d < (dim + 1) / 2; d++) {
21256f5f1567SToby Isaac       PetscInt tmp;
21266f5f1567SToby Isaac 
21276f5f1567SToby Isaac       tmp = pNewCount[d];
21286f5f1567SToby Isaac       pNewCount[d] = pNewCount[dim - d];
21296f5f1567SToby Isaac       pNewCount[dim - d] = tmp;
21306f5f1567SToby Isaac     }
21316f5f1567SToby Isaac 
21326f5f1567SToby Isaac     ierr = DMPlexCreateFromDAG(*ncdm,dim,pNewCount,newConeSizes,newCones,newOrientations,newVertexCoords);CHKERRQ(ierr);
21336f5f1567SToby Isaac     ierr = DMPlexSetReferenceTree(*ncdm,K);CHKERRQ(ierr);
21346f5f1567SToby Isaac     ierr = DMPlexSetTree(*ncdm,parentSection,parents,childIDs);CHKERRQ(ierr);
21356f5f1567SToby Isaac 
21366f5f1567SToby Isaac     /* clean up */
21376f5f1567SToby Isaac     ierr = DMPlexRestoreTransitiveClosure(dm,cell,PETSC_TRUE,&nc,&cellClosure);CHKERRQ(ierr);
21386f5f1567SToby Isaac     ierr = PetscFree5(pNewCount,pNewStart,pNewEnd,pOldStart,pOldEnd);CHKERRQ(ierr);
21396f5f1567SToby Isaac     ierr = PetscFree(newConeSizes);CHKERRQ(ierr);
21406f5f1567SToby Isaac     ierr = PetscFree2(newCones,newOrientations);CHKERRQ(ierr);
21416f5f1567SToby Isaac     ierr = PetscFree(newVertexCoords);CHKERRQ(ierr);
21426f5f1567SToby Isaac     ierr = PetscFree2(parents,childIDs);CHKERRQ(ierr);
21436f5f1567SToby Isaac     ierr = PetscFree4(Kembedding,perm,iperm,preOrient);CHKERRQ(ierr);
21446f5f1567SToby Isaac   }
21456f5f1567SToby Isaac   else {
21466f5f1567SToby Isaac     PetscInt    p, counts[4];
21476f5f1567SToby Isaac     PetscInt    *coneSizes, *cones, *orientations;
21486f5f1567SToby Isaac     Vec         coordVec;
21496f5f1567SToby Isaac     PetscScalar *coords;
21506f5f1567SToby Isaac 
21516f5f1567SToby Isaac     for (d = 0; d <= dim; d++) {
21526f5f1567SToby Isaac       PetscInt dStart, dEnd;
21536f5f1567SToby Isaac 
21546f5f1567SToby Isaac       ierr = DMPlexGetDepthStratum(dm,d,&dStart,&dEnd);CHKERRQ(ierr);
21556f5f1567SToby Isaac       counts[d] = dEnd - dStart;
21566f5f1567SToby Isaac     }
21576f5f1567SToby Isaac     ierr = PetscMalloc1(pEnd-pStart,&coneSizes);CHKERRQ(ierr);
21586f5f1567SToby Isaac     for (p = pStart; p < pEnd; p++) {
21596f5f1567SToby Isaac       ierr = DMPlexGetConeSize(dm,p,&coneSizes[p-pStart]);CHKERRQ(ierr);
21606f5f1567SToby Isaac     }
21616f5f1567SToby Isaac     ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
21626f5f1567SToby Isaac     ierr = DMPlexGetConeOrientations(dm, &orientations);CHKERRQ(ierr);
21636f5f1567SToby Isaac     ierr = DMGetCoordinatesLocal(dm,&coordVec);CHKERRQ(ierr);
21646f5f1567SToby Isaac     ierr = VecGetArray(coordVec,&coords);CHKERRQ(ierr);
21656f5f1567SToby Isaac 
21666f5f1567SToby Isaac     ierr = PetscSectionSetChart(parentSection,pStart,pEnd);CHKERRQ(ierr);
21676f5f1567SToby Isaac     ierr = PetscSectionSetUp(parentSection);CHKERRQ(ierr);
21686f5f1567SToby Isaac     ierr = DMPlexCreateFromDAG(*ncdm,dim,counts,coneSizes,cones,orientations,NULL);CHKERRQ(ierr);
21696f5f1567SToby Isaac     ierr = DMPlexSetReferenceTree(*ncdm,K);CHKERRQ(ierr);
21706f5f1567SToby Isaac     ierr = DMPlexSetTree(*ncdm,parentSection,NULL,NULL);CHKERRQ(ierr);
21716f5f1567SToby Isaac     ierr = VecRestoreArray(coordVec,&coords);CHKERRQ(ierr);
21726f5f1567SToby Isaac   }
21736f5f1567SToby Isaac   ierr = PetscSectionDestroy(&parentSection);CHKERRQ(ierr);
21746f5f1567SToby Isaac 
21756f5f1567SToby Isaac   PetscFunctionReturn(0);
21766f5f1567SToby Isaac }
21776ecaa68aSToby Isaac 
21786ecaa68aSToby Isaac PetscErrorCode DMPlexComputeInterpolatorTree(DM coarse, DM fine, PetscSF coarseToFine, PetscInt *childIds, Mat mat)
21796ecaa68aSToby Isaac {
21806ecaa68aSToby Isaac   PetscSF           coarseToFineEmbedded;
21816ecaa68aSToby Isaac   PetscSection      globalCoarse, globalFine;
21826ecaa68aSToby Isaac   PetscSection      localCoarse, localFine;
21836ecaa68aSToby Isaac   PetscSection      aSec, cSec;
21846ecaa68aSToby Isaac   PetscSection      rootIndicesSec, rootMatricesSec;
218546bdb399SToby Isaac   PetscSection      leafIndicesSec, leafMatricesSec;
218646bdb399SToby Isaac   PetscInt          *rootIndices, *leafIndices;
218746bdb399SToby Isaac   PetscScalar       *rootMatrices, *leafMatrices;
21886ecaa68aSToby Isaac   IS                aIS;
21896ecaa68aSToby Isaac   const PetscInt    *anchors;
21906ecaa68aSToby Isaac   Mat               cMat;
21914acb8e1eSToby Isaac   PetscInt          numFields, maxFields;
21926ecaa68aSToby Isaac   PetscInt          pStartC, pEndC, pStartF, pEndF, p;
21936ecaa68aSToby Isaac   PetscInt          aStart, aEnd, cStart, cEnd;
21941c58ffc4SToby Isaac   PetscInt          *maxChildIds;
2195e44e4e7fSToby Isaac   PetscInt          *offsets, *newOffsets, *offsetsCopy, *newOffsetsCopy, *rowOffsets, *numD, *numO;
21964acb8e1eSToby Isaac   const PetscInt    ***perms;
21974acb8e1eSToby Isaac   const PetscScalar ***flips;
2198*f489ac74SBarry Smith   PetscInt          ***iperms;
2199*f489ac74SBarry Smith   PetscScalar       ***iflips;
22006ecaa68aSToby Isaac   PetscErrorCode    ierr;
22016ecaa68aSToby Isaac 
22026ecaa68aSToby Isaac   PetscFunctionBegin;
22036ecaa68aSToby Isaac   ierr = DMPlexGetChart(coarse,&pStartC,&pEndC);CHKERRQ(ierr);
22046ecaa68aSToby Isaac   ierr = DMPlexGetChart(fine,&pStartF,&pEndF);CHKERRQ(ierr);
22056ecaa68aSToby Isaac   ierr = DMGetDefaultGlobalSection(fine,&globalFine);CHKERRQ(ierr);
22066ecaa68aSToby Isaac   { /* winnow fine points that don't have global dofs out of the sf */
220789698031SToby Isaac     PetscInt dof, cdof, numPointsWithDofs, offset, *pointsWithDofs, nleaves, l;
220889698031SToby Isaac     const PetscInt *leaves;
22096ecaa68aSToby Isaac 
221089698031SToby Isaac     ierr = PetscSFGetGraph(coarseToFine,NULL,&nleaves,&leaves,NULL);CHKERRQ(ierr);
221189698031SToby Isaac     for (l = 0, numPointsWithDofs = 0; l < nleaves; l++) {
221289698031SToby Isaac       p = leaves ? leaves[l] : l;
22136ecaa68aSToby Isaac       ierr = PetscSectionGetDof(globalFine,p,&dof);CHKERRQ(ierr);
22146ecaa68aSToby Isaac       ierr = PetscSectionGetConstraintDof(globalFine,p,&cdof);CHKERRQ(ierr);
22156ecaa68aSToby Isaac       if ((dof - cdof) > 0) {
22166ecaa68aSToby Isaac         numPointsWithDofs++;
22176ecaa68aSToby Isaac       }
22186ecaa68aSToby Isaac     }
22196ecaa68aSToby Isaac     ierr = PetscMalloc1(numPointsWithDofs,&pointsWithDofs);CHKERRQ(ierr);
22207cc7abc7SToby Isaac     for (l = 0, offset = 0; l < nleaves; l++) {
222189698031SToby Isaac       p = leaves ? leaves[l] : l;
22226ecaa68aSToby Isaac       ierr = PetscSectionGetDof(globalFine,p,&dof);CHKERRQ(ierr);
22236ecaa68aSToby Isaac       ierr = PetscSectionGetConstraintDof(globalFine,p,&cdof);CHKERRQ(ierr);
22246ecaa68aSToby Isaac       if ((dof - cdof) > 0) {
222589698031SToby Isaac         pointsWithDofs[offset++] = l;
22266ecaa68aSToby Isaac       }
22276ecaa68aSToby Isaac     }
22286ecaa68aSToby Isaac     ierr = PetscSFCreateEmbeddedLeafSF(coarseToFine, numPointsWithDofs, pointsWithDofs, &coarseToFineEmbedded);CHKERRQ(ierr);
2229ec92bd66SToby Isaac     ierr = PetscFree(pointsWithDofs);CHKERRQ(ierr);
22306ecaa68aSToby Isaac   }
22316ecaa68aSToby Isaac   /* communicate back to the coarse mesh which coarse points have children (that may require interpolation) */
22326ecaa68aSToby Isaac   ierr = PetscMalloc1(pEndC-pStartC,&maxChildIds);CHKERRQ(ierr);
22336ecaa68aSToby Isaac   for (p = pStartC; p < pEndC; p++) {
22348d2f55e7SToby Isaac     maxChildIds[p - pStartC] = -2;
22356ecaa68aSToby Isaac   }
22366ecaa68aSToby Isaac   ierr = PetscSFReduceBegin(coarseToFineEmbedded,MPIU_INT,childIds,maxChildIds,MPIU_MAX);CHKERRQ(ierr);
22376ecaa68aSToby Isaac   ierr = PetscSFReduceEnd(coarseToFineEmbedded,MPIU_INT,childIds,maxChildIds,MPIU_MAX);CHKERRQ(ierr);
223846bdb399SToby Isaac 
22396ecaa68aSToby Isaac   ierr = DMGetDefaultSection(coarse,&localCoarse);CHKERRQ(ierr);
22406ecaa68aSToby Isaac   ierr = DMGetDefaultGlobalSection(coarse,&globalCoarse);CHKERRQ(ierr);
224146bdb399SToby Isaac 
22426ecaa68aSToby Isaac   ierr = DMPlexGetAnchors(coarse,&aSec,&aIS);CHKERRQ(ierr);
22436ecaa68aSToby Isaac   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
22446ecaa68aSToby Isaac   ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
224546bdb399SToby Isaac 
22466ecaa68aSToby Isaac   ierr = DMGetDefaultConstraints(coarse,&cSec,&cMat);CHKERRQ(ierr);
22476ecaa68aSToby Isaac   ierr = PetscSectionGetChart(cSec,&cStart,&cEnd);CHKERRQ(ierr);
224846bdb399SToby Isaac 
224946bdb399SToby Isaac   /* create sections that will send to children the indices and matrices they will need to construct the interpolator */
22506ecaa68aSToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)coarse),&rootIndicesSec);CHKERRQ(ierr);
22516ecaa68aSToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)coarse),&rootMatricesSec);CHKERRQ(ierr);
22526ecaa68aSToby Isaac   ierr = PetscSectionSetChart(rootIndicesSec,pStartC,pEndC);CHKERRQ(ierr);
22536ecaa68aSToby Isaac   ierr = PetscSectionSetChart(rootMatricesSec,pStartC,pEndC);CHKERRQ(ierr);
2254708c7f19SToby Isaac   ierr = PetscSectionGetNumFields(localCoarse,&numFields);CHKERRQ(ierr);
2255713c1c5dSToby Isaac   maxFields = PetscMax(1,numFields);
2256713c1c5dSToby Isaac   ierr = PetscMalloc7(maxFields+1,&offsets,maxFields+1,&offsetsCopy,maxFields+1,&newOffsets,maxFields+1,&newOffsetsCopy,maxFields+1,&rowOffsets,maxFields+1,&numD,maxFields+1,&numO);CHKERRQ(ierr);
2257*f489ac74SBarry Smith   ierr = PetscMalloc2(maxFields+1,(PetscInt****)&perms,maxFields+1,(PetscScalar****)&flips);CHKERRQ(ierr);
2258713c1c5dSToby Isaac   ierr = PetscMemzero((void *) perms, (maxFields+1) * sizeof(const PetscInt **));CHKERRQ(ierr);
2259713c1c5dSToby Isaac   ierr = PetscMemzero((void *) flips, (maxFields+1) * sizeof(const PetscScalar **));CHKERRQ(ierr);
226046bdb399SToby Isaac 
226146bdb399SToby Isaac   for (p = pStartC; p < pEndC; p++) { /* count the sizes of the indices and matrices */
22628d2f55e7SToby Isaac     PetscInt dof, matSize   = 0;
22636ecaa68aSToby Isaac     PetscInt aDof           = 0;
22646ecaa68aSToby Isaac     PetscInt cDof           = 0;
22656ecaa68aSToby Isaac     PetscInt maxChildId     = maxChildIds[p - pStartC];
22666ecaa68aSToby Isaac     PetscInt numRowIndices  = 0;
22676ecaa68aSToby Isaac     PetscInt numColIndices  = 0;
2268f13f9184SToby Isaac     PetscInt f;
22696ecaa68aSToby Isaac 
22706ecaa68aSToby Isaac     ierr = PetscSectionGetDof(globalCoarse,p,&dof);CHKERRQ(ierr);
22711cfc5b76SToby Isaac     if (dof < 0) {
22721cfc5b76SToby Isaac       dof = -(dof + 1);
22731cfc5b76SToby Isaac     }
22746ecaa68aSToby Isaac     if (p >= aStart && p < aEnd) {
22756ecaa68aSToby Isaac       ierr = PetscSectionGetDof(aSec,p,&aDof);CHKERRQ(ierr);
22766ecaa68aSToby Isaac     }
22776ecaa68aSToby Isaac     if (p >= cStart && p < cEnd) {
22786ecaa68aSToby Isaac       ierr = PetscSectionGetDof(cSec,p,&cDof);CHKERRQ(ierr);
22796ecaa68aSToby Isaac     }
2280f13f9184SToby Isaac     for (f = 0; f <= numFields; f++) offsets[f] = 0;
2281f13f9184SToby Isaac     for (f = 0; f <= numFields; f++) newOffsets[f] = 0;
22826ecaa68aSToby Isaac     if (maxChildId >= 0) { /* this point has children (with dofs) that will need to be interpolated from the closure of p */
2283f13f9184SToby Isaac       PetscInt *closure = NULL, closureSize, cl;
22846ecaa68aSToby Isaac 
22856ecaa68aSToby Isaac       ierr = DMPlexGetTransitiveClosure(coarse,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
228646bdb399SToby Isaac       for (cl = 0; cl < closureSize; cl++) { /* get the closure */
22876ecaa68aSToby Isaac         PetscInt c = closure[2 * cl], clDof;
22886ecaa68aSToby Isaac 
22896ecaa68aSToby Isaac         ierr = PetscSectionGetDof(localCoarse,c,&clDof);CHKERRQ(ierr);
22906ecaa68aSToby Isaac         numRowIndices += clDof;
22916ecaa68aSToby Isaac         for (f = 0; f < numFields; f++) {
22926ecaa68aSToby Isaac           ierr = PetscSectionGetFieldDof(localCoarse,c,f,&clDof);CHKERRQ(ierr);
22936ecaa68aSToby Isaac           offsets[f + 1] += clDof;
22946ecaa68aSToby Isaac         }
22956ecaa68aSToby Isaac       }
22966ecaa68aSToby Isaac       for (f = 0; f < numFields; f++) {
22976ecaa68aSToby Isaac         offsets[f + 1]   += offsets[f];
22986ecaa68aSToby Isaac         newOffsets[f + 1] = offsets[f + 1];
22996ecaa68aSToby Isaac       }
230046bdb399SToby Isaac       /* get the number of indices needed and their field offsets */
23014acb8e1eSToby Isaac       ierr = DMPlexAnchorsModifyMat(coarse,localCoarse,closureSize,numRowIndices,closure,NULL,NULL,NULL,&numColIndices,NULL,NULL,newOffsets,PETSC_FALSE);CHKERRQ(ierr);
23026ecaa68aSToby Isaac       ierr = DMPlexRestoreTransitiveClosure(coarse,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
23036ecaa68aSToby Isaac       if (!numColIndices) { /* there are no hanging constraint modifications, so the matrix is just the identity: do not send it */
23046ecaa68aSToby Isaac         numColIndices = numRowIndices;
23056ecaa68aSToby Isaac         matSize = 0;
23066ecaa68aSToby Isaac       }
230746bdb399SToby Isaac       else if (numFields) { /* we send one submat for each field: sum their sizes */
23086ecaa68aSToby Isaac         matSize = 0;
23096ecaa68aSToby Isaac         for (f = 0; f < numFields; f++) {
23106ecaa68aSToby Isaac           PetscInt numRow, numCol;
23116ecaa68aSToby Isaac 
23126ecaa68aSToby Isaac           numRow = offsets[f + 1] - offsets[f];
2313f13f9184SToby Isaac           numCol = newOffsets[f + 1] - newOffsets[f];
23146ecaa68aSToby Isaac           matSize += numRow * numCol;
23156ecaa68aSToby Isaac         }
23166ecaa68aSToby Isaac       }
23176ecaa68aSToby Isaac       else {
23186ecaa68aSToby Isaac         matSize = numRowIndices * numColIndices;
23196ecaa68aSToby Isaac       }
2320f13f9184SToby Isaac     } else if (maxChildId == -1) {
23218d2f55e7SToby Isaac       if (cDof > 0) { /* this point's dofs are interpolated via cMat: get the submatrix of cMat */
2322f13f9184SToby Isaac         PetscInt aOff, a;
23236ecaa68aSToby Isaac 
23246ecaa68aSToby Isaac         ierr = PetscSectionGetOffset(aSec,p,&aOff);CHKERRQ(ierr);
23256ecaa68aSToby Isaac         for (f = 0; f < numFields; f++) {
23266ecaa68aSToby Isaac           PetscInt fDof;
23276ecaa68aSToby Isaac 
23286ecaa68aSToby Isaac           ierr = PetscSectionGetFieldDof(localCoarse,p,f,&fDof);CHKERRQ(ierr);
232921968bf8SToby Isaac           offsets[f+1] = fDof;
23306ecaa68aSToby Isaac         }
23316ecaa68aSToby Isaac         for (a = 0; a < aDof; a++) {
23326ecaa68aSToby Isaac           PetscInt anchor = anchors[a + aOff], aLocalDof;
23336ecaa68aSToby Isaac 
23346ecaa68aSToby Isaac           ierr = PetscSectionGetDof(localCoarse,anchor,&aLocalDof);CHKERRQ(ierr);
23356ecaa68aSToby Isaac           numColIndices += aLocalDof;
23366ecaa68aSToby Isaac           for (f = 0; f < numFields; f++) {
23376ecaa68aSToby Isaac             PetscInt fDof;
23386ecaa68aSToby Isaac 
23396ecaa68aSToby Isaac             ierr = PetscSectionGetFieldDof(localCoarse,anchor,f,&fDof);CHKERRQ(ierr);
234021968bf8SToby Isaac             newOffsets[f+1] += fDof;
23416ecaa68aSToby Isaac           }
23426ecaa68aSToby Isaac         }
23436ecaa68aSToby Isaac         if (numFields) {
23446ecaa68aSToby Isaac           matSize = 0;
23456ecaa68aSToby Isaac           for (f = 0; f < numFields; f++) {
234621968bf8SToby Isaac             matSize += offsets[f+1] * newOffsets[f+1];
23476ecaa68aSToby Isaac           }
23486ecaa68aSToby Isaac         }
23496ecaa68aSToby Isaac         else {
23506ecaa68aSToby Isaac           matSize = numColIndices * dof;
23516ecaa68aSToby Isaac         }
23526ecaa68aSToby Isaac       }
23536ecaa68aSToby Isaac       else { /* no children, and no constraints on dofs: just get the global indices */
23546ecaa68aSToby Isaac         numColIndices = dof;
23556ecaa68aSToby Isaac         matSize       = 0;
23566ecaa68aSToby Isaac       }
23578d2f55e7SToby Isaac     }
235846bdb399SToby Isaac     /* we will pack the column indices with the field offsets */
23596ecaa68aSToby Isaac     ierr = PetscSectionSetDof(rootIndicesSec,p,numColIndices ? numColIndices+2*numFields : 0);CHKERRQ(ierr);
23606ecaa68aSToby Isaac     ierr = PetscSectionSetDof(rootMatricesSec,p,matSize);CHKERRQ(ierr);
23616ecaa68aSToby Isaac   }
23626ecaa68aSToby Isaac   ierr = PetscSectionSetUp(rootIndicesSec);CHKERRQ(ierr);
23636ecaa68aSToby Isaac   ierr = PetscSectionSetUp(rootMatricesSec);CHKERRQ(ierr);
23646ecaa68aSToby Isaac   {
23656ecaa68aSToby Isaac     PetscInt numRootIndices, numRootMatrices;
23666ecaa68aSToby Isaac 
23676ecaa68aSToby Isaac     ierr = PetscSectionGetStorageSize(rootIndicesSec,&numRootIndices);CHKERRQ(ierr);
23686ecaa68aSToby Isaac     ierr = PetscSectionGetStorageSize(rootMatricesSec,&numRootMatrices);CHKERRQ(ierr);
23696ecaa68aSToby Isaac     ierr = PetscMalloc2(numRootIndices,&rootIndices,numRootMatrices,&rootMatrices);CHKERRQ(ierr);
23706ecaa68aSToby Isaac     for (p = pStartC; p < pEndC; p++) {
23716ecaa68aSToby Isaac       PetscInt    numRowIndices, numColIndices, matSize, dof;
2372f13f9184SToby Isaac       PetscInt    pIndOff, pMatOff, f;
23736ecaa68aSToby Isaac       PetscInt    *pInd;
23746ecaa68aSToby Isaac       PetscInt    maxChildId = maxChildIds[p - pStartC];
23756ecaa68aSToby Isaac       PetscScalar *pMat = NULL;
23766ecaa68aSToby Isaac 
23776ecaa68aSToby Isaac       ierr = PetscSectionGetDof(rootIndicesSec,p,&numColIndices);CHKERRQ(ierr);
23786ecaa68aSToby Isaac       if (!numColIndices) {
23796ecaa68aSToby Isaac         continue;
23806ecaa68aSToby Isaac       }
2381f13f9184SToby Isaac       for (f = 0; f <= numFields; f++) {
2382f13f9184SToby Isaac         offsets[f]        = 0;
2383f13f9184SToby Isaac         newOffsets[f]     = 0;
2384f13f9184SToby Isaac         offsetsCopy[f]    = 0;
2385f13f9184SToby Isaac         newOffsetsCopy[f] = 0;
2386f13f9184SToby Isaac       }
23876ecaa68aSToby Isaac       numColIndices -= 2 * numFields;
23886ecaa68aSToby Isaac       ierr = PetscSectionGetOffset(rootIndicesSec,p,&pIndOff);CHKERRQ(ierr);
23896ecaa68aSToby Isaac       pInd = &(rootIndices[pIndOff]);
23906ecaa68aSToby Isaac       ierr = PetscSectionGetDof(rootMatricesSec,p,&matSize);CHKERRQ(ierr);
23916ecaa68aSToby Isaac       if (matSize) {
23926ecaa68aSToby Isaac         ierr = PetscSectionGetOffset(rootMatricesSec,p,&pMatOff);CHKERRQ(ierr);
23936ecaa68aSToby Isaac         pMat = &rootMatrices[pMatOff];
23946ecaa68aSToby Isaac       }
23956ecaa68aSToby Isaac       ierr = PetscSectionGetDof(globalCoarse,p,&dof);CHKERRQ(ierr);
23961cfc5b76SToby Isaac       if (dof < 0) {
23971cfc5b76SToby Isaac         dof = -(dof + 1);
23981cfc5b76SToby Isaac       }
23996ecaa68aSToby Isaac       if (maxChildId >= 0) { /* build an identity matrix, apply matrix constraints on the right */
24006ecaa68aSToby Isaac         PetscInt i, j;
24016ecaa68aSToby Isaac         PetscInt numRowIndices = matSize / numColIndices;
24026ecaa68aSToby Isaac 
24036ecaa68aSToby Isaac         if (!numRowIndices) { /* don't need to calculate the mat, just the indices */
24046ecaa68aSToby Isaac           PetscInt numIndices, *indices;
24056ecaa68aSToby Isaac           ierr = DMPlexGetClosureIndices(coarse,localCoarse,globalCoarse,p,&numIndices,&indices,offsets);CHKERRQ(ierr);
24066ecaa68aSToby Isaac           if (numIndices != numColIndices) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"mismatching constraint indices calculations");
24076ecaa68aSToby Isaac           for (i = 0; i < numColIndices; i++) {
24086ecaa68aSToby Isaac             pInd[i] = indices[i];
24096ecaa68aSToby Isaac           }
24106ecaa68aSToby Isaac           for (i = 0; i < numFields; i++) {
241146bdb399SToby Isaac             pInd[numColIndices + i]             = offsets[i+1];
241246bdb399SToby Isaac             pInd[numColIndices + numFields + i] = offsets[i+1];
24136ecaa68aSToby Isaac           }
241446bdb399SToby Isaac           ierr = DMPlexRestoreClosureIndices(coarse,localCoarse,globalCoarse,p,&numIndices,&indices,offsets);CHKERRQ(ierr);
24156ecaa68aSToby Isaac         }
24166ecaa68aSToby Isaac         else {
24176ecaa68aSToby Isaac           PetscInt closureSize, *closure = NULL, cl;
24186ecaa68aSToby Isaac           PetscScalar *pMatIn, *pMatModified;
24196ecaa68aSToby Isaac           PetscInt numPoints,*points;
24206ecaa68aSToby Isaac 
24216ecaa68aSToby Isaac           ierr = DMGetWorkArray(coarse,numRowIndices * numRowIndices,PETSC_SCALAR,&pMatIn);CHKERRQ(ierr);
24226ecaa68aSToby Isaac           for (i = 0; i < numRowIndices; i++) { /* initialize to the identity */
24236ecaa68aSToby Isaac             for (j = 0; j < numRowIndices; j++) {
24246ecaa68aSToby Isaac               pMatIn[i * numRowIndices + j] = (i == j) ? 1. : 0.;
24256ecaa68aSToby Isaac             }
24266ecaa68aSToby Isaac           }
24276ecaa68aSToby Isaac           ierr = DMPlexGetTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
24284acb8e1eSToby Isaac           for (f = 0; f < maxFields; f++) {
24294acb8e1eSToby Isaac             if (numFields) {ierr = PetscSectionGetFieldPointSyms(localCoarse,f,closureSize,closure,&perms[f],&flips[f]);CHKERRQ(ierr);}
24304acb8e1eSToby Isaac             else           {ierr = PetscSectionGetPointSyms(localCoarse,closureSize,closure,&perms[f],&flips[f]);CHKERRQ(ierr);}
24314acb8e1eSToby Isaac           }
24326ecaa68aSToby Isaac           if (numFields) {
24336ecaa68aSToby Isaac             for (cl = 0; cl < closureSize; cl++) {
24346ecaa68aSToby Isaac               PetscInt c = closure[2 * cl];
24356ecaa68aSToby Isaac 
24366ecaa68aSToby Isaac               for (f = 0; f < numFields; f++) {
24376ecaa68aSToby Isaac                 PetscInt fDof;
24386ecaa68aSToby Isaac 
24396ecaa68aSToby Isaac                 ierr = PetscSectionGetFieldDof(localCoarse,c,f,&fDof);CHKERRQ(ierr);
24406ecaa68aSToby Isaac                 offsets[f + 1] += fDof;
24416ecaa68aSToby Isaac               }
24426ecaa68aSToby Isaac             }
24436ecaa68aSToby Isaac             for (f = 0; f < numFields; f++) {
24446ecaa68aSToby Isaac               offsets[f + 1]   += offsets[f];
24456ecaa68aSToby Isaac               newOffsets[f + 1] = offsets[f + 1];
24466ecaa68aSToby Isaac             }
24476ecaa68aSToby Isaac           }
24484acb8e1eSToby Isaac           /* TODO : flips here ? */
24496ecaa68aSToby Isaac           /* apply hanging node constraints on the right, get the new points and the new offsets */
24504acb8e1eSToby Isaac           ierr = DMPlexAnchorsModifyMat(coarse,localCoarse,closureSize,numRowIndices,closure,perms,pMatIn,&numPoints,NULL,&points,&pMatModified,newOffsets,PETSC_FALSE);CHKERRQ(ierr);
24514acb8e1eSToby Isaac           for (f = 0; f < maxFields; f++) {
24524acb8e1eSToby Isaac             if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(localCoarse,f,closureSize,closure,&perms[f],&flips[f]);CHKERRQ(ierr);}
24534acb8e1eSToby Isaac             else           {ierr = PetscSectionRestorePointSyms(localCoarse,closureSize,closure,&perms[f],&flips[f]);CHKERRQ(ierr);}
24544acb8e1eSToby Isaac           }
24554acb8e1eSToby Isaac           for (f = 0; f < maxFields; f++) {
24564acb8e1eSToby Isaac             if (numFields) {ierr = PetscSectionGetFieldPointSyms(localCoarse,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
24574acb8e1eSToby Isaac             else           {ierr = PetscSectionGetPointSyms(localCoarse,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
24584acb8e1eSToby Isaac           }
24596ecaa68aSToby Isaac           if (!numFields) {
24606ecaa68aSToby Isaac             for (i = 0; i < numRowIndices * numColIndices; i++) {
24616ecaa68aSToby Isaac               pMat[i] = pMatModified[i];
24626ecaa68aSToby Isaac             }
24636ecaa68aSToby Isaac           }
24646ecaa68aSToby Isaac           else {
2465f13f9184SToby Isaac             PetscInt i, j, count;
24666ecaa68aSToby Isaac             for (f = 0, count = 0; f < numFields; f++) {
24676ecaa68aSToby Isaac               for (i = offsets[f]; i < offsets[f+1]; i++) {
24686ecaa68aSToby Isaac                 for (j = newOffsets[f]; j < newOffsets[f+1]; j++, count++) {
24696ecaa68aSToby Isaac                   pMat[count] = pMatModified[i * numColIndices + j];
24706ecaa68aSToby Isaac                 }
24716ecaa68aSToby Isaac               }
24726ecaa68aSToby Isaac             }
24736ecaa68aSToby Isaac           }
24746ecaa68aSToby Isaac           ierr = DMRestoreWorkArray(coarse,numRowIndices * numColIndices,PETSC_SCALAR,&pMatModified);CHKERRQ(ierr);
24756ecaa68aSToby Isaac           ierr = DMPlexRestoreTransitiveClosure(coarse, p, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
24766ecaa68aSToby Isaac           ierr = DMRestoreWorkArray(coarse,numRowIndices * numColIndices,PETSC_SCALAR,&pMatIn);CHKERRQ(ierr);
24776ecaa68aSToby Isaac           if (numFields) {
247846bdb399SToby Isaac             for (f = 0; f < numFields; f++) {
247946bdb399SToby Isaac               pInd[numColIndices + f]             = offsets[f+1];
248046bdb399SToby Isaac               pInd[numColIndices + numFields + f] = newOffsets[f+1];
24816ecaa68aSToby Isaac             }
24824acb8e1eSToby Isaac             for (cl = 0; cl < numPoints; cl++) {
24834acb8e1eSToby Isaac               PetscInt globalOff, c = points[2*cl];
24846ecaa68aSToby Isaac               ierr = PetscSectionGetOffset(globalCoarse, c, &globalOff);CHKERRQ(ierr);
24854acb8e1eSToby Isaac               DMPlexGetIndicesPointFields_Internal(localCoarse, c, globalOff < 0 ? -(globalOff+1) : globalOff, newOffsets, PETSC_FALSE, perms, cl, pInd);
24866ecaa68aSToby Isaac             }
24876ecaa68aSToby Isaac           } else {
24884acb8e1eSToby Isaac             for (cl = 0; cl < numPoints; cl++) {
24894acb8e1eSToby Isaac               PetscInt c = points[2*cl], globalOff;
24904acb8e1eSToby Isaac               const PetscInt *perm = perms[0] ? perms[0][cl] : NULL;
24914acb8e1eSToby Isaac 
24926ecaa68aSToby Isaac               ierr = PetscSectionGetOffset(globalCoarse, c, &globalOff);CHKERRQ(ierr);
24934acb8e1eSToby Isaac               DMPlexGetIndicesPoint_Internal(localCoarse, c, globalOff < 0 ? -(globalOff+1) : globalOff, newOffsets, PETSC_FALSE, perm, pInd);
24946ecaa68aSToby Isaac             }
24956ecaa68aSToby Isaac           }
24964acb8e1eSToby Isaac           for (f = 0; f < maxFields; f++) {
24974acb8e1eSToby Isaac             if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(localCoarse,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
24984acb8e1eSToby Isaac             else           {ierr = PetscSectionRestorePointSyms(localCoarse,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
24994acb8e1eSToby Isaac           }
250028552ed4SToby Isaac           ierr = DMRestoreWorkArray(coarse,numPoints,PETSC_SCALAR,&points);CHKERRQ(ierr);
25016ecaa68aSToby Isaac         }
25026ecaa68aSToby Isaac       }
25036ecaa68aSToby Isaac       else if (matSize) {
25046ecaa68aSToby Isaac         PetscInt cOff;
25056ecaa68aSToby Isaac         PetscInt *rowIndices, *colIndices, a, aDof, aOff;
25066ecaa68aSToby Isaac 
25076ecaa68aSToby Isaac         numRowIndices = matSize / numColIndices;
2508628cbfb8SToby Isaac         if (numRowIndices != dof) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Miscounted dofs");
25096ecaa68aSToby Isaac         ierr = DMGetWorkArray(coarse,numRowIndices,PETSC_INT,&rowIndices);CHKERRQ(ierr);
25106ecaa68aSToby Isaac         ierr = DMGetWorkArray(coarse,numColIndices,PETSC_INT,&colIndices);CHKERRQ(ierr);
25116ecaa68aSToby Isaac         ierr = PetscSectionGetOffset(cSec,p,&cOff);CHKERRQ(ierr);
25126ecaa68aSToby Isaac         ierr = PetscSectionGetDof(aSec,p,&aDof);CHKERRQ(ierr);
25136ecaa68aSToby Isaac         ierr = PetscSectionGetOffset(aSec,p,&aOff);CHKERRQ(ierr);
25146ecaa68aSToby Isaac         if (numFields) {
25156ecaa68aSToby Isaac           for (f = 0; f < numFields; f++) {
25166ecaa68aSToby Isaac             PetscInt fDof;
2517f13f9184SToby Isaac 
25186ecaa68aSToby Isaac             ierr = PetscSectionGetFieldDof(cSec,p,f,&fDof);CHKERRQ(ierr);
25196ecaa68aSToby Isaac             offsets[f + 1] = fDof;
25206ecaa68aSToby Isaac             for (a = 0; a < aDof; a++) {
25216ecaa68aSToby Isaac               PetscInt anchor = anchors[a + aOff];
25226ecaa68aSToby Isaac               ierr = PetscSectionGetFieldDof(localCoarse,anchor,f,&fDof);CHKERRQ(ierr);
25236ecaa68aSToby Isaac               newOffsets[f + 1] += fDof;
25246ecaa68aSToby Isaac             }
25256ecaa68aSToby Isaac           }
25266ecaa68aSToby Isaac           for (f = 0; f < numFields; f++) {
25276ecaa68aSToby Isaac             offsets[f + 1]       += offsets[f];
25286ecaa68aSToby Isaac             offsetsCopy[f + 1]    = offsets[f + 1];
25296ecaa68aSToby Isaac             newOffsets[f + 1]    += newOffsets[f];
25306ecaa68aSToby Isaac             newOffsetsCopy[f + 1] = newOffsets[f + 1];
25316ecaa68aSToby Isaac           }
25324acb8e1eSToby Isaac           DMPlexGetIndicesPointFields_Internal(cSec,p,cOff,offsetsCopy,PETSC_TRUE,NULL,-1,rowIndices);CHKERRQ(ierr);
25336ecaa68aSToby Isaac           for (a = 0; a < aDof; a++) {
25346ecaa68aSToby Isaac             PetscInt anchor = anchors[a + aOff], lOff;
25356ecaa68aSToby Isaac             ierr = PetscSectionGetOffset(localCoarse,anchor,&lOff);CHKERRQ(ierr);
25364acb8e1eSToby Isaac             DMPlexGetIndicesPointFields_Internal(localCoarse,anchor,lOff,newOffsetsCopy,PETSC_TRUE,NULL,-1,colIndices);CHKERRQ(ierr);
25376ecaa68aSToby Isaac           }
25386ecaa68aSToby Isaac         }
25396ecaa68aSToby Isaac         else {
25404acb8e1eSToby Isaac           DMPlexGetIndicesPoint_Internal(cSec,p,cOff,offsetsCopy,PETSC_TRUE,NULL,rowIndices);CHKERRQ(ierr);
25416ecaa68aSToby Isaac           for (a = 0; a < aDof; a++) {
25426ecaa68aSToby Isaac             PetscInt anchor = anchors[a + aOff], lOff;
25436ecaa68aSToby Isaac             ierr = PetscSectionGetOffset(localCoarse,anchor,&lOff);CHKERRQ(ierr);
25444acb8e1eSToby Isaac             DMPlexGetIndicesPoint_Internal(localCoarse,anchor,lOff,newOffsetsCopy,PETSC_TRUE,NULL,colIndices);CHKERRQ(ierr);
25456ecaa68aSToby Isaac           }
25466ecaa68aSToby Isaac         }
25476ecaa68aSToby Isaac         if (numFields) {
2548f13f9184SToby Isaac           PetscInt count, a;
2549f13f9184SToby Isaac 
25506ecaa68aSToby Isaac           for (f = 0, count = 0; f < numFields; f++) {
25516ecaa68aSToby Isaac             PetscInt iSize = offsets[f + 1] - offsets[f];
25526ecaa68aSToby Isaac             PetscInt jSize = newOffsets[f + 1] - newOffsets[f];
25536ecaa68aSToby Isaac             ierr = MatGetValues(cMat,iSize,&rowIndices[offsets[f]],jSize,&colIndices[newOffsets[f]],&pMat[count]);CHKERRQ(ierr);
25546ecaa68aSToby Isaac             count += iSize * jSize;
255546bdb399SToby Isaac             pInd[numColIndices + f]             = offsets[f+1];
255646bdb399SToby Isaac             pInd[numColIndices + numFields + f] = newOffsets[f+1];
25576ecaa68aSToby Isaac           }
25586ecaa68aSToby Isaac           for (a = 0; a < aDof; a++) {
25596ecaa68aSToby Isaac             PetscInt anchor = anchors[a + aOff];
25606ecaa68aSToby Isaac             PetscInt gOff;
25616ecaa68aSToby Isaac             ierr = PetscSectionGetOffset(globalCoarse,anchor,&gOff);CHKERRQ(ierr);
25624acb8e1eSToby Isaac             DMPlexGetIndicesPointFields_Internal(localCoarse,anchor,gOff < 0 ? -(gOff + 1) : gOff,newOffsets,PETSC_FALSE,NULL,-1,pInd);CHKERRQ(ierr);
25636ecaa68aSToby Isaac           }
25646ecaa68aSToby Isaac         }
25656ecaa68aSToby Isaac         else {
25666ecaa68aSToby Isaac           PetscInt a;
25676ecaa68aSToby Isaac           ierr = MatGetValues(cMat,numRowIndices,rowIndices,numColIndices,colIndices,pMat);CHKERRQ(ierr);
25686ecaa68aSToby Isaac           for (a = 0; a < aDof; a++) {
25696ecaa68aSToby Isaac             PetscInt anchor = anchors[a + aOff];
25706ecaa68aSToby Isaac             PetscInt gOff;
25716ecaa68aSToby Isaac             ierr = PetscSectionGetOffset(globalCoarse,anchor,&gOff);CHKERRQ(ierr);
25724acb8e1eSToby Isaac             DMPlexGetIndicesPoint_Internal(localCoarse,anchor,gOff < 0 ? -(gOff + 1) : gOff,newOffsets,PETSC_FALSE,NULL,pInd);CHKERRQ(ierr);
25736ecaa68aSToby Isaac           }
25746ecaa68aSToby Isaac         }
25756ecaa68aSToby Isaac         ierr = DMRestoreWorkArray(coarse,numColIndices,PETSC_INT,&colIndices);CHKERRQ(ierr);
25766ecaa68aSToby Isaac         ierr = DMRestoreWorkArray(coarse,numRowIndices,PETSC_INT,&rowIndices);CHKERRQ(ierr);
25776ecaa68aSToby Isaac       }
25786ecaa68aSToby Isaac       else {
25796ecaa68aSToby Isaac         PetscInt gOff;
25806ecaa68aSToby Isaac 
25816ecaa68aSToby Isaac         ierr = PetscSectionGetOffset(globalCoarse,p,&gOff);CHKERRQ(ierr);
25826ecaa68aSToby Isaac         if (numFields) {
25836ecaa68aSToby Isaac           for (f = 0; f < numFields; f++) {
25846ecaa68aSToby Isaac             PetscInt fDof;
25856ecaa68aSToby Isaac             ierr = PetscSectionGetFieldDof(localCoarse,p,f,&fDof);CHKERRQ(ierr);
25866ecaa68aSToby Isaac             offsets[f + 1] = fDof + offsets[f];
25876ecaa68aSToby Isaac           }
25886ecaa68aSToby Isaac           for (f = 0; f < numFields; f++) {
258946bdb399SToby Isaac             pInd[numColIndices + f]             = offsets[f+1];
259046bdb399SToby Isaac             pInd[numColIndices + numFields + f] = offsets[f+1];
25916ecaa68aSToby Isaac           }
25924acb8e1eSToby Isaac           DMPlexGetIndicesPointFields_Internal(localCoarse,p,gOff < 0 ? -(gOff + 1) : gOff,offsets,PETSC_FALSE,NULL,-1,pInd);CHKERRQ(ierr);
25936ecaa68aSToby Isaac         }
25946ecaa68aSToby Isaac         else {
25954acb8e1eSToby Isaac           DMPlexGetIndicesPoint_Internal(localCoarse,p,gOff < 0 ? -(gOff + 1) : gOff,offsets,PETSC_FALSE,NULL,pInd);CHKERRQ(ierr);
25966ecaa68aSToby Isaac         }
25976ecaa68aSToby Isaac       }
25986ecaa68aSToby Isaac     }
2599e44e4e7fSToby Isaac     ierr = PetscFree(maxChildIds);CHKERRQ(ierr);
26006ecaa68aSToby Isaac   }
260146bdb399SToby Isaac   {
260246bdb399SToby Isaac     PetscSF  indicesSF, matricesSF;
260346bdb399SToby Isaac     PetscInt *remoteOffsetsIndices, *remoteOffsetsMatrices, numLeafIndices, numLeafMatrices;
260446bdb399SToby Isaac 
260546bdb399SToby Isaac     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)fine),&leafIndicesSec);CHKERRQ(ierr);
260646bdb399SToby Isaac     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)fine),&leafMatricesSec);CHKERRQ(ierr);
260746bdb399SToby Isaac     ierr = PetscSFDistributeSection(coarseToFineEmbedded,rootIndicesSec,&remoteOffsetsIndices,leafIndicesSec);CHKERRQ(ierr);
260846bdb399SToby Isaac     ierr = PetscSFDistributeSection(coarseToFineEmbedded,rootMatricesSec,&remoteOffsetsMatrices,leafMatricesSec);CHKERRQ(ierr);
260946bdb399SToby Isaac     ierr = PetscSFCreateSectionSF(coarseToFineEmbedded,rootIndicesSec,remoteOffsetsIndices,leafIndicesSec,&indicesSF);CHKERRQ(ierr);
261046bdb399SToby Isaac     ierr = PetscSFCreateSectionSF(coarseToFineEmbedded,rootMatricesSec,remoteOffsetsMatrices,leafMatricesSec,&matricesSF);CHKERRQ(ierr);
2611e44e4e7fSToby Isaac     ierr = PetscSFDestroy(&coarseToFineEmbedded);CHKERRQ(ierr);
261246bdb399SToby Isaac     ierr = PetscFree(remoteOffsetsIndices);CHKERRQ(ierr);
261346bdb399SToby Isaac     ierr = PetscFree(remoteOffsetsMatrices);CHKERRQ(ierr);
261446bdb399SToby Isaac     ierr = PetscSectionGetStorageSize(leafIndicesSec,&numLeafIndices);CHKERRQ(ierr);
261546bdb399SToby Isaac     ierr = PetscSectionGetStorageSize(leafMatricesSec,&numLeafMatrices);CHKERRQ(ierr);
261646bdb399SToby Isaac     ierr = PetscMalloc2(numLeafIndices,&leafIndices,numLeafMatrices,&leafMatrices);CHKERRQ(ierr);
261746bdb399SToby Isaac     ierr = PetscSFBcastBegin(indicesSF,MPIU_INT,rootIndices,leafIndices);CHKERRQ(ierr);
2618267d4f3fSToby Isaac     ierr = PetscSFBcastBegin(matricesSF,MPIU_SCALAR,rootMatrices,leafMatrices);CHKERRQ(ierr);
261946bdb399SToby Isaac     ierr = PetscSFBcastEnd(indicesSF,MPIU_INT,rootIndices,leafIndices);CHKERRQ(ierr);
2620267d4f3fSToby Isaac     ierr = PetscSFBcastEnd(matricesSF,MPIU_SCALAR,rootMatrices,leafMatrices);CHKERRQ(ierr);
262146bdb399SToby Isaac     ierr = PetscSFDestroy(&matricesSF);CHKERRQ(ierr);
262246bdb399SToby Isaac     ierr = PetscSFDestroy(&indicesSF);CHKERRQ(ierr);
262346bdb399SToby Isaac     ierr = PetscFree2(rootIndices,rootMatrices);CHKERRQ(ierr);
262446bdb399SToby Isaac     ierr = PetscSectionDestroy(&rootIndicesSec);CHKERRQ(ierr);
262546bdb399SToby Isaac     ierr = PetscSectionDestroy(&rootMatricesSec);CHKERRQ(ierr);
262646bdb399SToby Isaac   }
262746bdb399SToby Isaac   /* count to preallocate */
262846bdb399SToby Isaac   ierr = DMGetDefaultSection(fine,&localFine);CHKERRQ(ierr);
262946bdb399SToby Isaac   {
263046bdb399SToby Isaac     PetscInt    nGlobal;
263146bdb399SToby Isaac     PetscInt    *dnnz, *onnz;
2632b9a5774bSToby Isaac     PetscLayout rowMap, colMap;
2633b9a5774bSToby Isaac     PetscInt    rowStart, rowEnd, colStart, colEnd;
26341c58ffc4SToby Isaac     PetscInt    maxDof;
26351c58ffc4SToby Isaac     PetscInt    *rowIndices;
26361c58ffc4SToby Isaac     DM           refTree;
26371c58ffc4SToby Isaac     PetscInt     **refPointFieldN;
26381c58ffc4SToby Isaac     PetscScalar  ***refPointFieldMats;
26391c58ffc4SToby Isaac     PetscSection refConSec, refAnSec;
26400eb7e1eaSToby Isaac     PetscInt     pRefStart,pRefEnd,maxConDof,maxColumns,leafStart,leafEnd;
26411c58ffc4SToby Isaac     PetscScalar  *pointWork;
264246bdb399SToby Isaac 
264346bdb399SToby Isaac     ierr = PetscSectionGetConstrainedStorageSize(globalFine,&nGlobal);CHKERRQ(ierr);
264446bdb399SToby Isaac     ierr = PetscCalloc2(nGlobal,&dnnz,nGlobal,&onnz);CHKERRQ(ierr);
2645b9a5774bSToby Isaac     ierr = MatGetLayouts(mat,&rowMap,&colMap);CHKERRQ(ierr);
2646b9a5774bSToby Isaac     ierr = PetscLayoutSetUp(rowMap);CHKERRQ(ierr);
26471c58ffc4SToby Isaac     ierr = PetscLayoutSetUp(colMap);CHKERRQ(ierr);
2648b9a5774bSToby Isaac     ierr = PetscLayoutGetRange(rowMap,&rowStart,&rowEnd);CHKERRQ(ierr);
264946bdb399SToby Isaac     ierr = PetscLayoutGetRange(colMap,&colStart,&colEnd);CHKERRQ(ierr);
26501c58ffc4SToby Isaac     ierr = PetscSectionGetMaxDof(globalFine,&maxDof);CHKERRQ(ierr);
26510eb7e1eaSToby Isaac     ierr = PetscSectionGetChart(leafIndicesSec,&leafStart,&leafEnd);CHKERRQ(ierr);
26521c58ffc4SToby Isaac     ierr = DMGetWorkArray(fine,maxDof,PETSC_INT,&rowIndices);CHKERRQ(ierr);
26530eb7e1eaSToby Isaac     for (p = leafStart; p < leafEnd; p++) {
265446bdb399SToby Isaac       PetscInt    gDof, gcDof, gOff;
265546bdb399SToby Isaac       PetscInt    numColIndices, pIndOff, *pInd;
265646bdb399SToby Isaac       PetscInt    matSize;
265721968bf8SToby Isaac       PetscInt    i;
265846bdb399SToby Isaac 
265946bdb399SToby Isaac       ierr = PetscSectionGetDof(globalFine,p,&gDof);CHKERRQ(ierr);
266046bdb399SToby Isaac       ierr = PetscSectionGetConstraintDof(globalFine,p,&gcDof);CHKERRQ(ierr);
266146bdb399SToby Isaac       if ((gDof - gcDof) <= 0) {
266246bdb399SToby Isaac         continue;
266346bdb399SToby Isaac       }
266446bdb399SToby Isaac       ierr = PetscSectionGetOffset(globalFine,p,&gOff);CHKERRQ(ierr);
266546bdb399SToby Isaac       if (gOff < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"I though having global dofs meant a non-negative offset");
2666b9a5774bSToby Isaac       if ((gOff < rowStart) || ((gOff + gDof - gcDof) > rowEnd)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"I thought the row map would constrain the global dofs");
266746bdb399SToby Isaac       ierr = PetscSectionGetDof(leafIndicesSec,p,&numColIndices);CHKERRQ(ierr);
266846bdb399SToby Isaac       ierr = PetscSectionGetOffset(leafIndicesSec,p,&pIndOff);CHKERRQ(ierr);
266946bdb399SToby Isaac       numColIndices -= 2 * numFields;
26701c58ffc4SToby Isaac       if (numColIndices <= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"global fine dof with no dofs to interpolate from");
267146bdb399SToby Isaac       pInd = &leafIndices[pIndOff];
267221968bf8SToby Isaac       offsets[0]        = 0;
267321968bf8SToby Isaac       offsetsCopy[0]    = 0;
267421968bf8SToby Isaac       newOffsets[0]     = 0;
267521968bf8SToby Isaac       newOffsetsCopy[0] = 0;
267646bdb399SToby Isaac       if (numFields) {
267721968bf8SToby Isaac         PetscInt f;
267846bdb399SToby Isaac         for (f = 0; f < numFields; f++) {
267946bdb399SToby Isaac           PetscInt rowDof;
268046bdb399SToby Isaac 
268146bdb399SToby Isaac           ierr = PetscSectionGetFieldDof(localFine,p,f,&rowDof);CHKERRQ(ierr);
268221968bf8SToby Isaac           offsets[f + 1]        = offsets[f] + rowDof;
268321968bf8SToby Isaac           offsetsCopy[f + 1]    = offsets[f + 1];
268421968bf8SToby Isaac           newOffsets[f + 1]     = pInd[numColIndices + numFields + f];
268521968bf8SToby Isaac           numD[f] = 0;
268621968bf8SToby Isaac           numO[f] = 0;
268746bdb399SToby Isaac         }
26884acb8e1eSToby Isaac         ierr = DMPlexGetIndicesPointFields_Internal(localFine,p,gOff,offsetsCopy,PETSC_FALSE,NULL,-1,rowIndices);CHKERRQ(ierr);
268946bdb399SToby Isaac         for (f = 0; f < numFields; f++) {
269021968bf8SToby Isaac           PetscInt colOffset    = newOffsets[f];
269121968bf8SToby Isaac           PetscInt numFieldCols = newOffsets[f + 1] - newOffsets[f];
269246bdb399SToby Isaac 
269346bdb399SToby Isaac           for (i = 0; i < numFieldCols; i++) {
269446bdb399SToby Isaac             PetscInt gInd = pInd[i + colOffset];
269546bdb399SToby Isaac 
269646bdb399SToby Isaac             if (gInd >= colStart && gInd < colEnd) {
269721968bf8SToby Isaac               numD[f]++;
269846bdb399SToby Isaac             }
269946bdb399SToby Isaac             else if (gInd >= 0) { /* negative means non-entry */
270021968bf8SToby Isaac               numO[f]++;
270146bdb399SToby Isaac             }
270246bdb399SToby Isaac           }
270346bdb399SToby Isaac         }
270446bdb399SToby Isaac       }
270546bdb399SToby Isaac       else {
27064acb8e1eSToby Isaac         ierr = DMPlexGetIndicesPoint_Internal(localFine,p,gOff,offsetsCopy,PETSC_FALSE,NULL,rowIndices);CHKERRQ(ierr);
270721968bf8SToby Isaac         numD[0] = 0;
270821968bf8SToby Isaac         numO[0] = 0;
270946bdb399SToby Isaac         for (i = 0; i < numColIndices; i++) {
271046bdb399SToby Isaac           PetscInt gInd = pInd[i];
271146bdb399SToby Isaac 
271246bdb399SToby Isaac           if (gInd >= colStart && gInd < colEnd) {
271321968bf8SToby Isaac             numD[0]++;
271446bdb399SToby Isaac           }
271546bdb399SToby Isaac           else if (gInd >= 0) { /* negative means non-entry */
271621968bf8SToby Isaac             numO[0]++;
271746bdb399SToby Isaac           }
271846bdb399SToby Isaac         }
271946bdb399SToby Isaac       }
272046bdb399SToby Isaac       ierr = PetscSectionGetDof(leafMatricesSec,p,&matSize);CHKERRQ(ierr);
272146bdb399SToby Isaac       if (!matSize) { /* incoming matrix is identity */
272246bdb399SToby Isaac         PetscInt childId;
272346bdb399SToby Isaac 
272446bdb399SToby Isaac         childId = childIds[p-pStartF];
272521968bf8SToby Isaac         if (childId < 0) { /* no child interpolation: one nnz per */
272646bdb399SToby Isaac           if (numFields) {
2727b9a5774bSToby Isaac             PetscInt f;
2728b9a5774bSToby Isaac             for (f = 0; f < numFields; f++) {
272921968bf8SToby Isaac               PetscInt numRows = offsets[f+1] - offsets[f], row;
273046bdb399SToby Isaac               for (row = 0; row < numRows; row++) {
273121968bf8SToby Isaac                 PetscInt gIndCoarse = pInd[newOffsets[f] + row];
273221968bf8SToby Isaac                 PetscInt gIndFine   = rowIndices[offsets[f] + row];
273346bdb399SToby Isaac                 if (gIndCoarse >= colStart && gIndCoarse < colEnd) { /* local */
2734b9a5774bSToby Isaac                   if (gIndFine < rowStart || gIndFine >= rowEnd) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
2735b9a5774bSToby Isaac                   dnnz[gIndFine - rowStart] = 1;
273646bdb399SToby Isaac                 }
273746bdb399SToby Isaac                 else if (gIndCoarse >= 0) { /* remote */
2738b9a5774bSToby Isaac                   if (gIndFine < rowStart || gIndFine >= rowEnd) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
2739b9a5774bSToby Isaac                   onnz[gIndFine - rowStart] = 1;
274046bdb399SToby Isaac                 }
274146bdb399SToby Isaac                 else { /* constrained */
274246bdb399SToby Isaac                   if (gIndFine >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
274346bdb399SToby Isaac                 }
274446bdb399SToby Isaac               }
274546bdb399SToby Isaac             }
274646bdb399SToby Isaac           }
274746bdb399SToby Isaac           else {
2748b9a5774bSToby Isaac             PetscInt i;
2749b9a5774bSToby Isaac             for (i = 0; i < gDof; i++) {
275046bdb399SToby Isaac               PetscInt gIndCoarse = pInd[i];
275146bdb399SToby Isaac               PetscInt gIndFine   = rowIndices[i];
275246bdb399SToby Isaac               if (gIndCoarse >= colStart && gIndCoarse < colEnd) { /* local */
2753b9a5774bSToby Isaac                 if (gIndFine < rowStart || gIndFine >= rowEnd) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
2754b9a5774bSToby Isaac                 dnnz[gIndFine - rowStart] = 1;
275546bdb399SToby Isaac               }
275646bdb399SToby Isaac               else if (gIndCoarse >= 0) { /* remote */
2757b9a5774bSToby Isaac                 if (gIndFine < rowStart || gIndFine >= rowEnd) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
2758b9a5774bSToby Isaac                 onnz[gIndFine - rowStart] = 1;
275946bdb399SToby Isaac               }
276046bdb399SToby Isaac               else { /* constrained */
276146bdb399SToby Isaac                 if (gIndFine >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
276246bdb399SToby Isaac               }
276346bdb399SToby Isaac             }
276446bdb399SToby Isaac           }
276546bdb399SToby Isaac         }
276646bdb399SToby Isaac         else { /* interpolate from all */
276746bdb399SToby Isaac           if (numFields) {
2768b9a5774bSToby Isaac             PetscInt f;
2769b9a5774bSToby Isaac             for (f = 0; f < numFields; f++) {
277021968bf8SToby Isaac               PetscInt numRows = offsets[f+1] - offsets[f], row;
277146bdb399SToby Isaac               for (row = 0; row < numRows; row++) {
277221968bf8SToby Isaac                 PetscInt gIndFine = rowIndices[offsets[f] + row];
277346bdb399SToby Isaac                 if (gIndFine >= 0) {
2774b9a5774bSToby Isaac                   if (gIndFine < rowStart || gIndFine >= rowEnd) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
2775b9a5774bSToby Isaac                   dnnz[gIndFine - rowStart] = numD[f];
2776b9a5774bSToby Isaac                   onnz[gIndFine - rowStart] = numO[f];
277746bdb399SToby Isaac                 }
277846bdb399SToby Isaac               }
277946bdb399SToby Isaac             }
278046bdb399SToby Isaac           }
278146bdb399SToby Isaac           else {
2782b9a5774bSToby Isaac             PetscInt i;
2783b9a5774bSToby Isaac             for (i = 0; i < gDof; i++) {
278446bdb399SToby Isaac               PetscInt gIndFine = rowIndices[i];
278546bdb399SToby Isaac               if (gIndFine >= 0) {
2786b9a5774bSToby Isaac                 if (gIndFine < rowStart || gIndFine >= rowEnd) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
2787b9a5774bSToby Isaac                 dnnz[gIndFine - rowStart] = numD[0];
2788b9a5774bSToby Isaac                 onnz[gIndFine - rowStart] = numO[0];
278946bdb399SToby Isaac               }
279046bdb399SToby Isaac             }
279146bdb399SToby Isaac           }
279246bdb399SToby Isaac         }
279346bdb399SToby Isaac       }
279446bdb399SToby Isaac       else { /* interpolate from all */
279546bdb399SToby Isaac         if (numFields) {
2796b9a5774bSToby Isaac           PetscInt f;
2797b9a5774bSToby Isaac           for (f = 0; f < numFields; f++) {
279821968bf8SToby Isaac             PetscInt numRows = offsets[f+1] - offsets[f], row;
279946bdb399SToby Isaac             for (row = 0; row < numRows; row++) {
280021968bf8SToby Isaac               PetscInt gIndFine = rowIndices[offsets[f] + row];
280146bdb399SToby Isaac               if (gIndFine >= 0) {
2802b9a5774bSToby Isaac                 if (gIndFine < rowStart || gIndFine >= rowEnd) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
2803b9a5774bSToby Isaac                 dnnz[gIndFine - rowStart] = numD[f];
2804b9a5774bSToby Isaac                 onnz[gIndFine - rowStart] = numO[f];
280546bdb399SToby Isaac               }
280646bdb399SToby Isaac             }
280746bdb399SToby Isaac           }
280846bdb399SToby Isaac         }
280946bdb399SToby Isaac         else { /* every dof get a full row */
2810b9a5774bSToby Isaac           PetscInt i;
2811b9a5774bSToby Isaac           for (i = 0; i < gDof; i++) {
281246bdb399SToby Isaac             PetscInt gIndFine = rowIndices[i];
281346bdb399SToby Isaac             if (gIndFine >= 0) {
2814b9a5774bSToby Isaac               if (gIndFine < rowStart || gIndFine >= rowEnd) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Mismatched number of constrained dofs");
2815b9a5774bSToby Isaac               dnnz[gIndFine - rowStart] = numD[0];
2816b9a5774bSToby Isaac               onnz[gIndFine - rowStart] = numO[0];
281746bdb399SToby Isaac             }
281846bdb399SToby Isaac           }
281946bdb399SToby Isaac         }
282046bdb399SToby Isaac       }
282146bdb399SToby Isaac     }
282246bdb399SToby Isaac     ierr = MatXAIJSetPreallocation(mat,1,dnnz,onnz,NULL,NULL);CHKERRQ(ierr);
282346bdb399SToby Isaac     ierr = PetscFree2(dnnz,onnz);CHKERRQ(ierr);
282421968bf8SToby Isaac 
282521968bf8SToby Isaac     ierr = DMPlexGetReferenceTree(fine,&refTree);CHKERRQ(ierr);
282621968bf8SToby Isaac     ierr = DMPlexReferenceTreeGetChildrenMatrices(refTree,&refPointFieldMats,&refPointFieldN);CHKERRQ(ierr);
2827e44e4e7fSToby Isaac     ierr = DMGetDefaultConstraints(refTree,&refConSec,NULL);CHKERRQ(ierr);
2828e44e4e7fSToby Isaac     ierr = DMPlexGetAnchors(refTree,&refAnSec,NULL);CHKERRQ(ierr);
2829e44e4e7fSToby Isaac     ierr = PetscSectionGetChart(refConSec,&pRefStart,&pRefEnd);CHKERRQ(ierr);
28301c58ffc4SToby Isaac     ierr = PetscSectionGetMaxDof(refConSec,&maxConDof);CHKERRQ(ierr);
28317c0540e0SToby Isaac     ierr = PetscSectionGetMaxDof(leafIndicesSec,&maxColumns);CHKERRQ(ierr);
28327c0540e0SToby Isaac     ierr = PetscMalloc1(maxConDof*maxColumns,&pointWork);CHKERRQ(ierr);
28330eb7e1eaSToby Isaac     for (p = leafStart; p < leafEnd; p++) {
2834e44e4e7fSToby Isaac       PetscInt    gDof, gcDof, gOff;
2835e44e4e7fSToby Isaac       PetscInt    numColIndices, pIndOff, *pInd;
2836e44e4e7fSToby Isaac       PetscInt    matSize;
2837e44e4e7fSToby Isaac       PetscInt    childId;
2838e44e4e7fSToby Isaac 
2839e44e4e7fSToby Isaac 
2840e44e4e7fSToby Isaac       ierr = PetscSectionGetDof(globalFine,p,&gDof);CHKERRQ(ierr);
2841e44e4e7fSToby Isaac       ierr = PetscSectionGetConstraintDof(globalFine,p,&gcDof);CHKERRQ(ierr);
2842e44e4e7fSToby Isaac       if ((gDof - gcDof) <= 0) {
2843e44e4e7fSToby Isaac         continue;
2844e44e4e7fSToby Isaac       }
2845e44e4e7fSToby Isaac       childId = childIds[p-pStartF];
2846e44e4e7fSToby Isaac       ierr = PetscSectionGetOffset(globalFine,p,&gOff);CHKERRQ(ierr);
2847e44e4e7fSToby Isaac       ierr = PetscSectionGetDof(leafIndicesSec,p,&numColIndices);CHKERRQ(ierr);
2848e44e4e7fSToby Isaac       ierr = PetscSectionGetOffset(leafIndicesSec,p,&pIndOff);CHKERRQ(ierr);
2849e44e4e7fSToby Isaac       numColIndices -= 2 * numFields;
2850e44e4e7fSToby Isaac       pInd = &leafIndices[pIndOff];
2851e44e4e7fSToby Isaac       offsets[0]        = 0;
2852e44e4e7fSToby Isaac       offsetsCopy[0]    = 0;
2853e44e4e7fSToby Isaac       newOffsets[0]     = 0;
2854e44e4e7fSToby Isaac       newOffsetsCopy[0] = 0;
2855e44e4e7fSToby Isaac       rowOffsets[0]     = 0;
2856e44e4e7fSToby Isaac       if (numFields) {
2857e44e4e7fSToby Isaac         PetscInt f;
2858e44e4e7fSToby Isaac         for (f = 0; f < numFields; f++) {
2859e44e4e7fSToby Isaac           PetscInt rowDof;
2860e44e4e7fSToby Isaac 
2861e44e4e7fSToby Isaac           ierr = PetscSectionGetFieldDof(localFine,p,f,&rowDof);CHKERRQ(ierr);
2862e44e4e7fSToby Isaac           offsets[f + 1]     = offsets[f] + rowDof;
2863e44e4e7fSToby Isaac           offsetsCopy[f + 1] = offsets[f + 1];
2864e44e4e7fSToby Isaac           rowOffsets[f + 1]  = pInd[numColIndices + f];
2865e44e4e7fSToby Isaac           newOffsets[f + 1]  = pInd[numColIndices + numFields + f];
2866e44e4e7fSToby Isaac         }
28674acb8e1eSToby Isaac         ierr = DMPlexGetIndicesPointFields_Internal(localFine,p,gOff,offsetsCopy,PETSC_FALSE,NULL,-1,rowIndices);CHKERRQ(ierr);
2868e44e4e7fSToby Isaac       }
28691c58ffc4SToby Isaac       else {
28704acb8e1eSToby Isaac         ierr = DMPlexGetIndicesPoint_Internal(localFine,p,gOff,offsetsCopy,PETSC_FALSE,NULL,rowIndices);CHKERRQ(ierr);
28711c58ffc4SToby Isaac       }
2872e44e4e7fSToby Isaac       ierr = PetscSectionGetDof(leafMatricesSec,p,&matSize);CHKERRQ(ierr);
2873e44e4e7fSToby Isaac       if (!matSize) { /* incoming matrix is identity */
2874e44e4e7fSToby Isaac         if (childId < 0) { /* no child interpolation: scatter */
2875e44e4e7fSToby Isaac           if (numFields) {
2876e44e4e7fSToby Isaac             PetscInt f;
2877e44e4e7fSToby Isaac             for (f = 0; f < numFields; f++) {
2878e44e4e7fSToby Isaac               PetscInt numRows = offsets[f+1] - offsets[f], row;
2879e44e4e7fSToby Isaac               for (row = 0; row < numRows; row++) {
2880e44e4e7fSToby Isaac                 ierr = MatSetValue(mat,rowIndices[offsets[f]+row],pInd[newOffsets[f]+row],1.,INSERT_VALUES);CHKERRQ(ierr);
288121968bf8SToby Isaac               }
288221968bf8SToby Isaac             }
2883e44e4e7fSToby Isaac           }
2884e44e4e7fSToby Isaac           else {
2885e44e4e7fSToby Isaac             PetscInt numRows = gDof, row;
2886e44e4e7fSToby Isaac             for (row = 0; row < numRows; row++) {
2887e44e4e7fSToby Isaac               ierr = MatSetValue(mat,rowIndices[row],pInd[row],1.,INSERT_VALUES);CHKERRQ(ierr);
2888e44e4e7fSToby Isaac             }
2889e44e4e7fSToby Isaac           }
2890e44e4e7fSToby Isaac         }
2891e44e4e7fSToby Isaac         else { /* interpolate from all */
2892e44e4e7fSToby Isaac           if (numFields) {
2893e44e4e7fSToby Isaac             PetscInt f;
2894e44e4e7fSToby Isaac             for (f = 0; f < numFields; f++) {
2895e44e4e7fSToby Isaac               PetscInt numRows = offsets[f+1] - offsets[f];
2896e44e4e7fSToby Isaac               PetscInt numCols = newOffsets[f + 1] - newOffsets[f];
2897e44e4e7fSToby Isaac               ierr = MatSetValues(mat,numRows,&rowIndices[offsets[f]],numCols,&pInd[newOffsets[f]],refPointFieldMats[childId - pRefStart][f],INSERT_VALUES);CHKERRQ(ierr);
2898e44e4e7fSToby Isaac             }
2899e44e4e7fSToby Isaac           }
2900e44e4e7fSToby Isaac           else {
2901e44e4e7fSToby Isaac             ierr = MatSetValues(mat,gDof,rowIndices,numColIndices,pInd,refPointFieldMats[childId - pRefStart][0],INSERT_VALUES);CHKERRQ(ierr);
2902e44e4e7fSToby Isaac           }
2903e44e4e7fSToby Isaac         }
2904e44e4e7fSToby Isaac       }
2905e44e4e7fSToby Isaac       else { /* interpolate from all */
2906e44e4e7fSToby Isaac         PetscInt    pMatOff;
2907e44e4e7fSToby Isaac         PetscScalar *pMat;
2908e44e4e7fSToby Isaac 
2909e44e4e7fSToby Isaac         ierr = PetscSectionGetOffset(leafMatricesSec,p,&pMatOff);CHKERRQ(ierr);
2910e44e4e7fSToby Isaac         pMat = &leafMatrices[pMatOff];
2911e44e4e7fSToby Isaac         if (childId < 0) { /* copy the incoming matrix */
2912e44e4e7fSToby Isaac           if (numFields) {
2913e44e4e7fSToby Isaac             PetscInt f, count;
2914e44e4e7fSToby Isaac             for (f = 0, count = 0; f < numFields; f++) {
2915e44e4e7fSToby Isaac               PetscInt numRows = offsets[f+1]-offsets[f];
2916e44e4e7fSToby Isaac               PetscInt numCols = newOffsets[f+1]-newOffsets[f];
2917e44e4e7fSToby Isaac               PetscInt numInRows = rowOffsets[f+1]-rowOffsets[f];
2918e44e4e7fSToby Isaac               PetscScalar *inMat = &pMat[count];
2919e44e4e7fSToby Isaac 
2920e44e4e7fSToby Isaac               ierr = MatSetValues(mat,numRows,&rowIndices[offsets[f]],numCols,&pInd[newOffsets[f]],inMat,INSERT_VALUES);CHKERRQ(ierr);
2921e44e4e7fSToby Isaac               count += numCols * numInRows;
2922e44e4e7fSToby Isaac             }
2923e44e4e7fSToby Isaac           }
2924e44e4e7fSToby Isaac           else {
2925e44e4e7fSToby Isaac             ierr = MatSetValues(mat,gDof,rowIndices,numColIndices,pInd,pMat,INSERT_VALUES);CHKERRQ(ierr);
2926e44e4e7fSToby Isaac           }
2927e44e4e7fSToby Isaac         }
2928e44e4e7fSToby Isaac         else { /* multiply the incoming matrix by the child interpolation */
2929e44e4e7fSToby Isaac           if (numFields) {
2930e44e4e7fSToby Isaac             PetscInt f, count;
2931e44e4e7fSToby Isaac             for (f = 0, count = 0; f < numFields; f++) {
2932e44e4e7fSToby Isaac               PetscInt numRows = offsets[f+1]-offsets[f];
2933e44e4e7fSToby Isaac               PetscInt numCols = newOffsets[f+1]-newOffsets[f];
2934e44e4e7fSToby Isaac               PetscInt numInRows = rowOffsets[f+1]-rowOffsets[f];
2935e44e4e7fSToby Isaac               PetscScalar *inMat = &pMat[count];
2936e44e4e7fSToby Isaac               PetscInt i, j, k;
2937e44e4e7fSToby Isaac               if (refPointFieldN[childId - pRefStart][f] != numInRows) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Point constraint matrix multiply dimension mismatch");
2938e44e4e7fSToby Isaac               for (i = 0; i < numRows; i++) {
2939e44e4e7fSToby Isaac                 for (j = 0; j < numCols; j++) {
2940e44e4e7fSToby Isaac                   PetscScalar val = 0.;
2941e44e4e7fSToby Isaac                   for (k = 0; k < numInRows; k++) {
2942e44e4e7fSToby Isaac                     val += refPointFieldMats[childId - pRefStart][f][i * numInRows + k] * inMat[k * numCols + j];
2943e44e4e7fSToby Isaac                   }
2944e44e4e7fSToby Isaac                   pointWork[i * numCols + j] = val;
2945e44e4e7fSToby Isaac                 }
2946e44e4e7fSToby Isaac               }
2947e44e4e7fSToby Isaac               ierr = MatSetValues(mat,numRows,&rowIndices[offsets[f]],numCols,&pInd[newOffsets[f]],pointWork,INSERT_VALUES);CHKERRQ(ierr);
2948e44e4e7fSToby Isaac               count += numCols * numInRows;
2949e44e4e7fSToby Isaac             }
2950e44e4e7fSToby Isaac           }
2951267d4f3fSToby Isaac           else { /* every dof gets a full row */
2952e44e4e7fSToby Isaac             PetscInt numRows   = gDof;
2953e44e4e7fSToby Isaac             PetscInt numCols   = numColIndices;
2954e44e4e7fSToby Isaac             PetscInt numInRows = matSize / numColIndices;
2955e44e4e7fSToby Isaac             PetscInt i, j, k;
2956e44e4e7fSToby Isaac             if (refPointFieldN[childId - pRefStart][0] != numInRows) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Point constraint matrix multiply dimension mismatch");
2957e44e4e7fSToby Isaac             for (i = 0; i < numRows; i++) {
2958e44e4e7fSToby Isaac               for (j = 0; j < numCols; j++) {
2959e44e4e7fSToby Isaac                 PetscScalar val = 0.;
2960e44e4e7fSToby Isaac                 for (k = 0; k < numInRows; k++) {
2961e44e4e7fSToby Isaac                   val += refPointFieldMats[childId - pRefStart][0][i * numInRows + k] * pMat[k * numCols + j];
2962e44e4e7fSToby Isaac                 }
2963e44e4e7fSToby Isaac                 pointWork[i * numCols + j] = val;
2964e44e4e7fSToby Isaac               }
2965e44e4e7fSToby Isaac             }
2966e44e4e7fSToby Isaac             ierr = MatSetValues(mat,numRows,rowIndices,numCols,pInd,pointWork,INSERT_VALUES);CHKERRQ(ierr);
2967e44e4e7fSToby Isaac           }
2968e44e4e7fSToby Isaac         }
2969e44e4e7fSToby Isaac       }
2970e44e4e7fSToby Isaac     }
29711c58ffc4SToby Isaac     ierr = DMPlexReferenceTreeRestoreChildrenMatrices(refTree,&refPointFieldMats,&refPointFieldN);CHKERRQ(ierr);
29721c58ffc4SToby Isaac     ierr = DMRestoreWorkArray(fine,maxDof,PETSC_INT,&rowIndices);CHKERRQ(ierr);
2973e44e4e7fSToby Isaac     ierr = PetscFree(pointWork);CHKERRQ(ierr);
2974e44e4e7fSToby Isaac   }
2975e44e4e7fSToby Isaac   ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
2976e44e4e7fSToby Isaac   ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
2977e44e4e7fSToby Isaac   ierr = PetscSectionDestroy(&leafIndicesSec);CHKERRQ(ierr);
2978e44e4e7fSToby Isaac   ierr = PetscSectionDestroy(&leafMatricesSec);CHKERRQ(ierr);
2979e44e4e7fSToby Isaac   ierr = PetscFree2(leafIndices,leafMatrices);CHKERRQ(ierr);
2980*f489ac74SBarry Smith   iperms = (PetscInt***)perms;
2981*f489ac74SBarry Smith   iflips = (PetscScalar***)flips;
29824acb8e1eSToby Isaac   ierr = PetscFree2(perms,flips);CHKERRQ(ierr);
2983e44e4e7fSToby Isaac   ierr = PetscFree7(offsets,offsetsCopy,newOffsets,newOffsetsCopy,rowOffsets,numD,numO);CHKERRQ(ierr);
29846ecaa68aSToby Isaac   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
29856ecaa68aSToby Isaac   PetscFunctionReturn(0);
29866ecaa68aSToby Isaac }
2987154bca37SToby Isaac 
29888d2f55e7SToby Isaac /*
29898d2f55e7SToby Isaac  * Assuming a nodal basis (w.r.t. the dual basis) basis:
29908d2f55e7SToby Isaac  *
29918d2f55e7SToby Isaac  * for each coarse dof \phi^c_i:
29928d2f55e7SToby Isaac  *   for each quadrature point (w_l,x_l) in the dual basis definition of \phi^c_i:
29938d2f55e7SToby Isaac  *     for each fine dof \phi^f_j;
29948d2f55e7SToby Isaac  *       a_{i,j} = 0;
29958d2f55e7SToby Isaac  *       for each fine dof \phi^f_k:
29968d2f55e7SToby Isaac  *         a_{i,j} += interp_{i,k} * \phi^f_k(x_l) * \phi^f_j(x_l) * w_l
29978d2f55e7SToby Isaac  *                    [^^^ this is = \phi^c_i ^^^]
29988d2f55e7SToby Isaac  */
29998d2f55e7SToby Isaac PetscErrorCode DMPlexComputeInjectorReferenceTree(DM refTree, Mat *inj)
30008d2f55e7SToby Isaac {
30018d2f55e7SToby Isaac   PetscDS        ds;
30028d2f55e7SToby Isaac   PetscSection   section, cSection;
30038d2f55e7SToby Isaac   DMLabel        canonical, depth;
30048d2f55e7SToby Isaac   Mat            cMat, mat;
30058d2f55e7SToby Isaac   PetscInt       *nnz;
30068d2f55e7SToby Isaac   PetscInt       f, dim, numFields, numSecFields, p, pStart, pEnd, cStart, cEnd;
30078d2f55e7SToby Isaac   PetscInt       m, n;
30088d2f55e7SToby Isaac   PetscScalar    *pointScalar;
30098d2f55e7SToby Isaac   PetscReal      *v0, *v0parent, *vtmp, *J, *Jparent, *invJ, *pointRef, detJ, detJparent;
30108d2f55e7SToby Isaac   PetscErrorCode ierr;
30118d2f55e7SToby Isaac 
30128d2f55e7SToby Isaac   PetscFunctionBegin;
30138d2f55e7SToby Isaac   ierr = DMGetDefaultSection(refTree,&section);CHKERRQ(ierr);
30148d2f55e7SToby Isaac   ierr = DMGetDimension(refTree, &dim);CHKERRQ(ierr);
30158d2f55e7SToby Isaac   ierr = PetscMalloc6(dim,&v0,dim,&v0parent,dim,&vtmp,dim*dim,&J,dim*dim,&Jparent,dim*dim,&invJ);CHKERRQ(ierr);
30168d2f55e7SToby Isaac   ierr = PetscMalloc2(dim,&pointScalar,dim,&pointRef);CHKERRQ(ierr);
30178d2f55e7SToby Isaac   ierr = DMGetDS(refTree,&ds);CHKERRQ(ierr);
30188d2f55e7SToby Isaac   ierr = PetscDSGetNumFields(ds,&numFields);CHKERRQ(ierr);
30198d2f55e7SToby Isaac   ierr = PetscSectionGetNumFields(section,&numSecFields);CHKERRQ(ierr);
30208d2f55e7SToby Isaac   ierr = DMGetLabel(refTree,"canonical",&canonical);CHKERRQ(ierr);
30218d2f55e7SToby Isaac   ierr = DMGetLabel(refTree,"depth",&depth);CHKERRQ(ierr);
30228d2f55e7SToby Isaac   ierr = DMGetDefaultConstraints(refTree,&cSection,&cMat);CHKERRQ(ierr);
30238d2f55e7SToby Isaac   ierr = DMPlexGetChart(refTree, &pStart, &pEnd);CHKERRQ(ierr);
30248d2f55e7SToby Isaac   ierr = DMPlexGetHeightStratum(refTree, 0, &cStart, &cEnd);CHKERRQ(ierr);
30258d2f55e7SToby Isaac   ierr = MatGetSize(cMat,&n,&m);CHKERRQ(ierr); /* the injector has transpose sizes from the constraint matrix */
30268d2f55e7SToby Isaac   /* Step 1: compute non-zero pattern.  A proper subset of constraint matrix non-zero */
30278d2f55e7SToby Isaac   ierr = PetscCalloc1(m,&nnz);CHKERRQ(ierr);
30288d2f55e7SToby Isaac   for (p = pStart; p < pEnd; p++) { /* a point will have non-zeros if it is canonical, it has dofs, and its children have dofs */
30298d2f55e7SToby Isaac     const PetscInt *children;
30308d2f55e7SToby Isaac     PetscInt numChildren;
30318d2f55e7SToby Isaac     PetscInt i, numChildDof, numSelfDof;
30328d2f55e7SToby Isaac 
30338d2f55e7SToby Isaac     if (canonical) {
30348d2f55e7SToby Isaac       PetscInt pCanonical;
30358d2f55e7SToby Isaac       ierr = DMLabelGetValue(canonical,p,&pCanonical);CHKERRQ(ierr);
30368d2f55e7SToby Isaac       if (p != pCanonical) continue;
30378d2f55e7SToby Isaac     }
30388d2f55e7SToby Isaac     ierr = DMPlexGetTreeChildren(refTree,p,&numChildren,&children);CHKERRQ(ierr);
30398d2f55e7SToby Isaac     if (!numChildren) continue;
30408d2f55e7SToby Isaac     for (i = 0, numChildDof = 0; i < numChildren; i++) {
30418d2f55e7SToby Isaac       PetscInt child = children[i];
30428d2f55e7SToby Isaac       PetscInt dof;
30438d2f55e7SToby Isaac 
30448d2f55e7SToby Isaac       ierr = PetscSectionGetDof(section,child,&dof);CHKERRQ(ierr);
30458d2f55e7SToby Isaac       numChildDof += dof;
30468d2f55e7SToby Isaac     }
30478d2f55e7SToby Isaac     ierr = PetscSectionGetDof(section,p,&numSelfDof);CHKERRQ(ierr);
30488d2f55e7SToby Isaac     if (!numChildDof || !numSelfDof) continue;
30498d2f55e7SToby Isaac     for (f = 0; f < numFields; f++) {
30508d2f55e7SToby Isaac       PetscInt selfOff;
30518d2f55e7SToby Isaac 
30528d2f55e7SToby Isaac       if (numSecFields) { /* count the dofs for just this field */
30538d2f55e7SToby Isaac         for (i = 0, numChildDof = 0; i < numChildren; i++) {
30548d2f55e7SToby Isaac           PetscInt child = children[i];
30558d2f55e7SToby Isaac           PetscInt dof;
30568d2f55e7SToby Isaac 
30578d2f55e7SToby Isaac           ierr = PetscSectionGetFieldDof(section,child,f,&dof);CHKERRQ(ierr);
30588d2f55e7SToby Isaac           numChildDof += dof;
30598d2f55e7SToby Isaac         }
30608d2f55e7SToby Isaac         ierr = PetscSectionGetFieldDof(section,p,f,&numSelfDof);CHKERRQ(ierr);
30618d2f55e7SToby Isaac         ierr = PetscSectionGetFieldOffset(section,p,f,&selfOff);CHKERRQ(ierr);
30628d2f55e7SToby Isaac       }
30638d2f55e7SToby Isaac       else {
30648d2f55e7SToby Isaac         ierr = PetscSectionGetOffset(section,p,&selfOff);CHKERRQ(ierr);
30658d2f55e7SToby Isaac       }
30668d2f55e7SToby Isaac       for (i = 0; i < numSelfDof; i++) {
30678d2f55e7SToby Isaac         nnz[selfOff + i] = numChildDof;
30688d2f55e7SToby Isaac       }
30698d2f55e7SToby Isaac     }
30708d2f55e7SToby Isaac   }
30718d2f55e7SToby Isaac   ierr = MatCreateAIJ(PETSC_COMM_SELF,m,n,m,n,-1,nnz,-1,NULL,&mat);CHKERRQ(ierr);
30728d2f55e7SToby Isaac   ierr = PetscFree(nnz);CHKERRQ(ierr);
30738d2f55e7SToby Isaac   /* Setp 2: compute entries */
30748d2f55e7SToby Isaac   for (p = pStart; p < pEnd; p++) {
30758d2f55e7SToby Isaac     const PetscInt *children;
30768d2f55e7SToby Isaac     PetscInt numChildren;
30778d2f55e7SToby Isaac     PetscInt i, numChildDof, numSelfDof;
30788d2f55e7SToby Isaac 
30798d2f55e7SToby Isaac     /* same conditions about when entries occur */
30808d2f55e7SToby Isaac     if (canonical) {
30818d2f55e7SToby Isaac       PetscInt pCanonical;
30828d2f55e7SToby Isaac       ierr = DMLabelGetValue(canonical,p,&pCanonical);CHKERRQ(ierr);
30838d2f55e7SToby Isaac       if (p != pCanonical) continue;
30848d2f55e7SToby Isaac     }
30858d2f55e7SToby Isaac     ierr = DMPlexGetTreeChildren(refTree,p,&numChildren,&children);CHKERRQ(ierr);
30868d2f55e7SToby Isaac     if (!numChildren) continue;
30878d2f55e7SToby Isaac     for (i = 0, numChildDof = 0; i < numChildren; i++) {
30888d2f55e7SToby Isaac       PetscInt child = children[i];
30898d2f55e7SToby Isaac       PetscInt dof;
30908d2f55e7SToby Isaac 
30918d2f55e7SToby Isaac       ierr = PetscSectionGetDof(section,child,&dof);CHKERRQ(ierr);
30928d2f55e7SToby Isaac       numChildDof += dof;
30938d2f55e7SToby Isaac     }
30948d2f55e7SToby Isaac     ierr = PetscSectionGetDof(section,p,&numSelfDof);CHKERRQ(ierr);
30958d2f55e7SToby Isaac     if (!numChildDof || !numSelfDof) continue;
30968d2f55e7SToby Isaac 
30978d2f55e7SToby Isaac     for (f = 0; f < numFields; f++) {
309852a3aeb4SToby Isaac       PetscInt       selfOff, Nc, parentCell;
30998d2f55e7SToby Isaac       PetscInt       cellShapeOff;
31008d2f55e7SToby Isaac       PetscObject    disc;
31018d2f55e7SToby Isaac       PetscDualSpace dsp;
31028d2f55e7SToby Isaac       PetscClassId   classId;
31038d2f55e7SToby Isaac       PetscScalar    *pointMat;
31043b1c2a6aSToby Isaac       PetscInt       *matRows, *matCols;
31058d2f55e7SToby Isaac       PetscInt       pO = PETSC_MIN_INT;
31068d2f55e7SToby Isaac       const PetscInt *depthNumDof;
31078d2f55e7SToby Isaac 
31088d2f55e7SToby Isaac       if (numSecFields) {
31098d2f55e7SToby Isaac         for (i = 0, numChildDof = 0; i < numChildren; i++) {
31108d2f55e7SToby Isaac           PetscInt child = children[i];
31118d2f55e7SToby Isaac           PetscInt dof;
31128d2f55e7SToby Isaac 
31138d2f55e7SToby Isaac           ierr = PetscSectionGetFieldDof(section,child,f,&dof);CHKERRQ(ierr);
31148d2f55e7SToby Isaac           numChildDof += dof;
31158d2f55e7SToby Isaac         }
31168d2f55e7SToby Isaac         ierr = PetscSectionGetFieldDof(section,p,f,&numSelfDof);CHKERRQ(ierr);
31178d2f55e7SToby Isaac         ierr = PetscSectionGetFieldOffset(section,p,f,&selfOff);CHKERRQ(ierr);
31188d2f55e7SToby Isaac       }
31198d2f55e7SToby Isaac       else {
31208d2f55e7SToby Isaac         ierr = PetscSectionGetOffset(section,p,&selfOff);CHKERRQ(ierr);
31218d2f55e7SToby Isaac       }
31228d2f55e7SToby Isaac 
31233b1c2a6aSToby Isaac       /* find a cell whose closure contains p */
31248d2f55e7SToby Isaac       if (p >= cStart && p < cEnd) {
31258d2f55e7SToby Isaac         parentCell = p;
31268d2f55e7SToby Isaac       }
31278d2f55e7SToby Isaac       else {
31288d2f55e7SToby Isaac         PetscInt *star = NULL;
31298d2f55e7SToby Isaac         PetscInt numStar;
31308d2f55e7SToby Isaac 
31318d2f55e7SToby Isaac         parentCell = -1;
31328d2f55e7SToby Isaac         ierr = DMPlexGetTransitiveClosure(refTree,p,PETSC_FALSE,&numStar,&star);CHKERRQ(ierr);
31338d2f55e7SToby Isaac         for (i = numStar - 1; i >= 0; i--) {
31348d2f55e7SToby Isaac           PetscInt c = star[2 * i];
31358d2f55e7SToby Isaac 
31368d2f55e7SToby Isaac           if (c >= cStart && c < cEnd) {
31378d2f55e7SToby Isaac             parentCell = c;
31388d2f55e7SToby Isaac             break;
31398d2f55e7SToby Isaac           }
31408d2f55e7SToby Isaac         }
31418d2f55e7SToby Isaac         ierr = DMPlexRestoreTransitiveClosure(refTree,p,PETSC_FALSE,&numStar,&star);CHKERRQ(ierr);
31428d2f55e7SToby Isaac       }
31438d2f55e7SToby Isaac       /* determine the offset of p's shape functions withing parentCell's shape functions */
3144c5356c36SToby Isaac       ierr = PetscDSGetDiscretization(ds,f,&disc);CHKERRQ(ierr);
3145c5356c36SToby Isaac       ierr = PetscObjectGetClassId(disc,&classId);CHKERRQ(ierr);
3146c5356c36SToby Isaac       if (classId == PETSCFE_CLASSID) {
3147c5356c36SToby Isaac         ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
3148c5356c36SToby Isaac       }
3149c5356c36SToby Isaac       else if (classId == PETSCFV_CLASSID) {
3150c5356c36SToby Isaac         ierr = PetscFVGetDualSpace((PetscFV)disc,&dsp);CHKERRQ(ierr);
3151c5356c36SToby Isaac       }
3152c5356c36SToby Isaac       else {
31539b90b7cdSMatthew G. Knepley         SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Unsupported discretization object");
3154c5356c36SToby Isaac       }
31558d2f55e7SToby Isaac       ierr = PetscDualSpaceGetNumDof(dsp,&depthNumDof);CHKERRQ(ierr);
315652a3aeb4SToby Isaac       ierr = PetscDualSpaceGetNumComponents(dsp,&Nc);CHKERRQ(ierr);
31578d2f55e7SToby Isaac       {
31588d2f55e7SToby Isaac         PetscInt *closure = NULL;
31598d2f55e7SToby Isaac         PetscInt numClosure;
31608d2f55e7SToby Isaac 
31618d2f55e7SToby Isaac         ierr = DMPlexGetTransitiveClosure(refTree,parentCell,PETSC_TRUE,&numClosure,&closure);CHKERRQ(ierr);
31628d2f55e7SToby Isaac         for (i = 0, cellShapeOff = 0; i < numClosure; i++) {
31638d2f55e7SToby Isaac           PetscInt point = closure[2 * i], pointDepth;
31648d2f55e7SToby Isaac 
31658d2f55e7SToby Isaac           pO = closure[2 * i + 1];
31668d2f55e7SToby Isaac           if (point == p) break;
31678d2f55e7SToby Isaac           ierr = DMLabelGetValue(depth,point,&pointDepth);CHKERRQ(ierr);
31688d2f55e7SToby Isaac           cellShapeOff += depthNumDof[pointDepth];
31698d2f55e7SToby Isaac         }
31708d2f55e7SToby Isaac         ierr = DMPlexRestoreTransitiveClosure(refTree,parentCell,PETSC_TRUE,&numClosure,&closure);CHKERRQ(ierr);
31718d2f55e7SToby Isaac       }
31728d2f55e7SToby Isaac 
317352a3aeb4SToby Isaac       ierr = DMGetWorkArray(refTree, numSelfDof * numChildDof, PETSC_SCALAR,&pointMat);CHKERRQ(ierr);
317452a3aeb4SToby Isaac       ierr = DMGetWorkArray(refTree, numSelfDof + numChildDof, PETSC_INT,&matRows);CHKERRQ(ierr);
317552a3aeb4SToby Isaac       matCols = matRows + numSelfDof;
317652a3aeb4SToby Isaac       for (i = 0; i < numSelfDof; i++) {
317752a3aeb4SToby Isaac         matRows[i] = selfOff + i;
31783b1c2a6aSToby Isaac       }
317952a3aeb4SToby Isaac       for (i = 0; i < numSelfDof * numChildDof; i++) pointMat[i] = 0.;
31803b1c2a6aSToby Isaac       {
31813b1c2a6aSToby Isaac         PetscInt colOff = 0;
31823b1c2a6aSToby Isaac 
31833b1c2a6aSToby Isaac         for (i = 0; i < numChildren; i++) {
31843b1c2a6aSToby Isaac           PetscInt child = children[i];
31853b1c2a6aSToby Isaac           PetscInt dof, off, j;
31863b1c2a6aSToby Isaac 
31873b1c2a6aSToby Isaac           if (numSecFields) {
3188c5356c36SToby Isaac             ierr = PetscSectionGetFieldDof(cSection,child,f,&dof);CHKERRQ(ierr);
3189c5356c36SToby Isaac             ierr = PetscSectionGetFieldOffset(cSection,child,f,&off);CHKERRQ(ierr);
31903b1c2a6aSToby Isaac           }
31913b1c2a6aSToby Isaac           else {
3192c5356c36SToby Isaac             ierr = PetscSectionGetDof(cSection,child,&dof);CHKERRQ(ierr);
3193c5356c36SToby Isaac             ierr = PetscSectionGetOffset(cSection,child,&off);CHKERRQ(ierr);
31943b1c2a6aSToby Isaac           }
31953b1c2a6aSToby Isaac 
319652a3aeb4SToby Isaac           for (j = 0; j < dof; j++) {
319752a3aeb4SToby Isaac             matCols[colOff++] = off + j;
31983b1c2a6aSToby Isaac           }
31993b1c2a6aSToby Isaac         }
32003b1c2a6aSToby Isaac       }
32018d2f55e7SToby Isaac       if (classId == PETSCFE_CLASSID) {
32028d2f55e7SToby Isaac         PetscFE        fe = (PetscFE) disc;
32038d2f55e7SToby Isaac         PetscInt       fSize;
32048d2f55e7SToby Isaac 
32058d2f55e7SToby Isaac         ierr = PetscFEGetDualSpace(fe,&dsp);CHKERRQ(ierr);
32063b1c2a6aSToby Isaac         ierr = PetscDualSpaceGetDimension(dsp,&fSize);CHKERRQ(ierr);
320752a3aeb4SToby Isaac         for (i = 0; i < numSelfDof; i++) { /* for every shape function */
32088d2f55e7SToby Isaac           PetscQuadrature q;
320952a3aeb4SToby Isaac           PetscInt        dim, thisNc, numPoints, j, k;
32108d2f55e7SToby Isaac           const PetscReal *points;
32118d2f55e7SToby Isaac           const PetscReal *weights;
32128d2f55e7SToby Isaac           PetscInt        *closure = NULL;
32138d2f55e7SToby Isaac           PetscInt        numClosure;
321452a3aeb4SToby Isaac           PetscInt        parentCellShapeDof = cellShapeOff + (pO < 0 ? (numSelfDof - 1 - i) : i);
32158d2f55e7SToby Isaac           PetscReal       *Bparent;
32168d2f55e7SToby Isaac 
32173b1c2a6aSToby Isaac           ierr = PetscDualSpaceGetFunctional(dsp,parentCellShapeDof,&q);CHKERRQ(ierr);
321852a3aeb4SToby Isaac           ierr = PetscQuadratureGetData(q,&dim,&thisNc,&numPoints,&points,&weights);CHKERRQ(ierr);
321952a3aeb4SToby Isaac           if (thisNc != Nc) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Functional dim %D does not much basis dim %D\n",thisNc,Nc);
32203b1c2a6aSToby Isaac           ierr = PetscFEGetTabulation(fe,numPoints,points,&Bparent,NULL,NULL);CHKERRQ(ierr); /* I'm expecting a nodal basis: weights[:]' * Bparent[:,cellShapeDof] = 1. */
32213b1c2a6aSToby Isaac           for (j = 0; j < numPoints; j++) {
32228d2f55e7SToby Isaac             PetscInt          childCell = -1;
322352a3aeb4SToby Isaac             PetscReal         *parentValAtPoint;
32248d2f55e7SToby Isaac             const PetscReal   *pointReal = &points[dim * j];
32258d2f55e7SToby Isaac             const PetscScalar *point;
32268d2f55e7SToby Isaac             PetscReal         *Bchild;
32278d2f55e7SToby Isaac             PetscInt          childCellShapeOff, pointMatOff;
32288d2f55e7SToby Isaac #if defined(PETSC_USE_COMPLEX)
32298d2f55e7SToby Isaac             PetscInt          d;
32308d2f55e7SToby Isaac 
32318d2f55e7SToby Isaac             for (d = 0; d < dim; d++) {
32328d2f55e7SToby Isaac               pointScalar[d] = points[dim * j + d];
32338d2f55e7SToby Isaac             }
32348d2f55e7SToby Isaac             point = pointScalar;
32358d2f55e7SToby Isaac #else
32368d2f55e7SToby Isaac             point = pointReal;
32378d2f55e7SToby Isaac #endif
32388d2f55e7SToby Isaac 
323952a3aeb4SToby Isaac             parentValAtPoint = &Bparent[(fSize * j + parentCellShapeDof) * Nc];
32403b1c2a6aSToby Isaac 
32413b1c2a6aSToby Isaac             for (k = 0; k < numChildren; k++) { /* locate the point in a child's star cell*/
32428d2f55e7SToby Isaac               PetscInt child = children[k];
32438d2f55e7SToby Isaac               PetscInt *star = NULL;
32448d2f55e7SToby Isaac               PetscInt numStar, s;
32458d2f55e7SToby Isaac 
32468d2f55e7SToby Isaac               ierr = DMPlexGetTransitiveClosure(refTree,child,PETSC_FALSE,&numStar,&star);CHKERRQ(ierr);
32478d2f55e7SToby Isaac               for (s = numStar - 1; s >= 0; s--) {
32488d2f55e7SToby Isaac                 PetscInt c = star[2 * s];
32498d2f55e7SToby Isaac 
32508d2f55e7SToby Isaac                 if (c < cStart || c >= cEnd) continue;
32518d2f55e7SToby Isaac                 ierr = DMPlexLocatePoint_Internal(refTree,dim,point,c,&childCell);CHKERRQ(ierr);
32528d2f55e7SToby Isaac                 if (childCell >= 0) break;
32538d2f55e7SToby Isaac               }
32548d2f55e7SToby Isaac               ierr = DMPlexRestoreTransitiveClosure(refTree,child,PETSC_FALSE,&numStar,&star);CHKERRQ(ierr);
32558d2f55e7SToby Isaac               if (childCell >= 0) break;
32568d2f55e7SToby Isaac             }
32578d2f55e7SToby Isaac             if (childCell < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Could not locate quadrature point");CHKERRQ(ierr);
32588d2f55e7SToby Isaac             ierr = DMPlexComputeCellGeometryFEM(refTree, childCell, NULL, v0, J, invJ, &detJ);CHKERRQ(ierr);
32598d2f55e7SToby Isaac             ierr = DMPlexComputeCellGeometryFEM(refTree, parentCell, NULL, v0parent, Jparent, NULL, &detJparent);CHKERRQ(ierr);
32608d2f55e7SToby Isaac             CoordinatesRefToReal(dim, dim, v0parent, Jparent, pointReal, vtmp);
32618d2f55e7SToby Isaac             CoordinatesRealToRef(dim, dim, v0, invJ, vtmp, pointRef);
32628d2f55e7SToby Isaac 
32638d2f55e7SToby Isaac             ierr = PetscFEGetTabulation(fe,1,pointRef,&Bchild,NULL,NULL);CHKERRQ(ierr);
32648d2f55e7SToby Isaac             ierr = DMPlexGetTransitiveClosure(refTree,childCell,PETSC_TRUE,&numClosure,&closure);CHKERRQ(ierr);
32653b1c2a6aSToby Isaac             for (k = 0, pointMatOff = 0; k < numChildren; k++) { /* point is located in cell => child dofs support at point are in closure of cell */
3266c5356c36SToby Isaac               PetscInt child = children[k], childDepth, childDof, childO = PETSC_MIN_INT;
32678d2f55e7SToby Isaac               PetscInt l;
32688d2f55e7SToby Isaac 
32698d2f55e7SToby Isaac               ierr = DMLabelGetValue(depth,child,&childDepth);CHKERRQ(ierr);
32708d2f55e7SToby Isaac               childDof = depthNumDof[childDepth];
32718d2f55e7SToby Isaac               for (l = 0, childCellShapeOff = 0; l < numClosure; l++) {
32728d2f55e7SToby Isaac                 PetscInt point = closure[2 * l];
32738d2f55e7SToby Isaac                 PetscInt pointDepth;
32748d2f55e7SToby Isaac 
32758d2f55e7SToby Isaac                 childO = closure[2 * l + 1];
32768d2f55e7SToby Isaac                 if (point == child) break;
32778d2f55e7SToby Isaac                 ierr = DMLabelGetValue(depth,point,&pointDepth);CHKERRQ(ierr);
32788d2f55e7SToby Isaac                 childCellShapeOff += depthNumDof[pointDepth];
32798d2f55e7SToby Isaac               }
32808d2f55e7SToby Isaac               if (l == numClosure) {
32818d2f55e7SToby Isaac                 pointMatOff += childDof;
32828d2f55e7SToby Isaac                 continue; /* child is not in the closure of the cell: has nothing to contribute to this point */
32838d2f55e7SToby Isaac               }
32848d2f55e7SToby Isaac               for (l = 0; l < childDof; l++) {
328552a3aeb4SToby Isaac                 PetscInt    childCellDof = childCellShapeOff + (childO ? (childDof - 1 - l) : l), m;
328652a3aeb4SToby Isaac                 PetscReal   *childValAtPoint;
328752a3aeb4SToby Isaac                 PetscReal   val = 0.;
32888d2f55e7SToby Isaac 
328952a3aeb4SToby Isaac                 childValAtPoint = &Bchild[childCellDof * Nc];
329052a3aeb4SToby Isaac                 for (m = 0; m < Nc; m++) {
329152a3aeb4SToby Isaac                   val += weights[j * Nc + m] * parentValAtPoint[m] * childValAtPoint[m];
329252a3aeb4SToby Isaac                 }
329352a3aeb4SToby Isaac 
329452a3aeb4SToby Isaac                 pointMat[i * numChildDof + pointMatOff + l] += val;
32958d2f55e7SToby Isaac               }
32968d2f55e7SToby Isaac               pointMatOff += childDof;
32978d2f55e7SToby Isaac             }
32988d2f55e7SToby Isaac             ierr = DMPlexRestoreTransitiveClosure(refTree,childCell,PETSC_TRUE,&numClosure,&closure);CHKERRQ(ierr);
32998d2f55e7SToby Isaac             ierr = PetscFERestoreTabulation(fe,1,pointRef,&Bchild,NULL,NULL);CHKERRQ(ierr);
33008d2f55e7SToby Isaac           }
33018d2f55e7SToby Isaac           ierr = PetscFERestoreTabulation(fe,numPoints,points,&Bparent,NULL,NULL);CHKERRQ(ierr);
33028d2f55e7SToby Isaac         }
33038d2f55e7SToby Isaac       }
3304c5356c36SToby Isaac       else { /* just the volume-weighted averages of the children */
33053b1c2a6aSToby Isaac         PetscReal parentVol;
3306bfaa5bdcSToby Isaac         PetscInt  childCell;
33073b1c2a6aSToby Isaac 
33083b1c2a6aSToby Isaac         ierr = DMPlexComputeCellGeometryFVM(refTree, p, &parentVol, NULL, NULL);CHKERRQ(ierr);
3309bfaa5bdcSToby Isaac         for (i = 0, childCell = 0; i < numChildren; i++) {
331052a3aeb4SToby Isaac           PetscInt  child = children[i], j;
33113b1c2a6aSToby Isaac           PetscReal childVol;
33123b1c2a6aSToby Isaac 
33133b1c2a6aSToby Isaac           if (child < cStart || child >= cEnd) continue;
33143b1c2a6aSToby Isaac           ierr = DMPlexComputeCellGeometryFVM(refTree, child, &childVol, NULL, NULL);CHKERRQ(ierr);
331552a3aeb4SToby Isaac           for (j = 0; j < Nc; j++) {
3316bfaa5bdcSToby Isaac             pointMat[j * numChildDof + Nc * childCell + j] = childVol / parentVol;
331752a3aeb4SToby Isaac           }
3318bfaa5bdcSToby Isaac           childCell++;
33193b1c2a6aSToby Isaac         }
33208d2f55e7SToby Isaac       }
33213b1c2a6aSToby Isaac       /* Insert pointMat into mat */
332252a3aeb4SToby Isaac       ierr = MatSetValues(mat,numSelfDof,matRows,numChildDof,matCols,pointMat,INSERT_VALUES);CHKERRQ(ierr);
332352a3aeb4SToby Isaac       ierr = DMRestoreWorkArray(refTree, numSelfDof + numChildDof, PETSC_INT,&matRows);CHKERRQ(ierr);
332452a3aeb4SToby Isaac       ierr = DMRestoreWorkArray(refTree, numSelfDof * numChildDof, PETSC_SCALAR,&pointMat);CHKERRQ(ierr);
33258d2f55e7SToby Isaac     }
33268d2f55e7SToby Isaac   }
33273b1c2a6aSToby Isaac   ierr = PetscFree6(v0,v0parent,vtmp,J,Jparent,invJ);CHKERRQ(ierr);
33288d2f55e7SToby Isaac   ierr = PetscFree2(pointScalar,pointRef);CHKERRQ(ierr);
33293b1c2a6aSToby Isaac   ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
33303b1c2a6aSToby Isaac   ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
33318d2f55e7SToby Isaac   *inj = mat;
33328d2f55e7SToby Isaac   PetscFunctionReturn(0);
33338d2f55e7SToby Isaac }
33348d2f55e7SToby Isaac 
3335f30e825dSToby Isaac static PetscErrorCode DMPlexReferenceTreeGetChildrenMatrices_Injection(DM refTree, Mat inj, PetscScalar ****childrenMats)
3336f30e825dSToby Isaac {
3337f30e825dSToby Isaac   PetscDS        ds;
3338f30e825dSToby Isaac   PetscInt       numFields, f, pRefStart, pRefEnd, p, *rows, *cols, maxDof;
3339f30e825dSToby Isaac   PetscScalar    ***refPointFieldMats;
3340f30e825dSToby Isaac   PetscSection   refConSec, refSection;
3341f30e825dSToby Isaac   PetscErrorCode ierr;
3342f30e825dSToby Isaac 
3343f30e825dSToby Isaac   PetscFunctionBegin;
3344f30e825dSToby Isaac   ierr = DMGetDS(refTree,&ds);CHKERRQ(ierr);
3345f30e825dSToby Isaac   ierr = PetscDSGetNumFields(ds,&numFields);CHKERRQ(ierr);
3346f30e825dSToby Isaac   ierr = DMGetDefaultConstraints(refTree,&refConSec,NULL);CHKERRQ(ierr);
3347f30e825dSToby Isaac   ierr = DMGetDefaultSection(refTree,&refSection);CHKERRQ(ierr);
3348f30e825dSToby Isaac   ierr = PetscSectionGetChart(refConSec,&pRefStart,&pRefEnd);CHKERRQ(ierr);
3349f30e825dSToby Isaac   ierr = PetscMalloc1(pRefEnd-pRefStart,&refPointFieldMats);CHKERRQ(ierr);
3350f30e825dSToby Isaac   ierr = PetscSectionGetMaxDof(refConSec,&maxDof);CHKERRQ(ierr);
3351f30e825dSToby Isaac   ierr = PetscMalloc1(maxDof,&rows);CHKERRQ(ierr);
3352f30e825dSToby Isaac   ierr = PetscMalloc1(maxDof*maxDof,&cols);CHKERRQ(ierr);
3353f30e825dSToby Isaac   for (p = pRefStart; p < pRefEnd; p++) {
3354f30e825dSToby Isaac     PetscInt parent, pDof, parentDof;
3355f30e825dSToby Isaac 
3356f30e825dSToby Isaac     ierr = DMPlexGetTreeParent(refTree,p,&parent,NULL);CHKERRQ(ierr);
3357f30e825dSToby Isaac     ierr = PetscSectionGetDof(refConSec,p,&pDof);CHKERRQ(ierr);
3358c6154584SToby Isaac     ierr = PetscSectionGetDof(refSection,parent,&parentDof);CHKERRQ(ierr);
3359f30e825dSToby Isaac     if (!pDof || !parentDof || parent == p) continue;
3360f30e825dSToby Isaac 
3361f30e825dSToby Isaac     ierr = PetscMalloc1(numFields,&refPointFieldMats[p-pRefStart]);CHKERRQ(ierr);
3362f30e825dSToby Isaac     for (f = 0; f < numFields; f++) {
336352a3aeb4SToby Isaac       PetscInt cDof, cOff, numCols, r;
3364f30e825dSToby Isaac 
3365f30e825dSToby Isaac       if (numFields > 1) {
3366f30e825dSToby Isaac         ierr = PetscSectionGetFieldDof(refConSec,p,f,&cDof);CHKERRQ(ierr);
3367f30e825dSToby Isaac         ierr = PetscSectionGetFieldOffset(refConSec,p,f,&cOff);CHKERRQ(ierr);
3368f30e825dSToby Isaac       }
3369f30e825dSToby Isaac       else {
3370f30e825dSToby Isaac         ierr = PetscSectionGetDof(refConSec,p,&cDof);CHKERRQ(ierr);
3371f30e825dSToby Isaac         ierr = PetscSectionGetOffset(refConSec,p,&cOff);CHKERRQ(ierr);
3372f30e825dSToby Isaac       }
3373f30e825dSToby Isaac 
3374f30e825dSToby Isaac       for (r = 0; r < cDof; r++) {
3375f30e825dSToby Isaac         rows[r] = cOff + r;
3376f30e825dSToby Isaac       }
3377f30e825dSToby Isaac       numCols = 0;
3378f30e825dSToby Isaac       {
3379f30e825dSToby Isaac         PetscInt aDof, aOff, j;
3380f30e825dSToby Isaac 
3381f30e825dSToby Isaac         if (numFields > 1) {
3382f30e825dSToby Isaac           ierr = PetscSectionGetFieldDof(refSection,parent,f,&aDof);CHKERRQ(ierr);
3383f30e825dSToby Isaac           ierr = PetscSectionGetFieldOffset(refSection,parent,f,&aOff);CHKERRQ(ierr);
3384f30e825dSToby Isaac         }
3385f30e825dSToby Isaac         else {
3386f30e825dSToby Isaac           ierr = PetscSectionGetDof(refSection,parent,&aDof);CHKERRQ(ierr);
3387f30e825dSToby Isaac           ierr = PetscSectionGetOffset(refSection,parent,&aOff);CHKERRQ(ierr);
3388f30e825dSToby Isaac         }
3389f30e825dSToby Isaac 
3390f30e825dSToby Isaac         for (j = 0; j < aDof; j++) {
3391f30e825dSToby Isaac           cols[numCols++] = aOff + j;
3392f30e825dSToby Isaac         }
3393f30e825dSToby Isaac       }
3394f30e825dSToby Isaac       ierr = PetscMalloc1(cDof*numCols,&refPointFieldMats[p-pRefStart][f]);CHKERRQ(ierr);
3395f30e825dSToby Isaac       /* transpose of constraint matrix */
3396f30e825dSToby Isaac       ierr = MatGetValues(inj,numCols,cols,cDof,rows,refPointFieldMats[p-pRefStart][f]);CHKERRQ(ierr);
3397f30e825dSToby Isaac     }
3398f30e825dSToby Isaac   }
3399f30e825dSToby Isaac   *childrenMats = refPointFieldMats;
3400f30e825dSToby Isaac   ierr = PetscFree(rows);CHKERRQ(ierr);
3401f30e825dSToby Isaac   ierr = PetscFree(cols);CHKERRQ(ierr);
3402f30e825dSToby Isaac   PetscFunctionReturn(0);
3403f30e825dSToby Isaac }
3404f30e825dSToby Isaac 
3405f30e825dSToby Isaac static PetscErrorCode DMPlexReferenceTreeRestoreChildrenMatrices_Injection(DM refTree, Mat inj, PetscScalar ****childrenMats)
3406f30e825dSToby Isaac {
3407f30e825dSToby Isaac   PetscDS        ds;
3408f30e825dSToby Isaac   PetscScalar    ***refPointFieldMats;
3409f30e825dSToby Isaac   PetscInt       numFields, pRefStart, pRefEnd, p, f;
3410c6154584SToby Isaac   PetscSection   refConSec, refSection;
3411f30e825dSToby Isaac   PetscErrorCode ierr;
3412f30e825dSToby Isaac 
3413f30e825dSToby Isaac   PetscFunctionBegin;
3414f30e825dSToby Isaac   refPointFieldMats = *childrenMats;
3415f30e825dSToby Isaac   *childrenMats = NULL;
3416f30e825dSToby Isaac   ierr = DMGetDS(refTree,&ds);CHKERRQ(ierr);
3417c6154584SToby Isaac   ierr = DMGetDefaultSection(refTree,&refSection);CHKERRQ(ierr);
3418f30e825dSToby Isaac   ierr = PetscDSGetNumFields(ds,&numFields);CHKERRQ(ierr);
3419f30e825dSToby Isaac   ierr = DMGetDefaultConstraints(refTree,&refConSec,NULL);CHKERRQ(ierr);
3420f30e825dSToby Isaac   ierr = PetscSectionGetChart(refConSec,&pRefStart,&pRefEnd);CHKERRQ(ierr);
3421f30e825dSToby Isaac   for (p = pRefStart; p < pRefEnd; p++) {
3422f30e825dSToby Isaac     PetscInt parent, pDof, parentDof;
3423f30e825dSToby Isaac 
3424f30e825dSToby Isaac     ierr = DMPlexGetTreeParent(refTree,p,&parent,NULL);CHKERRQ(ierr);
3425f30e825dSToby Isaac     ierr = PetscSectionGetDof(refConSec,p,&pDof);CHKERRQ(ierr);
3426c6154584SToby Isaac     ierr = PetscSectionGetDof(refSection,parent,&parentDof);CHKERRQ(ierr);
3427f30e825dSToby Isaac     if (!pDof || !parentDof || parent == p) continue;
3428f30e825dSToby Isaac 
3429f30e825dSToby Isaac     for (f = 0; f < numFields; f++) {
3430f30e825dSToby Isaac       PetscInt cDof;
3431f30e825dSToby Isaac 
3432f30e825dSToby Isaac       if (numFields > 1) {
3433f30e825dSToby Isaac         ierr = PetscSectionGetFieldDof(refConSec,p,f,&cDof);CHKERRQ(ierr);
3434f30e825dSToby Isaac       }
3435f30e825dSToby Isaac       else {
3436f30e825dSToby Isaac         ierr = PetscSectionGetDof(refConSec,p,&cDof);CHKERRQ(ierr);
3437f30e825dSToby Isaac       }
3438f30e825dSToby Isaac 
3439f30e825dSToby Isaac       ierr = PetscFree(refPointFieldMats[p - pRefStart][f]);CHKERRQ(ierr);
3440f30e825dSToby Isaac     }
3441f30e825dSToby Isaac     ierr = PetscFree(refPointFieldMats[p - pRefStart]);CHKERRQ(ierr);
3442f30e825dSToby Isaac   }
3443f30e825dSToby Isaac   ierr = PetscFree(refPointFieldMats);CHKERRQ(ierr);
3444f30e825dSToby Isaac   PetscFunctionReturn(0);
3445f30e825dSToby Isaac }
3446f30e825dSToby Isaac 
3447ebf164c7SToby Isaac static PetscErrorCode DMPlexReferenceTreeGetInjector(DM refTree,Mat *injRef)
3448154bca37SToby Isaac {
3449ebf164c7SToby Isaac   Mat            cMatRef;
34506148253fSToby Isaac   PetscObject    injRefObj;
34518d2f55e7SToby Isaac   PetscErrorCode ierr;
34528d2f55e7SToby Isaac 
3453154bca37SToby Isaac   PetscFunctionBegin;
3454ebf164c7SToby Isaac   ierr = DMGetDefaultConstraints(refTree,NULL,&cMatRef);CHKERRQ(ierr);
34556148253fSToby Isaac   ierr = PetscObjectQuery((PetscObject)cMatRef,"DMPlexComputeInjectorTree_refTree",&injRefObj);CHKERRQ(ierr);
3456ebf164c7SToby Isaac   *injRef = (Mat) injRefObj;
3457ebf164c7SToby Isaac   if (!*injRef) {
3458ebf164c7SToby Isaac     ierr = DMPlexComputeInjectorReferenceTree(refTree,injRef);CHKERRQ(ierr);
3459ebf164c7SToby Isaac     ierr = PetscObjectCompose((PetscObject)cMatRef,"DMPlexComputeInjectorTree_refTree",(PetscObject)*injRef);CHKERRQ(ierr);
3460ec92bd66SToby Isaac     /* there is now a reference in cMatRef, which should be the only one for symmetry with the above case */
3461ebf164c7SToby Isaac     ierr = PetscObjectDereference((PetscObject)*injRef);CHKERRQ(ierr);
3462ebf164c7SToby Isaac   }
3463ebf164c7SToby Isaac   PetscFunctionReturn(0);
34646148253fSToby Isaac }
3465f30e825dSToby Isaac 
3466c921d74cSToby Isaac static PetscErrorCode DMPlexTransferInjectorTree(DM coarse, DM fine, PetscSF coarseToFine, const PetscInt *childIds, Vec fineVec, PetscInt numFields, PetscInt *offsets, PetscSection *rootMultiSec, PetscSection *multiLeafSec, PetscInt **gatheredIndices, PetscScalar **gatheredValues)
3467ebf164c7SToby Isaac {
3468c921d74cSToby Isaac   PetscInt       pStartF, pEndF, pStartC, pEndC, p, maxDof, numMulti;
3469ebf164c7SToby Isaac   PetscSection   globalCoarse, globalFine;
3470ebf164c7SToby Isaac   PetscSection   localCoarse, localFine, leafIndicesSec;
3471c921d74cSToby Isaac   PetscSection   multiRootSec, rootIndicesSec;
3472c921d74cSToby Isaac   PetscInt       *leafInds, *rootInds = NULL;
3473c921d74cSToby Isaac   const PetscInt *rootDegrees;
3474c921d74cSToby Isaac   PetscScalar    *leafVals = NULL, *rootVals = NULL;
3475ebf164c7SToby Isaac   PetscSF        coarseToFineEmbedded;
3476ebf164c7SToby Isaac   PetscErrorCode ierr;
3477ebf164c7SToby Isaac 
3478ebf164c7SToby Isaac   PetscFunctionBegin;
3479ebf164c7SToby Isaac   ierr = DMPlexGetChart(coarse,&pStartC,&pEndC);CHKERRQ(ierr);
34808d2f55e7SToby Isaac   ierr = DMPlexGetChart(fine,&pStartF,&pEndF);CHKERRQ(ierr);
3481f30e825dSToby Isaac   ierr = DMGetDefaultSection(fine,&localFine);CHKERRQ(ierr);
34828d2f55e7SToby Isaac   ierr = DMGetDefaultGlobalSection(fine,&globalFine);CHKERRQ(ierr);
3483f30e825dSToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)fine),&leafIndicesSec);CHKERRQ(ierr);
3484f30e825dSToby Isaac   ierr = PetscSectionSetChart(leafIndicesSec,pStartF, pEndF);CHKERRQ(ierr);
3485c921d74cSToby Isaac   ierr = PetscSectionGetMaxDof(localFine,&maxDof);CHKERRQ(ierr);
34868d2f55e7SToby Isaac   { /* winnow fine points that don't have global dofs out of the sf */
34877e96bdafSToby Isaac     PetscInt l, nleaves, dof, cdof, numPointsWithDofs, offset, *pointsWithDofs, numIndices;
34887e96bdafSToby Isaac     const PetscInt *leaves;
34898d2f55e7SToby Isaac 
34907e96bdafSToby Isaac     ierr = PetscSFGetGraph(coarseToFine,NULL,&nleaves,&leaves,NULL);CHKERRQ(ierr);
34917e96bdafSToby Isaac     for (l = 0, numPointsWithDofs = 0; l < nleaves; l++) {
34927e96bdafSToby Isaac       p    = leaves ? leaves[l] : l;
34938d2f55e7SToby Isaac       ierr = PetscSectionGetDof(globalFine,p,&dof);CHKERRQ(ierr);
34948d2f55e7SToby Isaac       ierr = PetscSectionGetConstraintDof(globalFine,p,&cdof);CHKERRQ(ierr);
34958d2f55e7SToby Isaac       if ((dof - cdof) > 0) {
34968d2f55e7SToby Isaac         numPointsWithDofs++;
3497f30e825dSToby Isaac 
3498f30e825dSToby Isaac         ierr = PetscSectionGetDof(localFine,p,&dof);CHKERRQ(ierr);
3499f30e825dSToby Isaac         ierr = PetscSectionSetDof(leafIndicesSec,p,dof + 1);CHKERRQ(ierr);
35008d2f55e7SToby Isaac       }
35018d2f55e7SToby Isaac     }
35028d2f55e7SToby Isaac     ierr = PetscMalloc1(numPointsWithDofs,&pointsWithDofs);CHKERRQ(ierr);
3503f30e825dSToby Isaac     ierr = PetscSectionSetUp(leafIndicesSec);CHKERRQ(ierr);
3504f30e825dSToby Isaac     ierr = PetscSectionGetStorageSize(leafIndicesSec,&numIndices);CHKERRQ(ierr);
3505c921d74cSToby Isaac     ierr = PetscMalloc1(gatheredIndices ? numIndices : (maxDof + 1),&leafInds);CHKERRQ(ierr);
3506c921d74cSToby Isaac     if (gatheredValues)  {ierr = PetscMalloc1(numIndices,&leafVals);CHKERRQ(ierr);}
35077e96bdafSToby Isaac     for (l = 0, offset = 0; l < nleaves; l++) {
35087e96bdafSToby Isaac       p    = leaves ? leaves[l] : l;
35098d2f55e7SToby Isaac       ierr = PetscSectionGetDof(globalFine,p,&dof);CHKERRQ(ierr);
35108d2f55e7SToby Isaac       ierr = PetscSectionGetConstraintDof(globalFine,p,&cdof);CHKERRQ(ierr);
35118d2f55e7SToby Isaac       if ((dof - cdof) > 0) {
3512f30e825dSToby Isaac         PetscInt    off, gOff;
3513f30e825dSToby Isaac         PetscInt    *pInd;
3514c921d74cSToby Isaac         PetscScalar *pVal = NULL;
3515f30e825dSToby Isaac 
35167e96bdafSToby Isaac         pointsWithDofs[offset++] = l;
3517f30e825dSToby Isaac 
3518f30e825dSToby Isaac         ierr = PetscSectionGetOffset(leafIndicesSec,p,&off);CHKERRQ(ierr);
3519f30e825dSToby Isaac 
3520c921d74cSToby Isaac         pInd = gatheredIndices ? (&leafInds[off + 1]) : leafInds;
3521c921d74cSToby Isaac         if (gatheredValues) {
3522c921d74cSToby Isaac           PetscInt i;
3523c921d74cSToby Isaac 
3524c921d74cSToby Isaac           pVal = &leafVals[off + 1];
3525c921d74cSToby Isaac           for (i = 0; i < dof; i++) pVal[i] = 0.;
3526c921d74cSToby Isaac         }
3527f30e825dSToby Isaac         ierr = PetscSectionGetOffset(globalFine,p,&gOff);CHKERRQ(ierr);
3528f30e825dSToby Isaac 
3529f30e825dSToby Isaac         offsets[0] = 0;
3530f30e825dSToby Isaac         if (numFields) {
3531f30e825dSToby Isaac           PetscInt f;
3532f30e825dSToby Isaac 
3533f30e825dSToby Isaac           for (f = 0; f < numFields; f++) {
3534f30e825dSToby Isaac             PetscInt fDof;
3535f30e825dSToby Isaac             ierr = PetscSectionGetFieldDof(localFine,p,f,&fDof);CHKERRQ(ierr);
3536f30e825dSToby Isaac             offsets[f + 1] = fDof + offsets[f];
3537f30e825dSToby Isaac           }
35384acb8e1eSToby Isaac           DMPlexGetIndicesPointFields_Internal(localFine,p,gOff < 0 ? -(gOff + 1) : gOff,offsets,PETSC_FALSE,NULL,-1,pInd);CHKERRQ(ierr);
3539f30e825dSToby Isaac         }
3540f30e825dSToby Isaac         else {
35414acb8e1eSToby Isaac           DMPlexGetIndicesPoint_Internal(localFine,p,gOff < 0 ? -(gOff + 1) : gOff,offsets,PETSC_FALSE,NULL,pInd);CHKERRQ(ierr);
3542f30e825dSToby Isaac         }
3543c921d74cSToby Isaac         if (gatheredValues) {ierr = VecGetValues(fineVec,dof,pInd,pVal);CHKERRQ(ierr);}
35448d2f55e7SToby Isaac       }
35458d2f55e7SToby Isaac     }
35468d2f55e7SToby Isaac     ierr = PetscSFCreateEmbeddedLeafSF(coarseToFine, numPointsWithDofs, pointsWithDofs, &coarseToFineEmbedded);CHKERRQ(ierr);
3547f30e825dSToby Isaac     ierr = PetscFree(pointsWithDofs);CHKERRQ(ierr);
35488d2f55e7SToby Isaac   }
3549f30e825dSToby Isaac 
3550f30e825dSToby Isaac   ierr = DMPlexGetChart(coarse,&pStartC,&pEndC);CHKERRQ(ierr);
3551f30e825dSToby Isaac   ierr = DMGetDefaultSection(coarse,&localCoarse);CHKERRQ(ierr);
3552f30e825dSToby Isaac   ierr = DMGetDefaultGlobalSection(coarse,&globalCoarse);CHKERRQ(ierr);
3553f30e825dSToby Isaac 
35546148253fSToby Isaac   { /* there may be the case where an sf root has a parent: broadcast parents back to children */
35556148253fSToby Isaac     MPI_Datatype threeInt;
35566148253fSToby Isaac     PetscMPIInt  rank;
35576148253fSToby Isaac     PetscInt     (*parentNodeAndIdCoarse)[3];
35586148253fSToby Isaac     PetscInt     (*parentNodeAndIdFine)[3];
35596148253fSToby Isaac     PetscInt     p, nleaves, nleavesToParents;
35606148253fSToby Isaac     PetscSF      pointSF, sfToParents;
35616148253fSToby Isaac     const PetscInt *ilocal;
35626148253fSToby Isaac     const PetscSFNode *iremote;
35636148253fSToby Isaac     PetscSFNode  *iremoteToParents;
35646148253fSToby Isaac     PetscInt     *ilocalToParents;
35656148253fSToby Isaac 
35666148253fSToby Isaac     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)coarse),&rank);CHKERRQ(ierr);
35676148253fSToby Isaac     ierr = MPI_Type_contiguous(3,MPIU_INT,&threeInt);CHKERRQ(ierr);
35686148253fSToby Isaac     ierr = MPI_Type_commit(&threeInt);CHKERRQ(ierr);
35696148253fSToby Isaac     ierr = PetscMalloc2(pEndC-pStartC,&parentNodeAndIdCoarse,pEndF-pStartF,&parentNodeAndIdFine);CHKERRQ(ierr);
35706148253fSToby Isaac     ierr = DMGetPointSF(coarse,&pointSF);CHKERRQ(ierr);
35716148253fSToby Isaac     ierr = PetscSFGetGraph(pointSF,NULL,&nleaves,&ilocal,&iremote);CHKERRQ(ierr);
35726148253fSToby Isaac     for (p = pStartC; p < pEndC; p++) {
35736148253fSToby Isaac       PetscInt parent, childId;
35746148253fSToby Isaac       ierr = DMPlexGetTreeParent(coarse,p,&parent,&childId);CHKERRQ(ierr);
35756148253fSToby Isaac       parentNodeAndIdCoarse[p - pStartC][0] = rank;
35766148253fSToby Isaac       parentNodeAndIdCoarse[p - pStartC][1] = parent - pStartC;
35776148253fSToby Isaac       parentNodeAndIdCoarse[p - pStartC][2] = (p == parent) ? -1 : childId;
35786148253fSToby Isaac       if (nleaves > 0) {
35796148253fSToby Isaac         PetscInt leaf = -1;
35806148253fSToby Isaac 
35816148253fSToby Isaac         if (ilocal) {
35826148253fSToby Isaac           ierr  = PetscFindInt(parent,nleaves,ilocal,&leaf);CHKERRQ(ierr);
35836148253fSToby Isaac         }
35846148253fSToby Isaac         else {
35856148253fSToby Isaac           leaf = p - pStartC;
35866148253fSToby Isaac         }
35876148253fSToby Isaac         if (leaf >= 0) {
35886148253fSToby Isaac           parentNodeAndIdCoarse[p - pStartC][0] = iremote[leaf].rank;
35896148253fSToby Isaac           parentNodeAndIdCoarse[p - pStartC][1] = iremote[leaf].index;
35906148253fSToby Isaac         }
35916148253fSToby Isaac       }
35926148253fSToby Isaac     }
35936148253fSToby Isaac     for (p = pStartF; p < pEndF; p++) {
35946148253fSToby Isaac       parentNodeAndIdFine[p - pStartF][0] = -1;
35956148253fSToby Isaac       parentNodeAndIdFine[p - pStartF][1] = -1;
35966148253fSToby Isaac       parentNodeAndIdFine[p - pStartF][2] = -1;
35976148253fSToby Isaac     }
35986148253fSToby Isaac     ierr = PetscSFBcastBegin(coarseToFineEmbedded,threeInt,parentNodeAndIdCoarse,parentNodeAndIdFine);CHKERRQ(ierr);
35996148253fSToby Isaac     ierr = PetscSFBcastEnd(coarseToFineEmbedded,threeInt,parentNodeAndIdCoarse,parentNodeAndIdFine);CHKERRQ(ierr);
36006148253fSToby Isaac     for (p = pStartF, nleavesToParents = 0; p < pEndF; p++) {
3601f30e825dSToby Isaac       PetscInt dof;
3602f30e825dSToby Isaac 
3603f30e825dSToby Isaac       ierr = PetscSectionGetDof(leafIndicesSec,p,&dof);CHKERRQ(ierr);
3604f30e825dSToby Isaac       if (dof) {
3605f30e825dSToby Isaac         PetscInt off;
3606f30e825dSToby Isaac 
3607f30e825dSToby Isaac         ierr = PetscSectionGetOffset(leafIndicesSec,p,&off);CHKERRQ(ierr);
3608c921d74cSToby Isaac         if (gatheredIndices) {
3609c921d74cSToby Isaac           leafInds[off] = PetscMax(childIds[p-pStartF],parentNodeAndIdFine[p-pStartF][2]);
3610c921d74cSToby Isaac         } else if (gatheredValues) {
3611c921d74cSToby Isaac           leafVals[off] = (PetscScalar) PetscMax(childIds[p-pStartF],parentNodeAndIdFine[p-pStartF][2]);
3612c921d74cSToby Isaac         }
3613f30e825dSToby Isaac       }
36146148253fSToby Isaac       if (parentNodeAndIdFine[p-pStartF][0] >= 0) {
36156148253fSToby Isaac         nleavesToParents++;
36166148253fSToby Isaac       }
36176148253fSToby Isaac     }
36186148253fSToby Isaac     ierr = PetscMalloc1(nleavesToParents,&ilocalToParents);CHKERRQ(ierr);
36196148253fSToby Isaac     ierr = PetscMalloc1(nleavesToParents,&iremoteToParents);CHKERRQ(ierr);
36206148253fSToby Isaac     for (p = pStartF, nleavesToParents = 0; p < pEndF; p++) {
36216148253fSToby Isaac       if (parentNodeAndIdFine[p-pStartF][0] >= 0) {
36226148253fSToby Isaac         ilocalToParents[nleavesToParents] = p - pStartF;
36236148253fSToby Isaac         iremoteToParents[nleavesToParents].rank  = parentNodeAndIdFine[p-pStartF][0];
36246148253fSToby Isaac         iremoteToParents[nleavesToParents].index = parentNodeAndIdFine[p-pStartF][1];
36256148253fSToby Isaac         nleavesToParents++;
36266148253fSToby Isaac       }
36276148253fSToby Isaac     }
36286148253fSToby Isaac     ierr = PetscSFCreate(PetscObjectComm((PetscObject)coarse),&sfToParents);CHKERRQ(ierr);
36296148253fSToby Isaac     ierr = PetscSFSetGraph(sfToParents,pEndC-pStartC,nleavesToParents,ilocalToParents,PETSC_OWN_POINTER,iremoteToParents,PETSC_OWN_POINTER);CHKERRQ(ierr);
36306148253fSToby Isaac     ierr = PetscSFDestroy(&coarseToFineEmbedded);CHKERRQ(ierr);
36316148253fSToby Isaac 
36326148253fSToby Isaac     coarseToFineEmbedded = sfToParents;
36336148253fSToby Isaac 
36346148253fSToby Isaac     ierr = PetscFree2(parentNodeAndIdCoarse,parentNodeAndIdFine);CHKERRQ(ierr);
36356148253fSToby Isaac     ierr = MPI_Type_free(&threeInt);CHKERRQ(ierr);
36366148253fSToby Isaac   }
3637f30e825dSToby Isaac 
36386148253fSToby Isaac   { /* winnow out coarse points that don't have dofs */
36396148253fSToby Isaac     PetscInt dof, cdof, numPointsWithDofs, offset, *pointsWithDofs;
36406148253fSToby Isaac     PetscSF  sfDofsOnly;
36416148253fSToby Isaac 
36426148253fSToby Isaac     for (p = pStartC, numPointsWithDofs = 0; p < pEndC; p++) {
36436148253fSToby Isaac       ierr = PetscSectionGetDof(globalCoarse,p,&dof);CHKERRQ(ierr);
36446148253fSToby Isaac       ierr = PetscSectionGetConstraintDof(globalCoarse,p,&cdof);CHKERRQ(ierr);
36456148253fSToby Isaac       if ((dof - cdof) > 0) {
36466148253fSToby Isaac         numPointsWithDofs++;
36476148253fSToby Isaac       }
36486148253fSToby Isaac     }
36496148253fSToby Isaac     ierr = PetscMalloc1(numPointsWithDofs,&pointsWithDofs);CHKERRQ(ierr);
36506148253fSToby Isaac     for (p = pStartC, offset = 0; p < pEndC; p++) {
36516148253fSToby Isaac       ierr = PetscSectionGetDof(globalCoarse,p,&dof);CHKERRQ(ierr);
36526148253fSToby Isaac       ierr = PetscSectionGetConstraintDof(globalCoarse,p,&cdof);CHKERRQ(ierr);
36536148253fSToby Isaac       if ((dof - cdof) > 0) {
3654e03d9830SToby Isaac         pointsWithDofs[offset++] = p - pStartC;
36556148253fSToby Isaac       }
36566148253fSToby Isaac     }
36576148253fSToby Isaac     ierr = PetscSFCreateEmbeddedSF(coarseToFineEmbedded, numPointsWithDofs, pointsWithDofs, &sfDofsOnly);CHKERRQ(ierr);
36586148253fSToby Isaac     ierr = PetscSFDestroy(&coarseToFineEmbedded);CHKERRQ(ierr);
3659f30e825dSToby Isaac     ierr = PetscFree(pointsWithDofs);CHKERRQ(ierr);
36606148253fSToby Isaac     coarseToFineEmbedded = sfDofsOnly;
36616148253fSToby Isaac   }
3662f30e825dSToby Isaac 
36636148253fSToby Isaac   /* communicate back to the coarse mesh which coarse points have children (that may require injection) */
3664f30e825dSToby Isaac   ierr = PetscSFComputeDegreeBegin(coarseToFineEmbedded,&rootDegrees);CHKERRQ(ierr);
3665f30e825dSToby Isaac   ierr = PetscSFComputeDegreeEnd(coarseToFineEmbedded,&rootDegrees);CHKERRQ(ierr);
3666f30e825dSToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)coarse),&multiRootSec);CHKERRQ(ierr);
3667f30e825dSToby Isaac   ierr = PetscSectionSetChart(multiRootSec,pStartC,pEndC);CHKERRQ(ierr);
36688d2f55e7SToby Isaac   for (p = pStartC; p < pEndC; p++) {
3669f30e825dSToby Isaac     ierr = PetscSectionSetDof(multiRootSec,p,rootDegrees[p-pStartC]);CHKERRQ(ierr);
36708d2f55e7SToby Isaac   }
3671f30e825dSToby Isaac   ierr = PetscSectionSetUp(multiRootSec);CHKERRQ(ierr);
3672f30e825dSToby Isaac   ierr = PetscSectionGetStorageSize(multiRootSec,&numMulti);CHKERRQ(ierr);
36738d2f55e7SToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)coarse),&rootIndicesSec);CHKERRQ(ierr);
3674f30e825dSToby Isaac   { /* distribute the leaf section */
3675f30e825dSToby Isaac     PetscSF multi, multiInv, indicesSF;
3676f30e825dSToby Isaac     PetscInt *remoteOffsets, numRootIndices;
36778d2f55e7SToby Isaac 
3678f30e825dSToby Isaac     ierr = PetscSFGetMultiSF(coarseToFineEmbedded,&multi);CHKERRQ(ierr);
3679f30e825dSToby Isaac     ierr = PetscSFCreateInverseSF(multi,&multiInv);CHKERRQ(ierr);
3680f30e825dSToby Isaac     ierr = PetscSFDistributeSection(multiInv,leafIndicesSec,&remoteOffsets,rootIndicesSec);CHKERRQ(ierr);
3681f30e825dSToby Isaac     ierr = PetscSFCreateSectionSF(multiInv,leafIndicesSec,remoteOffsets,rootIndicesSec,&indicesSF);CHKERRQ(ierr);
3682f30e825dSToby Isaac     ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
3683f30e825dSToby Isaac     ierr = PetscSFDestroy(&multiInv);CHKERRQ(ierr);
36848d2f55e7SToby Isaac     ierr = PetscSectionGetStorageSize(rootIndicesSec,&numRootIndices);CHKERRQ(ierr);
3685c921d74cSToby Isaac     if (gatheredIndices) {
3686c921d74cSToby Isaac       ierr = PetscMalloc1(numRootIndices,&rootInds);CHKERRQ(ierr);
3687c921d74cSToby Isaac       ierr = PetscSFBcastBegin(indicesSF,MPIU_INT,leafInds,rootInds);CHKERRQ(ierr);
3688c921d74cSToby Isaac       ierr = PetscSFBcastEnd(indicesSF,MPIU_INT,leafInds,rootInds);CHKERRQ(ierr);
3689c921d74cSToby Isaac     }
3690c921d74cSToby Isaac     if (gatheredValues) {
3691c921d74cSToby Isaac       ierr = PetscMalloc1(numRootIndices,&rootVals);CHKERRQ(ierr);
3692c921d74cSToby Isaac       ierr = PetscSFBcastBegin(indicesSF,MPIU_SCALAR,leafVals,rootVals);CHKERRQ(ierr);
3693c921d74cSToby Isaac       ierr = PetscSFBcastEnd(indicesSF,MPIU_SCALAR,leafVals,rootVals);CHKERRQ(ierr);
3694c921d74cSToby Isaac     }
36958d2f55e7SToby Isaac     ierr = PetscSFDestroy(&indicesSF);CHKERRQ(ierr);
36968d2f55e7SToby Isaac   }
3697ec92bd66SToby Isaac   ierr = PetscSectionDestroy(&leafIndicesSec);CHKERRQ(ierr);
3698c921d74cSToby Isaac   ierr = PetscFree(leafInds);CHKERRQ(ierr);
3699c921d74cSToby Isaac   ierr = PetscFree(leafVals);CHKERRQ(ierr);
3700f30e825dSToby Isaac   ierr = PetscSFDestroy(&coarseToFineEmbedded);CHKERRQ(ierr);
3701c921d74cSToby Isaac   *rootMultiSec = multiRootSec;
3702c921d74cSToby Isaac   *multiLeafSec = rootIndicesSec;
3703c921d74cSToby Isaac   if (gatheredIndices) *gatheredIndices = rootInds;
3704c921d74cSToby Isaac   if (gatheredValues)  *gatheredValues  = rootVals;
3705ebf164c7SToby Isaac   PetscFunctionReturn(0);
3706ebf164c7SToby Isaac }
3707ebf164c7SToby Isaac 
3708ebf164c7SToby Isaac PetscErrorCode DMPlexComputeInjectorTree(DM coarse, DM fine, PetscSF coarseToFine, PetscInt *childIds, Mat mat)
3709ebf164c7SToby Isaac {
3710ebf164c7SToby Isaac   DM             refTree;
3711c921d74cSToby Isaac   PetscSection   multiRootSec, rootIndicesSec;
3712ebf164c7SToby Isaac   PetscSection   globalCoarse, globalFine;
3713ebf164c7SToby Isaac   PetscSection   localCoarse, localFine;
3714ebf164c7SToby Isaac   PetscSection   cSecRef;
3715c921d74cSToby Isaac   PetscInt       *rootIndices, *parentIndices, pRefStart, pRefEnd;
3716ebf164c7SToby Isaac   Mat            injRef;
3717c921d74cSToby Isaac   PetscInt       numFields, maxDof;
3718ebf164c7SToby Isaac   PetscInt       pStartC, pEndC, pStartF, pEndF, p;
3719ebf164c7SToby Isaac   PetscInt       *offsets, *offsetsCopy, *rowOffsets;
3720ebf164c7SToby Isaac   PetscLayout    rowMap, colMap;
3721ebf164c7SToby Isaac   PetscInt       rowStart, rowEnd, colStart, colEnd, *nnzD, *nnzO;
3722ebf164c7SToby Isaac   PetscScalar    ***childrenMats=NULL ; /* gcc -O gives 'may be used uninitialized' warning'. Initializing to suppress this warning */
3723ebf164c7SToby Isaac   PetscErrorCode ierr;
3724ebf164c7SToby Isaac 
3725ebf164c7SToby Isaac   PetscFunctionBegin;
3726ebf164c7SToby Isaac 
3727ebf164c7SToby Isaac   /* get the templates for the fine-to-coarse injection from the reference tree */
3728ebf164c7SToby Isaac   ierr = DMPlexGetReferenceTree(coarse,&refTree);CHKERRQ(ierr);
3729ebf164c7SToby Isaac   ierr = DMGetDefaultConstraints(refTree,&cSecRef,NULL);CHKERRQ(ierr);
3730ebf164c7SToby Isaac   ierr = PetscSectionGetChart(cSecRef,&pRefStart,&pRefEnd);CHKERRQ(ierr);
3731ebf164c7SToby Isaac   ierr = DMPlexReferenceTreeGetInjector(refTree,&injRef);CHKERRQ(ierr);
3732ebf164c7SToby Isaac 
3733ebf164c7SToby Isaac   ierr = DMPlexGetChart(fine,&pStartF,&pEndF);CHKERRQ(ierr);
3734ebf164c7SToby Isaac   ierr = DMGetDefaultSection(fine,&localFine);CHKERRQ(ierr);
3735ebf164c7SToby Isaac   ierr = DMGetDefaultGlobalSection(fine,&globalFine);CHKERRQ(ierr);
3736ebf164c7SToby Isaac   ierr = PetscSectionGetNumFields(localFine,&numFields);CHKERRQ(ierr);
3737ebf164c7SToby Isaac   ierr = DMPlexGetChart(coarse,&pStartC,&pEndC);CHKERRQ(ierr);
3738ebf164c7SToby Isaac   ierr = DMGetDefaultSection(coarse,&localCoarse);CHKERRQ(ierr);
3739ebf164c7SToby Isaac   ierr = DMGetDefaultGlobalSection(coarse,&globalCoarse);CHKERRQ(ierr);
3740ebf164c7SToby Isaac   ierr = PetscSectionGetMaxDof(localCoarse,&maxDof);CHKERRQ(ierr);
3741ebf164c7SToby Isaac   {
3742ebf164c7SToby Isaac     PetscInt maxFields = PetscMax(1,numFields) + 1;
3743ebf164c7SToby Isaac     ierr = PetscMalloc3(maxFields,&offsets,maxFields,&offsetsCopy,maxFields,&rowOffsets);CHKERRQ(ierr);
3744ebf164c7SToby Isaac   }
3745ebf164c7SToby Isaac 
3746c921d74cSToby Isaac   ierr = DMPlexTransferInjectorTree(coarse,fine,coarseToFine,childIds,NULL,numFields,offsets,&multiRootSec,&rootIndicesSec,&rootIndices,NULL);CHKERRQ(ierr);
37478d2f55e7SToby Isaac 
3748f30e825dSToby Isaac   ierr = PetscMalloc1(maxDof,&parentIndices);CHKERRQ(ierr);
3749f30e825dSToby Isaac 
3750f30e825dSToby Isaac   /* count indices */
37518d2f55e7SToby Isaac   ierr = MatGetLayouts(mat,&rowMap,&colMap);CHKERRQ(ierr);
3752c6154584SToby Isaac   ierr = PetscLayoutSetUp(rowMap);CHKERRQ(ierr);
3753c6154584SToby Isaac   ierr = PetscLayoutSetUp(colMap);CHKERRQ(ierr);
37548d2f55e7SToby Isaac   ierr = PetscLayoutGetRange(rowMap,&rowStart,&rowEnd);CHKERRQ(ierr);
37558d2f55e7SToby Isaac   ierr = PetscLayoutGetRange(colMap,&colStart,&colEnd);CHKERRQ(ierr);
3756f30e825dSToby Isaac   ierr = PetscCalloc2(rowEnd-rowStart,&nnzD,rowEnd-rowStart,&nnzO);CHKERRQ(ierr);
3757f30e825dSToby Isaac   for (p = pStartC; p < pEndC; p++) {
3758f30e825dSToby Isaac     PetscInt numLeaves, leafStart, leafEnd, l, dof, cdof, gOff;
37598d2f55e7SToby Isaac 
3760f30e825dSToby Isaac     ierr = PetscSectionGetDof(globalCoarse,p,&dof);CHKERRQ(ierr);
3761f30e825dSToby Isaac     ierr = PetscSectionGetConstraintDof(globalCoarse,p,&cdof);CHKERRQ(ierr);
3762f30e825dSToby Isaac     if ((dof - cdof) <= 0) continue;
3763f30e825dSToby Isaac     ierr = PetscSectionGetOffset(globalCoarse,p,&gOff);CHKERRQ(ierr);
37648d2f55e7SToby Isaac 
37658d2f55e7SToby Isaac     rowOffsets[0] = 0;
3766f30e825dSToby Isaac     offsetsCopy[0] = 0;
37678d2f55e7SToby Isaac     if (numFields) {
37688d2f55e7SToby Isaac       PetscInt f;
37698d2f55e7SToby Isaac 
3770f30e825dSToby Isaac       for (f = 0; f < numFields; f++) {
3771f30e825dSToby Isaac         PetscInt fDof;
3772f30e825dSToby Isaac         ierr = PetscSectionGetFieldDof(localCoarse,p,f,&fDof);CHKERRQ(ierr);
3773f30e825dSToby Isaac         rowOffsets[f + 1] = offsetsCopy[f + 1] = fDof + rowOffsets[f];
37748d2f55e7SToby Isaac       }
37754acb8e1eSToby Isaac       DMPlexGetIndicesPointFields_Internal(localCoarse,p,gOff < 0 ? -(gOff + 1) : gOff,offsetsCopy,PETSC_FALSE,NULL,-1,parentIndices);CHKERRQ(ierr);
37768d2f55e7SToby Isaac     }
37778d2f55e7SToby Isaac     else {
37784acb8e1eSToby Isaac       DMPlexGetIndicesPoint_Internal(localCoarse,p,gOff < 0 ? -(gOff + 1) : gOff,offsetsCopy,PETSC_FALSE,NULL,parentIndices);CHKERRQ(ierr);
3779f30e825dSToby Isaac       rowOffsets[1] = offsetsCopy[0];
37808d2f55e7SToby Isaac     }
3781f30e825dSToby Isaac 
3782f30e825dSToby Isaac     ierr = PetscSectionGetDof(multiRootSec,p,&numLeaves);CHKERRQ(ierr);
3783f30e825dSToby Isaac     ierr = PetscSectionGetOffset(multiRootSec,p,&leafStart);CHKERRQ(ierr);
3784f30e825dSToby Isaac     leafEnd = leafStart + numLeaves;
3785f30e825dSToby Isaac     for (l = leafStart; l < leafEnd; l++) {
3786f30e825dSToby Isaac       PetscInt numIndices, childId, offset;
3787f30e825dSToby Isaac       const PetscInt *childIndices;
3788f30e825dSToby Isaac 
3789f30e825dSToby Isaac       ierr = PetscSectionGetDof(rootIndicesSec,l,&numIndices);CHKERRQ(ierr);
3790f30e825dSToby Isaac       ierr = PetscSectionGetOffset(rootIndicesSec,l,&offset);CHKERRQ(ierr);
3791f30e825dSToby Isaac       childId = rootIndices[offset++];
3792f30e825dSToby Isaac       childIndices = &rootIndices[offset];
3793f30e825dSToby Isaac       numIndices--;
3794f30e825dSToby Isaac 
3795f30e825dSToby Isaac       if (childId == -1) { /* equivalent points: scatter */
3796f30e825dSToby Isaac         PetscInt i;
3797f30e825dSToby Isaac 
3798f30e825dSToby Isaac         for (i = 0; i < numIndices; i++) {
3799f30e825dSToby Isaac           PetscInt colIndex = childIndices[i];
3800f30e825dSToby Isaac           PetscInt rowIndex = parentIndices[i];
3801f30e825dSToby Isaac           if (rowIndex < 0) continue;
3802f30e825dSToby Isaac           if (colIndex < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unconstrained fine and constrained coarse");
3803a47f92cbSToby Isaac           if (colIndex >= colStart && colIndex < colEnd) {
3804f30e825dSToby Isaac             nnzD[rowIndex - rowStart] = 1;
3805f30e825dSToby Isaac           }
3806f30e825dSToby Isaac           else {
3807f30e825dSToby Isaac             nnzO[rowIndex - rowStart] = 1;
3808f30e825dSToby Isaac           }
3809f30e825dSToby Isaac         }
3810f30e825dSToby Isaac       }
3811f30e825dSToby Isaac       else {
3812f30e825dSToby Isaac         PetscInt parentId, f, lim;
3813f30e825dSToby Isaac 
3814f30e825dSToby Isaac         ierr = DMPlexGetTreeParent(refTree,childId,&parentId,NULL);CHKERRQ(ierr);
3815f30e825dSToby Isaac 
3816f30e825dSToby Isaac         lim = PetscMax(1,numFields);
3817f30e825dSToby Isaac         offsets[0] = 0;
38188d2f55e7SToby Isaac         if (numFields) {
38198d2f55e7SToby Isaac           PetscInt f;
3820f30e825dSToby Isaac 
38218d2f55e7SToby Isaac           for (f = 0; f < numFields; f++) {
3822f30e825dSToby Isaac             PetscInt fDof;
3823f30e825dSToby Isaac             ierr = PetscSectionGetFieldDof(cSecRef,childId,f,&fDof);CHKERRQ(ierr);
3824f30e825dSToby Isaac 
3825f30e825dSToby Isaac             offsets[f + 1] = fDof + offsets[f];
38268d2f55e7SToby Isaac           }
38278d2f55e7SToby Isaac         }
38288d2f55e7SToby Isaac         else {
3829f30e825dSToby Isaac           PetscInt cDof;
3830f30e825dSToby Isaac 
3831f30e825dSToby Isaac           ierr = PetscSectionGetDof(cSecRef,childId,&cDof);CHKERRQ(ierr);
3832f30e825dSToby Isaac           offsets[1] = cDof;
3833f30e825dSToby Isaac         }
3834f30e825dSToby Isaac         for (f = 0; f < lim; f++) {
3835f30e825dSToby Isaac           PetscInt parentStart = rowOffsets[f], parentEnd = rowOffsets[f + 1];
3836f30e825dSToby Isaac           PetscInt childStart = offsets[f], childEnd = offsets[f + 1];
3837f30e825dSToby Isaac           PetscInt i, numD = 0, numO = 0;
3838f30e825dSToby Isaac 
3839f30e825dSToby Isaac           for (i = childStart; i < childEnd; i++) {
3840f30e825dSToby Isaac             PetscInt colIndex = childIndices[i];
3841f30e825dSToby Isaac 
3842f30e825dSToby Isaac             if (colIndex < 0) continue;
3843f30e825dSToby Isaac             if (colIndex >= colStart && colIndex < colEnd) {
3844f30e825dSToby Isaac               numD++;
3845f30e825dSToby Isaac             }
3846f30e825dSToby Isaac             else {
3847f30e825dSToby Isaac               numO++;
3848f30e825dSToby Isaac             }
3849f30e825dSToby Isaac           }
3850f30e825dSToby Isaac           for (i = parentStart; i < parentEnd; i++) {
3851f30e825dSToby Isaac             PetscInt rowIndex = parentIndices[i];
3852f30e825dSToby Isaac 
3853f30e825dSToby Isaac             if (rowIndex < 0) continue;
3854f30e825dSToby Isaac             nnzD[rowIndex - rowStart] += numD;
3855f30e825dSToby Isaac             nnzO[rowIndex - rowStart] += numO;
38568d2f55e7SToby Isaac           }
38578d2f55e7SToby Isaac         }
38588d2f55e7SToby Isaac       }
3859f30e825dSToby Isaac     }
3860f30e825dSToby Isaac   }
3861f30e825dSToby Isaac   /* preallocate */
3862f30e825dSToby Isaac   ierr = MatXAIJSetPreallocation(mat,1,nnzD,nnzO,NULL,NULL);CHKERRQ(ierr);
3863f30e825dSToby Isaac   ierr = PetscFree2(nnzD,nnzO);CHKERRQ(ierr);
3864f30e825dSToby Isaac   /* insert values */
3865f30e825dSToby Isaac   ierr = DMPlexReferenceTreeGetChildrenMatrices_Injection(refTree,injRef,&childrenMats);CHKERRQ(ierr);
3866f30e825dSToby Isaac   for (p = pStartC; p < pEndC; p++) {
3867f30e825dSToby Isaac     PetscInt numLeaves, leafStart, leafEnd, l, dof, cdof, gOff;
3868f30e825dSToby Isaac 
3869f30e825dSToby Isaac     ierr = PetscSectionGetDof(globalCoarse,p,&dof);CHKERRQ(ierr);
3870f30e825dSToby Isaac     ierr = PetscSectionGetConstraintDof(globalCoarse,p,&cdof);CHKERRQ(ierr);
3871f30e825dSToby Isaac     if ((dof - cdof) <= 0) continue;
3872f30e825dSToby Isaac     ierr = PetscSectionGetOffset(globalCoarse,p,&gOff);CHKERRQ(ierr);
3873f30e825dSToby Isaac 
3874f30e825dSToby Isaac     rowOffsets[0] = 0;
3875f30e825dSToby Isaac     offsetsCopy[0] = 0;
38768d2f55e7SToby Isaac     if (numFields) {
38778d2f55e7SToby Isaac       PetscInt f;
3878f30e825dSToby Isaac 
38798d2f55e7SToby Isaac       for (f = 0; f < numFields; f++) {
3880f30e825dSToby Isaac         PetscInt fDof;
3881f30e825dSToby Isaac         ierr = PetscSectionGetFieldDof(localCoarse,p,f,&fDof);CHKERRQ(ierr);
3882f30e825dSToby Isaac         rowOffsets[f + 1] = offsetsCopy[f + 1] = fDof + rowOffsets[f];
3883f30e825dSToby Isaac       }
38844acb8e1eSToby Isaac       DMPlexGetIndicesPointFields_Internal(localCoarse,p,gOff < 0 ? -(gOff + 1) : gOff,offsetsCopy,PETSC_FALSE,NULL,-1,parentIndices);CHKERRQ(ierr);
3885f30e825dSToby Isaac     }
3886f30e825dSToby Isaac     else {
38874acb8e1eSToby Isaac       DMPlexGetIndicesPoint_Internal(localCoarse,p,gOff < 0 ? -(gOff + 1) : gOff,offsetsCopy,PETSC_FALSE,NULL,parentIndices);CHKERRQ(ierr);
3888f30e825dSToby Isaac       rowOffsets[1] = offsetsCopy[0];
3889f30e825dSToby Isaac     }
3890f30e825dSToby Isaac 
3891f30e825dSToby Isaac     ierr = PetscSectionGetDof(multiRootSec,p,&numLeaves);CHKERRQ(ierr);
3892f30e825dSToby Isaac     ierr = PetscSectionGetOffset(multiRootSec,p,&leafStart);CHKERRQ(ierr);
3893f30e825dSToby Isaac     leafEnd = leafStart + numLeaves;
3894f30e825dSToby Isaac     for (l = leafStart; l < leafEnd; l++) {
3895f30e825dSToby Isaac       PetscInt numIndices, childId, offset;
3896f30e825dSToby Isaac       const PetscInt *childIndices;
3897f30e825dSToby Isaac 
3898f30e825dSToby Isaac       ierr = PetscSectionGetDof(rootIndicesSec,l,&numIndices);CHKERRQ(ierr);
3899f30e825dSToby Isaac       ierr = PetscSectionGetOffset(rootIndicesSec,l,&offset);CHKERRQ(ierr);
3900f30e825dSToby Isaac       childId = rootIndices[offset++];
3901f30e825dSToby Isaac       childIndices = &rootIndices[offset];
3902f30e825dSToby Isaac       numIndices--;
3903f30e825dSToby Isaac 
3904f30e825dSToby Isaac       if (childId == -1) { /* equivalent points: scatter */
3905f30e825dSToby Isaac         PetscInt i;
3906f30e825dSToby Isaac 
3907f30e825dSToby Isaac         for (i = 0; i < numIndices; i++) {
3908f30e825dSToby Isaac           ierr = MatSetValue(mat,parentIndices[i],childIndices[i],1.,INSERT_VALUES);CHKERRQ(ierr);
39098d2f55e7SToby Isaac         }
39108d2f55e7SToby Isaac       }
39118d2f55e7SToby Isaac       else {
3912f30e825dSToby Isaac         PetscInt parentId, f, lim;
39138d2f55e7SToby Isaac 
3914f30e825dSToby Isaac         ierr = DMPlexGetTreeParent(refTree,childId,&parentId,NULL);CHKERRQ(ierr);
3915f30e825dSToby Isaac 
3916f30e825dSToby Isaac         lim = PetscMax(1,numFields);
3917f30e825dSToby Isaac         offsets[0] = 0;
39188d2f55e7SToby Isaac         if (numFields) {
3919f30e825dSToby Isaac           PetscInt f;
39208d2f55e7SToby Isaac 
3921f30e825dSToby Isaac           for (f = 0; f < numFields; f++) {
3922f30e825dSToby Isaac             PetscInt fDof;
3923f30e825dSToby Isaac             ierr = PetscSectionGetFieldDof(cSecRef,childId,f,&fDof);CHKERRQ(ierr);
3924f30e825dSToby Isaac 
3925f30e825dSToby Isaac             offsets[f + 1] = fDof + offsets[f];
39268d2f55e7SToby Isaac           }
39278d2f55e7SToby Isaac         }
39288d2f55e7SToby Isaac         else {
3929f30e825dSToby Isaac           PetscInt cDof;
3930f30e825dSToby Isaac 
3931f30e825dSToby Isaac           ierr = PetscSectionGetDof(cSecRef,childId,&cDof);CHKERRQ(ierr);
3932f30e825dSToby Isaac           offsets[1] = cDof;
39338d2f55e7SToby Isaac         }
3934f30e825dSToby Isaac         for (f = 0; f < lim; f++) {
3935f30e825dSToby Isaac           PetscScalar    *childMat   = &childrenMats[childId - pRefStart][f][0];
3936f30e825dSToby Isaac           PetscInt       *rowIndices = &parentIndices[rowOffsets[f]];
3937f30e825dSToby Isaac           const PetscInt *colIndices = &childIndices[offsets[f]];
3938f30e825dSToby Isaac 
3939f30e825dSToby Isaac           ierr = MatSetValues(mat,rowOffsets[f+1]-rowOffsets[f],rowIndices,offsets[f+1]-offsets[f],colIndices,childMat,INSERT_VALUES);CHKERRQ(ierr);
39408d2f55e7SToby Isaac         }
39418d2f55e7SToby Isaac       }
39428d2f55e7SToby Isaac     }
39438d2f55e7SToby Isaac   }
3944ec92bd66SToby Isaac   ierr = PetscSectionDestroy(&multiRootSec);CHKERRQ(ierr);
3945ec92bd66SToby Isaac   ierr = PetscSectionDestroy(&rootIndicesSec);CHKERRQ(ierr);
3946ec92bd66SToby Isaac   ierr = PetscFree(parentIndices);CHKERRQ(ierr);
3947f30e825dSToby Isaac   ierr = DMPlexReferenceTreeRestoreChildrenMatrices_Injection(refTree,injRef,&childrenMats);CHKERRQ(ierr);
3948f30e825dSToby Isaac   ierr = PetscFree(rootIndices);CHKERRQ(ierr);
3949f30e825dSToby Isaac   ierr = PetscFree3(offsets,offsetsCopy,rowOffsets);CHKERRQ(ierr);
3950f30e825dSToby Isaac 
39518d2f55e7SToby Isaac   ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
39528d2f55e7SToby Isaac   ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
3953154bca37SToby Isaac   PetscFunctionReturn(0);
3954154bca37SToby Isaac }
395538fc2455SToby Isaac 
39560eb7e1eaSToby Isaac static PetscErrorCode DMPlexTransferVecTree_Interpolate(DM coarse, Vec vecCoarseLocal, DM fine, Vec vecFine, PetscSF coarseToFine, PetscInt *cids, Vec grad, Vec cellGeom)
3957ebf164c7SToby Isaac {
39584833aeb0SToby Isaac   PetscDS           ds;
395962095d54SToby Isaac   PetscSF           coarseToFineEmbedded;
396062095d54SToby Isaac   PetscSection      globalCoarse, globalFine;
396162095d54SToby Isaac   PetscSection      localCoarse, localFine;
396262095d54SToby Isaac   PetscSection      aSec, cSec;
396362095d54SToby Isaac   PetscSection      rootValuesSec;
396462095d54SToby Isaac   PetscSection      leafValuesSec;
396562095d54SToby Isaac   PetscScalar       *rootValues, *leafValues;
396662095d54SToby Isaac   IS                aIS;
396762095d54SToby Isaac   const PetscInt    *anchors;
396862095d54SToby Isaac   Mat               cMat;
396962095d54SToby Isaac   PetscInt          numFields;
397089698031SToby Isaac   PetscInt          pStartC, pEndC, pStartF, pEndF, p, cellStart, cellEnd, cellEndInterior;
397162095d54SToby Isaac   PetscInt          aStart, aEnd, cStart, cEnd;
397262095d54SToby Isaac   PetscInt          *maxChildIds;
397362095d54SToby Isaac   PetscInt          *offsets, *newOffsets, *offsetsCopy, *newOffsetsCopy, *rowOffsets, *numD, *numO;
39740eb7e1eaSToby Isaac   PetscFV           fv = NULL;
39750eb7e1eaSToby Isaac   PetscInt          dim, numFVcomps = -1, fvField = -1;
39760eb7e1eaSToby Isaac   DM                cellDM = NULL, gradDM = NULL;
39770eb7e1eaSToby Isaac   const PetscScalar *cellGeomArray = NULL;
39780eb7e1eaSToby Isaac   const PetscScalar *gradArray = NULL;
397962095d54SToby Isaac   PetscErrorCode    ierr;
398062095d54SToby Isaac 
3981ebf164c7SToby Isaac   PetscFunctionBegin;
3982708c7f19SToby Isaac   ierr = VecSetOption(vecFine,VEC_IGNORE_NEGATIVE_INDICES,PETSC_TRUE);CHKERRQ(ierr);
398362095d54SToby Isaac   ierr = DMPlexGetChart(coarse,&pStartC,&pEndC);CHKERRQ(ierr);
39840eb7e1eaSToby Isaac   ierr = DMPlexGetHeightStratum(coarse,0,&cellStart,&cellEnd);CHKERRQ(ierr);
398589698031SToby Isaac   ierr = DMPlexGetHybridBounds(coarse,&cellEndInterior,NULL,NULL,NULL);CHKERRQ(ierr);
398689698031SToby Isaac   cellEnd = (cellEndInterior < 0) ? cellEnd : cellEndInterior;
398762095d54SToby Isaac   ierr = DMPlexGetChart(fine,&pStartF,&pEndF);CHKERRQ(ierr);
398862095d54SToby Isaac   ierr = DMGetDefaultGlobalSection(fine,&globalFine);CHKERRQ(ierr);
39890eb7e1eaSToby Isaac   ierr = DMGetCoordinateDim(coarse,&dim);CHKERRQ(ierr);
399062095d54SToby Isaac   { /* winnow fine points that don't have global dofs out of the sf */
3991e4a60869SToby Isaac     PetscInt       nleaves, l;
3992e4a60869SToby Isaac     const PetscInt *leaves;
399362095d54SToby Isaac     PetscInt       dof, cdof, numPointsWithDofs, offset, *pointsWithDofs;
399462095d54SToby Isaac 
3995e4a60869SToby Isaac     ierr = PetscSFGetGraph(coarseToFine,NULL,&nleaves,&leaves,NULL);CHKERRQ(ierr);
3996e4a60869SToby Isaac 
3997e4a60869SToby Isaac     for (l = 0, numPointsWithDofs = 0; l < nleaves; l++) {
3998e4a60869SToby Isaac       PetscInt p = leaves ? leaves[l] : l;
3999e4a60869SToby Isaac 
400062095d54SToby Isaac       ierr = PetscSectionGetDof(globalFine,p,&dof);CHKERRQ(ierr);
400162095d54SToby Isaac       ierr = PetscSectionGetConstraintDof(globalFine,p,&cdof);CHKERRQ(ierr);
400262095d54SToby Isaac       if ((dof - cdof) > 0) {
400362095d54SToby Isaac         numPointsWithDofs++;
400462095d54SToby Isaac       }
400562095d54SToby Isaac     }
400662095d54SToby Isaac     ierr = PetscMalloc1(numPointsWithDofs,&pointsWithDofs);CHKERRQ(ierr);
40074833aeb0SToby Isaac     for (l = 0, offset = 0; l < nleaves; l++) {
4008e4a60869SToby Isaac       PetscInt p = leaves ? leaves[l] : l;
4009e4a60869SToby Isaac 
401062095d54SToby Isaac       ierr = PetscSectionGetDof(globalFine,p,&dof);CHKERRQ(ierr);
401162095d54SToby Isaac       ierr = PetscSectionGetConstraintDof(globalFine,p,&cdof);CHKERRQ(ierr);
401262095d54SToby Isaac       if ((dof - cdof) > 0) {
4013e4a60869SToby Isaac         pointsWithDofs[offset++] = l;
401462095d54SToby Isaac       }
401562095d54SToby Isaac     }
401662095d54SToby Isaac     ierr = PetscSFCreateEmbeddedLeafSF(coarseToFine, numPointsWithDofs, pointsWithDofs, &coarseToFineEmbedded);CHKERRQ(ierr);
401762095d54SToby Isaac     ierr = PetscFree(pointsWithDofs);CHKERRQ(ierr);
401862095d54SToby Isaac   }
401962095d54SToby Isaac   /* communicate back to the coarse mesh which coarse points have children (that may require interpolation) */
402062095d54SToby Isaac   ierr = PetscMalloc1(pEndC-pStartC,&maxChildIds);CHKERRQ(ierr);
402162095d54SToby Isaac   for (p = pStartC; p < pEndC; p++) {
402262095d54SToby Isaac     maxChildIds[p - pStartC] = -2;
402362095d54SToby Isaac   }
402462095d54SToby Isaac   ierr = PetscSFReduceBegin(coarseToFineEmbedded,MPIU_INT,cids,maxChildIds,MPIU_MAX);CHKERRQ(ierr);
402562095d54SToby Isaac   ierr = PetscSFReduceEnd(coarseToFineEmbedded,MPIU_INT,cids,maxChildIds,MPIU_MAX);CHKERRQ(ierr);
402662095d54SToby Isaac 
402762095d54SToby Isaac   ierr = DMGetDefaultSection(coarse,&localCoarse);CHKERRQ(ierr);
402862095d54SToby Isaac   ierr = DMGetDefaultGlobalSection(coarse,&globalCoarse);CHKERRQ(ierr);
402962095d54SToby Isaac 
403062095d54SToby Isaac   ierr = DMPlexGetAnchors(coarse,&aSec,&aIS);CHKERRQ(ierr);
403162095d54SToby Isaac   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
403262095d54SToby Isaac   ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
403362095d54SToby Isaac 
403462095d54SToby Isaac   ierr = DMGetDefaultConstraints(coarse,&cSec,&cMat);CHKERRQ(ierr);
403562095d54SToby Isaac   ierr = PetscSectionGetChart(cSec,&cStart,&cEnd);CHKERRQ(ierr);
403662095d54SToby Isaac 
403762095d54SToby Isaac   /* create sections that will send to children the indices and matrices they will need to construct the interpolator */
403862095d54SToby Isaac   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)coarse),&rootValuesSec);CHKERRQ(ierr);
403962095d54SToby Isaac   ierr = PetscSectionSetChart(rootValuesSec,pStartC,pEndC);CHKERRQ(ierr);
4040708c7f19SToby Isaac   ierr = PetscSectionGetNumFields(localCoarse,&numFields);CHKERRQ(ierr);
404162095d54SToby Isaac   {
404262095d54SToby Isaac     PetscInt maxFields = PetscMax(1,numFields) + 1;
404362095d54SToby Isaac     ierr = PetscMalloc7(maxFields,&offsets,maxFields,&offsetsCopy,maxFields,&newOffsets,maxFields,&newOffsetsCopy,maxFields,&rowOffsets,maxFields,&numD,maxFields,&numO);CHKERRQ(ierr);
404462095d54SToby Isaac   }
40450eb7e1eaSToby Isaac   if (grad) {
40460eb7e1eaSToby Isaac     PetscInt i;
40470eb7e1eaSToby Isaac 
40480eb7e1eaSToby Isaac     ierr = VecGetDM(cellGeom,&cellDM);CHKERRQ(ierr);
40490eb7e1eaSToby Isaac     ierr = VecGetArrayRead(cellGeom,&cellGeomArray);CHKERRQ(ierr);
40500eb7e1eaSToby Isaac     ierr = VecGetDM(grad,&gradDM);CHKERRQ(ierr);
40510eb7e1eaSToby Isaac     ierr = VecGetArrayRead(grad,&gradArray);CHKERRQ(ierr);
40520eb7e1eaSToby Isaac     for (i = 0; i < PetscMax(1,numFields); i++) {
40530eb7e1eaSToby Isaac       PetscObject  obj;
40540eb7e1eaSToby Isaac       PetscClassId id;
40550eb7e1eaSToby Isaac 
40560eb7e1eaSToby Isaac       ierr = DMGetField(coarse, i, &obj);CHKERRQ(ierr);
40570eb7e1eaSToby Isaac       ierr = PetscObjectGetClassId(obj,&id);CHKERRQ(ierr);
40580eb7e1eaSToby Isaac       if (id == PETSCFV_CLASSID) {
40590eb7e1eaSToby Isaac         fv      = (PetscFV) obj;
40600eb7e1eaSToby Isaac         ierr    = PetscFVGetNumComponents(fv,&numFVcomps);CHKERRQ(ierr);
40610eb7e1eaSToby Isaac         fvField = i;
40620eb7e1eaSToby Isaac         break;
40630eb7e1eaSToby Isaac       }
40640eb7e1eaSToby Isaac     }
40650eb7e1eaSToby Isaac   }
406662095d54SToby Isaac 
406762095d54SToby Isaac   for (p = pStartC; p < pEndC; p++) { /* count the sizes of the indices and matrices */
406862095d54SToby Isaac     PetscInt dof;
406962095d54SToby Isaac     PetscInt maxChildId     = maxChildIds[p - pStartC];
407062095d54SToby Isaac     PetscInt numValues      = 0;
407162095d54SToby Isaac 
407262095d54SToby Isaac     ierr = PetscSectionGetDof(globalCoarse,p,&dof);CHKERRQ(ierr);
407362095d54SToby Isaac     if (dof < 0) {
407462095d54SToby Isaac       dof = -(dof + 1);
407562095d54SToby Isaac     }
407662095d54SToby Isaac     offsets[0]    = 0;
407762095d54SToby Isaac     newOffsets[0] = 0;
407862095d54SToby Isaac     if (maxChildId >= 0) { /* this point has children (with dofs) that will need to be interpolated from the closure of p */
407962095d54SToby Isaac       PetscInt *closure = NULL, closureSize, cl;
408062095d54SToby Isaac 
40814833aeb0SToby Isaac       ierr = DMPlexGetTransitiveClosure(coarse,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
408262095d54SToby Isaac       for (cl = 0; cl < closureSize; cl++) { /* get the closure */
408362095d54SToby Isaac         PetscInt c = closure[2 * cl], clDof;
408462095d54SToby Isaac 
408562095d54SToby Isaac         ierr = PetscSectionGetDof(localCoarse,c,&clDof);CHKERRQ(ierr);
408662095d54SToby Isaac         numValues += clDof;
408762095d54SToby Isaac       }
40884833aeb0SToby Isaac       ierr = DMPlexRestoreTransitiveClosure(coarse,p,PETSC_TRUE,&closureSize,&closure);CHKERRQ(ierr);
408962095d54SToby Isaac     }
409062095d54SToby Isaac     else if (maxChildId == -1) {
409162095d54SToby Isaac       ierr = PetscSectionGetDof(localCoarse,p,&numValues);CHKERRQ(ierr);
409262095d54SToby Isaac     }
409362095d54SToby Isaac     /* we will pack the column indices with the field offsets */
409478b7adb5SToby Isaac     if (maxChildId >= 0 && grad && p >= cellStart && p < cellEnd) {
40950eb7e1eaSToby Isaac       /* also send the centroid, and the gradient */
40960eb7e1eaSToby Isaac       numValues += dim * (1 + numFVcomps);
40970eb7e1eaSToby Isaac     }
409862095d54SToby Isaac     ierr = PetscSectionSetDof(rootValuesSec,p,numValues);CHKERRQ(ierr);
409962095d54SToby Isaac   }
410062095d54SToby Isaac   ierr = PetscSectionSetUp(rootValuesSec);CHKERRQ(ierr);
410162095d54SToby Isaac   {
410262095d54SToby Isaac     PetscInt          numRootValues;
410362095d54SToby Isaac     const PetscScalar *coarseArray;
410462095d54SToby Isaac 
410562095d54SToby Isaac     ierr = PetscSectionGetStorageSize(rootValuesSec,&numRootValues);CHKERRQ(ierr);
410662095d54SToby Isaac     ierr = PetscMalloc1(numRootValues,&rootValues);CHKERRQ(ierr);
410762095d54SToby Isaac     ierr = VecGetArrayRead(vecCoarseLocal,&coarseArray);CHKERRQ(ierr);
410862095d54SToby Isaac     for (p = pStartC; p < pEndC; p++) {
410962095d54SToby Isaac       PetscInt    numValues;
411062095d54SToby Isaac       PetscInt    pValOff;
411162095d54SToby Isaac       PetscScalar *pVal;
411262095d54SToby Isaac       PetscInt    maxChildId = maxChildIds[p - pStartC];
411362095d54SToby Isaac 
411462095d54SToby Isaac       ierr = PetscSectionGetDof(rootValuesSec,p,&numValues);CHKERRQ(ierr);
411562095d54SToby Isaac       if (!numValues) {
411662095d54SToby Isaac         continue;
411762095d54SToby Isaac       }
411862095d54SToby Isaac       ierr = PetscSectionGetOffset(rootValuesSec,p,&pValOff);CHKERRQ(ierr);
411962095d54SToby Isaac       pVal = &(rootValues[pValOff]);
412062095d54SToby Isaac       if (maxChildId >= 0) { /* build an identity matrix, apply matrix constraints on the right */
41210eb7e1eaSToby Isaac         PetscInt closureSize = numValues;
41220eb7e1eaSToby Isaac         ierr = DMPlexVecGetClosure(coarse,NULL,vecCoarseLocal,p,&closureSize,&pVal);CHKERRQ(ierr);
41230eb7e1eaSToby Isaac         if (grad && p >= cellStart && p < cellEnd) {
4124193eb951SToby Isaac           PetscFVCellGeom *cg;
41256dd00756SToby Isaac           PetscScalar     *gradVals = NULL;
41260eb7e1eaSToby Isaac           PetscInt        i;
41270eb7e1eaSToby Isaac 
41280eb7e1eaSToby Isaac           pVal += (numValues - dim * (1 + numFVcomps));
41290eb7e1eaSToby Isaac 
4130193eb951SToby Isaac           ierr = DMPlexPointLocalRead(cellDM,p,cellGeomArray,(void *) &cg);CHKERRQ(ierr);
41310eb7e1eaSToby Isaac           for (i = 0; i < dim; i++) pVal[i] = cg->centroid[i];
41320eb7e1eaSToby Isaac           pVal += dim;
4133193eb951SToby Isaac           ierr = DMPlexPointGlobalRead(gradDM,p,gradArray,(void *) &gradVals);CHKERRQ(ierr);
41340eb7e1eaSToby Isaac           for (i = 0; i < dim * numFVcomps; i++) pVal[i] = gradVals[i];
41350eb7e1eaSToby Isaac         }
413662095d54SToby Isaac       }
413778b7adb5SToby Isaac       else if (maxChildId == -1) {
413878b7adb5SToby Isaac         PetscInt lDof, lOff, i;
413978b7adb5SToby Isaac 
414078b7adb5SToby Isaac         ierr = PetscSectionGetDof(localCoarse,p,&lDof);CHKERRQ(ierr);
414178b7adb5SToby Isaac         ierr = PetscSectionGetOffset(localCoarse,p,&lOff);CHKERRQ(ierr);
414278b7adb5SToby Isaac         for (i = 0; i < lDof; i++) pVal[i] = coarseArray[lOff + i];
414378b7adb5SToby Isaac       }
414478b7adb5SToby Isaac     }
414562095d54SToby Isaac     ierr = VecRestoreArrayRead(vecCoarseLocal,&coarseArray);CHKERRQ(ierr);
414662095d54SToby Isaac     ierr = PetscFree(maxChildIds);CHKERRQ(ierr);
414762095d54SToby Isaac   }
414862095d54SToby Isaac   {
414962095d54SToby Isaac     PetscSF  valuesSF;
415062095d54SToby Isaac     PetscInt *remoteOffsetsValues, numLeafValues;
415162095d54SToby Isaac 
415262095d54SToby Isaac     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)fine),&leafValuesSec);CHKERRQ(ierr);
415362095d54SToby Isaac     ierr = PetscSFDistributeSection(coarseToFineEmbedded,rootValuesSec,&remoteOffsetsValues,leafValuesSec);CHKERRQ(ierr);
415462095d54SToby Isaac     ierr = PetscSFCreateSectionSF(coarseToFineEmbedded,rootValuesSec,remoteOffsetsValues,leafValuesSec,&valuesSF);CHKERRQ(ierr);
415562095d54SToby Isaac     ierr = PetscSFDestroy(&coarseToFineEmbedded);CHKERRQ(ierr);
415662095d54SToby Isaac     ierr = PetscFree(remoteOffsetsValues);CHKERRQ(ierr);
415762095d54SToby Isaac     ierr = PetscSectionGetStorageSize(leafValuesSec,&numLeafValues);CHKERRQ(ierr);
415862095d54SToby Isaac     ierr = PetscMalloc1(numLeafValues,&leafValues);CHKERRQ(ierr);
415962095d54SToby Isaac     ierr = PetscSFBcastBegin(valuesSF,MPIU_SCALAR,rootValues,leafValues);CHKERRQ(ierr);
416062095d54SToby Isaac     ierr = PetscSFBcastEnd(valuesSF,MPIU_SCALAR,rootValues,leafValues);CHKERRQ(ierr);
416162095d54SToby Isaac     ierr = PetscSFDestroy(&valuesSF);CHKERRQ(ierr);
416262095d54SToby Isaac     ierr = PetscFree(rootValues);CHKERRQ(ierr);
416362095d54SToby Isaac     ierr = PetscSectionDestroy(&rootValuesSec);CHKERRQ(ierr);
416462095d54SToby Isaac   }
416562095d54SToby Isaac   ierr = DMGetDefaultSection(fine,&localFine);CHKERRQ(ierr);
416662095d54SToby Isaac   {
416762095d54SToby Isaac     PetscInt    maxDof;
416862095d54SToby Isaac     PetscInt    *rowIndices;
416962095d54SToby Isaac     DM           refTree;
417062095d54SToby Isaac     PetscInt     **refPointFieldN;
417162095d54SToby Isaac     PetscScalar  ***refPointFieldMats;
417262095d54SToby Isaac     PetscSection refConSec, refAnSec;
41730eb7e1eaSToby Isaac     PetscInt     pRefStart,pRefEnd,leafStart,leafEnd;
417462095d54SToby Isaac     PetscScalar  *pointWork;
417562095d54SToby Isaac 
417662095d54SToby Isaac     ierr = PetscSectionGetMaxDof(globalFine,&maxDof);CHKERRQ(ierr);
417762095d54SToby Isaac     ierr = DMGetWorkArray(fine,maxDof,PETSC_INT,&rowIndices);CHKERRQ(ierr);
417862095d54SToby Isaac     ierr = DMGetWorkArray(fine,maxDof,PETSC_SCALAR,&pointWork);CHKERRQ(ierr);
417962095d54SToby Isaac     ierr = DMPlexGetReferenceTree(fine,&refTree);CHKERRQ(ierr);
41804833aeb0SToby Isaac     ierr = DMGetDS(fine,&ds);CHKERRQ(ierr);
41814833aeb0SToby Isaac     ierr = DMSetDS(refTree,ds);CHKERRQ(ierr);
418262095d54SToby Isaac     ierr = DMPlexReferenceTreeGetChildrenMatrices(refTree,&refPointFieldMats,&refPointFieldN);CHKERRQ(ierr);
418362095d54SToby Isaac     ierr = DMGetDefaultConstraints(refTree,&refConSec,NULL);CHKERRQ(ierr);
418462095d54SToby Isaac     ierr = DMPlexGetAnchors(refTree,&refAnSec,NULL);CHKERRQ(ierr);
418562095d54SToby Isaac     ierr = PetscSectionGetChart(refConSec,&pRefStart,&pRefEnd);CHKERRQ(ierr);
41860eb7e1eaSToby Isaac     ierr = PetscSectionGetChart(leafValuesSec,&leafStart,&leafEnd);CHKERRQ(ierr);
41870eb7e1eaSToby Isaac     ierr = DMPlexGetHeightStratum(fine,0,&cellStart,&cellEnd);CHKERRQ(ierr);
418889698031SToby Isaac     ierr = DMPlexGetHybridBounds(fine,&cellEndInterior,NULL,NULL,NULL);CHKERRQ(ierr);
418989698031SToby Isaac     cellEnd = (cellEndInterior < 0) ? cellEnd : cellEndInterior;
41900eb7e1eaSToby Isaac     for (p = leafStart; p < leafEnd; p++) {
419162095d54SToby Isaac       PetscInt          gDof, gcDof, gOff, lDof;
419262095d54SToby Isaac       PetscInt          numValues, pValOff;
419362095d54SToby Isaac       PetscInt          childId;
419462095d54SToby Isaac       const PetscScalar *pVal;
41950eb7e1eaSToby Isaac       const PetscScalar *fvGradData = NULL;
419662095d54SToby Isaac 
419762095d54SToby Isaac       ierr = PetscSectionGetDof(globalFine,p,&gDof);CHKERRQ(ierr);
419862095d54SToby Isaac       ierr = PetscSectionGetDof(localFine,p,&lDof);CHKERRQ(ierr);
419962095d54SToby Isaac       ierr = PetscSectionGetConstraintDof(globalFine,p,&gcDof);CHKERRQ(ierr);
420062095d54SToby Isaac       if ((gDof - gcDof) <= 0) {
420162095d54SToby Isaac         continue;
420262095d54SToby Isaac       }
420362095d54SToby Isaac       ierr = PetscSectionGetOffset(globalFine,p,&gOff);CHKERRQ(ierr);
420462095d54SToby Isaac       ierr = PetscSectionGetDof(leafValuesSec,p,&numValues);CHKERRQ(ierr);
420562095d54SToby Isaac       if (!numValues) continue;
420662095d54SToby Isaac       ierr = PetscSectionGetOffset(leafValuesSec,p,&pValOff);CHKERRQ(ierr);
420762095d54SToby Isaac       pVal = &leafValues[pValOff];
420862095d54SToby Isaac       offsets[0]        = 0;
420962095d54SToby Isaac       offsetsCopy[0]    = 0;
421062095d54SToby Isaac       newOffsets[0]     = 0;
421162095d54SToby Isaac       newOffsetsCopy[0] = 0;
42124833aeb0SToby Isaac       childId           = cids[p - pStartF];
421362095d54SToby Isaac       if (numFields) {
421462095d54SToby Isaac         PetscInt f;
421562095d54SToby Isaac         for (f = 0; f < numFields; f++) {
421662095d54SToby Isaac           PetscInt rowDof;
421762095d54SToby Isaac 
421862095d54SToby Isaac           ierr = PetscSectionGetFieldDof(localFine,p,f,&rowDof);CHKERRQ(ierr);
421962095d54SToby Isaac           offsets[f + 1]        = offsets[f] + rowDof;
422062095d54SToby Isaac           offsetsCopy[f + 1]    = offsets[f + 1];
422162095d54SToby Isaac           /* TODO: closure indices */
42229f4e70e1SToby Isaac           newOffsets[f + 1]     = newOffsets[f] + ((childId == -1) ? rowDof : refPointFieldN[childId - pRefStart][f]);
422362095d54SToby Isaac         }
42244acb8e1eSToby Isaac         ierr = DMPlexGetIndicesPointFields_Internal(localFine,p,gOff,offsetsCopy,PETSC_FALSE,NULL,-1,rowIndices);CHKERRQ(ierr);
422562095d54SToby Isaac       }
422662095d54SToby Isaac       else {
42274833aeb0SToby Isaac         offsets[0]    = 0;
42284833aeb0SToby Isaac         offsets[1]    = lDof;
42294833aeb0SToby Isaac         newOffsets[0] = 0;
42304833aeb0SToby Isaac         newOffsets[1] = (childId == -1) ? lDof : refPointFieldN[childId - pRefStart][0];
42314acb8e1eSToby Isaac         ierr = DMPlexGetIndicesPoint_Internal(localFine,p,gOff,offsetsCopy,PETSC_FALSE,NULL,rowIndices);CHKERRQ(ierr);
423262095d54SToby Isaac       }
423362095d54SToby Isaac       if (childId == -1) { /* no child interpolation: one nnz per */
42342f65e181SToby Isaac         ierr = VecSetValues(vecFine,numValues,rowIndices,pVal,INSERT_VALUES);CHKERRQ(ierr);
423562095d54SToby Isaac       } else {
423662095d54SToby Isaac         PetscInt f;
423762095d54SToby Isaac 
423878b7adb5SToby Isaac         if (grad && p >= cellStart && p < cellEnd) {
423978b7adb5SToby Isaac           numValues -= (dim * (1 + numFVcomps));
424078b7adb5SToby Isaac           fvGradData = &pVal[numValues];
424178b7adb5SToby Isaac         }
424262095d54SToby Isaac         for (f = 0; f < PetscMax(1,numFields); f++) {
424362095d54SToby Isaac           const PetscScalar *childMat = refPointFieldMats[childId - pRefStart][f];
424462095d54SToby Isaac           PetscInt numRows = offsets[f+1] - offsets[f];
424562095d54SToby Isaac           PetscInt numCols = newOffsets[f + 1] - newOffsets[f];
424662095d54SToby Isaac           const PetscScalar *cVal = &pVal[newOffsets[f]];
424762095d54SToby Isaac           PetscScalar *rVal = &pointWork[offsets[f]];
424862095d54SToby Isaac           PetscInt i, j;
424962095d54SToby Isaac 
4250708c7f19SToby Isaac #if 0
4251708c7f19SToby Isaac           ierr = PetscInfo6(coarse,"field %D, numFields %D, childId %D, numRows %D, numCols %D, refPointFieldN %D\n",f,numFields,childId,numRows,numCols,refPointFieldN[childId - pRefStart][f]);CHKERRQ(ierr);
4252708c7f19SToby Isaac #endif
425362095d54SToby Isaac           for (i = 0; i < numRows; i++) {
425462095d54SToby Isaac             PetscScalar val = 0.;
425562095d54SToby Isaac             for (j = 0; j < numCols; j++) {
425662095d54SToby Isaac               val += childMat[i * numCols + j] * cVal[j];
425762095d54SToby Isaac             }
425862095d54SToby Isaac             rVal[i] = val;
425962095d54SToby Isaac           }
42600eb7e1eaSToby Isaac           if (f == fvField && p >= cellStart && p < cellEnd) {
42610eb7e1eaSToby Isaac             PetscReal   centroid[3];
42620eb7e1eaSToby Isaac             PetscScalar diff[3];
42630eb7e1eaSToby Isaac             const PetscScalar *parentCentroid = &fvGradData[0];
42640eb7e1eaSToby Isaac             const PetscScalar *gradient       = &fvGradData[dim];
42650eb7e1eaSToby Isaac 
42660eb7e1eaSToby Isaac             ierr = DMPlexComputeCellGeometryFVM(fine,p,NULL,centroid,NULL);CHKERRQ(ierr);
42670eb7e1eaSToby Isaac             for (i = 0; i < dim; i++) {
42680eb7e1eaSToby Isaac               diff[i] = centroid[i] - parentCentroid[i];
42690eb7e1eaSToby Isaac             }
42700eb7e1eaSToby Isaac             for (i = 0; i < numFVcomps; i++) {
42710eb7e1eaSToby Isaac               PetscScalar val = 0.;
42720eb7e1eaSToby Isaac 
427389698031SToby Isaac               for (j = 0; j < dim; j++) {
42740eb7e1eaSToby Isaac                 val += gradient[dim * i + j] * diff[j];
42750eb7e1eaSToby Isaac               }
42760eb7e1eaSToby Isaac               rVal[i] += val;
42770eb7e1eaSToby Isaac             }
42780eb7e1eaSToby Isaac           }
42792f65e181SToby Isaac           ierr = VecSetValues(vecFine,numRows,&rowIndices[offsets[f]],rVal,INSERT_VALUES);CHKERRQ(ierr);
428062095d54SToby Isaac         }
428162095d54SToby Isaac       }
428262095d54SToby Isaac     }
428362095d54SToby Isaac     ierr = DMPlexReferenceTreeRestoreChildrenMatrices(refTree,&refPointFieldMats,&refPointFieldN);CHKERRQ(ierr);
428462095d54SToby Isaac     ierr = DMRestoreWorkArray(fine,maxDof,PETSC_INT,&rowIndices);CHKERRQ(ierr);
428562095d54SToby Isaac     ierr = DMRestoreWorkArray(fine,maxDof,PETSC_SCALAR,&pointWork);CHKERRQ(ierr);
428662095d54SToby Isaac   }
42874fe3dfefSToby Isaac   ierr = PetscFree(leafValues);CHKERRQ(ierr);
428862095d54SToby Isaac   ierr = PetscSectionDestroy(&leafValuesSec);CHKERRQ(ierr);
428962095d54SToby Isaac   ierr = PetscFree7(offsets,offsetsCopy,newOffsets,newOffsetsCopy,rowOffsets,numD,numO);CHKERRQ(ierr);
429062095d54SToby Isaac   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
4291ebf164c7SToby Isaac   PetscFunctionReturn(0);
4292ebf164c7SToby Isaac }
4293ebf164c7SToby Isaac 
4294ebf164c7SToby Isaac static PetscErrorCode DMPlexTransferVecTree_Inject(DM fine, Vec vecFine, DM coarse, Vec vecCoarse, PetscSF coarseToFine, PetscInt *cids)
4295ebf164c7SToby Isaac {
42964833aeb0SToby Isaac   PetscDS        ds;
4297c921d74cSToby Isaac   DM             refTree;
4298c921d74cSToby Isaac   PetscSection   multiRootSec, rootIndicesSec;
4299c921d74cSToby Isaac   PetscSection   globalCoarse, globalFine;
4300c921d74cSToby Isaac   PetscSection   localCoarse, localFine;
4301c921d74cSToby Isaac   PetscSection   cSecRef;
4302c921d74cSToby Isaac   PetscInt       *parentIndices, pRefStart, pRefEnd;
4303d3bc4906SToby Isaac   PetscScalar    *rootValues, *parentValues;
4304c921d74cSToby Isaac   Mat            injRef;
4305c921d74cSToby Isaac   PetscInt       numFields, maxDof;
4306c921d74cSToby Isaac   PetscInt       pStartC, pEndC, pStartF, pEndF, p;
4307c921d74cSToby Isaac   PetscInt       *offsets, *offsetsCopy, *rowOffsets;
4308c921d74cSToby Isaac   PetscLayout    rowMap, colMap;
4309c921d74cSToby Isaac   PetscInt       rowStart, rowEnd, colStart, colEnd;
4310c921d74cSToby Isaac   PetscScalar    ***childrenMats=NULL ; /* gcc -O gives 'may be used uninitialized' warning'. Initializing to suppress this warning */
4311c921d74cSToby Isaac   PetscErrorCode ierr;
4312c921d74cSToby Isaac 
4313ebf164c7SToby Isaac   PetscFunctionBegin;
4314c921d74cSToby Isaac 
4315c921d74cSToby Isaac   /* get the templates for the fine-to-coarse injection from the reference tree */
4316708c7f19SToby Isaac   ierr = VecSetOption(vecFine,VEC_IGNORE_NEGATIVE_INDICES,PETSC_TRUE);CHKERRQ(ierr);
4317708c7f19SToby Isaac   ierr = VecSetOption(vecCoarse,VEC_IGNORE_NEGATIVE_INDICES,PETSC_TRUE);CHKERRQ(ierr);
4318c921d74cSToby Isaac   ierr = DMPlexGetReferenceTree(coarse,&refTree);CHKERRQ(ierr);
43194833aeb0SToby Isaac   ierr = DMGetDS(coarse,&ds);CHKERRQ(ierr);
43204833aeb0SToby Isaac   ierr = DMSetDS(refTree,ds);CHKERRQ(ierr);
4321c921d74cSToby Isaac   ierr = DMGetDefaultConstraints(refTree,&cSecRef,NULL);CHKERRQ(ierr);
4322c921d74cSToby Isaac   ierr = PetscSectionGetChart(cSecRef,&pRefStart,&pRefEnd);CHKERRQ(ierr);
4323c921d74cSToby Isaac   ierr = DMPlexReferenceTreeGetInjector(refTree,&injRef);CHKERRQ(ierr);
4324c921d74cSToby Isaac 
4325c921d74cSToby Isaac   ierr = DMPlexGetChart(fine,&pStartF,&pEndF);CHKERRQ(ierr);
4326c921d74cSToby Isaac   ierr = DMGetDefaultSection(fine,&localFine);CHKERRQ(ierr);
4327c921d74cSToby Isaac   ierr = DMGetDefaultGlobalSection(fine,&globalFine);CHKERRQ(ierr);
4328c921d74cSToby Isaac   ierr = PetscSectionGetNumFields(localFine,&numFields);CHKERRQ(ierr);
4329c921d74cSToby Isaac   ierr = DMPlexGetChart(coarse,&pStartC,&pEndC);CHKERRQ(ierr);
4330c921d74cSToby Isaac   ierr = DMGetDefaultSection(coarse,&localCoarse);CHKERRQ(ierr);
4331c921d74cSToby Isaac   ierr = DMGetDefaultGlobalSection(coarse,&globalCoarse);CHKERRQ(ierr);
4332c921d74cSToby Isaac   ierr = PetscSectionGetMaxDof(localCoarse,&maxDof);CHKERRQ(ierr);
4333c921d74cSToby Isaac   {
4334c921d74cSToby Isaac     PetscInt maxFields = PetscMax(1,numFields) + 1;
4335c921d74cSToby Isaac     ierr = PetscMalloc3(maxFields,&offsets,maxFields,&offsetsCopy,maxFields,&rowOffsets);CHKERRQ(ierr);
4336c921d74cSToby Isaac   }
4337c921d74cSToby Isaac 
4338c921d74cSToby Isaac   ierr = DMPlexTransferInjectorTree(coarse,fine,coarseToFine,cids,vecFine,numFields,offsets,&multiRootSec,&rootIndicesSec,NULL,&rootValues);CHKERRQ(ierr);
4339c921d74cSToby Isaac 
4340d3bc4906SToby Isaac   ierr = PetscMalloc2(maxDof,&parentIndices,maxDof,&parentValues);CHKERRQ(ierr);
4341c921d74cSToby Isaac 
4342c921d74cSToby Isaac   /* count indices */
434362095d54SToby Isaac   ierr = VecGetLayout(vecFine,&colMap);CHKERRQ(ierr);
434462095d54SToby Isaac   ierr = VecGetLayout(vecCoarse,&rowMap);CHKERRQ(ierr);
4345c921d74cSToby Isaac   ierr = PetscLayoutSetUp(rowMap);CHKERRQ(ierr);
4346c921d74cSToby Isaac   ierr = PetscLayoutSetUp(colMap);CHKERRQ(ierr);
4347c921d74cSToby Isaac   ierr = PetscLayoutGetRange(rowMap,&rowStart,&rowEnd);CHKERRQ(ierr);
4348c921d74cSToby Isaac   ierr = PetscLayoutGetRange(colMap,&colStart,&colEnd);CHKERRQ(ierr);
4349c921d74cSToby Isaac   /* insert values */
4350c921d74cSToby Isaac   ierr = DMPlexReferenceTreeGetChildrenMatrices_Injection(refTree,injRef,&childrenMats);CHKERRQ(ierr);
4351c921d74cSToby Isaac   for (p = pStartC; p < pEndC; p++) {
4352c921d74cSToby Isaac     PetscInt  numLeaves, leafStart, leafEnd, l, dof, cdof, gOff;
435378b7adb5SToby Isaac     PetscBool contribute = PETSC_FALSE;
4354c921d74cSToby Isaac 
4355c921d74cSToby Isaac     ierr = PetscSectionGetDof(globalCoarse,p,&dof);CHKERRQ(ierr);
4356c921d74cSToby Isaac     ierr = PetscSectionGetConstraintDof(globalCoarse,p,&cdof);CHKERRQ(ierr);
4357c921d74cSToby Isaac     if ((dof - cdof) <= 0) continue;
43582f65e181SToby Isaac     ierr = PetscSectionGetDof(localCoarse,p,&dof);CHKERRQ(ierr);
4359c921d74cSToby Isaac     ierr = PetscSectionGetOffset(globalCoarse,p,&gOff);CHKERRQ(ierr);
4360c921d74cSToby Isaac 
4361c921d74cSToby Isaac     rowOffsets[0] = 0;
4362c921d74cSToby Isaac     offsetsCopy[0] = 0;
4363c921d74cSToby Isaac     if (numFields) {
4364c921d74cSToby Isaac       PetscInt f;
4365c921d74cSToby Isaac 
4366c921d74cSToby Isaac       for (f = 0; f < numFields; f++) {
4367c921d74cSToby Isaac         PetscInt fDof;
4368c921d74cSToby Isaac         ierr = PetscSectionGetFieldDof(localCoarse,p,f,&fDof);CHKERRQ(ierr);
4369c921d74cSToby Isaac         rowOffsets[f + 1] = offsetsCopy[f + 1] = fDof + rowOffsets[f];
4370c921d74cSToby Isaac       }
43714acb8e1eSToby Isaac       DMPlexGetIndicesPointFields_Internal(localCoarse,p,gOff < 0 ? -(gOff + 1) : gOff,offsetsCopy,PETSC_FALSE,NULL,-1,parentIndices);CHKERRQ(ierr);
4372c921d74cSToby Isaac     }
4373c921d74cSToby Isaac     else {
43744acb8e1eSToby Isaac       DMPlexGetIndicesPoint_Internal(localCoarse,p,gOff < 0 ? -(gOff + 1) : gOff,offsetsCopy,PETSC_FALSE,NULL,parentIndices);CHKERRQ(ierr);
4375c921d74cSToby Isaac       rowOffsets[1] = offsetsCopy[0];
4376c921d74cSToby Isaac     }
4377c921d74cSToby Isaac 
4378c921d74cSToby Isaac     ierr = PetscSectionGetDof(multiRootSec,p,&numLeaves);CHKERRQ(ierr);
4379c921d74cSToby Isaac     ierr = PetscSectionGetOffset(multiRootSec,p,&leafStart);CHKERRQ(ierr);
4380c921d74cSToby Isaac     leafEnd = leafStart + numLeaves;
43812f65e181SToby Isaac     for (l = 0; l < dof; l++) parentValues[l] = 0.;
4382c921d74cSToby Isaac     for (l = leafStart; l < leafEnd; l++) {
4383c921d74cSToby Isaac       PetscInt numIndices, childId, offset;
4384c921d74cSToby Isaac       const PetscScalar *childValues;
4385c921d74cSToby Isaac 
4386c921d74cSToby Isaac       ierr = PetscSectionGetDof(rootIndicesSec,l,&numIndices);CHKERRQ(ierr);
4387c921d74cSToby Isaac       ierr = PetscSectionGetOffset(rootIndicesSec,l,&offset);CHKERRQ(ierr);
4388c921d74cSToby Isaac       childId = (PetscInt) PetscRealPart(rootValues[offset++]);
4389c921d74cSToby Isaac       childValues = &rootValues[offset];
4390c921d74cSToby Isaac       numIndices--;
4391c921d74cSToby Isaac 
4392c921d74cSToby Isaac       if (childId == -2) { /* skip */
4393c921d74cSToby Isaac         continue;
4394c921d74cSToby Isaac       } else if (childId == -1) { /* equivalent points: scatter */
43952f65e181SToby Isaac         PetscInt m;
43962f65e181SToby Isaac 
439778b7adb5SToby Isaac         contribute = PETSC_TRUE;
43982f65e181SToby Isaac         for (m = 0; m < numIndices; m++) parentValues[m] = childValues[m];
4399beedf8abSToby Isaac       } else { /* contributions from children: sum with injectors from reference tree */
4400d3bc4906SToby Isaac         PetscInt parentId, f, lim;
4401d3bc4906SToby Isaac 
440278b7adb5SToby Isaac         contribute = PETSC_TRUE;
4403d3bc4906SToby Isaac         ierr = DMPlexGetTreeParent(refTree,childId,&parentId,NULL);CHKERRQ(ierr);
4404d3bc4906SToby Isaac 
4405d3bc4906SToby Isaac         lim = PetscMax(1,numFields);
4406d3bc4906SToby Isaac         offsets[0] = 0;
4407d3bc4906SToby Isaac         if (numFields) {
4408d3bc4906SToby Isaac           PetscInt f;
4409d3bc4906SToby Isaac 
4410d3bc4906SToby Isaac           for (f = 0; f < numFields; f++) {
4411d3bc4906SToby Isaac             PetscInt fDof;
4412d3bc4906SToby Isaac             ierr = PetscSectionGetFieldDof(cSecRef,childId,f,&fDof);CHKERRQ(ierr);
4413d3bc4906SToby Isaac 
4414d3bc4906SToby Isaac             offsets[f + 1] = fDof + offsets[f];
4415d3bc4906SToby Isaac           }
4416d3bc4906SToby Isaac         }
4417d3bc4906SToby Isaac         else {
4418d3bc4906SToby Isaac           PetscInt cDof;
4419d3bc4906SToby Isaac 
4420d3bc4906SToby Isaac           ierr = PetscSectionGetDof(cSecRef,childId,&cDof);CHKERRQ(ierr);
4421d3bc4906SToby Isaac           offsets[1] = cDof;
4422d3bc4906SToby Isaac         }
4423d3bc4906SToby Isaac         for (f = 0; f < lim; f++) {
4424d3bc4906SToby Isaac           PetscScalar       *childMat   = &childrenMats[childId - pRefStart][f][0];
4425d3bc4906SToby Isaac           PetscInt          n           = offsets[f+1]-offsets[f];
4426e328ff09SToby Isaac           PetscInt          m           = rowOffsets[f+1]-rowOffsets[f];
4427d3bc4906SToby Isaac           PetscInt          i, j;
4428d3bc4906SToby Isaac           const PetscScalar *colValues  = &childValues[offsets[f]];
4429d3bc4906SToby Isaac 
4430e328ff09SToby Isaac           for (i = 0; i < m; i++) {
4431d3bc4906SToby Isaac             PetscScalar val = 0.;
4432d3bc4906SToby Isaac             for (j = 0; j < n; j++) {
4433d3bc4906SToby Isaac               val += childMat[n * i + j] * colValues[j];
4434d3bc4906SToby Isaac             }
4435e328ff09SToby Isaac             parentValues[rowOffsets[f] + i] += val;
4436d3bc4906SToby Isaac           }
4437d3bc4906SToby Isaac         }
4438c921d74cSToby Isaac       }
4439c921d74cSToby Isaac     }
444078b7adb5SToby Isaac     if (contribute) {ierr = VecSetValues(vecCoarse,dof,parentIndices,parentValues,INSERT_VALUES);CHKERRQ(ierr);}
4441c921d74cSToby Isaac   }
4442c921d74cSToby Isaac   ierr = PetscSectionDestroy(&multiRootSec);CHKERRQ(ierr);
4443c921d74cSToby Isaac   ierr = PetscSectionDestroy(&rootIndicesSec);CHKERRQ(ierr);
4444d3bc4906SToby Isaac   ierr = PetscFree2(parentIndices,parentValues);CHKERRQ(ierr);
4445c921d74cSToby Isaac   ierr = DMPlexReferenceTreeRestoreChildrenMatrices_Injection(refTree,injRef,&childrenMats);CHKERRQ(ierr);
4446c921d74cSToby Isaac   ierr = PetscFree(rootValues);CHKERRQ(ierr);
4447c921d74cSToby Isaac   ierr = PetscFree3(offsets,offsetsCopy,rowOffsets);CHKERRQ(ierr);
4448ebf164c7SToby Isaac   PetscFunctionReturn(0);
4449ebf164c7SToby Isaac }
4450ebf164c7SToby Isaac 
4451ff1f73f7SToby Isaac /*@
4452ff1f73f7SToby Isaac   DMPlexTransferVecTree - transfer a vector between two meshes that differ from each other by refinement/coarsening
4453ff1f73f7SToby Isaac   that can be represented by a common reference tree used by both.  This routine can be used for a combination of
4454ff1f73f7SToby Isaac   coarsening and refinement at the same time.
4455ff1f73f7SToby Isaac 
4456ff1f73f7SToby Isaac   collective
4457ff1f73f7SToby Isaac 
4458ff1f73f7SToby Isaac   Input Parameters:
4459ff1f73f7SToby Isaac + dmIn        - The DMPlex mesh for the input vector
4460ff1f73f7SToby Isaac . vecIn       - The input vector
4461ff1f73f7SToby Isaac . sfRefine    - A star forest indicating points in the mesh dmIn (roots in the star forest) that are parents to points in
4462ff1f73f7SToby Isaac                 the mesh dmOut (leaves in the star forest), i.e. where dmOut is more refined than dmIn
4463ff1f73f7SToby Isaac . sfCoarsen   - A star forest indicating points in the mesh dmOut (roots in the star forest) that are parents to points in
4464ff1f73f7SToby Isaac                 the mesh dmIn (leaves in the star forest), i.e. where dmOut is more coarsened than dmIn
4465ff1f73f7SToby Isaac . cidsRefine  - The childIds of the points in dmOut.  These childIds relate back to the reference tree: childid[j] = k implies
4466ff1f73f7SToby Isaac                 that mesh point j of dmOut was refined from a point in dmIn just as the mesh point k in the reference
4467ff1f73f7SToby Isaac                 tree was refined from its parent.  childid[j] = -1 indicates that the point j in dmOut is exactly
4468ff1f73f7SToby Isaac                 equivalent to its root in dmIn, so no interpolation is necessary.  childid[j] = -2 indicates that this
4469ff1f73f7SToby Isaac                 point j in dmOut is not a leaf of sfRefine.
4470ff1f73f7SToby Isaac . cidsCoarsen - The childIds of the points in dmIn.  These childIds relate back to the reference tree: childid[j] = k implies
4471ff1f73f7SToby Isaac                 that mesh point j of dmIn coarsens to a point in dmOut just as the mesh point k in the reference
4472ff1f73f7SToby Isaac                 tree coarsens to its parent.  childid[j] = -2 indicates that point j in dmOut is not a leaf in sfCoarsen.
4473ff1f73f7SToby Isaac . useBCs      - PETSC_TRUE indicates that boundary values should be inserted into vecIn before transfer.
4474ff1f73f7SToby Isaac - time        - Used if boundary values are time dependent.
4475ff1f73f7SToby Isaac 
4476ff1f73f7SToby Isaac   Output Parameters:
4477ff1f73f7SToby Isaac . vecOut      - Using interpolation and injection operators calculated on the reference tree, the transfered
4478ff1f73f7SToby Isaac                 projection of vecIn from dmIn to dmOut.  Note that any field discretized with a PetscFV finite volume
4479ff1f73f7SToby Isaac                 method that uses gradient reconstruction will use reconstructed gradients when interpolating from
4480ff1f73f7SToby Isaac                 coarse points to fine points.
4481ff1f73f7SToby Isaac 
4482ff1f73f7SToby Isaac   Level: developer
4483ff1f73f7SToby Isaac 
4484ff1f73f7SToby Isaac .seealso(): DMPlexSetReferenceTree(), DMPlexGetReferenceTree(), PetscFVGetComputeGradients()
4485ff1f73f7SToby Isaac @*/
4486ff1f73f7SToby Isaac PetscErrorCode DMPlexTransferVecTree(DM dmIn, Vec vecIn, DM dmOut, Vec vecOut, PetscSF sfRefine, PetscSF sfCoarsen, PetscInt *cidsRefine, PetscInt *cidsCoarsen, PetscBool useBCs, PetscReal time)
448738fc2455SToby Isaac {
4488ebf164c7SToby Isaac   PetscErrorCode ierr;
4489ebf164c7SToby Isaac 
449038fc2455SToby Isaac   PetscFunctionBegin;
449178b7adb5SToby Isaac   ierr = VecSet(vecOut,0.0);CHKERRQ(ierr);
4492ff1f73f7SToby Isaac   if (sfRefine) {
4493fbfa57b9SToby Isaac     Vec vecInLocal;
44940eb7e1eaSToby Isaac     DM  dmGrad = NULL;
44950eb7e1eaSToby Isaac     Vec faceGeom = NULL, cellGeom = NULL, grad = NULL;
4496fbfa57b9SToby Isaac 
4497fbfa57b9SToby Isaac     ierr = DMGetLocalVector(dmIn,&vecInLocal);CHKERRQ(ierr);
4498fbfa57b9SToby Isaac     ierr = VecSet(vecInLocal,0.0);CHKERRQ(ierr);
44990eb7e1eaSToby Isaac     {
45000eb7e1eaSToby Isaac       PetscInt  numFields, i;
45010eb7e1eaSToby Isaac 
45020eb7e1eaSToby Isaac       ierr = DMGetNumFields(dmIn, &numFields);CHKERRQ(ierr);
45030eb7e1eaSToby Isaac       for (i = 0; i < numFields; i++) {
45040eb7e1eaSToby Isaac         PetscObject  obj;
45050eb7e1eaSToby Isaac         PetscClassId classid;
45060eb7e1eaSToby Isaac 
45070eb7e1eaSToby Isaac         ierr = DMGetField(dmIn, i, &obj);CHKERRQ(ierr);
45080eb7e1eaSToby Isaac         ierr = PetscObjectGetClassId(obj, &classid);CHKERRQ(ierr);
45090eb7e1eaSToby Isaac         if (classid == PETSCFV_CLASSID) {
45100eb7e1eaSToby Isaac           ierr = DMPlexGetDataFVM(dmIn,(PetscFV)obj,&cellGeom,&faceGeom,&dmGrad);CHKERRQ(ierr);
45110eb7e1eaSToby Isaac           break;
45120eb7e1eaSToby Isaac         }
45130eb7e1eaSToby Isaac       }
45140eb7e1eaSToby Isaac     }
45150eb7e1eaSToby Isaac     if (useBCs) {
45160eb7e1eaSToby Isaac       ierr = DMPlexInsertBoundaryValues(dmIn,PETSC_TRUE,vecInLocal,time,faceGeom,cellGeom,NULL);CHKERRQ(ierr);
45170eb7e1eaSToby Isaac     }
4518fbfa57b9SToby Isaac     ierr = DMGlobalToLocalBegin(dmIn,vecIn,INSERT_VALUES,vecInLocal);CHKERRQ(ierr);
4519fbfa57b9SToby Isaac     ierr = DMGlobalToLocalEnd(dmIn,vecIn,INSERT_VALUES,vecInLocal);CHKERRQ(ierr);
45200eb7e1eaSToby Isaac     if (dmGrad) {
45210eb7e1eaSToby Isaac       ierr = DMGetGlobalVector(dmGrad,&grad);CHKERRQ(ierr);
45220eb7e1eaSToby Isaac       ierr = DMPlexReconstructGradientsFVM(dmIn,vecInLocal,grad);CHKERRQ(ierr);
45230eb7e1eaSToby Isaac     }
4524ff1f73f7SToby Isaac     ierr = DMPlexTransferVecTree_Interpolate(dmIn,vecInLocal,dmOut,vecOut,sfRefine,cidsRefine,grad,cellGeom);CHKERRQ(ierr);
4525fbfa57b9SToby Isaac     ierr = DMRestoreLocalVector(dmIn,&vecInLocal);CHKERRQ(ierr);
45260eb7e1eaSToby Isaac     if (dmGrad) {
45270eb7e1eaSToby Isaac       ierr = DMRestoreGlobalVector(dmGrad,&grad);CHKERRQ(ierr);
45280eb7e1eaSToby Isaac     }
4529ebf164c7SToby Isaac   }
4530ff1f73f7SToby Isaac   if (sfCoarsen) {
4531ff1f73f7SToby Isaac     ierr = DMPlexTransferVecTree_Inject(dmIn,vecIn,dmOut,vecOut,sfCoarsen,cidsCoarsen);CHKERRQ(ierr);
4532ebf164c7SToby Isaac   }
45332f65e181SToby Isaac   ierr = VecAssemblyBegin(vecOut);CHKERRQ(ierr);
45342f65e181SToby Isaac   ierr = VecAssemblyEnd(vecOut);CHKERRQ(ierr);
453538fc2455SToby Isaac   PetscFunctionReturn(0);
453638fc2455SToby Isaac }
4537