xref: /petsc/src/dm/impls/plex/plex.c (revision 97d8846ca695d7e93a80b03b34bde28fe6a79ca9)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
17 
18   Input Parameter:
19 + dm     - The DMPlex object
20 - height - The cell height in the Plex, 0 is the default
21 
22   Output Parameters:
23 + cStart - The first "normal" cell
24 - cEnd   - The upper bound on "normal"" cells
25 
26   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
27 
28   Level: developer
29 
30 .seealso DMPlexConstructGhostCells(), DMPlexSetGhostCellStratum()
31 @*/
32 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
33 {
34   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
35   PetscInt       cS, cE, c;
36   PetscErrorCode ierr;
37 
38   PetscFunctionBegin;
39   ierr = DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE);CHKERRQ(ierr);
40   for (c = cS; c < cE; ++c) {
41     DMPolytopeType cct;
42 
43     ierr = DMPlexGetCellType(dm, c, &cct);CHKERRQ(ierr);
44     if ((PetscInt) cct < 0) break;
45     switch (cct) {
46       case DM_POLYTOPE_POINT:
47       case DM_POLYTOPE_SEGMENT:
48       case DM_POLYTOPE_TRIANGLE:
49       case DM_POLYTOPE_QUADRILATERAL:
50       case DM_POLYTOPE_TETRAHEDRON:
51       case DM_POLYTOPE_HEXAHEDRON:
52         ct = cct;
53         break;
54       default: break;
55     }
56     if (ct != DM_POLYTOPE_UNKNOWN) break;
57   }
58   if (ct != DM_POLYTOPE_UNKNOWN) {
59     DMLabel ctLabel;
60 
61     ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
62     ierr = DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE);CHKERRQ(ierr);
63   }
64   if (cStart) *cStart = cS;
65   if (cEnd)   *cEnd   = cE;
66   PetscFunctionReturn(0);
67 }
68 
69 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
70 {
71   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
72   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
73   PetscErrorCode ierr;
74 
75   PetscFunctionBegin;
76   *ft  = PETSC_VTK_INVALID;
77   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
78   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
79   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
80   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
81   if (field >= 0) {
82     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
83     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
84   } else {
85     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
86     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
87   }
88   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (globalvcdof[0]) {
90     *sStart = vStart;
91     *sEnd   = vEnd;
92     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
93     else                        *ft = PETSC_VTK_POINT_FIELD;
94   } else if (globalvcdof[1]) {
95     *sStart = cStart;
96     *sEnd   = cEnd;
97     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
98     else                        *ft = PETSC_VTK_CELL_FIELD;
99   } else {
100     if (field >= 0) {
101       const char *fieldname;
102 
103       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
104       ierr = PetscInfo2((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
105     } else {
106       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output typp of section\"%s\"\n");CHKERRQ(ierr);
107     }
108   }
109   PetscFunctionReturn(0);
110 }
111 
112 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
113 {
114   DM                 dm;
115   PetscSection       s;
116   PetscDraw          draw, popup;
117   DM                 cdm;
118   PetscSection       coordSection;
119   Vec                coordinates;
120   const PetscScalar *coords, *array;
121   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
122   PetscReal          vbound[2], time;
123   PetscBool          isnull, flg;
124   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
125   const char        *name;
126   char               title[PETSC_MAX_PATH_LEN];
127   PetscErrorCode     ierr;
128 
129   PetscFunctionBegin;
130   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
131   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
132   if (isnull) PetscFunctionReturn(0);
133 
134   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
135   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
136   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
137   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
138   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
139   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
140   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
141   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
142   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
143   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
144   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
145 
146   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
147   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
148 
149   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
150   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
151   for (c = 0; c < N; c += dim) {
152     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
153     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
154   }
155   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
156   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
157 
158   /* Could implement something like DMDASelectFields() */
159   for (f = 0; f < Nf; ++f) {
160     DM   fdm = dm;
161     Vec  fv  = v;
162     IS   fis;
163     char prefix[PETSC_MAX_PATH_LEN];
164     const char *fname;
165 
166     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
167     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
168 
169     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
170     else               {prefix[0] = '\0';}
171     if (Nf > 1) {
172       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
173       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
174       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
175       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
176     }
177     for (comp = 0; comp < Nc; ++comp, ++w) {
178       PetscInt nmax = 2;
179 
180       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
181       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
182       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
183       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
184 
185       /* TODO Get max and min only for this component */
186       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
187       if (!flg) {
188         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
189         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
190         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
191       }
192       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
193       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
194       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
195 
196       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
197       for (c = cStart; c < cEnd; ++c) {
198         PetscScalar *coords = NULL, *a = NULL;
199         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
200 
201         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
202         if (a) {
203           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
204           color[1] = color[2] = color[3] = color[0];
205         } else {
206           PetscScalar *vals = NULL;
207           PetscInt     numVals, va;
208 
209           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
210           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
211           switch (numVals/Nc) {
212           case 3: /* P1 Triangle */
213           case 4: /* P1 Quadrangle */
214             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
215             break;
216           case 6: /* P2 Triangle */
217           case 8: /* P2 Quadrangle */
218             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
219             break;
220           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
221           }
222           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
223         }
224         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
225         switch (numCoords) {
226         case 6:
227           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
228           break;
229         case 8:
230           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
231           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
232           break;
233         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
234         }
235         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
236       }
237       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
238       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
239       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
240       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
241     }
242     if (Nf > 1) {
243       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
244       ierr = ISDestroy(&fis);CHKERRQ(ierr);
245       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
246     }
247   }
248   PetscFunctionReturn(0);
249 }
250 
251 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
252 {
253   DM                      dm;
254   Vec                     locv;
255   const char              *name;
256   PetscSection            section;
257   PetscInt                pStart, pEnd;
258   PetscInt                numFields;
259   PetscViewerVTKFieldType ft;
260   PetscErrorCode          ierr;
261 
262   PetscFunctionBegin;
263   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
264   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
265   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
266   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
267   ierr = VecCopy(v, locv);CHKERRQ(ierr);
268   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
269   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
270   if (!numFields) {
271     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
272     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
273   } else {
274     PetscInt f;
275 
276     for (f = 0; f < numFields; f++) {
277       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
278       if (ft == PETSC_VTK_INVALID) continue;
279       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
280       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
281     }
282     ierr = VecDestroy(&locv);CHKERRQ(ierr);
283   }
284   PetscFunctionReturn(0);
285 }
286 
287 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
288 {
289   DM             dm;
290   PetscBool      isvtk, ishdf5, isdraw, isglvis;
291   PetscErrorCode ierr;
292 
293   PetscFunctionBegin;
294   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
295   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
296   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
297   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
298   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
299   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
300   if (isvtk || ishdf5 || isdraw || isglvis) {
301     PetscInt    i,numFields;
302     PetscObject fe;
303     PetscBool   fem = PETSC_FALSE;
304     Vec         locv = v;
305     const char  *name;
306     PetscInt    step;
307     PetscReal   time;
308 
309     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
310     for (i=0; i<numFields; i++) {
311       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
312       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
313     }
314     if (fem) {
315       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
316       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
317       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
318       ierr = VecCopy(v, locv);CHKERRQ(ierr);
319       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
320       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
321     }
322     if (isvtk) {
323       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
324     } else if (ishdf5) {
325 #if defined(PETSC_HAVE_HDF5)
326       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
327 #else
328       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
329 #endif
330     } else if (isdraw) {
331       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
332     } else if (isglvis) {
333       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
334       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
335       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
336     }
337     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
338   } else {
339     PetscBool isseq;
340 
341     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
342     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
343     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
344   }
345   PetscFunctionReturn(0);
346 }
347 
348 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
349 {
350   DM             dm;
351   PetscBool      isvtk, ishdf5, isdraw, isglvis;
352   PetscErrorCode ierr;
353 
354   PetscFunctionBegin;
355   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
356   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
357   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
358   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
359   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
360   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
361   if (isvtk || isdraw || isglvis) {
362     Vec         locv;
363     const char *name;
364 
365     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
366     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
367     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
368     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
369     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
370     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
371     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
372   } else if (ishdf5) {
373 #if defined(PETSC_HAVE_HDF5)
374     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
375 #else
376     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
377 #endif
378   } else {
379     PetscBool isseq;
380 
381     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
382     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
383     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
384   }
385   PetscFunctionReturn(0);
386 }
387 
388 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
389 {
390   DM                dm;
391   MPI_Comm          comm;
392   PetscViewerFormat format;
393   Vec               v;
394   PetscBool         isvtk, ishdf5;
395   PetscErrorCode    ierr;
396 
397   PetscFunctionBegin;
398   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
399   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
400   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
401   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
402   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
403   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
404   if (format == PETSC_VIEWER_NATIVE) {
405     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
406     /* this need a better fix */
407     if (dm->useNatural) {
408       if (dm->sfNatural) {
409         const char *vecname;
410         PetscInt    n, nroots;
411 
412         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
413         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
414         if (n == nroots) {
415           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
416           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
417           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
418           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
419           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
420         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
421       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
422     } else v = originalv;
423   } else v = originalv;
424 
425   if (ishdf5) {
426 #if defined(PETSC_HAVE_HDF5)
427     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
428 #else
429     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
430 #endif
431   } else if (isvtk) {
432     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
433   } else {
434     PetscBool isseq;
435 
436     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
437     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
438     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
439   }
440   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
441   PetscFunctionReturn(0);
442 }
443 
444 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
445 {
446   DM             dm;
447   PetscBool      ishdf5;
448   PetscErrorCode ierr;
449 
450   PetscFunctionBegin;
451   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
452   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
453   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
454   if (ishdf5) {
455     DM          dmBC;
456     Vec         gv;
457     const char *name;
458 
459     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
460     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
461     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
462     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
463     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
464     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
465     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
466     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
467   } else {
468     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
469   }
470   PetscFunctionReturn(0);
471 }
472 
473 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
474 {
475   DM             dm;
476   PetscBool      ishdf5;
477   PetscErrorCode ierr;
478 
479   PetscFunctionBegin;
480   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
481   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
482   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
483   if (ishdf5) {
484 #if defined(PETSC_HAVE_HDF5)
485     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
486 #else
487     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
488 #endif
489   } else {
490     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
491   }
492   PetscFunctionReturn(0);
493 }
494 
495 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
496 {
497   DM                dm;
498   PetscViewerFormat format;
499   PetscBool         ishdf5;
500   PetscErrorCode    ierr;
501 
502   PetscFunctionBegin;
503   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
504   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
505   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
506   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
507   if (format == PETSC_VIEWER_NATIVE) {
508     if (dm->useNatural) {
509       if (dm->sfNatural) {
510         if (ishdf5) {
511 #if defined(PETSC_HAVE_HDF5)
512           Vec         v;
513           const char *vecname;
514 
515           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
516           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
517           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
518           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
519           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
520           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
521           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
522 #else
523           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
524 #endif
525         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
526       }
527     } else {
528       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
529     }
530   }
531   PetscFunctionReturn(0);
532 }
533 
534 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
535 {
536   PetscSection       coordSection;
537   Vec                coordinates;
538   DMLabel            depthLabel, celltypeLabel;
539   const char        *name[4];
540   const PetscScalar *a;
541   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
542   PetscErrorCode     ierr;
543 
544   PetscFunctionBegin;
545   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
546   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
547   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
548   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
549   ierr = DMPlexGetCellTypeLabel(dm, &celltypeLabel);CHKERRQ(ierr);
550   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
551   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
552   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
553   name[0]     = "vertex";
554   name[1]     = "edge";
555   name[dim-1] = "face";
556   name[dim]   = "cell";
557   for (c = cStart; c < cEnd; ++c) {
558     PetscInt *closure = NULL;
559     PetscInt  closureSize, cl, ct;
560 
561     ierr = DMLabelGetValue(celltypeLabel, c, &ct);CHKERRQ(ierr);
562     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D polytope type %s:\n", c, DMPolytopeTypes[ct]);CHKERRQ(ierr);
563     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
564     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
565     for (cl = 0; cl < closureSize*2; cl += 2) {
566       PetscInt point = closure[cl], depth, dof, off, d, p;
567 
568       if ((point < pStart) || (point >= pEnd)) continue;
569       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
570       if (!dof) continue;
571       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
572       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
573       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
574       for (p = 0; p < dof/dim; ++p) {
575         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
576         for (d = 0; d < dim; ++d) {
577           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
578           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
579         }
580         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
581       }
582       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
583     }
584     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
585     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
586   }
587   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
588   PetscFunctionReturn(0);
589 }
590 
591 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
592 {
593   DM_Plex          *mesh = (DM_Plex*) dm->data;
594   DM                cdm;
595   DMLabel           markers, celltypes;
596   PetscSection      coordSection;
597   Vec               coordinates;
598   PetscViewerFormat format;
599   PetscErrorCode    ierr;
600 
601   PetscFunctionBegin;
602   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
603   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
604   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
605   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
606   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
607     const char *name;
608     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
609     PetscInt    pStart, pEnd, p;
610     PetscMPIInt rank, size;
611 
612     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
613     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
614     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
615     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
616     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
617     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
618     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
619     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
620     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
621     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
622     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
623     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
624     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
625     for (p = pStart; p < pEnd; ++p) {
626       PetscInt dof, off, s;
627 
628       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
629       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
630       for (s = off; s < off+dof; ++s) {
631         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
632       }
633     }
634     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
635     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
636     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
637     for (p = pStart; p < pEnd; ++p) {
638       PetscInt dof, off, c;
639 
640       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
641       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
642       for (c = off; c < off+dof; ++c) {
643         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
644       }
645     }
646     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
647     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
648     if (coordSection && coordinates) {
649       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
650     }
651     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
652     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
653     ierr = DMPlexGetCellTypeLabel(dm, &celltypes);CHKERRQ(ierr);
654     if (celltypes) {ierr = DMLabelView(celltypes, viewer);CHKERRQ(ierr);}
655     if (size > 1) {
656       PetscSF sf;
657 
658       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
659       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
660     }
661     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
662   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
663     const char  *name, *color;
664     const char  *defcolors[3]  = {"gray", "orange", "green"};
665     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
666     char         lname[PETSC_MAX_PATH_LEN];
667     PetscReal    scale         = 2.0;
668     PetscReal    tikzscale     = 1.0;
669     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
670     double       tcoords[3];
671     PetscScalar *coords;
672     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
673     PetscMPIInt  rank, size;
674     char         **names, **colors, **lcolors;
675     PetscBool    plotEdges, flg, lflg;
676     PetscBT      wp = NULL;
677     PetscInt     pEnd, pStart;
678 
679     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
680     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
681     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
682     numLabels  = PetscMax(numLabels, 10);
683     numColors  = 10;
684     numLColors = 10;
685     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
686     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
687     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
688     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
689     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
690     if (!useLabels) numLabels = 0;
691     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
692     if (!useColors) {
693       numColors = 3;
694       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
695     }
696     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
697     if (!useColors) {
698       numLColors = 4;
699       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
700     }
701     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, PETSC_MAX_PATH_LEN, &lflg);CHKERRQ(ierr);
702     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
703     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
704     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
705     if (depth < dim) plotEdges = PETSC_FALSE;
706 
707     /* filter points with labelvalue != labeldefaultvalue */
708     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
709     if (lflg) {
710       DMLabel lbl;
711 
712       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
713       if (lbl) {
714         PetscInt val, defval;
715 
716         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
717         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
718         for (c = pStart;  c < pEnd; c++) {
719           PetscInt *closure = NULL;
720           PetscInt  closureSize;
721 
722           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
723           if (val == defval) continue;
724 
725           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
726           for (p = 0; p < closureSize*2; p += 2) {
727             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
728           }
729           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
730         }
731       }
732     }
733 
734     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
735     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
736     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
737     ierr = PetscViewerASCIIPrintf(viewer, "\
738 \\documentclass[tikz]{standalone}\n\n\
739 \\usepackage{pgflibraryshapes}\n\
740 \\usetikzlibrary{backgrounds}\n\
741 \\usetikzlibrary{arrows}\n\
742 \\begin{document}\n");CHKERRQ(ierr);
743     if (size > 1) {
744       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
745       for (p = 0; p < size; ++p) {
746         if (p > 0 && p == size-1) {
747           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
748         } else if (p > 0) {
749           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
750         }
751         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
752       }
753       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
754     }
755     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
756 
757     /* Plot vertices */
758     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
759     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
760     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
761     for (v = vStart; v < vEnd; ++v) {
762       PetscInt  off, dof, d;
763       PetscBool isLabeled = PETSC_FALSE;
764 
765       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
766       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
767       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
768       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
769       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
770       for (d = 0; d < dof; ++d) {
771         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
772         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
773       }
774       /* Rotate coordinates since PGF makes z point out of the page instead of up */
775       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
776       for (d = 0; d < dof; ++d) {
777         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
778         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
779       }
780       color = colors[rank%numColors];
781       for (l = 0; l < numLabels; ++l) {
782         PetscInt val;
783         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
784         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
785       }
786       if (useNumbers) {
787         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
788       } else {
789         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
790       }
791     }
792     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
793     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
794     /* Plot cells */
795     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
796     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
797     if (dim == 3 || !useNumbers) {
798       for (e = eStart; e < eEnd; ++e) {
799         const PetscInt *cone;
800 
801         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
802         color = colors[rank%numColors];
803         for (l = 0; l < numLabels; ++l) {
804           PetscInt val;
805           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
806           if (val >= 0) {color = lcolors[l%numLColors]; break;}
807         }
808         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
809         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
810       }
811     } else {
812       for (c = cStart; c < cEnd; ++c) {
813         PetscInt *closure = NULL;
814         PetscInt  closureSize, firstPoint = -1;
815 
816         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
817         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
818         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
819         for (p = 0; p < closureSize*2; p += 2) {
820           const PetscInt point = closure[p];
821 
822           if ((point < vStart) || (point >= vEnd)) continue;
823           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
824           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
825           if (firstPoint < 0) firstPoint = point;
826         }
827         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
828         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
829         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
830       }
831     }
832     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
833     for (c = cStart; c < cEnd; ++c) {
834       double    ccoords[3] = {0.0, 0.0, 0.0};
835       PetscBool isLabeled  = PETSC_FALSE;
836       PetscInt *closure    = NULL;
837       PetscInt  closureSize, dof, d, n = 0;
838 
839       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
840       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
841       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
842       for (p = 0; p < closureSize*2; p += 2) {
843         const PetscInt point = closure[p];
844         PetscInt       off;
845 
846         if ((point < vStart) || (point >= vEnd)) continue;
847         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
848         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
849         for (d = 0; d < dof; ++d) {
850           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
851           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
852         }
853         /* Rotate coordinates since PGF makes z point out of the page instead of up */
854         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
855         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
856         ++n;
857       }
858       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
859       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
860       for (d = 0; d < dof; ++d) {
861         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
862         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
863       }
864       color = colors[rank%numColors];
865       for (l = 0; l < numLabels; ++l) {
866         PetscInt val;
867         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
868         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
869       }
870       if (useNumbers) {
871         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
872       } else {
873         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
874       }
875     }
876     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
877     /* Plot edges */
878     if (plotEdges) {
879       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
880       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
881       for (e = eStart; e < eEnd; ++e) {
882         const PetscInt *cone;
883         PetscInt        coneSize, offA, offB, dof, d;
884 
885         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
886         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
887         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
888         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
889         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
890         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
891         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
892         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
893         for (d = 0; d < dof; ++d) {
894           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
895           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
896         }
897         /* Rotate coordinates since PGF makes z point out of the page instead of up */
898         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
899         for (d = 0; d < dof; ++d) {
900           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
901           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
902         }
903         color = colors[rank%numColors];
904         for (l = 0; l < numLabels; ++l) {
905           PetscInt val;
906           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
907           if (val >= 0) {color = lcolors[l%numLColors]; break;}
908         }
909         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
910       }
911       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
912       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
913       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
914     }
915     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
916     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
917     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
918     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
919     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
920     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
921     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
922     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
923     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
924   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
925     Vec                    cown,acown;
926     VecScatter             sct;
927     ISLocalToGlobalMapping g2l;
928     IS                     gid,acis;
929     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
930     MPI_Group              ggroup,ngroup;
931     PetscScalar            *array,nid;
932     const PetscInt         *idxs;
933     PetscInt               *idxs2,*start,*adjacency,*work;
934     PetscInt64             lm[3],gm[3];
935     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
936     PetscMPIInt            d1,d2,rank;
937 
938     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
939     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
940 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
941     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
942 #endif
943     if (ncomm != MPI_COMM_NULL) {
944       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
945       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
946       d1   = 0;
947       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
948       nid  = d2;
949       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
950       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
951       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
952     } else nid = 0.0;
953 
954     /* Get connectivity */
955     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
956     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
957 
958     /* filter overlapped local cells */
959     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
960     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
961     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
962     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
963     for (c = cStart, cum = 0; c < cEnd; c++) {
964       if (idxs[c-cStart] < 0) continue;
965       idxs2[cum++] = idxs[c-cStart];
966     }
967     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
968     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
969     ierr = ISDestroy(&gid);CHKERRQ(ierr);
970     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
971 
972     /* support for node-aware cell locality */
973     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
974     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
975     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
976     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
977     for (c = 0; c < numVertices; c++) array[c] = nid;
978     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
979     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
980     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
981     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
982     ierr = ISDestroy(&acis);CHKERRQ(ierr);
983     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
984     ierr = VecDestroy(&cown);CHKERRQ(ierr);
985 
986     /* compute edgeCut */
987     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
988     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
989     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
990     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
991     ierr = ISDestroy(&gid);CHKERRQ(ierr);
992     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
993     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
994       PetscInt totl;
995 
996       totl = start[c+1]-start[c];
997       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
998       for (i = 0; i < totl; i++) {
999         if (work[i] < 0) {
1000           ect  += 1;
1001           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1002         }
1003       }
1004     }
1005     ierr  = PetscFree(work);CHKERRQ(ierr);
1006     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1007     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1008     lm[1] = -numVertices;
1009     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
1010     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1011     lm[0] = ect; /* edgeCut */
1012     lm[1] = ectn; /* node-aware edgeCut */
1013     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1014     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
1015     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1016 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1017     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1018 #else
1019     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1020 #endif
1021     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1022     ierr  = PetscFree(start);CHKERRQ(ierr);
1023     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1024     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1025   } else {
1026     const char    *name;
1027     PetscInt      *sizes, *hybsizes, *ghostsizes;
1028     PetscInt       locDepth, depth, cellHeight, dim, d;
1029     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1030     PetscInt       numLabels, l;
1031     DMPolytopeType ct0;
1032     MPI_Comm       comm;
1033     PetscMPIInt    size, rank;
1034 
1035     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
1036     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1037     ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1038     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1039     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1040     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1041     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1042     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1043     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1044     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1045     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1046     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1047     gcNum = gcEnd - gcStart;
1048     ierr = PetscCalloc3(size,&sizes,size,&hybsizes,size,&ghostsizes);CHKERRQ(ierr);
1049     for (d = 0; d <= depth; d++) {
1050       PetscInt Nc[2] = {0, 0}, ict;
1051 
1052       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1053       ierr = DMPlexGetCellType(dm, pStart, &ct0);CHKERRQ(ierr);
1054       ict  = ct0;
1055       ierr = MPI_Bcast(&ict, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1056       ct0  = (DMPolytopeType) ict;
1057       for (p = pStart; p < pEnd; ++p) {
1058         DMPolytopeType ct;
1059 
1060         ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
1061         if (ct == ct0) ++Nc[0];
1062         else           ++Nc[1];
1063       }
1064       ierr = MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1065       ierr = MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1066       if (d == depth) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);}
1067       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1068       for (p = 0; p < size; ++p) {
1069         if (!rank) {
1070           ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]+hybsizes[p]);CHKERRQ(ierr);
1071           if (hybsizes[p]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", hybsizes[p]);CHKERRQ(ierr);}
1072           if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1073         }
1074       }
1075       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1076     }
1077     ierr = PetscFree3(sizes,hybsizes,ghostsizes);CHKERRQ(ierr);
1078     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1079     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1080     for (l = 0; l < numLabels; ++l) {
1081       DMLabel         label;
1082       const char     *name;
1083       IS              valueIS;
1084       const PetscInt *values;
1085       PetscInt        numValues, v;
1086 
1087       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1088       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1089       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1090       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1091       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1092       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1093       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1094       for (v = 0; v < numValues; ++v) {
1095         PetscInt size;
1096 
1097         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1098         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1099         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1100       }
1101       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1102       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1103       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1104       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1105     }
1106     /* If no fields are specified, people do not want to see adjacency */
1107     if (dm->Nf) {
1108       PetscInt f;
1109 
1110       for (f = 0; f < dm->Nf; ++f) {
1111         const char *name;
1112 
1113         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1114         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1115         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1116         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1117         if (dm->fields[f].adjacency[0]) {
1118           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1119           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1120         } else {
1121           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1122           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1123         }
1124         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1125       }
1126     }
1127     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1128     if (cdm) {
1129       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1130       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1131       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1132     }
1133   }
1134   PetscFunctionReturn(0);
1135 }
1136 
1137 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1138 {
1139   PetscDraw          draw;
1140   DM                 cdm;
1141   PetscSection       coordSection;
1142   Vec                coordinates;
1143   const PetscScalar *coords;
1144   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1145   PetscBool          isnull;
1146   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1147   PetscMPIInt        rank;
1148   PetscErrorCode     ierr;
1149 
1150   PetscFunctionBegin;
1151   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1152   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1153   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1154   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1155   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1156   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1157   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1158 
1159   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1160   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1161   if (isnull) PetscFunctionReturn(0);
1162   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1163 
1164   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1165   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1166   for (c = 0; c < N; c += dim) {
1167     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1168     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1169   }
1170   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1171   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1172   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1173   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1174   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1175 
1176   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1177   for (c = cStart; c < cEnd; ++c) {
1178     PetscScalar   *coords = NULL;
1179     DMPolytopeType ct;
1180     PetscInt       numCoords;
1181 
1182     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
1183     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1184     switch (ct) {
1185     case DM_POLYTOPE_TRIANGLE:
1186       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1187                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1188                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1189                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1190       break;
1191     case DM_POLYTOPE_QUADRILATERAL:
1192       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1193                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1194                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1195                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1196       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1197                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1198                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1199                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1200       break;
1201     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1202     }
1203     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1204   }
1205   for (c = cStart; c < cEnd; ++c) {
1206     PetscScalar   *coords = NULL;
1207     DMPolytopeType ct;
1208     PetscInt       numCoords;
1209 
1210     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
1211     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1212     switch (ct) {
1213     case DM_POLYTOPE_TRIANGLE:
1214       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1215       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1216       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1217       break;
1218     case DM_POLYTOPE_QUADRILATERAL:
1219       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1220       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1221       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1222       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1223       break;
1224     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1225     }
1226     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1227   }
1228   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1229   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1230   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1231   PetscFunctionReturn(0);
1232 }
1233 
1234 #if defined(PETSC_HAVE_EXODUSII)
1235 #include <exodusII.h>
1236 #endif
1237 
1238 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1239 {
1240   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1241   char           name[PETSC_MAX_PATH_LEN];
1242   PetscErrorCode ierr;
1243 
1244   PetscFunctionBegin;
1245   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1246   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1247   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii);CHKERRQ(ierr);
1248   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
1249   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
1250   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
1251   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
1252   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus);CHKERRQ(ierr);
1253   if (iascii) {
1254     PetscViewerFormat format;
1255     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1256     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1257       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1258     } else {
1259       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1260     }
1261   } else if (ishdf5) {
1262 #if defined(PETSC_HAVE_HDF5)
1263     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1264 #else
1265     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1266 #endif
1267   } else if (isvtk) {
1268     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1269   } else if (isdraw) {
1270     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1271   } else if (isglvis) {
1272     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1273 #if defined(PETSC_HAVE_EXODUSII)
1274   } else if (isexodus) {
1275     int exoid;
1276     PetscInt cStart, cEnd, c;
1277 
1278     ierr = DMCreateLabel(dm, "Cell Sets");CHKERRQ(ierr);
1279     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1280     for (c = cStart; c < cEnd; ++c) {ierr = DMSetLabelValue(dm, "Cell Sets", c, 1);CHKERRQ(ierr);}
1281     ierr = PetscViewerExodusIIGetId(viewer, &exoid);CHKERRQ(ierr);
1282     ierr = DMPlexView_ExodusII_Internal(dm, exoid, 1);CHKERRQ(ierr);
1283 #endif
1284   } else {
1285     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1286   }
1287   /* Optionally view the partition */
1288   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1289   if (flg) {
1290     Vec ranks;
1291     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1292     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1293     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1294   }
1295   /* Optionally view a label */
1296   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1297   if (flg) {
1298     DMLabel label;
1299     Vec     val;
1300 
1301     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1302     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1303     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1304     ierr = VecView(val, viewer);CHKERRQ(ierr);
1305     ierr = VecDestroy(&val);CHKERRQ(ierr);
1306   }
1307   PetscFunctionReturn(0);
1308 }
1309 
1310 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1311 {
1312   PetscBool      ishdf5;
1313   PetscErrorCode ierr;
1314 
1315   PetscFunctionBegin;
1316   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1317   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1318   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1319   if (ishdf5) {
1320 #if defined(PETSC_HAVE_HDF5)
1321     PetscViewerFormat format;
1322     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1323     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1324       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1325     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1326       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1327     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1328 #else
1329     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1330 #endif
1331   } else {
1332     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1333   }
1334   PetscFunctionReturn(0);
1335 }
1336 
1337 PetscErrorCode DMDestroy_Plex(DM dm)
1338 {
1339   DM_Plex       *mesh = (DM_Plex*) dm->data;
1340   PetscErrorCode ierr;
1341 
1342   PetscFunctionBegin;
1343   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1344   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1345   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
1346   if (--mesh->refct > 0) PetscFunctionReturn(0);
1347   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1348   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1349   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1350   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1351   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1352   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1353   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1354   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1355   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1356   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1357   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1358   ierr = ISDestroy(&mesh->subpointIS);CHKERRQ(ierr);
1359   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1360   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1361   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1362   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1363   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1364   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1365   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1366   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1367   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1368   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1369   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1370   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1371   ierr = PetscFree(mesh);CHKERRQ(ierr);
1372   PetscFunctionReturn(0);
1373 }
1374 
1375 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1376 {
1377   PetscSection           sectionGlobal;
1378   PetscInt               bs = -1, mbs;
1379   PetscInt               localSize;
1380   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1381   PetscErrorCode         ierr;
1382   MatType                mtype;
1383   ISLocalToGlobalMapping ltog;
1384 
1385   PetscFunctionBegin;
1386   ierr = MatInitializePackage();CHKERRQ(ierr);
1387   mtype = dm->mattype;
1388   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1389   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1390   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1391   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1392   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1393   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1394   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1395   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1396   if (mbs > 1) bs = mbs;
1397   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1398   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1399   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1400   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1401   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1402   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1403   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1404   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1405   if (!isShell) {
1406     PetscSection subSection;
1407     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1408     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1409     PetscInt     pStart, pEnd, p, dof, cdof;
1410 
1411     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1412     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1413       PetscSection section;
1414       PetscInt     size;
1415 
1416       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
1417       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1418       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1419       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1420     } else {
1421       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1422     }
1423     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1424     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1425       PetscInt bdof;
1426 
1427       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1428       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1429       dof  = dof < 0 ? -(dof+1) : dof;
1430       bdof = cdof && (dof-cdof) ? 1 : dof;
1431       if (dof) {
1432         if (bs < 0)          {bs = bdof;}
1433         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1434       }
1435       if (isMatIS) {
1436         PetscInt loff,c,off;
1437         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1438         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1439         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1440       }
1441     }
1442     /* Must have same blocksize on all procs (some might have no points) */
1443     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1444     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1445     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1446     else                            {bs = bsMinMax[0];}
1447     bs = PetscMax(1,bs);
1448     if (isMatIS) { /* Must reduce indices by blocksize */
1449       PetscInt l;
1450 
1451       lsize = lsize/bs;
1452       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
1453       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1454     }
1455     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1456     if (isMatIS) {
1457       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1458     }
1459     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1460     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1461     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1462   }
1463   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1464   PetscFunctionReturn(0);
1465 }
1466 
1467 /*@
1468   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1469 
1470   Not collective
1471 
1472   Input Parameter:
1473 . mesh - The DMPlex
1474 
1475   Output Parameters:
1476 . subsection - The subdomain section
1477 
1478   Level: developer
1479 
1480 .seealso:
1481 @*/
1482 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1483 {
1484   DM_Plex       *mesh = (DM_Plex*) dm->data;
1485   PetscErrorCode ierr;
1486 
1487   PetscFunctionBegin;
1488   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1489   if (!mesh->subdomainSection) {
1490     PetscSection section;
1491     PetscSF      sf;
1492 
1493     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1494     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
1495     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1496     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1497   }
1498   *subsection = mesh->subdomainSection;
1499   PetscFunctionReturn(0);
1500 }
1501 
1502 /*@
1503   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1504 
1505   Not collective
1506 
1507   Input Parameter:
1508 . mesh - The DMPlex
1509 
1510   Output Parameters:
1511 + pStart - The first mesh point
1512 - pEnd   - The upper bound for mesh points
1513 
1514   Level: beginner
1515 
1516 .seealso: DMPlexCreate(), DMPlexSetChart()
1517 @*/
1518 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1519 {
1520   DM_Plex       *mesh = (DM_Plex*) dm->data;
1521   PetscErrorCode ierr;
1522 
1523   PetscFunctionBegin;
1524   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1525   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1526   PetscFunctionReturn(0);
1527 }
1528 
1529 /*@
1530   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1531 
1532   Not collective
1533 
1534   Input Parameters:
1535 + mesh - The DMPlex
1536 . pStart - The first mesh point
1537 - pEnd   - The upper bound for mesh points
1538 
1539   Output Parameters:
1540 
1541   Level: beginner
1542 
1543 .seealso: DMPlexCreate(), DMPlexGetChart()
1544 @*/
1545 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1546 {
1547   DM_Plex       *mesh = (DM_Plex*) dm->data;
1548   PetscErrorCode ierr;
1549 
1550   PetscFunctionBegin;
1551   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1552   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1553   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1554   PetscFunctionReturn(0);
1555 }
1556 
1557 /*@
1558   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1559 
1560   Not collective
1561 
1562   Input Parameters:
1563 + mesh - The DMPlex
1564 - p - The point, which must lie in the chart set with DMPlexSetChart()
1565 
1566   Output Parameter:
1567 . size - The cone size for point p
1568 
1569   Level: beginner
1570 
1571 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1572 @*/
1573 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1574 {
1575   DM_Plex       *mesh = (DM_Plex*) dm->data;
1576   PetscErrorCode ierr;
1577 
1578   PetscFunctionBegin;
1579   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1580   PetscValidPointer(size, 3);
1581   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1582   PetscFunctionReturn(0);
1583 }
1584 
1585 /*@
1586   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1587 
1588   Not collective
1589 
1590   Input Parameters:
1591 + mesh - The DMPlex
1592 . p - The point, which must lie in the chart set with DMPlexSetChart()
1593 - size - The cone size for point p
1594 
1595   Output Parameter:
1596 
1597   Note:
1598   This should be called after DMPlexSetChart().
1599 
1600   Level: beginner
1601 
1602 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1603 @*/
1604 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1605 {
1606   DM_Plex       *mesh = (DM_Plex*) dm->data;
1607   PetscErrorCode ierr;
1608 
1609   PetscFunctionBegin;
1610   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1611   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1612 
1613   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1614   PetscFunctionReturn(0);
1615 }
1616 
1617 /*@
1618   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1619 
1620   Not collective
1621 
1622   Input Parameters:
1623 + mesh - The DMPlex
1624 . p - The point, which must lie in the chart set with DMPlexSetChart()
1625 - size - The additional cone size for point p
1626 
1627   Output Parameter:
1628 
1629   Note:
1630   This should be called after DMPlexSetChart().
1631 
1632   Level: beginner
1633 
1634 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1635 @*/
1636 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1637 {
1638   DM_Plex       *mesh = (DM_Plex*) dm->data;
1639   PetscInt       csize;
1640   PetscErrorCode ierr;
1641 
1642   PetscFunctionBegin;
1643   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1644   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1645   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1646 
1647   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1648   PetscFunctionReturn(0);
1649 }
1650 
1651 /*@C
1652   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1653 
1654   Not collective
1655 
1656   Input Parameters:
1657 + dm - The DMPlex
1658 - p - The point, which must lie in the chart set with DMPlexSetChart()
1659 
1660   Output Parameter:
1661 . cone - An array of points which are on the in-edges for point p
1662 
1663   Level: beginner
1664 
1665   Fortran Notes:
1666   Since it returns an array, this routine is only available in Fortran 90, and you must
1667   include petsc.h90 in your code.
1668   You must also call DMPlexRestoreCone() after you finish using the returned array.
1669   DMPlexRestoreCone() is not needed/available in C.
1670 
1671 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1672 @*/
1673 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1674 {
1675   DM_Plex       *mesh = (DM_Plex*) dm->data;
1676   PetscInt       off;
1677   PetscErrorCode ierr;
1678 
1679   PetscFunctionBegin;
1680   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1681   PetscValidPointer(cone, 3);
1682   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1683   *cone = &mesh->cones[off];
1684   PetscFunctionReturn(0);
1685 }
1686 
1687 /*@C
1688   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1689 
1690   Not collective
1691 
1692   Input Parameters:
1693 + dm - The DMPlex
1694 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1695 
1696   Output Parameter:
1697 + pConesSection - PetscSection describing the layout of pCones
1698 - pCones - An array of points which are on the in-edges for the point set p
1699 
1700   Level: intermediate
1701 
1702 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1703 @*/
1704 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1705 {
1706   PetscSection        cs, newcs;
1707   PetscInt            *cones;
1708   PetscInt            *newarr=NULL;
1709   PetscInt            n;
1710   PetscErrorCode      ierr;
1711 
1712   PetscFunctionBegin;
1713   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1714   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1715   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1716   if (pConesSection) *pConesSection = newcs;
1717   if (pCones) {
1718     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1719     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1720   }
1721   PetscFunctionReturn(0);
1722 }
1723 
1724 /*@
1725   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
1726 
1727   Not collective
1728 
1729   Input Parameters:
1730 + dm - The DMPlex
1731 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1732 
1733   Output Parameter:
1734 . expandedPoints - An array of vertices recursively expanded from input points
1735 
1736   Level: advanced
1737 
1738   Notes:
1739   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
1740   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
1741 
1742 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
1743 @*/
1744 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
1745 {
1746   IS                  *expandedPointsAll;
1747   PetscInt            depth;
1748   PetscErrorCode      ierr;
1749 
1750   PetscFunctionBegin;
1751   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1752   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1753   PetscValidPointer(expandedPoints, 3);
1754   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1755   *expandedPoints = expandedPointsAll[0];
1756   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);
1757   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1758   PetscFunctionReturn(0);
1759 }
1760 
1761 /*@
1762   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
1763 
1764   Not collective
1765 
1766   Input Parameters:
1767 + dm - The DMPlex
1768 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1769 
1770   Output Parameter:
1771 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1772 . expandedPoints - (optional) An array of index sets with recursively expanded cones
1773 - sections - (optional) An array of sections which describe mappings from points to their cone points
1774 
1775   Level: advanced
1776 
1777   Notes:
1778   Like DMPlexGetConeTuple() but recursive.
1779 
1780   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
1781   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
1782 
1783   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
1784   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
1785   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
1786 
1787 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1788 @*/
1789 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1790 {
1791   const PetscInt      *arr0=NULL, *cone=NULL;
1792   PetscInt            *arr=NULL, *newarr=NULL;
1793   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
1794   IS                  *expandedPoints_;
1795   PetscSection        *sections_;
1796   PetscErrorCode      ierr;
1797 
1798   PetscFunctionBegin;
1799   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1800   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1801   if (depth) PetscValidIntPointer(depth, 3);
1802   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
1803   if (sections) PetscValidPointer(sections, 5);
1804   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
1805   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
1806   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1807   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
1808   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
1809   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
1810   for (d=depth_-1; d>=0; d--) {
1811     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
1812     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
1813     for (i=0; i<n; i++) {
1814       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
1815       if (arr[i] >= start && arr[i] < end) {
1816         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
1817         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
1818       } else {
1819         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
1820       }
1821     }
1822     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
1823     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
1824     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
1825     for (i=0; i<n; i++) {
1826       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
1827       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
1828       if (cn > 1) {
1829         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
1830         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
1831       } else {
1832         newarr[co] = arr[i];
1833       }
1834     }
1835     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
1836     arr = newarr;
1837     n = newn;
1838   }
1839   ierr = ISRestoreIndices(points, &arr0);CHKERRQ(ierr);
1840   *depth = depth_;
1841   if (expandedPoints) *expandedPoints = expandedPoints_;
1842   else {
1843     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
1844     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
1845   }
1846   if (sections) *sections = sections_;
1847   else {
1848     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
1849     ierr = PetscFree(sections_);CHKERRQ(ierr);
1850   }
1851   PetscFunctionReturn(0);
1852 }
1853 
1854 /*@
1855   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
1856 
1857   Not collective
1858 
1859   Input Parameters:
1860 + dm - The DMPlex
1861 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1862 
1863   Output Parameter:
1864 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1865 . expandedPoints - (optional) An array of recursively expanded cones
1866 - sections - (optional) An array of sections which describe mappings from points to their cone points
1867 
1868   Level: advanced
1869 
1870   Notes:
1871   See DMPlexGetConeRecursive() for details.
1872 
1873 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1874 @*/
1875 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1876 {
1877   PetscInt            d, depth_;
1878   PetscErrorCode      ierr;
1879 
1880   PetscFunctionBegin;
1881   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1882   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
1883   if (depth) *depth = 0;
1884   if (expandedPoints) {
1885     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
1886     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
1887   }
1888   if (sections)  {
1889     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
1890     ierr = PetscFree(*sections);CHKERRQ(ierr);
1891   }
1892   PetscFunctionReturn(0);
1893 }
1894 
1895 /*@
1896   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1897 
1898   Not collective
1899 
1900   Input Parameters:
1901 + mesh - The DMPlex
1902 . p - The point, which must lie in the chart set with DMPlexSetChart()
1903 - cone - An array of points which are on the in-edges for point p
1904 
1905   Output Parameter:
1906 
1907   Note:
1908   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1909 
1910   Developer Note: Why not call this DMPlexSetCover()
1911 
1912   Level: beginner
1913 
1914 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1915 @*/
1916 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1917 {
1918   DM_Plex       *mesh = (DM_Plex*) dm->data;
1919   PetscInt       pStart, pEnd;
1920   PetscInt       dof, off, c;
1921   PetscErrorCode ierr;
1922 
1923   PetscFunctionBegin;
1924   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1925   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1926   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1927   if (dof) PetscValidPointer(cone, 3);
1928   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1929   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1930   for (c = 0; c < dof; ++c) {
1931     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1932     mesh->cones[off+c] = cone[c];
1933   }
1934   PetscFunctionReturn(0);
1935 }
1936 
1937 /*@C
1938   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1939 
1940   Not collective
1941 
1942   Input Parameters:
1943 + mesh - The DMPlex
1944 - p - The point, which must lie in the chart set with DMPlexSetChart()
1945 
1946   Output Parameter:
1947 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1948                     integer giving the prescription for cone traversal. If it is negative, the cone is
1949                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1950                     the index of the cone point on which to start.
1951 
1952   Level: beginner
1953 
1954   Fortran Notes:
1955   Since it returns an array, this routine is only available in Fortran 90, and you must
1956   include petsc.h90 in your code.
1957   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1958   DMPlexRestoreConeOrientation() is not needed/available in C.
1959 
1960 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1961 @*/
1962 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1963 {
1964   DM_Plex       *mesh = (DM_Plex*) dm->data;
1965   PetscInt       off;
1966   PetscErrorCode ierr;
1967 
1968   PetscFunctionBegin;
1969   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1970 #if defined(PETSC_USE_DEBUG)
1971   {
1972     PetscInt dof;
1973     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1974     if (dof) PetscValidPointer(coneOrientation, 3);
1975   }
1976 #endif
1977   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1978 
1979   *coneOrientation = &mesh->coneOrientations[off];
1980   PetscFunctionReturn(0);
1981 }
1982 
1983 /*@
1984   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1985 
1986   Not collective
1987 
1988   Input Parameters:
1989 + mesh - The DMPlex
1990 . p - The point, which must lie in the chart set with DMPlexSetChart()
1991 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1992                     integer giving the prescription for cone traversal. If it is negative, the cone is
1993                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1994                     the index of the cone point on which to start.
1995 
1996   Output Parameter:
1997 
1998   Note:
1999   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2000 
2001   Level: beginner
2002 
2003 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2004 @*/
2005 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2006 {
2007   DM_Plex       *mesh = (DM_Plex*) dm->data;
2008   PetscInt       pStart, pEnd;
2009   PetscInt       dof, off, c;
2010   PetscErrorCode ierr;
2011 
2012   PetscFunctionBegin;
2013   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2014   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2015   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2016   if (dof) PetscValidPointer(coneOrientation, 3);
2017   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2018   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2019   for (c = 0; c < dof; ++c) {
2020     PetscInt cdof, o = coneOrientation[c];
2021 
2022     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2023     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2024     mesh->coneOrientations[off+c] = o;
2025   }
2026   PetscFunctionReturn(0);
2027 }
2028 
2029 /*@
2030   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2031 
2032   Not collective
2033 
2034   Input Parameters:
2035 + mesh - The DMPlex
2036 . p - The point, which must lie in the chart set with DMPlexSetChart()
2037 . conePos - The local index in the cone where the point should be put
2038 - conePoint - The mesh point to insert
2039 
2040   Level: beginner
2041 
2042 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2043 @*/
2044 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2045 {
2046   DM_Plex       *mesh = (DM_Plex*) dm->data;
2047   PetscInt       pStart, pEnd;
2048   PetscInt       dof, off;
2049   PetscErrorCode ierr;
2050 
2051   PetscFunctionBegin;
2052   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2053   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2054   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2055   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2056   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2057   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2058   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2059   mesh->cones[off+conePos] = conePoint;
2060   PetscFunctionReturn(0);
2061 }
2062 
2063 /*@
2064   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2065 
2066   Not collective
2067 
2068   Input Parameters:
2069 + mesh - The DMPlex
2070 . p - The point, which must lie in the chart set with DMPlexSetChart()
2071 . conePos - The local index in the cone where the point should be put
2072 - coneOrientation - The point orientation to insert
2073 
2074   Level: beginner
2075 
2076 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2077 @*/
2078 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2079 {
2080   DM_Plex       *mesh = (DM_Plex*) dm->data;
2081   PetscInt       pStart, pEnd;
2082   PetscInt       dof, off;
2083   PetscErrorCode ierr;
2084 
2085   PetscFunctionBegin;
2086   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2087   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2088   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2089   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2090   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2091   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2092   mesh->coneOrientations[off+conePos] = coneOrientation;
2093   PetscFunctionReturn(0);
2094 }
2095 
2096 /*@
2097   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2098 
2099   Not collective
2100 
2101   Input Parameters:
2102 + mesh - The DMPlex
2103 - p - The point, which must lie in the chart set with DMPlexSetChart()
2104 
2105   Output Parameter:
2106 . size - The support size for point p
2107 
2108   Level: beginner
2109 
2110 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2111 @*/
2112 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2113 {
2114   DM_Plex       *mesh = (DM_Plex*) dm->data;
2115   PetscErrorCode ierr;
2116 
2117   PetscFunctionBegin;
2118   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2119   PetscValidPointer(size, 3);
2120   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2121   PetscFunctionReturn(0);
2122 }
2123 
2124 /*@
2125   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2126 
2127   Not collective
2128 
2129   Input Parameters:
2130 + mesh - The DMPlex
2131 . p - The point, which must lie in the chart set with DMPlexSetChart()
2132 - size - The support size for point p
2133 
2134   Output Parameter:
2135 
2136   Note:
2137   This should be called after DMPlexSetChart().
2138 
2139   Level: beginner
2140 
2141 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2142 @*/
2143 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2144 {
2145   DM_Plex       *mesh = (DM_Plex*) dm->data;
2146   PetscErrorCode ierr;
2147 
2148   PetscFunctionBegin;
2149   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2150   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2151 
2152   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2153   PetscFunctionReturn(0);
2154 }
2155 
2156 /*@C
2157   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2158 
2159   Not collective
2160 
2161   Input Parameters:
2162 + mesh - The DMPlex
2163 - p - The point, which must lie in the chart set with DMPlexSetChart()
2164 
2165   Output Parameter:
2166 . support - An array of points which are on the out-edges for point p
2167 
2168   Level: beginner
2169 
2170   Fortran Notes:
2171   Since it returns an array, this routine is only available in Fortran 90, and you must
2172   include petsc.h90 in your code.
2173   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2174   DMPlexRestoreSupport() is not needed/available in C.
2175 
2176 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2177 @*/
2178 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2179 {
2180   DM_Plex       *mesh = (DM_Plex*) dm->data;
2181   PetscInt       off;
2182   PetscErrorCode ierr;
2183 
2184   PetscFunctionBegin;
2185   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2186   PetscValidPointer(support, 3);
2187   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2188   *support = &mesh->supports[off];
2189   PetscFunctionReturn(0);
2190 }
2191 
2192 /*@
2193   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2194 
2195   Not collective
2196 
2197   Input Parameters:
2198 + mesh - The DMPlex
2199 . p - The point, which must lie in the chart set with DMPlexSetChart()
2200 - support - An array of points which are on the out-edges for point p
2201 
2202   Output Parameter:
2203 
2204   Note:
2205   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2206 
2207   Level: beginner
2208 
2209 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2210 @*/
2211 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2212 {
2213   DM_Plex       *mesh = (DM_Plex*) dm->data;
2214   PetscInt       pStart, pEnd;
2215   PetscInt       dof, off, c;
2216   PetscErrorCode ierr;
2217 
2218   PetscFunctionBegin;
2219   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2220   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2221   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2222   if (dof) PetscValidPointer(support, 3);
2223   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2224   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2225   for (c = 0; c < dof; ++c) {
2226     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2227     mesh->supports[off+c] = support[c];
2228   }
2229   PetscFunctionReturn(0);
2230 }
2231 
2232 /*@
2233   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2234 
2235   Not collective
2236 
2237   Input Parameters:
2238 + mesh - The DMPlex
2239 . p - The point, which must lie in the chart set with DMPlexSetChart()
2240 . supportPos - The local index in the cone where the point should be put
2241 - supportPoint - The mesh point to insert
2242 
2243   Level: beginner
2244 
2245 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2246 @*/
2247 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2248 {
2249   DM_Plex       *mesh = (DM_Plex*) dm->data;
2250   PetscInt       pStart, pEnd;
2251   PetscInt       dof, off;
2252   PetscErrorCode ierr;
2253 
2254   PetscFunctionBegin;
2255   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2256   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2257   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2258   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2259   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2260   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2261   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2262   mesh->supports[off+supportPos] = supportPoint;
2263   PetscFunctionReturn(0);
2264 }
2265 
2266 /*@C
2267   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2268 
2269   Not collective
2270 
2271   Input Parameters:
2272 + mesh - The DMPlex
2273 . p - The point, which must lie in the chart set with DMPlexSetChart()
2274 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2275 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2276 
2277   Output Parameters:
2278 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2279 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2280 
2281   Note:
2282   If using internal storage (points is NULL on input), each call overwrites the last output.
2283 
2284   Fortran Notes:
2285   Since it returns an array, this routine is only available in Fortran 90, and you must
2286   include petsc.h90 in your code.
2287 
2288   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2289 
2290   Level: beginner
2291 
2292 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2293 @*/
2294 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2295 {
2296   DM_Plex        *mesh = (DM_Plex*) dm->data;
2297   PetscInt       *closure, *fifo;
2298   const PetscInt *tmp = NULL, *tmpO = NULL;
2299   PetscInt        tmpSize, t;
2300   PetscInt        depth       = 0, maxSize;
2301   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2302   PetscErrorCode  ierr;
2303 
2304   PetscFunctionBegin;
2305   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2306   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2307   /* This is only 1-level */
2308   if (useCone) {
2309     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2310     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2311     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2312   } else {
2313     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2314     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2315   }
2316   if (depth == 1) {
2317     if (*points) {
2318       closure = *points;
2319     } else {
2320       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2321       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2322     }
2323     closure[0] = p; closure[1] = 0;
2324     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2325       closure[closureSize]   = tmp[t];
2326       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2327     }
2328     if (numPoints) *numPoints = closureSize/2;
2329     if (points)    *points    = closure;
2330     PetscFunctionReturn(0);
2331   }
2332   {
2333     PetscInt c, coneSeries, s,supportSeries;
2334 
2335     c = mesh->maxConeSize;
2336     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2337     s = mesh->maxSupportSize;
2338     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2339     maxSize = 2*PetscMax(coneSeries,supportSeries);
2340   }
2341   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2342   if (*points) {
2343     closure = *points;
2344   } else {
2345     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2346   }
2347   closure[0] = p; closure[1] = 0;
2348   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2349     const PetscInt cp = tmp[t];
2350     const PetscInt co = tmpO ? tmpO[t] : 0;
2351 
2352     closure[closureSize]   = cp;
2353     closure[closureSize+1] = co;
2354     fifo[fifoSize]         = cp;
2355     fifo[fifoSize+1]       = co;
2356   }
2357   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2358   while (fifoSize - fifoStart) {
2359     const PetscInt q   = fifo[fifoStart];
2360     const PetscInt o   = fifo[fifoStart+1];
2361     const PetscInt rev = o >= 0 ? 0 : 1;
2362     const PetscInt off = rev ? -(o+1) : o;
2363 
2364     if (useCone) {
2365       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2366       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2367       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2368     } else {
2369       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2370       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2371       tmpO = NULL;
2372     }
2373     for (t = 0; t < tmpSize; ++t) {
2374       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2375       const PetscInt cp = tmp[i];
2376       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2377       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2378        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2379       PetscInt       co = tmpO ? tmpO[i] : 0;
2380       PetscInt       c;
2381 
2382       if (rev) {
2383         PetscInt childSize, coff;
2384         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2385         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2386         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2387       }
2388       /* Check for duplicate */
2389       for (c = 0; c < closureSize; c += 2) {
2390         if (closure[c] == cp) break;
2391       }
2392       if (c == closureSize) {
2393         closure[closureSize]   = cp;
2394         closure[closureSize+1] = co;
2395         fifo[fifoSize]         = cp;
2396         fifo[fifoSize+1]       = co;
2397         closureSize           += 2;
2398         fifoSize              += 2;
2399       }
2400     }
2401     fifoStart += 2;
2402   }
2403   if (numPoints) *numPoints = closureSize/2;
2404   if (points)    *points    = closure;
2405   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2406   PetscFunctionReturn(0);
2407 }
2408 
2409 /*@C
2410   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2411 
2412   Not collective
2413 
2414   Input Parameters:
2415 + mesh - The DMPlex
2416 . p - The point, which must lie in the chart set with DMPlexSetChart()
2417 . orientation - The orientation of the point
2418 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2419 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2420 
2421   Output Parameters:
2422 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2423 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2424 
2425   Note:
2426   If using internal storage (points is NULL on input), each call overwrites the last output.
2427 
2428   Fortran Notes:
2429   Since it returns an array, this routine is only available in Fortran 90, and you must
2430   include petsc.h90 in your code.
2431 
2432   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2433 
2434   Level: beginner
2435 
2436 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2437 @*/
2438 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2439 {
2440   DM_Plex        *mesh = (DM_Plex*) dm->data;
2441   PetscInt       *closure, *fifo;
2442   const PetscInt *tmp = NULL, *tmpO = NULL;
2443   PetscInt        tmpSize, t;
2444   PetscInt        depth       = 0, maxSize;
2445   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2446   PetscErrorCode  ierr;
2447 
2448   PetscFunctionBegin;
2449   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2450   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2451   /* This is only 1-level */
2452   if (useCone) {
2453     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2454     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2455     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2456   } else {
2457     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2458     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2459   }
2460   if (depth == 1) {
2461     if (*points) {
2462       closure = *points;
2463     } else {
2464       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2465       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2466     }
2467     closure[0] = p; closure[1] = ornt;
2468     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2469       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2470       closure[closureSize]   = tmp[i];
2471       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2472     }
2473     if (numPoints) *numPoints = closureSize/2;
2474     if (points)    *points    = closure;
2475     PetscFunctionReturn(0);
2476   }
2477   {
2478     PetscInt c, coneSeries, s,supportSeries;
2479 
2480     c = mesh->maxConeSize;
2481     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2482     s = mesh->maxSupportSize;
2483     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2484     maxSize = 2*PetscMax(coneSeries,supportSeries);
2485   }
2486   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2487   if (*points) {
2488     closure = *points;
2489   } else {
2490     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2491   }
2492   closure[0] = p; closure[1] = ornt;
2493   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2494     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2495     const PetscInt cp = tmp[i];
2496     PetscInt       co = tmpO ? tmpO[i] : 0;
2497 
2498     if (ornt < 0) {
2499       PetscInt childSize, coff;
2500       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2501       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2502       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2503     }
2504     closure[closureSize]   = cp;
2505     closure[closureSize+1] = co;
2506     fifo[fifoSize]         = cp;
2507     fifo[fifoSize+1]       = co;
2508   }
2509   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2510   while (fifoSize - fifoStart) {
2511     const PetscInt q   = fifo[fifoStart];
2512     const PetscInt o   = fifo[fifoStart+1];
2513     const PetscInt rev = o >= 0 ? 0 : 1;
2514     const PetscInt off = rev ? -(o+1) : o;
2515 
2516     if (useCone) {
2517       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2518       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2519       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2520     } else {
2521       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2522       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2523       tmpO = NULL;
2524     }
2525     for (t = 0; t < tmpSize; ++t) {
2526       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2527       const PetscInt cp = tmp[i];
2528       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2529       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2530        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2531       PetscInt       co = tmpO ? tmpO[i] : 0;
2532       PetscInt       c;
2533 
2534       if (rev) {
2535         PetscInt childSize, coff;
2536         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2537         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2538         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2539       }
2540       /* Check for duplicate */
2541       for (c = 0; c < closureSize; c += 2) {
2542         if (closure[c] == cp) break;
2543       }
2544       if (c == closureSize) {
2545         closure[closureSize]   = cp;
2546         closure[closureSize+1] = co;
2547         fifo[fifoSize]         = cp;
2548         fifo[fifoSize+1]       = co;
2549         closureSize           += 2;
2550         fifoSize              += 2;
2551       }
2552     }
2553     fifoStart += 2;
2554   }
2555   if (numPoints) *numPoints = closureSize/2;
2556   if (points)    *points    = closure;
2557   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2558   PetscFunctionReturn(0);
2559 }
2560 
2561 /*@C
2562   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2563 
2564   Not collective
2565 
2566   Input Parameters:
2567 + mesh - The DMPlex
2568 . p - The point, which must lie in the chart set with DMPlexSetChart()
2569 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2570 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2571 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2572 
2573   Note:
2574   If not using internal storage (points is not NULL on input), this call is unnecessary
2575 
2576   Fortran Notes:
2577   Since it returns an array, this routine is only available in Fortran 90, and you must
2578   include petsc.h90 in your code.
2579 
2580   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2581 
2582   Level: beginner
2583 
2584 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2585 @*/
2586 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2587 {
2588   PetscErrorCode ierr;
2589 
2590   PetscFunctionBegin;
2591   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2592   if (numPoints) PetscValidIntPointer(numPoints,4);
2593   if (points) PetscValidPointer(points,5);
2594   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2595   if (numPoints) *numPoints = 0;
2596   PetscFunctionReturn(0);
2597 }
2598 
2599 /*@
2600   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2601 
2602   Not collective
2603 
2604   Input Parameter:
2605 . mesh - The DMPlex
2606 
2607   Output Parameters:
2608 + maxConeSize - The maximum number of in-edges
2609 - maxSupportSize - The maximum number of out-edges
2610 
2611   Level: beginner
2612 
2613 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2614 @*/
2615 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2616 {
2617   DM_Plex *mesh = (DM_Plex*) dm->data;
2618 
2619   PetscFunctionBegin;
2620   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2621   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2622   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2623   PetscFunctionReturn(0);
2624 }
2625 
2626 PetscErrorCode DMSetUp_Plex(DM dm)
2627 {
2628   DM_Plex       *mesh = (DM_Plex*) dm->data;
2629   PetscInt       size;
2630   PetscErrorCode ierr;
2631 
2632   PetscFunctionBegin;
2633   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2634   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2635   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2636   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2637   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2638   if (mesh->maxSupportSize) {
2639     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2640     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2641     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2642   }
2643   PetscFunctionReturn(0);
2644 }
2645 
2646 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2647 {
2648   PetscErrorCode ierr;
2649 
2650   PetscFunctionBegin;
2651   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2652   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2653   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2654   if (dm->useNatural && dm->sfMigration) {
2655     PetscSF        sfMigrationInv,sfNatural;
2656     PetscSection   section, sectionSeq;
2657 
2658     (*subdm)->sfMigration = dm->sfMigration;
2659     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2660     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
2661     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2662     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2663     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2664 
2665     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2666     (*subdm)->sfNatural = sfNatural;
2667     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2668     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2669   }
2670   PetscFunctionReturn(0);
2671 }
2672 
2673 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2674 {
2675   PetscErrorCode ierr;
2676   PetscInt       i = 0;
2677 
2678   PetscFunctionBegin;
2679   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2680   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2681   (*superdm)->useNatural = PETSC_FALSE;
2682   for (i = 0; i < len; i++){
2683     if (dms[i]->useNatural && dms[i]->sfMigration) {
2684       PetscSF        sfMigrationInv,sfNatural;
2685       PetscSection   section, sectionSeq;
2686 
2687       (*superdm)->sfMigration = dms[i]->sfMigration;
2688       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2689       (*superdm)->useNatural = PETSC_TRUE;
2690       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
2691       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2692       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2693       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2694 
2695       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2696       (*superdm)->sfNatural = sfNatural;
2697       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2698       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2699       break;
2700     }
2701   }
2702   PetscFunctionReturn(0);
2703 }
2704 
2705 /*@
2706   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2707 
2708   Not collective
2709 
2710   Input Parameter:
2711 . mesh - The DMPlex
2712 
2713   Output Parameter:
2714 
2715   Note:
2716   This should be called after all calls to DMPlexSetCone()
2717 
2718   Level: beginner
2719 
2720 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2721 @*/
2722 PetscErrorCode DMPlexSymmetrize(DM dm)
2723 {
2724   DM_Plex       *mesh = (DM_Plex*) dm->data;
2725   PetscInt      *offsets;
2726   PetscInt       supportSize;
2727   PetscInt       pStart, pEnd, p;
2728   PetscErrorCode ierr;
2729 
2730   PetscFunctionBegin;
2731   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2732   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2733   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2734   /* Calculate support sizes */
2735   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2736   for (p = pStart; p < pEnd; ++p) {
2737     PetscInt dof, off, c;
2738 
2739     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2740     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2741     for (c = off; c < off+dof; ++c) {
2742       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2743     }
2744   }
2745   for (p = pStart; p < pEnd; ++p) {
2746     PetscInt dof;
2747 
2748     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2749 
2750     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2751   }
2752   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2753   /* Calculate supports */
2754   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2755   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2756   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2757   for (p = pStart; p < pEnd; ++p) {
2758     PetscInt dof, off, c;
2759 
2760     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2761     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2762     for (c = off; c < off+dof; ++c) {
2763       const PetscInt q = mesh->cones[c];
2764       PetscInt       offS;
2765 
2766       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2767 
2768       mesh->supports[offS+offsets[q]] = p;
2769       ++offsets[q];
2770     }
2771   }
2772   ierr = PetscFree(offsets);CHKERRQ(ierr);
2773   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2774   PetscFunctionReturn(0);
2775 }
2776 
2777 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2778 {
2779   IS             stratumIS;
2780   PetscErrorCode ierr;
2781 
2782   PetscFunctionBegin;
2783   if (pStart >= pEnd) PetscFunctionReturn(0);
2784 #if defined(PETSC_USE_DEBUG)
2785   {
2786     PetscInt  qStart, qEnd, numLevels, level;
2787     PetscBool overlap = PETSC_FALSE;
2788     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2789     for (level = 0; level < numLevels; level++) {
2790       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2791       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2792     }
2793     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2794   }
2795 #endif
2796   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2797   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2798   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2799   PetscFunctionReturn(0);
2800 }
2801 
2802 /*@
2803   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
2804   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2805   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2806   the DAG.
2807 
2808   Collective on dm
2809 
2810   Input Parameter:
2811 . mesh - The DMPlex
2812 
2813   Output Parameter:
2814 
2815   Notes:
2816   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
2817   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
2818   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2819   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2820   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2821 
2822   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
2823   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
2824   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
2825   to interpolate only that one (e0), so that
2826 $  cone(c0) = {e0, v2}
2827 $  cone(e0) = {v0, v1}
2828   If DMPlexStratify() is run on this mesh, it will give depths
2829 $  depth 0 = {v0, v1, v2}
2830 $  depth 1 = {e0, c0}
2831   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
2832 
2833   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2834 
2835   Level: beginner
2836 
2837 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexComputeCellTypes()
2838 @*/
2839 PetscErrorCode DMPlexStratify(DM dm)
2840 {
2841   DM_Plex       *mesh = (DM_Plex*) dm->data;
2842   DMLabel        label;
2843   PetscInt       pStart, pEnd, p;
2844   PetscInt       numRoots = 0, numLeaves = 0;
2845   PetscErrorCode ierr;
2846 
2847   PetscFunctionBegin;
2848   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2849   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2850 
2851   /* Create depth label */
2852   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2853   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2854   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2855 
2856   {
2857     /* Initialize roots and count leaves */
2858     PetscInt sMin = PETSC_MAX_INT;
2859     PetscInt sMax = PETSC_MIN_INT;
2860     PetscInt coneSize, supportSize;
2861 
2862     for (p = pStart; p < pEnd; ++p) {
2863       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2864       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2865       if (!coneSize && supportSize) {
2866         sMin = PetscMin(p, sMin);
2867         sMax = PetscMax(p, sMax);
2868         ++numRoots;
2869       } else if (!supportSize && coneSize) {
2870         ++numLeaves;
2871       } else if (!supportSize && !coneSize) {
2872         /* Isolated points */
2873         sMin = PetscMin(p, sMin);
2874         sMax = PetscMax(p, sMax);
2875       }
2876     }
2877     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2878   }
2879 
2880   if (numRoots + numLeaves == (pEnd - pStart)) {
2881     PetscInt sMin = PETSC_MAX_INT;
2882     PetscInt sMax = PETSC_MIN_INT;
2883     PetscInt coneSize, supportSize;
2884 
2885     for (p = pStart; p < pEnd; ++p) {
2886       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2887       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2888       if (!supportSize && coneSize) {
2889         sMin = PetscMin(p, sMin);
2890         sMax = PetscMax(p, sMax);
2891       }
2892     }
2893     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2894   } else {
2895     PetscInt level = 0;
2896     PetscInt qStart, qEnd, q;
2897 
2898     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2899     while (qEnd > qStart) {
2900       PetscInt sMin = PETSC_MAX_INT;
2901       PetscInt sMax = PETSC_MIN_INT;
2902 
2903       for (q = qStart; q < qEnd; ++q) {
2904         const PetscInt *support;
2905         PetscInt        supportSize, s;
2906 
2907         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2908         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2909         for (s = 0; s < supportSize; ++s) {
2910           sMin = PetscMin(support[s], sMin);
2911           sMax = PetscMax(support[s], sMax);
2912         }
2913       }
2914       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2915       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2916       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2917     }
2918   }
2919   { /* just in case there is an empty process */
2920     PetscInt numValues, maxValues = 0, v;
2921 
2922     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2923     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2924     for (v = numValues; v < maxValues; v++) {
2925       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2926     }
2927   }
2928   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2929   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2930   PetscFunctionReturn(0);
2931 }
2932 
2933 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
2934 {
2935   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
2936   PetscInt       dim, depth, pheight, coneSize;
2937   PetscErrorCode ierr;
2938 
2939   PetscFunctionBeginHot;
2940   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2941   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2942   ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2943   pheight = depth - pdepth;
2944   if (depth <= 1) {
2945     switch (pdepth) {
2946       case 0: ct = DM_POLYTOPE_POINT;break;
2947       case 1:
2948         switch (coneSize) {
2949           case 2: ct = DM_POLYTOPE_SEGMENT;break;
2950           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
2951           case 4:
2952           switch (dim) {
2953             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
2954             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
2955             default: break;
2956           }
2957           break;
2958         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
2959         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
2960         default: break;
2961       }
2962     }
2963   } else {
2964     if (pdepth == 0) {
2965       ct = DM_POLYTOPE_POINT;
2966     } else if (pheight == 0) {
2967       switch (dim) {
2968         case 1:
2969           switch (coneSize) {
2970             case 2: ct = DM_POLYTOPE_SEGMENT;break;
2971             default: break;
2972           }
2973           break;
2974         case 2:
2975           switch (coneSize) {
2976             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
2977             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
2978             default: break;
2979           }
2980           break;
2981         case 3:
2982           switch (coneSize) {
2983             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
2984             case 5: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
2985             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
2986             default: break;
2987           }
2988           break;
2989         default: break;
2990       }
2991     } else if (pheight > 0) {
2992       switch (coneSize) {
2993         case 2: ct = DM_POLYTOPE_SEGMENT;break;
2994         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
2995         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
2996         default: break;
2997       }
2998     }
2999   }
3000   *pt = ct;
3001   PetscFunctionReturn(0);
3002 }
3003 
3004 /*@
3005   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
3006 
3007   Collective on dm
3008 
3009   Input Parameter:
3010 . mesh - The DMPlex
3011 
3012   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
3013 
3014   Level: developer
3015 
3016   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
3017   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
3018   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
3019 
3020 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexStratify(), DMGetLabel(), DMCreateLabel()
3021 @*/
3022 PetscErrorCode DMPlexComputeCellTypes(DM dm)
3023 {
3024   DM_Plex       *mesh;
3025   DMLabel        ctLabel;
3026   PetscInt       pStart, pEnd, p;
3027   PetscErrorCode ierr;
3028 
3029   PetscFunctionBegin;
3030   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3031   mesh = (DM_Plex *) dm->data;
3032   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
3033   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
3034   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3035   for (p = pStart; p < pEnd; ++p) {
3036     DMPolytopeType ct;
3037     PetscInt       pdepth;
3038 
3039     ierr = DMPlexGetPointDepth(dm, p, &pdepth);CHKERRQ(ierr);
3040     ierr = DMPlexComputeCellType_Internal(dm, p, pdepth, &ct);CHKERRQ(ierr);
3041     if (ct == DM_POLYTOPE_UNKNOWN) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %D is screwed up", p);
3042     ierr = DMLabelSetValue(ctLabel, p, ct);CHKERRQ(ierr);
3043   }
3044   ierr = PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState);CHKERRQ(ierr);
3045   ierr = PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view");CHKERRQ(ierr);
3046   PetscFunctionReturn(0);
3047 }
3048 
3049 /*@C
3050   DMPlexGetJoin - Get an array for the join of the set of points
3051 
3052   Not Collective
3053 
3054   Input Parameters:
3055 + dm - The DMPlex object
3056 . numPoints - The number of input points for the join
3057 - points - The input points
3058 
3059   Output Parameters:
3060 + numCoveredPoints - The number of points in the join
3061 - coveredPoints - The points in the join
3062 
3063   Level: intermediate
3064 
3065   Note: Currently, this is restricted to a single level join
3066 
3067   Fortran Notes:
3068   Since it returns an array, this routine is only available in Fortran 90, and you must
3069   include petsc.h90 in your code.
3070 
3071   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3072 
3073 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
3074 @*/
3075 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3076 {
3077   DM_Plex       *mesh = (DM_Plex*) dm->data;
3078   PetscInt      *join[2];
3079   PetscInt       joinSize, i = 0;
3080   PetscInt       dof, off, p, c, m;
3081   PetscErrorCode ierr;
3082 
3083   PetscFunctionBegin;
3084   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3085   PetscValidIntPointer(points, 3);
3086   PetscValidIntPointer(numCoveredPoints, 4);
3087   PetscValidPointer(coveredPoints, 5);
3088   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3089   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3090   /* Copy in support of first point */
3091   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
3092   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
3093   for (joinSize = 0; joinSize < dof; ++joinSize) {
3094     join[i][joinSize] = mesh->supports[off+joinSize];
3095   }
3096   /* Check each successive support */
3097   for (p = 1; p < numPoints; ++p) {
3098     PetscInt newJoinSize = 0;
3099 
3100     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
3101     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
3102     for (c = 0; c < dof; ++c) {
3103       const PetscInt point = mesh->supports[off+c];
3104 
3105       for (m = 0; m < joinSize; ++m) {
3106         if (point == join[i][m]) {
3107           join[1-i][newJoinSize++] = point;
3108           break;
3109         }
3110       }
3111     }
3112     joinSize = newJoinSize;
3113     i        = 1-i;
3114   }
3115   *numCoveredPoints = joinSize;
3116   *coveredPoints    = join[i];
3117   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3118   PetscFunctionReturn(0);
3119 }
3120 
3121 /*@C
3122   DMPlexRestoreJoin - Restore an array for the join of the set of points
3123 
3124   Not Collective
3125 
3126   Input Parameters:
3127 + dm - The DMPlex object
3128 . numPoints - The number of input points for the join
3129 - points - The input points
3130 
3131   Output Parameters:
3132 + numCoveredPoints - The number of points in the join
3133 - coveredPoints - The points in the join
3134 
3135   Fortran Notes:
3136   Since it returns an array, this routine is only available in Fortran 90, and you must
3137   include petsc.h90 in your code.
3138 
3139   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3140 
3141   Level: intermediate
3142 
3143 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3144 @*/
3145 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3146 {
3147   PetscErrorCode ierr;
3148 
3149   PetscFunctionBegin;
3150   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3151   if (points) PetscValidIntPointer(points,3);
3152   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3153   PetscValidPointer(coveredPoints, 5);
3154   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3155   if (numCoveredPoints) *numCoveredPoints = 0;
3156   PetscFunctionReturn(0);
3157 }
3158 
3159 /*@C
3160   DMPlexGetFullJoin - Get an array for the join of the set of points
3161 
3162   Not Collective
3163 
3164   Input Parameters:
3165 + dm - The DMPlex object
3166 . numPoints - The number of input points for the join
3167 - points - The input points
3168 
3169   Output Parameters:
3170 + numCoveredPoints - The number of points in the join
3171 - coveredPoints - The points in the join
3172 
3173   Fortran Notes:
3174   Since it returns an array, this routine is only available in Fortran 90, and you must
3175   include petsc.h90 in your code.
3176 
3177   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3178 
3179   Level: intermediate
3180 
3181 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3182 @*/
3183 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3184 {
3185   DM_Plex       *mesh = (DM_Plex*) dm->data;
3186   PetscInt      *offsets, **closures;
3187   PetscInt      *join[2];
3188   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3189   PetscInt       p, d, c, m, ms;
3190   PetscErrorCode ierr;
3191 
3192   PetscFunctionBegin;
3193   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3194   PetscValidIntPointer(points, 3);
3195   PetscValidIntPointer(numCoveredPoints, 4);
3196   PetscValidPointer(coveredPoints, 5);
3197 
3198   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3199   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3200   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3201   ms      = mesh->maxSupportSize;
3202   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3203   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3204   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3205 
3206   for (p = 0; p < numPoints; ++p) {
3207     PetscInt closureSize;
3208 
3209     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3210 
3211     offsets[p*(depth+2)+0] = 0;
3212     for (d = 0; d < depth+1; ++d) {
3213       PetscInt pStart, pEnd, i;
3214 
3215       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
3216       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
3217         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3218           offsets[p*(depth+2)+d+1] = i;
3219           break;
3220         }
3221       }
3222       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
3223     }
3224     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
3225   }
3226   for (d = 0; d < depth+1; ++d) {
3227     PetscInt dof;
3228 
3229     /* Copy in support of first point */
3230     dof = offsets[d+1] - offsets[d];
3231     for (joinSize = 0; joinSize < dof; ++joinSize) {
3232       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
3233     }
3234     /* Check each successive cone */
3235     for (p = 1; p < numPoints && joinSize; ++p) {
3236       PetscInt newJoinSize = 0;
3237 
3238       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
3239       for (c = 0; c < dof; ++c) {
3240         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
3241 
3242         for (m = 0; m < joinSize; ++m) {
3243           if (point == join[i][m]) {
3244             join[1-i][newJoinSize++] = point;
3245             break;
3246           }
3247         }
3248       }
3249       joinSize = newJoinSize;
3250       i        = 1-i;
3251     }
3252     if (joinSize) break;
3253   }
3254   *numCoveredPoints = joinSize;
3255   *coveredPoints    = join[i];
3256   for (p = 0; p < numPoints; ++p) {
3257     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
3258   }
3259   ierr = PetscFree(closures);CHKERRQ(ierr);
3260   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3261   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3262   PetscFunctionReturn(0);
3263 }
3264 
3265 /*@C
3266   DMPlexGetMeet - Get an array for the meet of the set of points
3267 
3268   Not Collective
3269 
3270   Input Parameters:
3271 + dm - The DMPlex object
3272 . numPoints - The number of input points for the meet
3273 - points - The input points
3274 
3275   Output Parameters:
3276 + numCoveredPoints - The number of points in the meet
3277 - coveredPoints - The points in the meet
3278 
3279   Level: intermediate
3280 
3281   Note: Currently, this is restricted to a single level meet
3282 
3283   Fortran Notes:
3284   Since it returns an array, this routine is only available in Fortran 90, and you must
3285   include petsc.h90 in your code.
3286 
3287   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3288 
3289 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3290 @*/
3291 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3292 {
3293   DM_Plex       *mesh = (DM_Plex*) dm->data;
3294   PetscInt      *meet[2];
3295   PetscInt       meetSize, i = 0;
3296   PetscInt       dof, off, p, c, m;
3297   PetscErrorCode ierr;
3298 
3299   PetscFunctionBegin;
3300   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3301   PetscValidPointer(points, 2);
3302   PetscValidPointer(numCoveringPoints, 3);
3303   PetscValidPointer(coveringPoints, 4);
3304   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3305   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3306   /* Copy in cone of first point */
3307   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3308   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3309   for (meetSize = 0; meetSize < dof; ++meetSize) {
3310     meet[i][meetSize] = mesh->cones[off+meetSize];
3311   }
3312   /* Check each successive cone */
3313   for (p = 1; p < numPoints; ++p) {
3314     PetscInt newMeetSize = 0;
3315 
3316     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3317     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3318     for (c = 0; c < dof; ++c) {
3319       const PetscInt point = mesh->cones[off+c];
3320 
3321       for (m = 0; m < meetSize; ++m) {
3322         if (point == meet[i][m]) {
3323           meet[1-i][newMeetSize++] = point;
3324           break;
3325         }
3326       }
3327     }
3328     meetSize = newMeetSize;
3329     i        = 1-i;
3330   }
3331   *numCoveringPoints = meetSize;
3332   *coveringPoints    = meet[i];
3333   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3334   PetscFunctionReturn(0);
3335 }
3336 
3337 /*@C
3338   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3339 
3340   Not Collective
3341 
3342   Input Parameters:
3343 + dm - The DMPlex object
3344 . numPoints - The number of input points for the meet
3345 - points - The input points
3346 
3347   Output Parameters:
3348 + numCoveredPoints - The number of points in the meet
3349 - coveredPoints - The points in the meet
3350 
3351   Level: intermediate
3352 
3353   Fortran Notes:
3354   Since it returns an array, this routine is only available in Fortran 90, and you must
3355   include petsc.h90 in your code.
3356 
3357   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3358 
3359 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3360 @*/
3361 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3362 {
3363   PetscErrorCode ierr;
3364 
3365   PetscFunctionBegin;
3366   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3367   if (points) PetscValidIntPointer(points,3);
3368   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3369   PetscValidPointer(coveredPoints,5);
3370   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3371   if (numCoveredPoints) *numCoveredPoints = 0;
3372   PetscFunctionReturn(0);
3373 }
3374 
3375 /*@C
3376   DMPlexGetFullMeet - Get an array for the meet of the set of points
3377 
3378   Not Collective
3379 
3380   Input Parameters:
3381 + dm - The DMPlex object
3382 . numPoints - The number of input points for the meet
3383 - points - The input points
3384 
3385   Output Parameters:
3386 + numCoveredPoints - The number of points in the meet
3387 - coveredPoints - The points in the meet
3388 
3389   Level: intermediate
3390 
3391   Fortran Notes:
3392   Since it returns an array, this routine is only available in Fortran 90, and you must
3393   include petsc.h90 in your code.
3394 
3395   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3396 
3397 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3398 @*/
3399 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3400 {
3401   DM_Plex       *mesh = (DM_Plex*) dm->data;
3402   PetscInt      *offsets, **closures;
3403   PetscInt      *meet[2];
3404   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3405   PetscInt       p, h, c, m, mc;
3406   PetscErrorCode ierr;
3407 
3408   PetscFunctionBegin;
3409   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3410   PetscValidPointer(points, 2);
3411   PetscValidPointer(numCoveredPoints, 3);
3412   PetscValidPointer(coveredPoints, 4);
3413 
3414   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3415   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3416   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3417   mc      = mesh->maxConeSize;
3418   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3419   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3420   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3421 
3422   for (p = 0; p < numPoints; ++p) {
3423     PetscInt closureSize;
3424 
3425     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3426 
3427     offsets[p*(height+2)+0] = 0;
3428     for (h = 0; h < height+1; ++h) {
3429       PetscInt pStart, pEnd, i;
3430 
3431       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3432       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3433         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3434           offsets[p*(height+2)+h+1] = i;
3435           break;
3436         }
3437       }
3438       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3439     }
3440     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3441   }
3442   for (h = 0; h < height+1; ++h) {
3443     PetscInt dof;
3444 
3445     /* Copy in cone of first point */
3446     dof = offsets[h+1] - offsets[h];
3447     for (meetSize = 0; meetSize < dof; ++meetSize) {
3448       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3449     }
3450     /* Check each successive cone */
3451     for (p = 1; p < numPoints && meetSize; ++p) {
3452       PetscInt newMeetSize = 0;
3453 
3454       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3455       for (c = 0; c < dof; ++c) {
3456         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3457 
3458         for (m = 0; m < meetSize; ++m) {
3459           if (point == meet[i][m]) {
3460             meet[1-i][newMeetSize++] = point;
3461             break;
3462           }
3463         }
3464       }
3465       meetSize = newMeetSize;
3466       i        = 1-i;
3467     }
3468     if (meetSize) break;
3469   }
3470   *numCoveredPoints = meetSize;
3471   *coveredPoints    = meet[i];
3472   for (p = 0; p < numPoints; ++p) {
3473     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3474   }
3475   ierr = PetscFree(closures);CHKERRQ(ierr);
3476   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3477   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3478   PetscFunctionReturn(0);
3479 }
3480 
3481 /*@C
3482   DMPlexEqual - Determine if two DMs have the same topology
3483 
3484   Not Collective
3485 
3486   Input Parameters:
3487 + dmA - A DMPlex object
3488 - dmB - A DMPlex object
3489 
3490   Output Parameters:
3491 . equal - PETSC_TRUE if the topologies are identical
3492 
3493   Level: intermediate
3494 
3495   Notes:
3496   We are not solving graph isomorphism, so we do not permutation.
3497 
3498 .seealso: DMPlexGetCone()
3499 @*/
3500 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3501 {
3502   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3503   PetscErrorCode ierr;
3504 
3505   PetscFunctionBegin;
3506   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3507   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3508   PetscValidPointer(equal, 3);
3509 
3510   *equal = PETSC_FALSE;
3511   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3512   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3513   if (depth != depthB) PetscFunctionReturn(0);
3514   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3515   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3516   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3517   for (p = pStart; p < pEnd; ++p) {
3518     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3519     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3520 
3521     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3522     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3523     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3524     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3525     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3526     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3527     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3528     for (c = 0; c < coneSize; ++c) {
3529       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3530       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3531     }
3532     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3533     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3534     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3535     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3536     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3537     for (s = 0; s < supportSize; ++s) {
3538       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3539     }
3540   }
3541   *equal = PETSC_TRUE;
3542   PetscFunctionReturn(0);
3543 }
3544 
3545 /*@C
3546   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3547 
3548   Not Collective
3549 
3550   Input Parameters:
3551 + dm         - The DMPlex
3552 . cellDim    - The cell dimension
3553 - numCorners - The number of vertices on a cell
3554 
3555   Output Parameters:
3556 . numFaceVertices - The number of vertices on a face
3557 
3558   Level: developer
3559 
3560   Notes:
3561   Of course this can only work for a restricted set of symmetric shapes
3562 
3563 .seealso: DMPlexGetCone()
3564 @*/
3565 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3566 {
3567   MPI_Comm       comm;
3568   PetscErrorCode ierr;
3569 
3570   PetscFunctionBegin;
3571   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3572   PetscValidPointer(numFaceVertices,3);
3573   switch (cellDim) {
3574   case 0:
3575     *numFaceVertices = 0;
3576     break;
3577   case 1:
3578     *numFaceVertices = 1;
3579     break;
3580   case 2:
3581     switch (numCorners) {
3582     case 3: /* triangle */
3583       *numFaceVertices = 2; /* Edge has 2 vertices */
3584       break;
3585     case 4: /* quadrilateral */
3586       *numFaceVertices = 2; /* Edge has 2 vertices */
3587       break;
3588     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3589       *numFaceVertices = 3; /* Edge has 3 vertices */
3590       break;
3591     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3592       *numFaceVertices = 3; /* Edge has 3 vertices */
3593       break;
3594     default:
3595       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3596     }
3597     break;
3598   case 3:
3599     switch (numCorners) {
3600     case 4: /* tetradehdron */
3601       *numFaceVertices = 3; /* Face has 3 vertices */
3602       break;
3603     case 6: /* tet cohesive cells */
3604       *numFaceVertices = 4; /* Face has 4 vertices */
3605       break;
3606     case 8: /* hexahedron */
3607       *numFaceVertices = 4; /* Face has 4 vertices */
3608       break;
3609     case 9: /* tet cohesive Lagrange cells */
3610       *numFaceVertices = 6; /* Face has 6 vertices */
3611       break;
3612     case 10: /* quadratic tetrahedron */
3613       *numFaceVertices = 6; /* Face has 6 vertices */
3614       break;
3615     case 12: /* hex cohesive Lagrange cells */
3616       *numFaceVertices = 6; /* Face has 6 vertices */
3617       break;
3618     case 18: /* quadratic tet cohesive Lagrange cells */
3619       *numFaceVertices = 6; /* Face has 6 vertices */
3620       break;
3621     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3622       *numFaceVertices = 9; /* Face has 9 vertices */
3623       break;
3624     default:
3625       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3626     }
3627     break;
3628   default:
3629     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3630   }
3631   PetscFunctionReturn(0);
3632 }
3633 
3634 /*@
3635   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3636 
3637   Not Collective
3638 
3639   Input Parameter:
3640 . dm    - The DMPlex object
3641 
3642   Output Parameter:
3643 . depthLabel - The DMLabel recording point depth
3644 
3645   Level: developer
3646 
3647 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3648 @*/
3649 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3650 {
3651   PetscFunctionBegin;
3652   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3653   PetscValidPointer(depthLabel, 2);
3654   *depthLabel = dm->depthLabel;
3655   PetscFunctionReturn(0);
3656 }
3657 
3658 /*@
3659   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3660 
3661   Not Collective
3662 
3663   Input Parameter:
3664 . dm    - The DMPlex object
3665 
3666   Output Parameter:
3667 . depth - The number of strata (breadth first levels) in the DAG
3668 
3669   Level: developer
3670 
3671   Notes:
3672   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
3673   The point depth is described more in detail in DMPlexSymmetrize().
3674 
3675 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(), DMPlexGetPointHeight(), DMPlexSymmetrize()
3676 @*/
3677 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3678 {
3679   DMLabel        label;
3680   PetscInt       d = 0;
3681   PetscErrorCode ierr;
3682 
3683   PetscFunctionBegin;
3684   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3685   PetscValidPointer(depth, 2);
3686   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3687   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3688   *depth = d-1;
3689   PetscFunctionReturn(0);
3690 }
3691 
3692 /*@
3693   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3694 
3695   Not Collective
3696 
3697   Input Parameters:
3698 + dm           - The DMPlex object
3699 - stratumValue - The requested depth
3700 
3701   Output Parameters:
3702 + start - The first point at this depth
3703 - end   - One beyond the last point at this depth
3704 
3705   Notes:
3706   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3707   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3708   higher dimension, e.g., "edges".
3709 
3710   Level: developer
3711 
3712 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth(), DMPlexGetPointDepth()
3713 @*/
3714 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3715 {
3716   DMLabel        label;
3717   PetscInt       pStart, pEnd;
3718   PetscErrorCode ierr;
3719 
3720   PetscFunctionBegin;
3721   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3722   if (start) {PetscValidPointer(start, 3); *start = 0;}
3723   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3724   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3725   if (pStart == pEnd) PetscFunctionReturn(0);
3726   if (stratumValue < 0) {
3727     if (start) *start = pStart;
3728     if (end)   *end   = pEnd;
3729     PetscFunctionReturn(0);
3730   }
3731   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3732   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3733   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3734   PetscFunctionReturn(0);
3735 }
3736 
3737 /*@
3738   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3739 
3740   Not Collective
3741 
3742   Input Parameters:
3743 + dm           - The DMPlex object
3744 - stratumValue - The requested height
3745 
3746   Output Parameters:
3747 + start - The first point at this height
3748 - end   - One beyond the last point at this height
3749 
3750   Notes:
3751   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3752   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3753   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3754 
3755   Level: developer
3756 
3757 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth(), DMPlexGetPointHeight()
3758 @*/
3759 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3760 {
3761   DMLabel        label;
3762   PetscInt       depth, pStart, pEnd;
3763   PetscErrorCode ierr;
3764 
3765   PetscFunctionBegin;
3766   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3767   if (start) {PetscValidPointer(start, 3); *start = 0;}
3768   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3769   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3770   if (pStart == pEnd) PetscFunctionReturn(0);
3771   if (stratumValue < 0) {
3772     if (start) *start = pStart;
3773     if (end)   *end   = pEnd;
3774     PetscFunctionReturn(0);
3775   }
3776   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3777   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3778   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3779   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3780   PetscFunctionReturn(0);
3781 }
3782 
3783 /*@
3784   DMPlexGetPointDepth - Get the depth of a given point
3785 
3786   Not Collective
3787 
3788   Input Parameter:
3789 + dm    - The DMPlex object
3790 - point - The point
3791 
3792   Output Parameter:
3793 . depth - The depth of the point
3794 
3795   Level: intermediate
3796 
3797 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointHeight()
3798 @*/
3799 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
3800 {
3801   PetscErrorCode ierr;
3802 
3803   PetscFunctionBegin;
3804   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3805   PetscValidIntPointer(depth, 3);
3806   ierr = DMLabelGetValue(dm->depthLabel, point, depth);CHKERRQ(ierr);
3807   PetscFunctionReturn(0);
3808 }
3809 
3810 /*@
3811   DMPlexGetPointHeight - Get the height of a given point
3812 
3813   Not Collective
3814 
3815   Input Parameter:
3816 + dm    - The DMPlex object
3817 - point - The point
3818 
3819   Output Parameter:
3820 . height - The height of the point
3821 
3822   Level: intermediate
3823 
3824 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointDepth()
3825 @*/
3826 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
3827 {
3828   PetscInt       n, pDepth;
3829   PetscErrorCode ierr;
3830 
3831   PetscFunctionBegin;
3832   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3833   PetscValidIntPointer(height, 3);
3834   ierr = DMLabelGetNumValues(dm->depthLabel, &n);CHKERRQ(ierr);
3835   ierr = DMLabelGetValue(dm->depthLabel, point, &pDepth);CHKERRQ(ierr);
3836   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
3837   PetscFunctionReturn(0);
3838 }
3839 
3840 /*@
3841   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
3842 
3843   Not Collective
3844 
3845   Input Parameter:
3846 . dm - The DMPlex object
3847 
3848   Output Parameter:
3849 . celltypeLabel - The DMLabel recording cell polytope type
3850 
3851   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
3852   DMCreateLabel(dm, "celltype") beforehand.
3853 
3854   Level: developer
3855 
3856 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMCreateLabel()
3857 @*/
3858 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
3859 {
3860   PetscErrorCode ierr;
3861 
3862   PetscFunctionBegin;
3863   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3864   PetscValidPointer(celltypeLabel, 2);
3865   if (!dm->celltypeLabel) {ierr = DMPlexComputeCellTypes(dm);CHKERRQ(ierr);}
3866   *celltypeLabel = dm->celltypeLabel;
3867   PetscFunctionReturn(0);
3868 }
3869 
3870 /*@
3871   DMPlexGetCellType - Get the polytope type of a given cell
3872 
3873   Not Collective
3874 
3875   Input Parameter:
3876 + dm   - The DMPlex object
3877 - cell - The cell
3878 
3879   Output Parameter:
3880 . celltype - The polytope type of the cell
3881 
3882   Level: intermediate
3883 
3884 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth()
3885 @*/
3886 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
3887 {
3888   DMLabel        label;
3889   PetscInt       ct;
3890   PetscErrorCode ierr;
3891 
3892   PetscFunctionBegin;
3893   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3894   PetscValidPointer(celltype, 3);
3895   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
3896   ierr = DMLabelGetValue(label, cell, &ct);CHKERRQ(ierr);
3897   *celltype = (DMPolytopeType) ct;
3898   PetscFunctionReturn(0);
3899 }
3900 
3901 /*@
3902   DMPlexSetCellType - Set the polytope type of a given cell
3903 
3904   Not Collective
3905 
3906   Input Parameters:
3907 + dm   - The DMPlex object
3908 . cell - The cell
3909 - celltype - The polytope type of the cell
3910 
3911   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
3912   is executed. This function will override the computed type. However, if automatic classification will not succeed
3913   and a user wants to manually specify all types, the classification must be disabled by calling
3914   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
3915 
3916   Level: advanced
3917 
3918 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexComputeCellTypes(), DMCreateLabel()
3919 @*/
3920 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
3921 {
3922   DMLabel        label;
3923   PetscErrorCode ierr;
3924 
3925   PetscFunctionBegin;
3926   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3927   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
3928   ierr = DMLabelSetValue(label, cell, celltype);CHKERRQ(ierr);
3929   PetscFunctionReturn(0);
3930 }
3931 
3932 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3933 {
3934   PetscSection   section, s;
3935   Mat            m;
3936   PetscInt       maxHeight;
3937   PetscErrorCode ierr;
3938 
3939   PetscFunctionBegin;
3940   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3941   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3942   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3943   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3944   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
3945   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3946   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3947   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3948   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3949   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3950   ierr = MatDestroy(&m);CHKERRQ(ierr);
3951 
3952   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3953   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3954   PetscFunctionReturn(0);
3955 }
3956 
3957 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3958 {
3959   Vec            coordsLocal;
3960   DM             coordsDM;
3961   PetscErrorCode ierr;
3962 
3963   PetscFunctionBegin;
3964   *field = NULL;
3965   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3966   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3967   if (coordsLocal && coordsDM) {
3968     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3969   }
3970   PetscFunctionReturn(0);
3971 }
3972 
3973 /*@C
3974   DMPlexGetConeSection - Return a section which describes the layout of cone data
3975 
3976   Not Collective
3977 
3978   Input Parameters:
3979 . dm        - The DMPlex object
3980 
3981   Output Parameter:
3982 . section - The PetscSection object
3983 
3984   Level: developer
3985 
3986 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3987 @*/
3988 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3989 {
3990   DM_Plex *mesh = (DM_Plex*) dm->data;
3991 
3992   PetscFunctionBegin;
3993   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3994   if (section) *section = mesh->coneSection;
3995   PetscFunctionReturn(0);
3996 }
3997 
3998 /*@C
3999   DMPlexGetSupportSection - Return a section which describes the layout of support data
4000 
4001   Not Collective
4002 
4003   Input Parameters:
4004 . dm        - The DMPlex object
4005 
4006   Output Parameter:
4007 . section - The PetscSection object
4008 
4009   Level: developer
4010 
4011 .seealso: DMPlexGetConeSection()
4012 @*/
4013 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4014 {
4015   DM_Plex *mesh = (DM_Plex*) dm->data;
4016 
4017   PetscFunctionBegin;
4018   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4019   if (section) *section = mesh->supportSection;
4020   PetscFunctionReturn(0);
4021 }
4022 
4023 /*@C
4024   DMPlexGetCones - Return cone data
4025 
4026   Not Collective
4027 
4028   Input Parameters:
4029 . dm        - The DMPlex object
4030 
4031   Output Parameter:
4032 . cones - The cone for each point
4033 
4034   Level: developer
4035 
4036 .seealso: DMPlexGetConeSection()
4037 @*/
4038 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4039 {
4040   DM_Plex *mesh = (DM_Plex*) dm->data;
4041 
4042   PetscFunctionBegin;
4043   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4044   if (cones) *cones = mesh->cones;
4045   PetscFunctionReturn(0);
4046 }
4047 
4048 /*@C
4049   DMPlexGetConeOrientations - Return cone orientation data
4050 
4051   Not Collective
4052 
4053   Input Parameters:
4054 . dm        - The DMPlex object
4055 
4056   Output Parameter:
4057 . coneOrientations - The cone orientation for each point
4058 
4059   Level: developer
4060 
4061 .seealso: DMPlexGetConeSection()
4062 @*/
4063 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4064 {
4065   DM_Plex *mesh = (DM_Plex*) dm->data;
4066 
4067   PetscFunctionBegin;
4068   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4069   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4070   PetscFunctionReturn(0);
4071 }
4072 
4073 /******************************** FEM Support **********************************/
4074 
4075 /*
4076  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
4077  representing a line in the section.
4078 */
4079 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
4080 {
4081   PetscErrorCode ierr;
4082 
4083   PetscFunctionBeginHot;
4084   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
4085   if (line < 0) {
4086     *k = 0;
4087     *Nc = 0;
4088   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
4089     *k = 1;
4090   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
4091     /* An order k SEM disc has k-1 dofs on an edge */
4092     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
4093     *k = *k / *Nc + 1;
4094   }
4095   PetscFunctionReturn(0);
4096 }
4097 
4098 /*@
4099 
4100   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
4101   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
4102   section provided (or the section of the DM).
4103 
4104   Input Parameters:
4105 + dm      - The DM
4106 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
4107 - section - The PetscSection to reorder, or NULL for the default section
4108 
4109   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
4110   degree of the basis.
4111 
4112   Example:
4113   A typical interpolated single-quad mesh might order points as
4114 .vb
4115   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
4116 
4117   v4 -- e6 -- v3
4118   |           |
4119   e7    c0    e8
4120   |           |
4121   v1 -- e5 -- v2
4122 .ve
4123 
4124   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
4125   dofs in the order of points, e.g.,
4126 .vb
4127     c0 -> [0,1,2,3]
4128     v1 -> [4]
4129     ...
4130     e5 -> [8, 9]
4131 .ve
4132 
4133   which corresponds to the dofs
4134 .vb
4135     6   10  11  7
4136     13  2   3   15
4137     12  0   1   14
4138     4   8   9   5
4139 .ve
4140 
4141   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
4142 .vb
4143   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
4144 .ve
4145 
4146   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
4147 .vb
4148    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
4149 .ve
4150 
4151   Level: developer
4152 
4153 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
4154 @*/
4155 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
4156 {
4157   DMLabel        label;
4158   PetscInt      *perm;
4159   PetscInt       dim, depth = -1, eStart = -1, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
4160   PetscBool      vertexchart;
4161   PetscErrorCode ierr;
4162 
4163   PetscFunctionBegin;
4164   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4165   if (dim < 1) PetscFunctionReturn(0);
4166   if (point < 0) {
4167     PetscInt sStart,sEnd;
4168 
4169     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
4170     point = sEnd-sStart ? sStart : point;
4171   }
4172   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4173   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
4174   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4175   if (depth == 1) {eStart = point;}
4176   else if  (depth == dim) {
4177     const PetscInt *cone;
4178 
4179     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4180     if (dim == 2) eStart = cone[0];
4181     else if (dim == 3) {
4182       const PetscInt *cone2;
4183       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
4184       eStart = cone2[0];
4185     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
4186   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
4187   {                             /* Determine whether the chart covers all points or just vertices. */
4188     PetscInt pStart,pEnd,cStart,cEnd;
4189     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
4190     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
4191     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
4192     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
4193   }
4194   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
4195   for (f = 0; f < Nf; ++f) {
4196     ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4197     size += PetscPowInt(k+1, dim)*Nc;
4198   }
4199   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
4200   for (f = 0; f < Nf; ++f) {
4201     switch (dim) {
4202     case 1:
4203       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4204       /*
4205         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
4206         We want              [ vtx0; edge of length k-1; vtx1 ]
4207       */
4208       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
4209       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
4210       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
4211       foffset = offset;
4212       break;
4213     case 2:
4214       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
4215       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4216       /* The SEM order is
4217 
4218          v_lb, {e_b}, v_rb,
4219          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
4220          v_lt, reverse {e_t}, v_rt
4221       */
4222       {
4223         const PetscInt of   = 0;
4224         const PetscInt oeb  = of   + PetscSqr(k-1);
4225         const PetscInt oer  = oeb  + (k-1);
4226         const PetscInt oet  = oer  + (k-1);
4227         const PetscInt oel  = oet  + (k-1);
4228         const PetscInt ovlb = oel  + (k-1);
4229         const PetscInt ovrb = ovlb + 1;
4230         const PetscInt ovrt = ovrb + 1;
4231         const PetscInt ovlt = ovrt + 1;
4232         PetscInt       o;
4233 
4234         /* bottom */
4235         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
4236         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4237         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
4238         /* middle */
4239         for (i = 0; i < k-1; ++i) {
4240           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
4241           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4242           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
4243         }
4244         /* top */
4245         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
4246         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4247         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
4248         foffset = offset;
4249       }
4250       break;
4251     case 3:
4252       /* The original hex closure is
4253 
4254          {c,
4255           f_b, f_t, f_f, f_b, f_r, f_l,
4256           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
4257           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
4258       */
4259       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4260       /* The SEM order is
4261          Bottom Slice
4262          v_blf, {e^{(k-1)-n}_bf}, v_brf,
4263          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
4264          v_blb, {e_bb}, v_brb,
4265 
4266          Middle Slice (j)
4267          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
4268          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
4269          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
4270 
4271          Top Slice
4272          v_tlf, {e_tf}, v_trf,
4273          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
4274          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
4275       */
4276       {
4277         const PetscInt oc    = 0;
4278         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
4279         const PetscInt oft   = ofb   + PetscSqr(k-1);
4280         const PetscInt off   = oft   + PetscSqr(k-1);
4281         const PetscInt ofk   = off   + PetscSqr(k-1);
4282         const PetscInt ofr   = ofk   + PetscSqr(k-1);
4283         const PetscInt ofl   = ofr   + PetscSqr(k-1);
4284         const PetscInt oebl  = ofl   + PetscSqr(k-1);
4285         const PetscInt oebb  = oebl  + (k-1);
4286         const PetscInt oebr  = oebb  + (k-1);
4287         const PetscInt oebf  = oebr  + (k-1);
4288         const PetscInt oetf  = oebf  + (k-1);
4289         const PetscInt oetr  = oetf  + (k-1);
4290         const PetscInt oetb  = oetr  + (k-1);
4291         const PetscInt oetl  = oetb  + (k-1);
4292         const PetscInt oerf  = oetl  + (k-1);
4293         const PetscInt oelf  = oerf  + (k-1);
4294         const PetscInt oelb  = oelf  + (k-1);
4295         const PetscInt oerb  = oelb  + (k-1);
4296         const PetscInt ovblf = oerb  + (k-1);
4297         const PetscInt ovblb = ovblf + 1;
4298         const PetscInt ovbrb = ovblb + 1;
4299         const PetscInt ovbrf = ovbrb + 1;
4300         const PetscInt ovtlf = ovbrf + 1;
4301         const PetscInt ovtrf = ovtlf + 1;
4302         const PetscInt ovtrb = ovtrf + 1;
4303         const PetscInt ovtlb = ovtrb + 1;
4304         PetscInt       o, n;
4305 
4306         /* Bottom Slice */
4307         /*   bottom */
4308         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
4309         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4310         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
4311         /*   middle */
4312         for (i = 0; i < k-1; ++i) {
4313           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
4314           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
4315           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
4316         }
4317         /*   top */
4318         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
4319         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4320         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
4321 
4322         /* Middle Slice */
4323         for (j = 0; j < k-1; ++j) {
4324           /*   bottom */
4325           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
4326           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4327           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
4328           /*   middle */
4329           for (i = 0; i < k-1; ++i) {
4330             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
4331             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
4332             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
4333           }
4334           /*   top */
4335           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
4336           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4337           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
4338         }
4339 
4340         /* Top Slice */
4341         /*   bottom */
4342         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
4343         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4344         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
4345         /*   middle */
4346         for (i = 0; i < k-1; ++i) {
4347           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
4348           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
4349           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
4350         }
4351         /*   top */
4352         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
4353         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4354         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
4355 
4356         foffset = offset;
4357       }
4358       break;
4359     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
4360     }
4361   }
4362   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
4363   /* Check permutation */
4364   {
4365     PetscInt *check;
4366 
4367     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
4368     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
4369     for (i = 0; i < size; ++i) check[perm[i]] = i;
4370     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
4371     ierr = PetscFree(check);CHKERRQ(ierr);
4372   }
4373   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
4374   PetscFunctionReturn(0);
4375 }
4376 
4377 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
4378 {
4379   PetscDS        prob;
4380   PetscInt       depth, Nf, h;
4381   DMLabel        label;
4382   PetscErrorCode ierr;
4383 
4384   PetscFunctionBeginHot;
4385   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
4386   Nf      = prob->Nf;
4387   label   = dm->depthLabel;
4388   *dspace = NULL;
4389   if (field < Nf) {
4390     PetscObject disc = prob->disc[field];
4391 
4392     if (disc->classid == PETSCFE_CLASSID) {
4393       PetscDualSpace dsp;
4394 
4395       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
4396       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
4397       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
4398       h    = depth - 1 - h;
4399       if (h) {
4400         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
4401       } else {
4402         *dspace = dsp;
4403       }
4404     }
4405   }
4406   PetscFunctionReturn(0);
4407 }
4408 
4409 
4410 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4411 {
4412   PetscScalar    *array, *vArray;
4413   const PetscInt *cone, *coneO;
4414   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4415   PetscErrorCode  ierr;
4416 
4417   PetscFunctionBeginHot;
4418   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4419   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4420   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4421   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4422   if (!values || !*values) {
4423     if ((point >= pStart) && (point < pEnd)) {
4424       PetscInt dof;
4425 
4426       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4427       size += dof;
4428     }
4429     for (p = 0; p < numPoints; ++p) {
4430       const PetscInt cp = cone[p];
4431       PetscInt       dof;
4432 
4433       if ((cp < pStart) || (cp >= pEnd)) continue;
4434       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4435       size += dof;
4436     }
4437     if (!values) {
4438       if (csize) *csize = size;
4439       PetscFunctionReturn(0);
4440     }
4441     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4442   } else {
4443     array = *values;
4444   }
4445   size = 0;
4446   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4447   if ((point >= pStart) && (point < pEnd)) {
4448     PetscInt     dof, off, d;
4449     PetscScalar *varr;
4450 
4451     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4452     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4453     varr = &vArray[off];
4454     for (d = 0; d < dof; ++d, ++offset) {
4455       array[offset] = varr[d];
4456     }
4457     size += dof;
4458   }
4459   for (p = 0; p < numPoints; ++p) {
4460     const PetscInt cp = cone[p];
4461     PetscInt       o  = coneO[p];
4462     PetscInt       dof, off, d;
4463     PetscScalar   *varr;
4464 
4465     if ((cp < pStart) || (cp >= pEnd)) continue;
4466     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4467     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4468     varr = &vArray[off];
4469     if (o >= 0) {
4470       for (d = 0; d < dof; ++d, ++offset) {
4471         array[offset] = varr[d];
4472       }
4473     } else {
4474       for (d = dof-1; d >= 0; --d, ++offset) {
4475         array[offset] = varr[d];
4476       }
4477     }
4478     size += dof;
4479   }
4480   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4481   if (!*values) {
4482     if (csize) *csize = size;
4483     *values = array;
4484   } else {
4485     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4486     *csize = size;
4487   }
4488   PetscFunctionReturn(0);
4489 }
4490 
4491 /* Compressed closure does not apply closure permutation */
4492 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4493 {
4494   const PetscInt *cla;
4495   PetscInt       np, *pts = NULL;
4496   PetscErrorCode ierr;
4497 
4498   PetscFunctionBeginHot;
4499   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4500   if (!*clPoints) {
4501     PetscInt pStart, pEnd, p, q;
4502 
4503     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4504     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4505     /* Compress out points not in the section */
4506     for (p = 0, q = 0; p < np; p++) {
4507       PetscInt r = pts[2*p];
4508       if ((r >= pStart) && (r < pEnd)) {
4509         pts[q*2]   = r;
4510         pts[q*2+1] = pts[2*p+1];
4511         ++q;
4512       }
4513     }
4514     np = q;
4515     cla = NULL;
4516   } else {
4517     PetscInt dof, off;
4518 
4519     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4520     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4521     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4522     np   = dof/2;
4523     pts  = (PetscInt *) &cla[off];
4524   }
4525   *numPoints = np;
4526   *points    = pts;
4527   *clp       = cla;
4528 
4529   PetscFunctionReturn(0);
4530 }
4531 
4532 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4533 {
4534   PetscErrorCode ierr;
4535 
4536   PetscFunctionBeginHot;
4537   if (!*clPoints) {
4538     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4539   } else {
4540     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4541   }
4542   *numPoints = 0;
4543   *points    = NULL;
4544   *clSec     = NULL;
4545   *clPoints  = NULL;
4546   *clp       = NULL;
4547   PetscFunctionReturn(0);
4548 }
4549 
4550 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4551 {
4552   PetscInt          offset = 0, p;
4553   const PetscInt    **perms = NULL;
4554   const PetscScalar **flips = NULL;
4555   PetscErrorCode    ierr;
4556 
4557   PetscFunctionBeginHot;
4558   *size = 0;
4559   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4560   for (p = 0; p < numPoints; p++) {
4561     const PetscInt    point = points[2*p];
4562     const PetscInt    *perm = perms ? perms[p] : NULL;
4563     const PetscScalar *flip = flips ? flips[p] : NULL;
4564     PetscInt          dof, off, d;
4565     const PetscScalar *varr;
4566 
4567     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4568     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4569     varr = &vArray[off];
4570     if (clperm) {
4571       if (perm) {
4572         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4573       } else {
4574         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4575       }
4576       if (flip) {
4577         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4578       }
4579     } else {
4580       if (perm) {
4581         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4582       } else {
4583         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4584       }
4585       if (flip) {
4586         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4587       }
4588     }
4589     offset += dof;
4590   }
4591   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4592   *size = offset;
4593   PetscFunctionReturn(0);
4594 }
4595 
4596 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4597 {
4598   PetscInt          offset = 0, f;
4599   PetscErrorCode    ierr;
4600 
4601   PetscFunctionBeginHot;
4602   *size = 0;
4603   for (f = 0; f < numFields; ++f) {
4604     PetscInt          p;
4605     const PetscInt    **perms = NULL;
4606     const PetscScalar **flips = NULL;
4607 
4608     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4609     for (p = 0; p < numPoints; p++) {
4610       const PetscInt    point = points[2*p];
4611       PetscInt          fdof, foff, b;
4612       const PetscScalar *varr;
4613       const PetscInt    *perm = perms ? perms[p] : NULL;
4614       const PetscScalar *flip = flips ? flips[p] : NULL;
4615 
4616       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4617       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4618       varr = &vArray[foff];
4619       if (clperm) {
4620         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4621         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4622         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4623       } else {
4624         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4625         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4626         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4627       }
4628       offset += fdof;
4629     }
4630     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4631   }
4632   *size = offset;
4633   PetscFunctionReturn(0);
4634 }
4635 
4636 /*@C
4637   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4638 
4639   Not collective
4640 
4641   Input Parameters:
4642 + dm - The DM
4643 . section - The section describing the layout in v, or NULL to use the default section
4644 . v - The local vector
4645 . point - The point in the DM
4646 . csize - The size of the input values array, or NULL
4647 - values - An array to use for the values, or NULL to have it allocated automatically
4648 
4649   Output Parameters:
4650 + csize - The number of values in the closure
4651 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4652 
4653 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4654 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4655 $ assembly function, and a user may already have allocated storage for this operation.
4656 $
4657 $ A typical use could be
4658 $
4659 $  values = NULL;
4660 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4661 $  for (cl = 0; cl < clSize; ++cl) {
4662 $    <Compute on closure>
4663 $  }
4664 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4665 $
4666 $ or
4667 $
4668 $  PetscMalloc1(clMaxSize, &values);
4669 $  for (p = pStart; p < pEnd; ++p) {
4670 $    clSize = clMaxSize;
4671 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4672 $    for (cl = 0; cl < clSize; ++cl) {
4673 $      <Compute on closure>
4674 $    }
4675 $  }
4676 $  PetscFree(values);
4677 
4678   Fortran Notes:
4679   Since it returns an array, this routine is only available in Fortran 90, and you must
4680   include petsc.h90 in your code.
4681 
4682   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4683 
4684   Level: intermediate
4685 
4686 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4687 @*/
4688 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4689 {
4690   PetscSection       clSection;
4691   IS                 clPoints;
4692   PetscScalar       *array;
4693   const PetscScalar *vArray;
4694   PetscInt          *points = NULL;
4695   const PetscInt    *clp, *perm;
4696   PetscInt           depth, numFields, numPoints, size;
4697   PetscErrorCode     ierr;
4698 
4699   PetscFunctionBeginHot;
4700   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4701   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4702   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4703   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4704   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4705   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4706   if (depth == 1 && numFields < 2) {
4707     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4708     PetscFunctionReturn(0);
4709   }
4710   /* Get points */
4711   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4712   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4713   /* Get array */
4714   if (!values || !*values) {
4715     PetscInt asize = 0, dof, p;
4716 
4717     for (p = 0; p < numPoints*2; p += 2) {
4718       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4719       asize += dof;
4720     }
4721     if (!values) {
4722       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4723       if (csize) *csize = asize;
4724       PetscFunctionReturn(0);
4725     }
4726     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4727   } else {
4728     array = *values;
4729   }
4730   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4731   /* Get values */
4732   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4733   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4734   /* Cleanup points */
4735   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4736   /* Cleanup array */
4737   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4738   if (!*values) {
4739     if (csize) *csize = size;
4740     *values = array;
4741   } else {
4742     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4743     *csize = size;
4744   }
4745   PetscFunctionReturn(0);
4746 }
4747 
4748 /*@C
4749   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4750 
4751   Not collective
4752 
4753   Input Parameters:
4754 + dm - The DM
4755 . section - The section describing the layout in v, or NULL to use the default section
4756 . v - The local vector
4757 . point - The point in the DM
4758 . csize - The number of values in the closure, or NULL
4759 - values - The array of values, which is a borrowed array and should not be freed
4760 
4761   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4762 
4763   Fortran Notes:
4764   Since it returns an array, this routine is only available in Fortran 90, and you must
4765   include petsc.h90 in your code.
4766 
4767   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4768 
4769   Level: intermediate
4770 
4771 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4772 @*/
4773 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4774 {
4775   PetscInt       size = 0;
4776   PetscErrorCode ierr;
4777 
4778   PetscFunctionBegin;
4779   /* Should work without recalculating size */
4780   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4781   *values = NULL;
4782   PetscFunctionReturn(0);
4783 }
4784 
4785 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4786 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4787 
4788 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4789 {
4790   PetscInt        cdof;   /* The number of constraints on this point */
4791   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4792   PetscScalar    *a;
4793   PetscInt        off, cind = 0, k;
4794   PetscErrorCode  ierr;
4795 
4796   PetscFunctionBegin;
4797   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4798   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4799   a    = &array[off];
4800   if (!cdof || setBC) {
4801     if (clperm) {
4802       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4803       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4804     } else {
4805       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4806       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4807     }
4808   } else {
4809     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4810     if (clperm) {
4811       if (perm) {for (k = 0; k < dof; ++k) {
4812           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4813           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4814         }
4815       } else {
4816         for (k = 0; k < dof; ++k) {
4817           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4818           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4819         }
4820       }
4821     } else {
4822       if (perm) {
4823         for (k = 0; k < dof; ++k) {
4824           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4825           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4826         }
4827       } else {
4828         for (k = 0; k < dof; ++k) {
4829           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4830           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4831         }
4832       }
4833     }
4834   }
4835   PetscFunctionReturn(0);
4836 }
4837 
4838 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4839 {
4840   PetscInt        cdof;   /* The number of constraints on this point */
4841   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4842   PetscScalar    *a;
4843   PetscInt        off, cind = 0, k;
4844   PetscErrorCode  ierr;
4845 
4846   PetscFunctionBegin;
4847   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4848   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4849   a    = &array[off];
4850   if (cdof) {
4851     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4852     if (clperm) {
4853       if (perm) {
4854         for (k = 0; k < dof; ++k) {
4855           if ((cind < cdof) && (k == cdofs[cind])) {
4856             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4857             cind++;
4858           }
4859         }
4860       } else {
4861         for (k = 0; k < dof; ++k) {
4862           if ((cind < cdof) && (k == cdofs[cind])) {
4863             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4864             cind++;
4865           }
4866         }
4867       }
4868     } else {
4869       if (perm) {
4870         for (k = 0; k < dof; ++k) {
4871           if ((cind < cdof) && (k == cdofs[cind])) {
4872             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4873             cind++;
4874           }
4875         }
4876       } else {
4877         for (k = 0; k < dof; ++k) {
4878           if ((cind < cdof) && (k == cdofs[cind])) {
4879             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4880             cind++;
4881           }
4882         }
4883       }
4884     }
4885   }
4886   PetscFunctionReturn(0);
4887 }
4888 
4889 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4890 {
4891   PetscScalar    *a;
4892   PetscInt        fdof, foff, fcdof, foffset = *offset;
4893   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4894   PetscInt        cind = 0, b;
4895   PetscErrorCode  ierr;
4896 
4897   PetscFunctionBegin;
4898   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4899   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4900   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4901   a    = &array[foff];
4902   if (!fcdof || setBC) {
4903     if (clperm) {
4904       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4905       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4906     } else {
4907       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4908       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4909     }
4910   } else {
4911     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4912     if (clperm) {
4913       if (perm) {
4914         for (b = 0; b < fdof; b++) {
4915           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4916           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4917         }
4918       } else {
4919         for (b = 0; b < fdof; b++) {
4920           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4921           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4922         }
4923       }
4924     } else {
4925       if (perm) {
4926         for (b = 0; b < fdof; b++) {
4927           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4928           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4929         }
4930       } else {
4931         for (b = 0; b < fdof; b++) {
4932           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4933           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4934         }
4935       }
4936     }
4937   }
4938   *offset += fdof;
4939   PetscFunctionReturn(0);
4940 }
4941 
4942 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4943 {
4944   PetscScalar    *a;
4945   PetscInt        fdof, foff, fcdof, foffset = *offset;
4946   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4947   PetscInt        Nc, cind = 0, ncind = 0, b;
4948   PetscBool       ncSet, fcSet;
4949   PetscErrorCode  ierr;
4950 
4951   PetscFunctionBegin;
4952   ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
4953   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4954   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4955   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4956   a    = &array[foff];
4957   if (fcdof) {
4958     /* We just override fcdof and fcdofs with Ncc and comps */
4959     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4960     if (clperm) {
4961       if (perm) {
4962         if (comps) {
4963           for (b = 0; b < fdof; b++) {
4964             ncSet = fcSet = PETSC_FALSE;
4965             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
4966             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4967             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4968           }
4969         } else {
4970           for (b = 0; b < fdof; b++) {
4971             if ((cind < fcdof) && (b == fcdofs[cind])) {
4972               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4973               ++cind;
4974             }
4975           }
4976         }
4977       } else {
4978         if (comps) {
4979           for (b = 0; b < fdof; b++) {
4980             ncSet = fcSet = PETSC_FALSE;
4981             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
4982             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4983             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4984           }
4985         } else {
4986           for (b = 0; b < fdof; b++) {
4987             if ((cind < fcdof) && (b == fcdofs[cind])) {
4988               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4989               ++cind;
4990             }
4991           }
4992         }
4993       }
4994     } else {
4995       if (perm) {
4996         if (comps) {
4997           for (b = 0; b < fdof; b++) {
4998             ncSet = fcSet = PETSC_FALSE;
4999             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5000             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5001             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
5002           }
5003         } else {
5004           for (b = 0; b < fdof; b++) {
5005             if ((cind < fcdof) && (b == fcdofs[cind])) {
5006               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
5007               ++cind;
5008             }
5009           }
5010         }
5011       } else {
5012         if (comps) {
5013           for (b = 0; b < fdof; b++) {
5014             ncSet = fcSet = PETSC_FALSE;
5015             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5016             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5017             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
5018           }
5019         } else {
5020           for (b = 0; b < fdof; b++) {
5021             if ((cind < fcdof) && (b == fcdofs[cind])) {
5022               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
5023               ++cind;
5024             }
5025           }
5026         }
5027       }
5028     }
5029   }
5030   *offset += fdof;
5031   PetscFunctionReturn(0);
5032 }
5033 
5034 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5035 {
5036   PetscScalar    *array;
5037   const PetscInt *cone, *coneO;
5038   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5039   PetscErrorCode  ierr;
5040 
5041   PetscFunctionBeginHot;
5042   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5043   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5044   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5045   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5046   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5047   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5048     const PetscInt cp = !p ? point : cone[p-1];
5049     const PetscInt o  = !p ? 0     : coneO[p-1];
5050 
5051     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5052     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5053     /* ADD_VALUES */
5054     {
5055       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5056       PetscScalar    *a;
5057       PetscInt        cdof, coff, cind = 0, k;
5058 
5059       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5060       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5061       a    = &array[coff];
5062       if (!cdof) {
5063         if (o >= 0) {
5064           for (k = 0; k < dof; ++k) {
5065             a[k] += values[off+k];
5066           }
5067         } else {
5068           for (k = 0; k < dof; ++k) {
5069             a[k] += values[off+dof-k-1];
5070           }
5071         }
5072       } else {
5073         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
5074         if (o >= 0) {
5075           for (k = 0; k < dof; ++k) {
5076             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5077             a[k] += values[off+k];
5078           }
5079         } else {
5080           for (k = 0; k < dof; ++k) {
5081             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5082             a[k] += values[off+dof-k-1];
5083           }
5084         }
5085       }
5086     }
5087   }
5088   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5089   PetscFunctionReturn(0);
5090 }
5091 
5092 /*@C
5093   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
5094 
5095   Not collective
5096 
5097   Input Parameters:
5098 + dm - The DM
5099 . section - The section describing the layout in v, or NULL to use the default section
5100 . v - The local vector
5101 . point - The point in the DM
5102 . values - The array of values
5103 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
5104          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
5105 
5106   Fortran Notes:
5107   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5108 
5109   Level: intermediate
5110 
5111 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
5112 @*/
5113 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5114 {
5115   PetscSection    clSection;
5116   IS              clPoints;
5117   PetscScalar    *array;
5118   PetscInt       *points = NULL;
5119   const PetscInt *clp, *clperm;
5120   PetscInt        depth, numFields, numPoints, p;
5121   PetscErrorCode  ierr;
5122 
5123   PetscFunctionBeginHot;
5124   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5125   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5126   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5127   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5128   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5129   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5130   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
5131     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
5132     PetscFunctionReturn(0);
5133   }
5134   /* Get points */
5135   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5136   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5137   /* Get array */
5138   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5139   /* Get values */
5140   if (numFields > 0) {
5141     PetscInt offset = 0, f;
5142     for (f = 0; f < numFields; ++f) {
5143       const PetscInt    **perms = NULL;
5144       const PetscScalar **flips = NULL;
5145 
5146       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5147       switch (mode) {
5148       case INSERT_VALUES:
5149         for (p = 0; p < numPoints; p++) {
5150           const PetscInt    point = points[2*p];
5151           const PetscInt    *perm = perms ? perms[p] : NULL;
5152           const PetscScalar *flip = flips ? flips[p] : NULL;
5153           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
5154         } break;
5155       case INSERT_ALL_VALUES:
5156         for (p = 0; p < numPoints; p++) {
5157           const PetscInt    point = points[2*p];
5158           const PetscInt    *perm = perms ? perms[p] : NULL;
5159           const PetscScalar *flip = flips ? flips[p] : NULL;
5160           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
5161         } break;
5162       case INSERT_BC_VALUES:
5163         for (p = 0; p < numPoints; p++) {
5164           const PetscInt    point = points[2*p];
5165           const PetscInt    *perm = perms ? perms[p] : NULL;
5166           const PetscScalar *flip = flips ? flips[p] : NULL;
5167           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
5168         } break;
5169       case ADD_VALUES:
5170         for (p = 0; p < numPoints; p++) {
5171           const PetscInt    point = points[2*p];
5172           const PetscInt    *perm = perms ? perms[p] : NULL;
5173           const PetscScalar *flip = flips ? flips[p] : NULL;
5174           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
5175         } break;
5176       case ADD_ALL_VALUES:
5177         for (p = 0; p < numPoints; p++) {
5178           const PetscInt    point = points[2*p];
5179           const PetscInt    *perm = perms ? perms[p] : NULL;
5180           const PetscScalar *flip = flips ? flips[p] : NULL;
5181           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
5182         } break;
5183       case ADD_BC_VALUES:
5184         for (p = 0; p < numPoints; p++) {
5185           const PetscInt    point = points[2*p];
5186           const PetscInt    *perm = perms ? perms[p] : NULL;
5187           const PetscScalar *flip = flips ? flips[p] : NULL;
5188           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
5189         } break;
5190       default:
5191         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5192       }
5193       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5194     }
5195   } else {
5196     PetscInt dof, off;
5197     const PetscInt    **perms = NULL;
5198     const PetscScalar **flips = NULL;
5199 
5200     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5201     switch (mode) {
5202     case INSERT_VALUES:
5203       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
5204         const PetscInt    point = points[2*p];
5205         const PetscInt    *perm = perms ? perms[p] : NULL;
5206         const PetscScalar *flip = flips ? flips[p] : NULL;
5207         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5208         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
5209       } break;
5210     case INSERT_ALL_VALUES:
5211       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
5212         const PetscInt    point = points[2*p];
5213         const PetscInt    *perm = perms ? perms[p] : NULL;
5214         const PetscScalar *flip = flips ? flips[p] : NULL;
5215         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5216         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
5217       } break;
5218     case INSERT_BC_VALUES:
5219       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
5220         const PetscInt    point = points[2*p];
5221         const PetscInt    *perm = perms ? perms[p] : NULL;
5222         const PetscScalar *flip = flips ? flips[p] : NULL;
5223         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5224         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
5225       } break;
5226     case ADD_VALUES:
5227       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
5228         const PetscInt    point = points[2*p];
5229         const PetscInt    *perm = perms ? perms[p] : NULL;
5230         const PetscScalar *flip = flips ? flips[p] : NULL;
5231         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5232         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
5233       } break;
5234     case ADD_ALL_VALUES:
5235       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
5236         const PetscInt    point = points[2*p];
5237         const PetscInt    *perm = perms ? perms[p] : NULL;
5238         const PetscScalar *flip = flips ? flips[p] : NULL;
5239         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5240         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
5241       } break;
5242     case ADD_BC_VALUES:
5243       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
5244         const PetscInt    point = points[2*p];
5245         const PetscInt    *perm = perms ? perms[p] : NULL;
5246         const PetscScalar *flip = flips ? flips[p] : NULL;
5247         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5248         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
5249       } break;
5250     default:
5251       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5252     }
5253     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5254   }
5255   /* Cleanup points */
5256   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5257   /* Cleanup array */
5258   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5259   PetscFunctionReturn(0);
5260 }
5261 
5262 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
5263 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
5264 {
5265   PetscSection      clSection;
5266   IS                clPoints;
5267   PetscScalar       *array;
5268   PetscInt          *points = NULL;
5269   const PetscInt    *clp;
5270   PetscInt          numFields, numPoints, p;
5271   PetscInt          offset = 0, f;
5272   PetscErrorCode    ierr;
5273 
5274   PetscFunctionBeginHot;
5275   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5276   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5277   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5278   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5279   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5280   /* Get points */
5281   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5282   /* Get array */
5283   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5284   /* Get values */
5285   for (f = 0; f < numFields; ++f) {
5286     const PetscInt    **perms = NULL;
5287     const PetscScalar **flips = NULL;
5288 
5289     if (!fieldActive[f]) {
5290       for (p = 0; p < numPoints*2; p += 2) {
5291         PetscInt fdof;
5292         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5293         offset += fdof;
5294       }
5295       continue;
5296     }
5297     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5298     switch (mode) {
5299     case INSERT_VALUES:
5300       for (p = 0; p < numPoints; p++) {
5301         const PetscInt    point = points[2*p];
5302         const PetscInt    *perm = perms ? perms[p] : NULL;
5303         const PetscScalar *flip = flips ? flips[p] : NULL;
5304         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array);
5305       } break;
5306     case INSERT_ALL_VALUES:
5307       for (p = 0; p < numPoints; p++) {
5308         const PetscInt    point = points[2*p];
5309         const PetscInt    *perm = perms ? perms[p] : NULL;
5310         const PetscScalar *flip = flips ? flips[p] : NULL;
5311         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array);
5312         } break;
5313     case INSERT_BC_VALUES:
5314       for (p = 0; p < numPoints; p++) {
5315         const PetscInt    point = points[2*p];
5316         const PetscInt    *perm = perms ? perms[p] : NULL;
5317         const PetscScalar *flip = flips ? flips[p] : NULL;
5318         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array);
5319       } break;
5320     case ADD_VALUES:
5321       for (p = 0; p < numPoints; p++) {
5322         const PetscInt    point = points[2*p];
5323         const PetscInt    *perm = perms ? perms[p] : NULL;
5324         const PetscScalar *flip = flips ? flips[p] : NULL;
5325         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array);
5326       } break;
5327     case ADD_ALL_VALUES:
5328       for (p = 0; p < numPoints; p++) {
5329         const PetscInt    point = points[2*p];
5330         const PetscInt    *perm = perms ? perms[p] : NULL;
5331         const PetscScalar *flip = flips ? flips[p] : NULL;
5332         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array);
5333       } break;
5334     default:
5335       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5336     }
5337     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5338   }
5339   /* Cleanup points */
5340   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5341   /* Cleanup array */
5342   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5343   PetscFunctionReturn(0);
5344 }
5345 
5346 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5347 {
5348   PetscMPIInt    rank;
5349   PetscInt       i, j;
5350   PetscErrorCode ierr;
5351 
5352   PetscFunctionBegin;
5353   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5354   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
5355   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5356   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5357   numCIndices = numCIndices ? numCIndices : numRIndices;
5358   for (i = 0; i < numRIndices; i++) {
5359     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
5360     for (j = 0; j < numCIndices; j++) {
5361 #if defined(PETSC_USE_COMPLEX)
5362       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5363 #else
5364       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5365 #endif
5366     }
5367     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5368   }
5369   PetscFunctionReturn(0);
5370 }
5371 
5372 /*
5373   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
5374 
5375   Input Parameters:
5376 + section - The section for this data layout
5377 . islocal - Is the section (and thus indices being requested) local or global?
5378 . point   - The point contributing dofs with these indices
5379 . off     - The global offset of this point
5380 . loff    - The local offset of each field
5381 . setBC   - The flag determining whether to include indices of bounsary values
5382 . perm    - A permutation of the dofs on this point, or NULL
5383 - indperm - A permutation of the entire indices array, or NULL
5384 
5385   Output Parameter:
5386 . indices - Indices for dofs on this point
5387 
5388   Level: developer
5389 
5390   Note: The indices could be local or global, depending on the value of 'off'.
5391 */
5392 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
5393 {
5394   PetscInt        dof;   /* The number of unknowns on this point */
5395   PetscInt        cdof;  /* The number of constraints on this point */
5396   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5397   PetscInt        cind = 0, k;
5398   PetscErrorCode  ierr;
5399 
5400   PetscFunctionBegin;
5401   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
5402   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5403   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5404   if (!cdof || setBC) {
5405     for (k = 0; k < dof; ++k) {
5406       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5407       const PetscInt ind    = indperm ? indperm[preind] : preind;
5408 
5409       indices[ind] = off + k;
5410     }
5411   } else {
5412     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5413     for (k = 0; k < dof; ++k) {
5414       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5415       const PetscInt ind    = indperm ? indperm[preind] : preind;
5416 
5417       if ((cind < cdof) && (k == cdofs[cind])) {
5418         /* Insert check for returning constrained indices */
5419         indices[ind] = -(off+k+1);
5420         ++cind;
5421       } else {
5422         indices[ind] = off + k - (islocal ? 0 : cind);
5423       }
5424     }
5425   }
5426   *loff += dof;
5427   PetscFunctionReturn(0);
5428 }
5429 
5430 /*
5431  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
5432 
5433  Input Parameters:
5434 + section - a section (global or local)
5435 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
5436 . point - point within section
5437 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
5438 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
5439 . setBC - identify constrained (boundary condition) points via involution.
5440 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
5441 . permsoff - offset
5442 - indperm - index permutation
5443 
5444  Output Parameter:
5445 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
5446 . indices - array to hold indices (as defined by section) of each dof associated with point
5447 
5448  Notes:
5449  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
5450  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
5451  in the local vector.
5452 
5453  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
5454  significant).  It is invalid to call with a global section and setBC=true.
5455 
5456  Developer Note:
5457  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
5458  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
5459  offset could be obtained from the section instead of passing it explicitly as we do now.
5460 
5461  Example:
5462  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
5463  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
5464  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
5465  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
5466 
5467  Level: developer
5468 */
5469 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5470 {
5471   PetscInt       numFields, foff, f;
5472   PetscErrorCode ierr;
5473 
5474   PetscFunctionBegin;
5475   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
5476   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5477   for (f = 0, foff = 0; f < numFields; ++f) {
5478     PetscInt        fdof, cfdof;
5479     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5480     PetscInt        cind = 0, b;
5481     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5482 
5483     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5484     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5485     if (!cfdof || setBC) {
5486       for (b = 0; b < fdof; ++b) {
5487         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5488         const PetscInt ind    = indperm ? indperm[preind] : preind;
5489 
5490         indices[ind] = off+foff+b;
5491       }
5492     } else {
5493       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5494       for (b = 0; b < fdof; ++b) {
5495         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5496         const PetscInt ind    = indperm ? indperm[preind] : preind;
5497 
5498         if ((cind < cfdof) && (b == fcdofs[cind])) {
5499           indices[ind] = -(off+foff+b+1);
5500           ++cind;
5501         } else {
5502           indices[ind] = off + foff + b - (islocal ? 0 : cind);
5503         }
5504       }
5505     }
5506     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
5507     foffs[f] += fdof;
5508   }
5509   PetscFunctionReturn(0);
5510 }
5511 
5512 /*
5513   This version believes the globalSection offsets for each field, rather than just the point offset
5514 
5515  . foffs - The offset into 'indices' for each field, since it is segregated by field
5516 
5517  Notes:
5518  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
5519  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
5520 */
5521 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5522 {
5523   PetscInt       numFields, foff, f;
5524   PetscErrorCode ierr;
5525 
5526   PetscFunctionBegin;
5527   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5528   for (f = 0; f < numFields; ++f) {
5529     PetscInt        fdof, cfdof;
5530     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5531     PetscInt        cind = 0, b;
5532     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5533 
5534     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5535     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5536     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5537     if (!cfdof) {
5538       for (b = 0; b < fdof; ++b) {
5539         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5540         const PetscInt ind    = indperm ? indperm[preind] : preind;
5541 
5542         indices[ind] = foff+b;
5543       }
5544     } else {
5545       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5546       for (b = 0; b < fdof; ++b) {
5547         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5548         const PetscInt ind    = indperm ? indperm[preind] : preind;
5549 
5550         if ((cind < cfdof) && (b == fcdofs[cind])) {
5551           indices[ind] = -(foff+b+1);
5552           ++cind;
5553         } else {
5554           indices[ind] = foff+b-cind;
5555         }
5556       }
5557     }
5558     foffs[f] += fdof;
5559   }
5560   PetscFunctionReturn(0);
5561 }
5562 
5563 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5564 {
5565   Mat             cMat;
5566   PetscSection    aSec, cSec;
5567   IS              aIS;
5568   PetscInt        aStart = -1, aEnd = -1;
5569   const PetscInt  *anchors;
5570   PetscInt        numFields, f, p, q, newP = 0;
5571   PetscInt        newNumPoints = 0, newNumIndices = 0;
5572   PetscInt        *newPoints, *indices, *newIndices;
5573   PetscInt        maxAnchor, maxDof;
5574   PetscInt        newOffsets[32];
5575   PetscInt        *pointMatOffsets[32];
5576   PetscInt        *newPointOffsets[32];
5577   PetscScalar     *pointMat[32];
5578   PetscScalar     *newValues=NULL,*tmpValues;
5579   PetscBool       anyConstrained = PETSC_FALSE;
5580   PetscErrorCode  ierr;
5581 
5582   PetscFunctionBegin;
5583   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5584   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5585   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5586 
5587   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5588   /* if there are point-to-point constraints */
5589   if (aSec) {
5590     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
5591     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5592     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5593     /* figure out how many points are going to be in the new element matrix
5594      * (we allow double counting, because it's all just going to be summed
5595      * into the global matrix anyway) */
5596     for (p = 0; p < 2*numPoints; p+=2) {
5597       PetscInt b    = points[p];
5598       PetscInt bDof = 0, bSecDof;
5599 
5600       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5601       if (!bSecDof) {
5602         continue;
5603       }
5604       if (b >= aStart && b < aEnd) {
5605         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5606       }
5607       if (bDof) {
5608         /* this point is constrained */
5609         /* it is going to be replaced by its anchors */
5610         PetscInt bOff, q;
5611 
5612         anyConstrained = PETSC_TRUE;
5613         newNumPoints  += bDof;
5614         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5615         for (q = 0; q < bDof; q++) {
5616           PetscInt a = anchors[bOff + q];
5617           PetscInt aDof;
5618 
5619           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5620           newNumIndices += aDof;
5621           for (f = 0; f < numFields; ++f) {
5622             PetscInt fDof;
5623 
5624             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5625             newOffsets[f+1] += fDof;
5626           }
5627         }
5628       }
5629       else {
5630         /* this point is not constrained */
5631         newNumPoints++;
5632         newNumIndices += bSecDof;
5633         for (f = 0; f < numFields; ++f) {
5634           PetscInt fDof;
5635 
5636           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5637           newOffsets[f+1] += fDof;
5638         }
5639       }
5640     }
5641   }
5642   if (!anyConstrained) {
5643     if (outNumPoints)  *outNumPoints  = 0;
5644     if (outNumIndices) *outNumIndices = 0;
5645     if (outPoints)     *outPoints     = NULL;
5646     if (outValues)     *outValues     = NULL;
5647     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5648     PetscFunctionReturn(0);
5649   }
5650 
5651   if (outNumPoints)  *outNumPoints  = newNumPoints;
5652   if (outNumIndices) *outNumIndices = newNumIndices;
5653 
5654   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5655 
5656   if (!outPoints && !outValues) {
5657     if (offsets) {
5658       for (f = 0; f <= numFields; f++) {
5659         offsets[f] = newOffsets[f];
5660       }
5661     }
5662     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5663     PetscFunctionReturn(0);
5664   }
5665 
5666   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5667 
5668   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5669 
5670   /* workspaces */
5671   if (numFields) {
5672     for (f = 0; f < numFields; f++) {
5673       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5674       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5675     }
5676   }
5677   else {
5678     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5679     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5680   }
5681 
5682   /* get workspaces for the point-to-point matrices */
5683   if (numFields) {
5684     PetscInt totalOffset, totalMatOffset;
5685 
5686     for (p = 0; p < numPoints; p++) {
5687       PetscInt b    = points[2*p];
5688       PetscInt bDof = 0, bSecDof;
5689 
5690       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5691       if (!bSecDof) {
5692         for (f = 0; f < numFields; f++) {
5693           newPointOffsets[f][p + 1] = 0;
5694           pointMatOffsets[f][p + 1] = 0;
5695         }
5696         continue;
5697       }
5698       if (b >= aStart && b < aEnd) {
5699         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5700       }
5701       if (bDof) {
5702         for (f = 0; f < numFields; f++) {
5703           PetscInt fDof, q, bOff, allFDof = 0;
5704 
5705           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5706           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5707           for (q = 0; q < bDof; q++) {
5708             PetscInt a = anchors[bOff + q];
5709             PetscInt aFDof;
5710 
5711             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5712             allFDof += aFDof;
5713           }
5714           newPointOffsets[f][p+1] = allFDof;
5715           pointMatOffsets[f][p+1] = fDof * allFDof;
5716         }
5717       }
5718       else {
5719         for (f = 0; f < numFields; f++) {
5720           PetscInt fDof;
5721 
5722           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5723           newPointOffsets[f][p+1] = fDof;
5724           pointMatOffsets[f][p+1] = 0;
5725         }
5726       }
5727     }
5728     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5729       newPointOffsets[f][0] = totalOffset;
5730       pointMatOffsets[f][0] = totalMatOffset;
5731       for (p = 0; p < numPoints; p++) {
5732         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5733         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5734       }
5735       totalOffset    = newPointOffsets[f][numPoints];
5736       totalMatOffset = pointMatOffsets[f][numPoints];
5737       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5738     }
5739   }
5740   else {
5741     for (p = 0; p < numPoints; p++) {
5742       PetscInt b    = points[2*p];
5743       PetscInt bDof = 0, bSecDof;
5744 
5745       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5746       if (!bSecDof) {
5747         newPointOffsets[0][p + 1] = 0;
5748         pointMatOffsets[0][p + 1] = 0;
5749         continue;
5750       }
5751       if (b >= aStart && b < aEnd) {
5752         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5753       }
5754       if (bDof) {
5755         PetscInt bOff, q, allDof = 0;
5756 
5757         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5758         for (q = 0; q < bDof; q++) {
5759           PetscInt a = anchors[bOff + q], aDof;
5760 
5761           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5762           allDof += aDof;
5763         }
5764         newPointOffsets[0][p+1] = allDof;
5765         pointMatOffsets[0][p+1] = bSecDof * allDof;
5766       }
5767       else {
5768         newPointOffsets[0][p+1] = bSecDof;
5769         pointMatOffsets[0][p+1] = 0;
5770       }
5771     }
5772     newPointOffsets[0][0] = 0;
5773     pointMatOffsets[0][0] = 0;
5774     for (p = 0; p < numPoints; p++) {
5775       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5776       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5777     }
5778     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5779   }
5780 
5781   /* output arrays */
5782   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5783 
5784   /* get the point-to-point matrices; construct newPoints */
5785   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5786   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5787   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5788   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5789   if (numFields) {
5790     for (p = 0, newP = 0; p < numPoints; p++) {
5791       PetscInt b    = points[2*p];
5792       PetscInt o    = points[2*p+1];
5793       PetscInt bDof = 0, bSecDof;
5794 
5795       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5796       if (!bSecDof) {
5797         continue;
5798       }
5799       if (b >= aStart && b < aEnd) {
5800         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5801       }
5802       if (bDof) {
5803         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5804 
5805         fStart[0] = 0;
5806         fEnd[0]   = 0;
5807         for (f = 0; f < numFields; f++) {
5808           PetscInt fDof;
5809 
5810           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5811           fStart[f+1] = fStart[f] + fDof;
5812           fEnd[f+1]   = fStart[f+1];
5813         }
5814         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5815         ierr = DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5816 
5817         fAnchorStart[0] = 0;
5818         fAnchorEnd[0]   = 0;
5819         for (f = 0; f < numFields; f++) {
5820           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5821 
5822           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5823           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5824         }
5825         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5826         for (q = 0; q < bDof; q++) {
5827           PetscInt a = anchors[bOff + q], aOff;
5828 
5829           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5830           newPoints[2*(newP + q)]     = a;
5831           newPoints[2*(newP + q) + 1] = 0;
5832           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5833           ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5834         }
5835         newP += bDof;
5836 
5837         if (outValues) {
5838           /* get the point-to-point submatrix */
5839           for (f = 0; f < numFields; f++) {
5840             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5841           }
5842         }
5843       }
5844       else {
5845         newPoints[2 * newP]     = b;
5846         newPoints[2 * newP + 1] = o;
5847         newP++;
5848       }
5849     }
5850   } else {
5851     for (p = 0; p < numPoints; p++) {
5852       PetscInt b    = points[2*p];
5853       PetscInt o    = points[2*p+1];
5854       PetscInt bDof = 0, bSecDof;
5855 
5856       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5857       if (!bSecDof) {
5858         continue;
5859       }
5860       if (b >= aStart && b < aEnd) {
5861         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5862       }
5863       if (bDof) {
5864         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5865 
5866         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5867         ierr = DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5868 
5869         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5870         for (q = 0; q < bDof; q++) {
5871           PetscInt a = anchors[bOff + q], aOff;
5872 
5873           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5874 
5875           newPoints[2*(newP + q)]     = a;
5876           newPoints[2*(newP + q) + 1] = 0;
5877           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5878           ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5879         }
5880         newP += bDof;
5881 
5882         /* get the point-to-point submatrix */
5883         if (outValues) {
5884           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5885         }
5886       }
5887       else {
5888         newPoints[2 * newP]     = b;
5889         newPoints[2 * newP + 1] = o;
5890         newP++;
5891       }
5892     }
5893   }
5894 
5895   if (outValues) {
5896     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5897     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
5898     /* multiply constraints on the right */
5899     if (numFields) {
5900       for (f = 0; f < numFields; f++) {
5901         PetscInt oldOff = offsets[f];
5902 
5903         for (p = 0; p < numPoints; p++) {
5904           PetscInt cStart = newPointOffsets[f][p];
5905           PetscInt b      = points[2 * p];
5906           PetscInt c, r, k;
5907           PetscInt dof;
5908 
5909           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5910           if (!dof) {
5911             continue;
5912           }
5913           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5914             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5915             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5916 
5917             for (r = 0; r < numIndices; r++) {
5918               for (c = 0; c < nCols; c++) {
5919                 for (k = 0; k < dof; k++) {
5920                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5921                 }
5922               }
5923             }
5924           }
5925           else {
5926             /* copy this column as is */
5927             for (r = 0; r < numIndices; r++) {
5928               for (c = 0; c < dof; c++) {
5929                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5930               }
5931             }
5932           }
5933           oldOff += dof;
5934         }
5935       }
5936     }
5937     else {
5938       PetscInt oldOff = 0;
5939       for (p = 0; p < numPoints; p++) {
5940         PetscInt cStart = newPointOffsets[0][p];
5941         PetscInt b      = points[2 * p];
5942         PetscInt c, r, k;
5943         PetscInt dof;
5944 
5945         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5946         if (!dof) {
5947           continue;
5948         }
5949         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5950           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5951           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5952 
5953           for (r = 0; r < numIndices; r++) {
5954             for (c = 0; c < nCols; c++) {
5955               for (k = 0; k < dof; k++) {
5956                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5957               }
5958             }
5959           }
5960         }
5961         else {
5962           /* copy this column as is */
5963           for (r = 0; r < numIndices; r++) {
5964             for (c = 0; c < dof; c++) {
5965               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5966             }
5967           }
5968         }
5969         oldOff += dof;
5970       }
5971     }
5972 
5973     if (multiplyLeft) {
5974       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5975       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
5976       /* multiply constraints transpose on the left */
5977       if (numFields) {
5978         for (f = 0; f < numFields; f++) {
5979           PetscInt oldOff = offsets[f];
5980 
5981           for (p = 0; p < numPoints; p++) {
5982             PetscInt rStart = newPointOffsets[f][p];
5983             PetscInt b      = points[2 * p];
5984             PetscInt c, r, k;
5985             PetscInt dof;
5986 
5987             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5988             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5989               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5990               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5991 
5992               for (r = 0; r < nRows; r++) {
5993                 for (c = 0; c < newNumIndices; c++) {
5994                   for (k = 0; k < dof; k++) {
5995                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5996                   }
5997                 }
5998               }
5999             }
6000             else {
6001               /* copy this row as is */
6002               for (r = 0; r < dof; r++) {
6003                 for (c = 0; c < newNumIndices; c++) {
6004                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
6005                 }
6006               }
6007             }
6008             oldOff += dof;
6009           }
6010         }
6011       }
6012       else {
6013         PetscInt oldOff = 0;
6014 
6015         for (p = 0; p < numPoints; p++) {
6016           PetscInt rStart = newPointOffsets[0][p];
6017           PetscInt b      = points[2 * p];
6018           PetscInt c, r, k;
6019           PetscInt dof;
6020 
6021           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
6022           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
6023             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
6024             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
6025 
6026             for (r = 0; r < nRows; r++) {
6027               for (c = 0; c < newNumIndices; c++) {
6028                 for (k = 0; k < dof; k++) {
6029                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
6030                 }
6031               }
6032             }
6033           }
6034           else {
6035             /* copy this row as is */
6036             for (r = 0; r < dof; r++) {
6037               for (c = 0; c < newNumIndices; c++) {
6038                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
6039               }
6040             }
6041           }
6042           oldOff += dof;
6043         }
6044       }
6045 
6046       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
6047     }
6048     else {
6049       newValues = tmpValues;
6050     }
6051   }
6052 
6053   /* clean up */
6054   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6055   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
6056 
6057   if (numFields) {
6058     for (f = 0; f < numFields; f++) {
6059       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
6060       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
6061       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
6062     }
6063   }
6064   else {
6065     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
6066     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
6067     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
6068   }
6069   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
6070 
6071   /* output */
6072   if (outPoints) {
6073     *outPoints = newPoints;
6074   }
6075   else {
6076     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6077   }
6078   if (outValues) {
6079     *outValues = newValues;
6080   }
6081   for (f = 0; f <= numFields; f++) {
6082     offsets[f] = newOffsets[f];
6083   }
6084   PetscFunctionReturn(0);
6085 }
6086 
6087 /*@C
6088   DMPlexGetClosureIndices - Get the global indices for all local points in the closure of the given point
6089 
6090   Not collective
6091 
6092   Input Parameters:
6093 + dm - The DM
6094 . section - The section describing the points (a local section)
6095 . idxSection - The section on which to obtain indices (may be local or global)
6096 - point - The mesh point
6097 
6098   Output parameters:
6099 + numIndices - The number of indices
6100 . indices - The indices
6101 - outOffsets - Field offset if not NULL
6102 
6103   Notes:
6104   Must call DMPlexRestoreClosureIndices() to free allocated memory
6105 
6106   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
6107   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
6108   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
6109   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
6110   indices (with the above semantics) are implied.
6111 
6112   Level: advanced
6113 
6114 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
6115 @*/
6116 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
6117 {
6118   PetscBool       isLocal = (PetscBool)(section == idxSection);
6119   PetscSection    clSection;
6120   IS              clPoints;
6121   const PetscInt *clp, *clperm;
6122   const PetscInt  **perms[32] = {NULL};
6123   PetscInt       *points = NULL, *pointsNew;
6124   PetscInt        numPoints, numPointsNew;
6125   PetscInt        offsets[32];
6126   PetscInt        Nf, Nind, NindNew, off, idxOff, f, p;
6127   PetscErrorCode  ierr;
6128 
6129   PetscFunctionBegin;
6130   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6131   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6132   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
6133   if (numIndices) PetscValidPointer(numIndices, 4);
6134   PetscValidPointer(indices, 5);
6135   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
6136   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
6137   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
6138   /* Get points in closure */
6139   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6140   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
6141   /* Get number of indices and indices per field */
6142   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
6143     PetscInt dof, fdof;
6144 
6145     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6146     for (f = 0; f < Nf; ++f) {
6147       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6148       offsets[f+1] += fdof;
6149     }
6150     Nind += dof;
6151   }
6152   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
6153   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
6154   if (!Nf) offsets[1] = Nind;
6155   /* Get dual space symmetries */
6156   for (f = 0; f < PetscMax(1,Nf); f++) {
6157     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
6158     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
6159   }
6160   /* Correct for hanging node constraints */
6161   {
6162     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
6163     if (numPointsNew) {
6164       for (f = 0; f < PetscMax(1,Nf); f++) {
6165         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
6166         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
6167       }
6168       for (f = 0; f < PetscMax(1,Nf); f++) {
6169         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
6170         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
6171       }
6172       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6173       numPoints = numPointsNew;
6174       Nind      = NindNew;
6175       points    = pointsNew;
6176     }
6177   }
6178   /* Calculate indices */
6179   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
6180   if (Nf) {
6181     if (outOffsets) {
6182       PetscInt f;
6183 
6184       for (f = 0; f <= Nf; f++) {
6185         outOffsets[f] = offsets[f];
6186       }
6187     }
6188     for (p = 0; p < numPoints; p++) {
6189       ierr = PetscSectionGetOffset(idxSection, points[2*p], &idxOff);CHKERRQ(ierr);
6190       ierr = DMPlexGetIndicesPointFields_Internal(section, isLocal, points[2*p], idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);CHKERRQ(ierr);
6191     }
6192   } else {
6193     for (p = 0, off = 0; p < numPoints; p++) {
6194       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
6195 
6196       ierr = PetscSectionGetOffset(idxSection, points[2*p], &idxOff);CHKERRQ(ierr);
6197       ierr = DMPlexGetIndicesPoint_Internal(section, isLocal, points[2*p], idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, *indices);CHKERRQ(ierr);
6198     }
6199   }
6200   /* Cleanup points */
6201   for (f = 0; f < PetscMax(1,Nf); f++) {
6202     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
6203     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
6204   }
6205   if (numPointsNew) {
6206     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
6207   } else {
6208     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6209   }
6210   if (numIndices) *numIndices = Nind;
6211   PetscFunctionReturn(0);
6212 }
6213 
6214 /*@C
6215   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
6216 
6217   Not collective
6218 
6219   Input Parameters:
6220 + dm - The DM
6221 . section - The section describing the layout in v, or NULL to use the default section
6222 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
6223 . point - The mesh point
6224 . numIndices - The number of indices
6225 . indices - The indices
6226 - outOffsets - Field offset if not NULL
6227 
6228   Level: advanced
6229 
6230 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
6231 @*/
6232 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
6233 {
6234   PetscErrorCode ierr;
6235 
6236   PetscFunctionBegin;
6237   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6238   PetscValidPointer(indices, 5);
6239   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
6240   PetscFunctionReturn(0);
6241 }
6242 
6243 /*@C
6244   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
6245 
6246   Not collective
6247 
6248   Input Parameters:
6249 + dm - The DM
6250 . section - The section describing the layout in v, or NULL to use the default section
6251 . globalSection - The section describing the layout in v, or NULL to use the default global section
6252 . A - The matrix
6253 . point - The point in the DM
6254 . values - The array of values
6255 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
6256 
6257   Fortran Notes:
6258   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6259 
6260   Level: intermediate
6261 
6262 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
6263 @*/
6264 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
6265 {
6266   DM_Plex            *mesh   = (DM_Plex*) dm->data;
6267   PetscSection        clSection;
6268   IS                  clPoints;
6269   PetscInt           *points = NULL, *newPoints;
6270   const PetscInt     *clp, *clperm;
6271   PetscInt           *indices;
6272   PetscInt            offsets[32];
6273   const PetscInt    **perms[32] = {NULL};
6274   const PetscScalar **flips[32] = {NULL};
6275   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
6276   PetscScalar        *valCopy = NULL;
6277   PetscScalar        *newValues;
6278   PetscErrorCode      ierr;
6279 
6280   PetscFunctionBegin;
6281   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6282   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6283   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6284   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
6285   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
6286   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
6287   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6288   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6289   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
6290   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
6291   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6292   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
6293     PetscInt fdof;
6294 
6295     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6296     for (f = 0; f < numFields; ++f) {
6297       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6298       offsets[f+1] += fdof;
6299     }
6300     numIndices += dof;
6301   }
6302   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
6303 
6304   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
6305   /* Get symmetries */
6306   for (f = 0; f < PetscMax(1,numFields); f++) {
6307     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6308     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6309     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
6310       PetscInt foffset = offsets[f];
6311 
6312       for (p = 0; p < numPoints; p++) {
6313         PetscInt point          = points[2*p], fdof;
6314         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
6315 
6316         if (!numFields) {
6317           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
6318         } else {
6319           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
6320         }
6321         if (flip) {
6322           PetscInt i, j, k;
6323 
6324           if (!valCopy) {
6325             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6326             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
6327             values = valCopy;
6328           }
6329           for (i = 0; i < fdof; i++) {
6330             PetscScalar fval = flip[i];
6331 
6332             for (k = 0; k < numIndices; k++) {
6333               valCopy[numIndices * (foffset + i) + k] *= fval;
6334               valCopy[numIndices * k + (foffset + i)] *= fval;
6335             }
6336           }
6337         }
6338         foffset += fdof;
6339       }
6340     }
6341   }
6342   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
6343   if (newNumPoints) {
6344     if (valCopy) {
6345       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6346     }
6347     for (f = 0; f < PetscMax(1,numFields); f++) {
6348       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6349       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6350     }
6351     for (f = 0; f < PetscMax(1,numFields); f++) {
6352       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6353       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6354     }
6355     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6356     numPoints  = newNumPoints;
6357     numIndices = newNumIndices;
6358     points     = newPoints;
6359     values     = newValues;
6360   }
6361   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6362   if (numFields) {
6363     PetscBool useFieldOffsets;
6364 
6365     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
6366     if (useFieldOffsets) {
6367       for (p = 0; p < numPoints; p++) {
6368         ierr = DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, perms, p, clperm, indices);CHKERRQ(ierr);
6369       }
6370     } else {
6371       for (p = 0; p < numPoints; p++) {
6372         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6373         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
6374          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
6375          * global section. */
6376         ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_FALSE, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);CHKERRQ(ierr);
6377       }
6378     }
6379   } else {
6380     for (p = 0, off = 0; p < numPoints; p++) {
6381       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
6382       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6383       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
6384        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
6385        * global section. */
6386       ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_FALSE, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);CHKERRQ(ierr);
6387     }
6388   }
6389   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
6390   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);CHKERRQ(ierr);
6391   if (mesh->printFEM > 1) {
6392     PetscInt i;
6393     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
6394     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
6395     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6396   }
6397   if (ierr) {
6398     PetscMPIInt    rank;
6399     PetscErrorCode ierr2;
6400 
6401     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6402     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6403     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
6404     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
6405     CHKERRQ(ierr);
6406   }
6407   for (f = 0; f < PetscMax(1,numFields); f++) {
6408     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6409     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6410   }
6411   if (newNumPoints) {
6412     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6413     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6414   }
6415   else {
6416     if (valCopy) {
6417       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6418     }
6419     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6420   }
6421   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6422   PetscFunctionReturn(0);
6423 }
6424 
6425 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
6426 {
6427   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
6428   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
6429   PetscInt       *cpoints = NULL;
6430   PetscInt       *findices, *cindices;
6431   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
6432   PetscInt        foffsets[32], coffsets[32];
6433   DMPolytopeType  ct;
6434   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6435   PetscErrorCode  ierr;
6436 
6437   PetscFunctionBegin;
6438   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6439   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6440   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6441   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6442   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6443   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6444   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6445   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6446   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6447   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6448   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
6449   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6450   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6451   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6452   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6453   /* Column indices */
6454   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6455   maxFPoints = numCPoints;
6456   /* Compress out points not in the section */
6457   /*   TODO: Squeeze out points with 0 dof as well */
6458   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6459   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6460     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6461       cpoints[q*2]   = cpoints[p];
6462       cpoints[q*2+1] = cpoints[p+1];
6463       ++q;
6464     }
6465   }
6466   numCPoints = q;
6467   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6468     PetscInt fdof;
6469 
6470     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6471     if (!dof) continue;
6472     for (f = 0; f < numFields; ++f) {
6473       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6474       coffsets[f+1] += fdof;
6475     }
6476     numCIndices += dof;
6477   }
6478   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6479   /* Row indices */
6480   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
6481   {
6482     DMPlexCellRefiner cr;
6483     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
6484     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6485     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
6486   }
6487   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6488   for (r = 0, q = 0; r < numSubcells; ++r) {
6489     /* TODO Map from coarse to fine cells */
6490     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6491     /* Compress out points not in the section */
6492     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6493     for (p = 0; p < numFPoints*2; p += 2) {
6494       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6495         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6496         if (!dof) continue;
6497         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6498         if (s < q) continue;
6499         ftotpoints[q*2]   = fpoints[p];
6500         ftotpoints[q*2+1] = fpoints[p+1];
6501         ++q;
6502       }
6503     }
6504     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6505   }
6506   numFPoints = q;
6507   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6508     PetscInt fdof;
6509 
6510     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6511     if (!dof) continue;
6512     for (f = 0; f < numFields; ++f) {
6513       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6514       foffsets[f+1] += fdof;
6515     }
6516     numFIndices += dof;
6517   }
6518   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6519 
6520   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6521   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6522   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6523   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6524   if (numFields) {
6525     const PetscInt **permsF[32] = {NULL};
6526     const PetscInt **permsC[32] = {NULL};
6527 
6528     for (f = 0; f < numFields; f++) {
6529       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6530       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6531     }
6532     for (p = 0; p < numFPoints; p++) {
6533       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6534       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6535     }
6536     for (p = 0; p < numCPoints; p++) {
6537       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6538       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6539     }
6540     for (f = 0; f < numFields; f++) {
6541       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6542       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6543     }
6544   } else {
6545     const PetscInt **permsF = NULL;
6546     const PetscInt **permsC = NULL;
6547 
6548     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6549     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6550     for (p = 0, off = 0; p < numFPoints; p++) {
6551       const PetscInt *perm = permsF ? permsF[p] : NULL;
6552 
6553       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6554       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6555     }
6556     for (p = 0, off = 0; p < numCPoints; p++) {
6557       const PetscInt *perm = permsC ? permsC[p] : NULL;
6558 
6559       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6560       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6561     }
6562     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6563     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6564   }
6565   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6566   /* TODO: flips */
6567   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6568   if (ierr) {
6569     PetscMPIInt    rank;
6570     PetscErrorCode ierr2;
6571 
6572     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6573     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6574     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6575     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6576     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6577     CHKERRQ(ierr);
6578   }
6579   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6580   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6581   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6582   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6583   PetscFunctionReturn(0);
6584 }
6585 
6586 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6587 {
6588   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6589   PetscInt      *cpoints = NULL;
6590   PetscInt       foffsets[32], coffsets[32];
6591   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
6592   DMPolytopeType ct;
6593   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6594   PetscErrorCode ierr;
6595 
6596   PetscFunctionBegin;
6597   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6598   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6599   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6600   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6601   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6602   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6603   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6604   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6605   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6606   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6607   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6608   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6609   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6610   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6611   /* Column indices */
6612   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6613   maxFPoints = numCPoints;
6614   /* Compress out points not in the section */
6615   /*   TODO: Squeeze out points with 0 dof as well */
6616   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6617   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6618     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6619       cpoints[q*2]   = cpoints[p];
6620       cpoints[q*2+1] = cpoints[p+1];
6621       ++q;
6622     }
6623   }
6624   numCPoints = q;
6625   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6626     PetscInt fdof;
6627 
6628     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6629     if (!dof) continue;
6630     for (f = 0; f < numFields; ++f) {
6631       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6632       coffsets[f+1] += fdof;
6633     }
6634     numCIndices += dof;
6635   }
6636   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6637   /* Row indices */
6638   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
6639   {
6640     DMPlexCellRefiner cr;
6641     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
6642     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6643     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
6644   }
6645   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6646   for (r = 0, q = 0; r < numSubcells; ++r) {
6647     /* TODO Map from coarse to fine cells */
6648     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6649     /* Compress out points not in the section */
6650     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6651     for (p = 0; p < numFPoints*2; p += 2) {
6652       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6653         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6654         if (!dof) continue;
6655         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6656         if (s < q) continue;
6657         ftotpoints[q*2]   = fpoints[p];
6658         ftotpoints[q*2+1] = fpoints[p+1];
6659         ++q;
6660       }
6661     }
6662     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6663   }
6664   numFPoints = q;
6665   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6666     PetscInt fdof;
6667 
6668     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6669     if (!dof) continue;
6670     for (f = 0; f < numFields; ++f) {
6671       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6672       foffsets[f+1] += fdof;
6673     }
6674     numFIndices += dof;
6675   }
6676   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6677 
6678   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6679   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6680   if (numFields) {
6681     const PetscInt **permsF[32] = {NULL};
6682     const PetscInt **permsC[32] = {NULL};
6683 
6684     for (f = 0; f < numFields; f++) {
6685       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6686       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6687     }
6688     for (p = 0; p < numFPoints; p++) {
6689       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6690       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6691     }
6692     for (p = 0; p < numCPoints; p++) {
6693       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6694       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6695     }
6696     for (f = 0; f < numFields; f++) {
6697       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6698       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6699     }
6700   } else {
6701     const PetscInt **permsF = NULL;
6702     const PetscInt **permsC = NULL;
6703 
6704     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6705     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6706     for (p = 0, off = 0; p < numFPoints; p++) {
6707       const PetscInt *perm = permsF ? permsF[p] : NULL;
6708 
6709       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6710       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6711     }
6712     for (p = 0, off = 0; p < numCPoints; p++) {
6713       const PetscInt *perm = permsC ? permsC[p] : NULL;
6714 
6715       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6716       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6717     }
6718     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6719     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6720   }
6721   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6722   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6723   PetscFunctionReturn(0);
6724 }
6725 
6726 /*@C
6727   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6728 
6729   Input Parameter:
6730 . dm   - The DMPlex object
6731 
6732   Output Parameter:
6733 . cellHeight - The height of a cell
6734 
6735   Level: developer
6736 
6737 .seealso DMPlexSetVTKCellHeight()
6738 @*/
6739 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6740 {
6741   DM_Plex *mesh = (DM_Plex*) dm->data;
6742 
6743   PetscFunctionBegin;
6744   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6745   PetscValidPointer(cellHeight, 2);
6746   *cellHeight = mesh->vtkCellHeight;
6747   PetscFunctionReturn(0);
6748 }
6749 
6750 /*@C
6751   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6752 
6753   Input Parameters:
6754 + dm   - The DMPlex object
6755 - cellHeight - The height of a cell
6756 
6757   Level: developer
6758 
6759 .seealso DMPlexGetVTKCellHeight()
6760 @*/
6761 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6762 {
6763   DM_Plex *mesh = (DM_Plex*) dm->data;
6764 
6765   PetscFunctionBegin;
6766   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6767   mesh->vtkCellHeight = cellHeight;
6768   PetscFunctionReturn(0);
6769 }
6770 
6771 /*@
6772   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
6773 
6774   Input Parameter:
6775 . dm - The DMPlex object
6776 
6777   Output Parameters:
6778 + gcStart - The first ghost cell, or NULL
6779 - gcEnd   - The upper bound on ghost cells, or NULL
6780 
6781   Level: advanced
6782 
6783 .seealso DMPlexConstructGhostCells(), DMPlexSetGhostCellStratum()
6784 @*/
6785 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
6786 {
6787   DMLabel        ctLabel;
6788   PetscErrorCode ierr;
6789 
6790   PetscFunctionBegin;
6791   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6792   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
6793   ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd);CHKERRQ(ierr);
6794   PetscFunctionReturn(0);
6795 }
6796 
6797 /* We can easily have a form that takes an IS instead */
6798 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6799 {
6800   PetscSection   section, globalSection;
6801   PetscInt      *numbers, p;
6802   PetscErrorCode ierr;
6803 
6804   PetscFunctionBegin;
6805   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6806   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6807   for (p = pStart; p < pEnd; ++p) {
6808     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6809   }
6810   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6811   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6812   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6813   for (p = pStart; p < pEnd; ++p) {
6814     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6815     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6816     else                       numbers[p-pStart] += shift;
6817   }
6818   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6819   if (globalSize) {
6820     PetscLayout layout;
6821     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6822     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6823     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6824   }
6825   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6826   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6827   PetscFunctionReturn(0);
6828 }
6829 
6830 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6831 {
6832   PetscInt       cellHeight, cStart, cEnd;
6833   PetscErrorCode ierr;
6834 
6835   PetscFunctionBegin;
6836   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6837   if (includeHybrid) {ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
6838   else               {ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
6839   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6840   PetscFunctionReturn(0);
6841 }
6842 
6843 /*@
6844   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6845 
6846   Input Parameter:
6847 . dm   - The DMPlex object
6848 
6849   Output Parameter:
6850 . globalCellNumbers - Global cell numbers for all cells on this process
6851 
6852   Level: developer
6853 
6854 .seealso DMPlexGetVertexNumbering()
6855 @*/
6856 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6857 {
6858   DM_Plex       *mesh = (DM_Plex*) dm->data;
6859   PetscErrorCode ierr;
6860 
6861   PetscFunctionBegin;
6862   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6863   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6864   *globalCellNumbers = mesh->globalCellNumbers;
6865   PetscFunctionReturn(0);
6866 }
6867 
6868 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6869 {
6870   PetscInt       vStart, vEnd;
6871   PetscErrorCode ierr;
6872 
6873   PetscFunctionBegin;
6874   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6875   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6876   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6877   PetscFunctionReturn(0);
6878 }
6879 
6880 /*@
6881   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
6882 
6883   Input Parameter:
6884 . dm   - The DMPlex object
6885 
6886   Output Parameter:
6887 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6888 
6889   Level: developer
6890 
6891 .seealso DMPlexGetCellNumbering()
6892 @*/
6893 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6894 {
6895   DM_Plex       *mesh = (DM_Plex*) dm->data;
6896   PetscErrorCode ierr;
6897 
6898   PetscFunctionBegin;
6899   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6900   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6901   *globalVertexNumbers = mesh->globalVertexNumbers;
6902   PetscFunctionReturn(0);
6903 }
6904 
6905 /*@
6906   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6907 
6908   Input Parameter:
6909 . dm   - The DMPlex object
6910 
6911   Output Parameter:
6912 . globalPointNumbers - Global numbers for all points on this process
6913 
6914   Level: developer
6915 
6916 .seealso DMPlexGetCellNumbering()
6917 @*/
6918 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6919 {
6920   IS             nums[4];
6921   PetscInt       depths[4], gdepths[4], starts[4];
6922   PetscInt       depth, d, shift = 0;
6923   PetscErrorCode ierr;
6924 
6925   PetscFunctionBegin;
6926   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6927   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6928   /* For unstratified meshes use dim instead of depth */
6929   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6930   for (d = 0; d <= depth; ++d) {
6931     PetscInt end;
6932 
6933     depths[d] = depth-d;
6934     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6935     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6936   }
6937   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6938   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6939   for (d = 0; d <= depth; ++d) {
6940     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6941   }
6942   for (d = 0; d <= depth; ++d) {
6943     PetscInt pStart, pEnd, gsize;
6944 
6945     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6946     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6947     shift += gsize;
6948   }
6949   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6950   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6951   PetscFunctionReturn(0);
6952 }
6953 
6954 
6955 /*@
6956   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6957 
6958   Input Parameter:
6959 . dm - The DMPlex object
6960 
6961   Output Parameter:
6962 . ranks - The rank field
6963 
6964   Options Database Keys:
6965 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6966 
6967   Level: intermediate
6968 
6969 .seealso: DMView()
6970 @*/
6971 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6972 {
6973   DM             rdm;
6974   PetscFE        fe;
6975   PetscScalar   *r;
6976   PetscMPIInt    rank;
6977   PetscInt       dim, cStart, cEnd, c;
6978   PetscErrorCode ierr;
6979 
6980   PetscFunctionBeginUser;
6981   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6982   PetscValidPointer(ranks, 2);
6983   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6984   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6985   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6986   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6987   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6988   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6989   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6990   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6991   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6992   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6993   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6994   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6995   for (c = cStart; c < cEnd; ++c) {
6996     PetscScalar *lr;
6997 
6998     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6999     if (lr) *lr = rank;
7000   }
7001   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
7002   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
7003   PetscFunctionReturn(0);
7004 }
7005 
7006 /*@
7007   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
7008 
7009   Input Parameters:
7010 + dm    - The DMPlex
7011 - label - The DMLabel
7012 
7013   Output Parameter:
7014 . val - The label value field
7015 
7016   Options Database Keys:
7017 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
7018 
7019   Level: intermediate
7020 
7021 .seealso: DMView()
7022 @*/
7023 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
7024 {
7025   DM             rdm;
7026   PetscFE        fe;
7027   PetscScalar   *v;
7028   PetscInt       dim, cStart, cEnd, c;
7029   PetscErrorCode ierr;
7030 
7031   PetscFunctionBeginUser;
7032   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7033   PetscValidPointer(label, 2);
7034   PetscValidPointer(val, 3);
7035   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
7036   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
7037   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
7038   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
7039   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
7040   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
7041   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
7042   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7043   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
7044   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
7045   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
7046   for (c = cStart; c < cEnd; ++c) {
7047     PetscScalar *lv;
7048     PetscInt     cval;
7049 
7050     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
7051     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
7052     *lv = cval;
7053   }
7054   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
7055   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
7056   PetscFunctionReturn(0);
7057 }
7058 
7059 /*@
7060   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
7061 
7062   Input Parameter:
7063 . dm - The DMPlex object
7064 
7065   Notes:
7066   This is a useful diagnostic when creating meshes programmatically.
7067 
7068   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7069 
7070   Level: developer
7071 
7072 .seealso: DMCreate(), DMSetFromOptions()
7073 @*/
7074 PetscErrorCode DMPlexCheckSymmetry(DM dm)
7075 {
7076   PetscSection    coneSection, supportSection;
7077   const PetscInt *cone, *support;
7078   PetscInt        coneSize, c, supportSize, s;
7079   PetscInt        pStart, pEnd, p, pp, csize, ssize;
7080   PetscBool       storagecheck = PETSC_TRUE;
7081   PetscErrorCode  ierr;
7082 
7083   PetscFunctionBegin;
7084   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7085   ierr = DMViewFromOptions(dm, NULL, "-sym_dm_view");CHKERRQ(ierr);
7086   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
7087   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
7088   /* Check that point p is found in the support of its cone points, and vice versa */
7089   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
7090   for (p = pStart; p < pEnd; ++p) {
7091     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
7092     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
7093     for (c = 0; c < coneSize; ++c) {
7094       PetscBool dup = PETSC_FALSE;
7095       PetscInt  d;
7096       for (d = c-1; d >= 0; --d) {
7097         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
7098       }
7099       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
7100       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
7101       for (s = 0; s < supportSize; ++s) {
7102         if (support[s] == p) break;
7103       }
7104       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
7105         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
7106         for (s = 0; s < coneSize; ++s) {
7107           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
7108         }
7109         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7110         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
7111         for (s = 0; s < supportSize; ++s) {
7112           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
7113         }
7114         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7115         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
7116         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
7117       }
7118     }
7119     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
7120     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
7121     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
7122     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
7123     for (s = 0; s < supportSize; ++s) {
7124       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
7125       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
7126       for (c = 0; c < coneSize; ++c) {
7127         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
7128         if (cone[c] != pp) { c = 0; break; }
7129         if (cone[c] == p) break;
7130       }
7131       if (c >= coneSize) {
7132         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
7133         for (c = 0; c < supportSize; ++c) {
7134           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
7135         }
7136         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7137         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
7138         for (c = 0; c < coneSize; ++c) {
7139           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
7140         }
7141         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7142         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
7143       }
7144     }
7145   }
7146   if (storagecheck) {
7147     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
7148     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
7149     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
7150   }
7151   PetscFunctionReturn(0);
7152 }
7153 
7154 /*
7155   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
7156 */
7157 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
7158 {
7159   DMPolytopeType  cct;
7160   PetscInt        ptpoints[4];
7161   const PetscInt *cone, *ccone, *ptcone;
7162   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
7163   PetscErrorCode  ierr;
7164 
7165   PetscFunctionBegin;
7166   *unsplit = 0;
7167   switch (ct) {
7168     case DM_POLYTOPE_SEG_PRISM_TENSOR:
7169       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
7170       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
7171       for (cp = 0; cp < coneSize; ++cp) {
7172         ierr = DMPlexGetCellType(dm, cone[cp], &cct);CHKERRQ(ierr);
7173         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
7174       }
7175       break;
7176     case DM_POLYTOPE_TRI_PRISM_TENSOR:
7177     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
7178       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
7179       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
7180       for (cp = 0; cp < coneSize; ++cp) {
7181         ierr = DMPlexGetCone(dm, cone[cp], &ccone);CHKERRQ(ierr);
7182         ierr = DMPlexGetConeSize(dm, cone[cp], &cconeSize);CHKERRQ(ierr);
7183         for (ccp = 0; ccp < cconeSize; ++ccp) {
7184           ierr = DMPlexGetCellType(dm, ccone[ccp], &cct);CHKERRQ(ierr);
7185           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
7186             PetscInt p;
7187             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
7188             if (p == npt) ptpoints[npt++] = ccone[ccp];
7189           }
7190         }
7191       }
7192       break;
7193     default: break;
7194   }
7195   for (pt = 0; pt < npt; ++pt) {
7196     ierr = DMPlexGetCone(dm, ptpoints[pt], &ptcone);CHKERRQ(ierr);
7197     if (ptcone[0] == ptcone[1]) ++(*unsplit);
7198   }
7199   PetscFunctionReturn(0);
7200 }
7201 
7202 /*@
7203   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
7204 
7205   Input Parameters:
7206 + dm - The DMPlex object
7207 - cellHeight - Normally 0
7208 
7209   Notes:
7210   This is a useful diagnostic when creating meshes programmatically.
7211   Currently applicable only to homogeneous simplex or tensor meshes.
7212 
7213   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7214 
7215   Level: developer
7216 
7217 .seealso: DMCreate(), DMSetFromOptions()
7218 @*/
7219 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
7220 {
7221   DMPlexInterpolatedFlag interp;
7222   DMPolytopeType         ct;
7223   PetscInt               vStart, vEnd, cStart, cEnd, c;
7224   PetscErrorCode         ierr;
7225 
7226   PetscFunctionBegin;
7227   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7228   ierr = DMPlexIsInterpolated(dm, &interp);CHKERRQ(ierr);
7229   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
7230   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7231   for (c = cStart; c < cEnd; ++c) {
7232     PetscInt *closure = NULL;
7233     PetscInt  coneSize, closureSize, cl, Nv = 0;
7234 
7235     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
7236     if ((PetscInt) ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has no cell type", c);
7237     if (ct == DM_POLYTOPE_UNKNOWN) continue;
7238     if (interp == DMPLEX_INTERPOLATED_FULL) {
7239       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
7240       if (coneSize != DMPolytopeTypeGetConeSize(ct)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has cone size %D != %D", c, coneSize, DMPolytopeTypeGetConeSize(ct));
7241     }
7242     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7243     for (cl = 0; cl < closureSize*2; cl += 2) {
7244       const PetscInt p = closure[cl];
7245       if ((p >= vStart) && (p < vEnd)) ++Nv;
7246     }
7247     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7248     /* Special Case: Tensor faces with identified vertices */
7249     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
7250       PetscInt unsplit;
7251 
7252       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
7253       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
7254     }
7255     if (Nv != DMPolytopeTypeGetNumVertices(ct)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D vertices != %D", c, Nv, DMPolytopeTypeGetNumVertices(ct));
7256   }
7257   PetscFunctionReturn(0);
7258 }
7259 
7260 /*@
7261   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
7262 
7263   Not Collective
7264 
7265   Input Parameters:
7266 + dm - The DMPlex object
7267 - cellHeight - Normally 0
7268 
7269   Notes:
7270   This is a useful diagnostic when creating meshes programmatically.
7271   This routine is only relevant for meshes that are fully interpolated across all ranks.
7272   It will error out if a partially interpolated mesh is given on some rank.
7273   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
7274 
7275   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7276 
7277   Level: developer
7278 
7279 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
7280 @*/
7281 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
7282 {
7283   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
7284   PetscErrorCode ierr;
7285   DMPlexInterpolatedFlag interpEnum;
7286 
7287   PetscFunctionBegin;
7288   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7289   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
7290   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
7291   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
7292     PetscMPIInt	rank;
7293     MPI_Comm	comm;
7294 
7295     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
7296     ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
7297     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
7298   }
7299 
7300   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7301   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7302   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7303   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
7304     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
7305     for (c = cStart; c < cEnd; ++c) {
7306       const PetscInt      *cone, *ornt, *faceSizes, *faces;
7307       const DMPolytopeType *faceTypes;
7308       DMPolytopeType        ct;
7309       PetscInt              numFaces, coneSize, f;
7310       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
7311 
7312       ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
7313       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
7314       if (unsplit) continue;
7315       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
7316       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
7317       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
7318       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7319       for (cl = 0; cl < closureSize*2; cl += 2) {
7320         const PetscInt p = closure[cl];
7321         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
7322       }
7323       ierr = DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
7324       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
7325       for (f = 0; f < numFaces; ++f) {
7326         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
7327 
7328         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
7329         for (cl = 0; cl < fclosureSize*2; cl += 2) {
7330           const PetscInt p = fclosure[cl];
7331           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
7332         }
7333         if (fnumCorners != faceSizes[f]) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSizes[f]);
7334         for (v = 0; v < fnumCorners; ++v) {
7335           if (fclosure[v] != faces[fOff+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[fOff+v]);
7336         }
7337         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
7338         fOff += faceSizes[f];
7339       }
7340       ierr = DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
7341       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7342     }
7343   }
7344   PetscFunctionReturn(0);
7345 }
7346 
7347 /*@
7348   DMPlexCheckGeometry - Check the geometry of mesh cells
7349 
7350   Input Parameter:
7351 . dm - The DMPlex object
7352 
7353   Notes:
7354   This is a useful diagnostic when creating meshes programmatically.
7355 
7356   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7357 
7358   Level: developer
7359 
7360 .seealso: DMCreate(), DMSetFromOptions()
7361 @*/
7362 PetscErrorCode DMPlexCheckGeometry(DM dm)
7363 {
7364   PetscReal      detJ, J[9], refVol = 1.0;
7365   PetscReal      vol;
7366   PetscBool      periodic;
7367   PetscInt       dim, depth, d, cStart, cEnd, c;
7368   PetscErrorCode ierr;
7369 
7370   PetscFunctionBegin;
7371   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7372   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7373   ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
7374   for (d = 0; d < dim; ++d) refVol *= 2.0;
7375   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7376   for (c = cStart; c < cEnd; ++c) {
7377     DMPolytopeType ct;
7378     PetscInt       unsplit;
7379     PetscBool      ignoreZeroVol = PETSC_FALSE;
7380 
7381     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
7382     switch (ct) {
7383       case DM_POLYTOPE_SEG_PRISM_TENSOR:
7384       case DM_POLYTOPE_TRI_PRISM_TENSOR:
7385       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
7386         ignoreZeroVol = PETSC_TRUE; break;
7387       default: break;
7388     }
7389     switch (ct) {
7390       case DM_POLYTOPE_TRI_PRISM:
7391       case DM_POLYTOPE_TRI_PRISM_TENSOR:
7392       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
7393         continue;
7394       default: break;
7395     }
7396     ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
7397     if (unsplit) continue;
7398     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
7399     if (detJ < -PETSC_SMALL || (detJ <= 0.0 && !ignoreZeroVol)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
7400     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
7401     if (depth > 1 && !periodic) {
7402       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
7403       if (vol < -PETSC_SMALL || (vol <= 0.0 && !ignoreZeroVol)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
7404       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
7405     }
7406   }
7407   PetscFunctionReturn(0);
7408 }
7409 
7410 /*@
7411   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
7412 
7413   Input Parameters:
7414 . dm - The DMPlex object
7415 
7416   Notes:
7417   This is mainly intended for debugging/testing purposes.
7418   It currently checks only meshes with no partition overlapping.
7419 
7420   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7421 
7422   Level: developer
7423 
7424 .seealso: DMGetPointSF(), DMSetFromOptions()
7425 @*/
7426 PetscErrorCode DMPlexCheckPointSF(DM dm)
7427 {
7428   PetscSF         pointSF;
7429   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
7430   const PetscInt *locals, *rootdegree;
7431   PetscBool       distributed;
7432   PetscErrorCode  ierr;
7433 
7434   PetscFunctionBegin;
7435   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7436   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
7437   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
7438   if (!distributed) PetscFunctionReturn(0);
7439   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
7440   if (overlap) {
7441     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");
7442     PetscFunctionReturn(0);
7443   }
7444   if (!pointSF) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
7445   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
7446   if (nroots < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
7447   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
7448   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
7449 
7450   /* 1) check there are no faces in 2D, cells in 3D, in interface */
7451   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
7452   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
7453   for (l = 0; l < nleaves; ++l) {
7454     const PetscInt point = locals[l];
7455 
7456     if (point >= cStart && point < cEnd) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
7457   }
7458 
7459   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
7460   for (l = 0; l < nleaves; ++l) {
7461     const PetscInt  point = locals[l];
7462     const PetscInt *cone;
7463     PetscInt        coneSize, c, idx;
7464 
7465     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
7466     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
7467     for (c = 0; c < coneSize; ++c) {
7468       if (!rootdegree[cone[c]]) {
7469         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
7470         if (idx < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
7471       }
7472     }
7473   }
7474   PetscFunctionReturn(0);
7475 }
7476 
7477 typedef struct cell_stats
7478 {
7479   PetscReal min, max, sum, squaresum;
7480   PetscInt  count;
7481 } cell_stats_t;
7482 
7483 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
7484 {
7485   PetscInt i, N = *len;
7486 
7487   for (i = 0; i < N; i++) {
7488     cell_stats_t *A = (cell_stats_t *) a;
7489     cell_stats_t *B = (cell_stats_t *) b;
7490 
7491     B->min = PetscMin(A->min,B->min);
7492     B->max = PetscMax(A->max,B->max);
7493     B->sum += A->sum;
7494     B->squaresum += A->squaresum;
7495     B->count += A->count;
7496   }
7497 }
7498 
7499 /*@
7500   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
7501 
7502   Collective on dm
7503 
7504   Input Parameters:
7505 + dm        - The DMPlex object
7506 . output    - If true, statistics will be displayed on stdout
7507 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
7508 
7509   Notes:
7510   This is mainly intended for debugging/testing purposes.
7511 
7512   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7513 
7514   Level: developer
7515 
7516 .seealso: DMSetFromOptions()
7517 @*/
7518 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
7519 {
7520   DM             dmCoarse;
7521   cell_stats_t   stats, globalStats;
7522   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
7523   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
7524   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
7525   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
7526   PetscMPIInt    rank,size;
7527   PetscErrorCode ierr;
7528 
7529   PetscFunctionBegin;
7530   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7531   stats.min   = PETSC_MAX_REAL;
7532   stats.max   = PETSC_MIN_REAL;
7533   stats.sum   = stats.squaresum = 0.;
7534   stats.count = 0;
7535 
7536   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
7537   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
7538   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
7539   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
7540   ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
7541   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
7542   for (c = cStart; c < cEnd; c++) {
7543     PetscInt  i;
7544     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
7545 
7546     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7547     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7548     for (i = 0; i < PetscSqr(cdim); ++i) {
7549       frobJ    += J[i] * J[i];
7550       frobInvJ += invJ[i] * invJ[i];
7551     }
7552     cond2 = frobJ * frobInvJ;
7553     cond  = PetscSqrtReal(cond2);
7554 
7555     stats.min        = PetscMin(stats.min,cond);
7556     stats.max        = PetscMax(stats.max,cond);
7557     stats.sum       += cond;
7558     stats.squaresum += cond2;
7559     stats.count++;
7560     if (output && cond > limit) {
7561       PetscSection coordSection;
7562       Vec          coordsLocal;
7563       PetscScalar *coords = NULL;
7564       PetscInt     Nv, d, clSize, cl, *closure = NULL;
7565 
7566       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
7567       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
7568       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7569       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
7570       for (i = 0; i < Nv/cdim; ++i) {
7571         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
7572         for (d = 0; d < cdim; ++d) {
7573           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
7574           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
7575         }
7576         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
7577       }
7578       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7579       for (cl = 0; cl < clSize*2; cl += 2) {
7580         const PetscInt edge = closure[cl];
7581 
7582         if ((edge >= eStart) && (edge < eEnd)) {
7583           PetscReal len;
7584 
7585           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
7586           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
7587         }
7588       }
7589       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7590       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7591     }
7592   }
7593   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
7594 
7595   if (size > 1) {
7596     PetscMPIInt   blockLengths[2] = {4,1};
7597     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7598     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7599     MPI_Op        statReduce;
7600 
7601     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7602     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7603     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7604     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7605     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7606     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7607   } else {
7608     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
7609   }
7610   if (!rank) {
7611     count = globalStats.count;
7612     min   = globalStats.min;
7613     max   = globalStats.max;
7614     mean  = globalStats.sum / globalStats.count;
7615     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7616   }
7617 
7618   if (output) {
7619     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7620   }
7621   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7622 
7623   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7624   if (dmCoarse) {
7625     PetscBool isplex;
7626 
7627     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7628     if (isplex) {
7629       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
7630     }
7631   }
7632   PetscFunctionReturn(0);
7633 }
7634 
7635 /* Pointwise interpolation
7636      Just code FEM for now
7637      u^f = I u^c
7638      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7639      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7640      I_{ij} = psi^f_i phi^c_j
7641 */
7642 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7643 {
7644   PetscSection   gsc, gsf;
7645   PetscInt       m, n;
7646   void          *ctx;
7647   DM             cdm;
7648   PetscBool      regular, ismatis;
7649   PetscErrorCode ierr;
7650 
7651   PetscFunctionBegin;
7652   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7653   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7654   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7655   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7656 
7657   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7658   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7659   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7660   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7661   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7662 
7663   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7664   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7665   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7666   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7667   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7668   if (scaling) {
7669     /* Use naive scaling */
7670     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7671   }
7672   PetscFunctionReturn(0);
7673 }
7674 
7675 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7676 {
7677   PetscErrorCode ierr;
7678   VecScatter     ctx;
7679 
7680   PetscFunctionBegin;
7681   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7682   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7683   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7684   PetscFunctionReturn(0);
7685 }
7686 
7687 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7688 {
7689   PetscSection   gsc, gsf;
7690   PetscInt       m, n;
7691   void          *ctx;
7692   DM             cdm;
7693   PetscBool      regular;
7694   PetscErrorCode ierr;
7695 
7696   PetscFunctionBegin;
7697   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7698   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7699   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7700   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7701 
7702   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7703   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7704   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7705   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7706 
7707   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7708   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7709   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7710   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7711   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7712   PetscFunctionReturn(0);
7713 }
7714 
7715 /*@
7716   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7717 
7718   Input Parameter:
7719 . dm - The DMPlex object
7720 
7721   Output Parameter:
7722 . regular - The flag
7723 
7724   Level: intermediate
7725 
7726 .seealso: DMPlexSetRegularRefinement()
7727 @*/
7728 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7729 {
7730   PetscFunctionBegin;
7731   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7732   PetscValidPointer(regular, 2);
7733   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7734   PetscFunctionReturn(0);
7735 }
7736 
7737 /*@
7738   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7739 
7740   Input Parameters:
7741 + dm - The DMPlex object
7742 - regular - The flag
7743 
7744   Level: intermediate
7745 
7746 .seealso: DMPlexGetRegularRefinement()
7747 @*/
7748 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7749 {
7750   PetscFunctionBegin;
7751   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7752   ((DM_Plex *) dm->data)->regularRefinement = regular;
7753   PetscFunctionReturn(0);
7754 }
7755 
7756 /*@
7757   DMPlexGetCellRefinerType - Get the strategy for refining a cell
7758 
7759   Input Parameter:
7760 . dm - The DMPlex object
7761 
7762   Output Parameter:
7763 . cr - The strategy number
7764 
7765   Level: intermediate
7766 
7767 .seealso: DMPlexSetCellRefinerType(), DMPlexSetRegularRefinement()
7768 @*/
7769 PetscErrorCode DMPlexGetCellRefinerType(DM dm, DMPlexCellRefinerType *cr)
7770 {
7771   PetscFunctionBegin;
7772   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7773   PetscValidPointer(cr, 2);
7774   *cr = ((DM_Plex *) dm->data)->cellRefiner;
7775   PetscFunctionReturn(0);
7776 }
7777 
7778 /*@
7779   DMPlexSetCellRefinerType - Set the strategy for refining a cell
7780 
7781   Input Parameters:
7782 + dm - The DMPlex object
7783 - cr - The strategy number
7784 
7785   Level: intermediate
7786 
7787 .seealso: DMPlexGetCellRefinerType(), DMPlexGetRegularRefinement()
7788 @*/
7789 PetscErrorCode DMPlexSetCellRefinerType(DM dm, DMPlexCellRefinerType cr)
7790 {
7791   PetscFunctionBegin;
7792   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7793   ((DM_Plex *) dm->data)->cellRefiner = cr;
7794   PetscFunctionReturn(0);
7795 }
7796 
7797 /* anchors */
7798 /*@
7799   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7800   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7801 
7802   not collective
7803 
7804   Input Parameters:
7805 . dm - The DMPlex object
7806 
7807   Output Parameters:
7808 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7809 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7810 
7811 
7812   Level: intermediate
7813 
7814 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7815 @*/
7816 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7817 {
7818   DM_Plex *plex = (DM_Plex *)dm->data;
7819   PetscErrorCode ierr;
7820 
7821   PetscFunctionBegin;
7822   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7823   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7824   if (anchorSection) *anchorSection = plex->anchorSection;
7825   if (anchorIS) *anchorIS = plex->anchorIS;
7826   PetscFunctionReturn(0);
7827 }
7828 
7829 /*@
7830   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7831   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7832   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7833 
7834   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7835   DMGetConstraints() and filling in the entries in the constraint matrix.
7836 
7837   collective on dm
7838 
7839   Input Parameters:
7840 + dm - The DMPlex object
7841 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7842 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7843 
7844   The reference counts of anchorSection and anchorIS are incremented.
7845 
7846   Level: intermediate
7847 
7848 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7849 @*/
7850 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7851 {
7852   DM_Plex        *plex = (DM_Plex *)dm->data;
7853   PetscMPIInt    result;
7854   PetscErrorCode ierr;
7855 
7856   PetscFunctionBegin;
7857   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7858   if (anchorSection) {
7859     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7860     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7861     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7862   }
7863   if (anchorIS) {
7864     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7865     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7866     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7867   }
7868 
7869   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7870   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7871   plex->anchorSection = anchorSection;
7872 
7873   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7874   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7875   plex->anchorIS = anchorIS;
7876 
7877 #if defined(PETSC_USE_DEBUG)
7878   if (anchorIS && anchorSection) {
7879     PetscInt size, a, pStart, pEnd;
7880     const PetscInt *anchors;
7881 
7882     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7883     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7884     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7885     for (a = 0; a < size; a++) {
7886       PetscInt p;
7887 
7888       p = anchors[a];
7889       if (p >= pStart && p < pEnd) {
7890         PetscInt dof;
7891 
7892         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7893         if (dof) {
7894           PetscErrorCode ierr2;
7895 
7896           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7897           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7898         }
7899       }
7900     }
7901     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7902   }
7903 #endif
7904   /* reset the generic constraints */
7905   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7906   PetscFunctionReturn(0);
7907 }
7908 
7909 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7910 {
7911   PetscSection anchorSection;
7912   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7913   PetscErrorCode ierr;
7914 
7915   PetscFunctionBegin;
7916   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7917   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7918   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7919   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7920   if (numFields) {
7921     PetscInt f;
7922     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7923 
7924     for (f = 0; f < numFields; f++) {
7925       PetscInt numComp;
7926 
7927       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7928       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7929     }
7930   }
7931   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7932   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7933   pStart = PetscMax(pStart,sStart);
7934   pEnd   = PetscMin(pEnd,sEnd);
7935   pEnd   = PetscMax(pStart,pEnd);
7936   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7937   for (p = pStart; p < pEnd; p++) {
7938     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7939     if (dof) {
7940       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7941       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7942       for (f = 0; f < numFields; f++) {
7943         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7944         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7945       }
7946     }
7947   }
7948   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7949   PetscFunctionReturn(0);
7950 }
7951 
7952 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7953 {
7954   PetscSection aSec;
7955   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7956   const PetscInt *anchors;
7957   PetscInt numFields, f;
7958   IS aIS;
7959   PetscErrorCode ierr;
7960 
7961   PetscFunctionBegin;
7962   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7963   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7964   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7965   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7966   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7967   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7968   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7969   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7970   /* cSec will be a subset of aSec and section */
7971   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7972   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7973   i[0] = 0;
7974   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7975   for (p = pStart; p < pEnd; p++) {
7976     PetscInt rDof, rOff, r;
7977 
7978     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7979     if (!rDof) continue;
7980     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7981     if (numFields) {
7982       for (f = 0; f < numFields; f++) {
7983         annz = 0;
7984         for (r = 0; r < rDof; r++) {
7985           a = anchors[rOff + r];
7986           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7987           annz += aDof;
7988         }
7989         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7990         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7991         for (q = 0; q < dof; q++) {
7992           i[off + q + 1] = i[off + q] + annz;
7993         }
7994       }
7995     }
7996     else {
7997       annz = 0;
7998       for (q = 0; q < dof; q++) {
7999         a = anchors[off + q];
8000         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
8001         annz += aDof;
8002       }
8003       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
8004       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
8005       for (q = 0; q < dof; q++) {
8006         i[off + q + 1] = i[off + q] + annz;
8007       }
8008     }
8009   }
8010   nnz = i[m];
8011   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
8012   offset = 0;
8013   for (p = pStart; p < pEnd; p++) {
8014     if (numFields) {
8015       for (f = 0; f < numFields; f++) {
8016         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
8017         for (q = 0; q < dof; q++) {
8018           PetscInt rDof, rOff, r;
8019           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
8020           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
8021           for (r = 0; r < rDof; r++) {
8022             PetscInt s;
8023 
8024             a = anchors[rOff + r];
8025             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
8026             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
8027             for (s = 0; s < aDof; s++) {
8028               j[offset++] = aOff + s;
8029             }
8030           }
8031         }
8032       }
8033     }
8034     else {
8035       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
8036       for (q = 0; q < dof; q++) {
8037         PetscInt rDof, rOff, r;
8038         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
8039         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
8040         for (r = 0; r < rDof; r++) {
8041           PetscInt s;
8042 
8043           a = anchors[rOff + r];
8044           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
8045           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
8046           for (s = 0; s < aDof; s++) {
8047             j[offset++] = aOff + s;
8048           }
8049         }
8050       }
8051     }
8052   }
8053   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
8054   ierr = PetscFree(i);CHKERRQ(ierr);
8055   ierr = PetscFree(j);CHKERRQ(ierr);
8056   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
8057   PetscFunctionReturn(0);
8058 }
8059 
8060 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
8061 {
8062   DM_Plex        *plex = (DM_Plex *)dm->data;
8063   PetscSection   anchorSection, section, cSec;
8064   Mat            cMat;
8065   PetscErrorCode ierr;
8066 
8067   PetscFunctionBegin;
8068   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8069   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
8070   if (anchorSection) {
8071     PetscInt Nf;
8072 
8073     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
8074     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
8075     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
8076     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
8077     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
8078     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
8079     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
8080     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
8081   }
8082   PetscFunctionReturn(0);
8083 }
8084 
8085 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
8086 {
8087   IS             subis;
8088   PetscSection   section, subsection;
8089   PetscErrorCode ierr;
8090 
8091   PetscFunctionBegin;
8092   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
8093   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
8094   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
8095   /* Create subdomain */
8096   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
8097   /* Create submodel */
8098   ierr = DMPlexGetSubpointIS(*subdm, &subis);CHKERRQ(ierr);
8099   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
8100   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
8101   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
8102   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
8103   /* Create map from submodel to global model */
8104   if (is) {
8105     PetscSection    sectionGlobal, subsectionGlobal;
8106     IS              spIS;
8107     const PetscInt *spmap;
8108     PetscInt       *subIndices;
8109     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
8110     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
8111 
8112     ierr = DMPlexGetSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
8113     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
8114     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
8115     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
8116     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
8117     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
8118     for (p = pStart; p < pEnd; ++p) {
8119       PetscInt gdof, pSubSize  = 0;
8120 
8121       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
8122       if (gdof > 0) {
8123         for (f = 0; f < Nf; ++f) {
8124           PetscInt fdof, fcdof;
8125 
8126           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
8127           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
8128           pSubSize += fdof-fcdof;
8129         }
8130         subSize += pSubSize;
8131         if (pSubSize) {
8132           if (bs < 0) {
8133             bs = pSubSize;
8134           } else if (bs != pSubSize) {
8135             /* Layout does not admit a pointwise block size */
8136             bs = 1;
8137           }
8138         }
8139       }
8140     }
8141     /* Must have same blocksize on all procs (some might have no points) */
8142     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
8143     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
8144     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
8145     else                            {bs = bsMinMax[0];}
8146     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
8147     for (p = pStart; p < pEnd; ++p) {
8148       PetscInt gdof, goff;
8149 
8150       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
8151       if (gdof > 0) {
8152         const PetscInt point = spmap[p];
8153 
8154         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
8155         for (f = 0; f < Nf; ++f) {
8156           PetscInt fdof, fcdof, fc, f2, poff = 0;
8157 
8158           /* Can get rid of this loop by storing field information in the global section */
8159           for (f2 = 0; f2 < f; ++f2) {
8160             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
8161             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
8162             poff += fdof-fcdof;
8163           }
8164           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
8165           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
8166           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
8167             subIndices[subOff] = goff+poff+fc;
8168           }
8169         }
8170       }
8171     }
8172     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
8173     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
8174     if (bs > 1) {
8175       /* We need to check that the block size does not come from non-contiguous fields */
8176       PetscInt i, j, set = 1;
8177       for (i = 0; i < subSize; i += bs) {
8178         for (j = 0; j < bs; ++j) {
8179           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
8180         }
8181       }
8182       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
8183     }
8184     /* Attach nullspace */
8185     for (f = 0; f < Nf; ++f) {
8186       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
8187       if ((*subdm)->nullspaceConstructors[f]) break;
8188     }
8189     if (f < Nf) {
8190       MatNullSpace nullSpace;
8191 
8192       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
8193       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
8194       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
8195     }
8196   }
8197   PetscFunctionReturn(0);
8198 }
8199 
8200 /*@
8201   DMPlexMonitorThroughput - Report the cell throughput of FE integration
8202 
8203   Input Parameter:
8204 - dm - The DM
8205 
8206   Level: developer
8207 
8208   Options Database Keys:
8209 . -dm_plex_monitor_throughput - Activate the monitor
8210 
8211 .seealso: DMSetFromOptions(), DMPlexCreate()
8212 @*/
8213 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
8214 {
8215   PetscStageLog      stageLog;
8216   PetscLogEvent      event;
8217   PetscLogStage      stage;
8218   PetscEventPerfInfo eventInfo;
8219   PetscReal          cellRate, flopRate;
8220   PetscInt           cStart, cEnd, Nf, N;
8221   const char        *name;
8222   PetscErrorCode     ierr;
8223 
8224   PetscFunctionBegin;
8225   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8226 #if defined(PETSC_USE_LOG)
8227   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
8228   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8229   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
8230   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
8231   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
8232   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
8233   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
8234   N        = (cEnd - cStart)*Nf*eventInfo.count;
8235   flopRate = eventInfo.flops/eventInfo.time;
8236   cellRate = N/eventInfo.time;
8237   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6));CHKERRQ(ierr);
8238 #else
8239   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
8240 #endif
8241   PetscFunctionReturn(0);
8242 }
8243