xref: /petsc/src/dm/impls/plex/plex.c (revision ee14c7024d937659fa3550ba1b9a77d7ae2cc83e) 
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF,DMPLEX_LocatePoints;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
17 
18   Input Parameter:
19 . dm      - The DMPlex object
20 
21   Output Parameter:
22 . simplex - Flag checking for a simplex
23 
24   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
25   If the mesh has no cells, this returns PETSC_FALSE.
26 
27   Level: intermediate
28 
29 .seealso DMPlexGetSimplexOrBoxCells(), DMPlexGetCellType(), DMPlexGetHeightStratum(), DMPolytopeTypeGetNumVertices()
30 @*/
31 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
32 {
33   DMPolytopeType ct;
34   PetscInt       cStart, cEnd;
35   PetscErrorCode ierr;
36 
37   PetscFunctionBegin;
38   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
39   if (cEnd <= cStart) {*simplex = PETSC_FALSE; PetscFunctionReturn(0);}
40   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
41   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
42   PetscFunctionReturn(0);
43 }
44 
45 /*@
46   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
47 
48   Input Parameter:
49 + dm     - The DMPlex object
50 - height - The cell height in the Plex, 0 is the default
51 
52   Output Parameters:
53 + cStart - The first "normal" cell
54 - cEnd   - The upper bound on "normal"" cells
55 
56   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
57 
58   Level: developer
59 
60 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
61 @*/
62 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
63 {
64   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
65   PetscInt       cS, cE, c;
66   PetscErrorCode ierr;
67 
68   PetscFunctionBegin;
69   ierr = DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE);CHKERRQ(ierr);
70   for (c = cS; c < cE; ++c) {
71     DMPolytopeType cct;
72 
73     ierr = DMPlexGetCellType(dm, c, &cct);CHKERRQ(ierr);
74     if ((PetscInt) cct < 0) break;
75     switch (cct) {
76       case DM_POLYTOPE_POINT:
77       case DM_POLYTOPE_SEGMENT:
78       case DM_POLYTOPE_TRIANGLE:
79       case DM_POLYTOPE_QUADRILATERAL:
80       case DM_POLYTOPE_TETRAHEDRON:
81       case DM_POLYTOPE_HEXAHEDRON:
82         ct = cct;
83         break;
84       default: break;
85     }
86     if (ct != DM_POLYTOPE_UNKNOWN) break;
87   }
88   if (ct != DM_POLYTOPE_UNKNOWN) {
89     DMLabel ctLabel;
90 
91     ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
92     ierr = DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE);CHKERRQ(ierr);
93   }
94   if (cStart) *cStart = cS;
95   if (cEnd)   *cEnd   = cE;
96   PetscFunctionReturn(0);
97 }
98 
99 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
100 {
101   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
102   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
103   PetscErrorCode ierr;
104 
105   PetscFunctionBegin;
106   *ft  = PETSC_VTK_INVALID;
107   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
108   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
109   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
110   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
111   if (field >= 0) {
112     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
113     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
114   } else {
115     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
116     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
117   }
118   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
119   if (globalvcdof[0]) {
120     *sStart = vStart;
121     *sEnd   = vEnd;
122     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
123     else                        *ft = PETSC_VTK_POINT_FIELD;
124   } else if (globalvcdof[1]) {
125     *sStart = cStart;
126     *sEnd   = cEnd;
127     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
128     else                        *ft = PETSC_VTK_CELL_FIELD;
129   } else {
130     if (field >= 0) {
131       const char *fieldname;
132 
133       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
134       ierr = PetscInfo2((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
135     } else {
136       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section\"%s\"\n");CHKERRQ(ierr);
137     }
138   }
139   PetscFunctionReturn(0);
140 }
141 
142 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
143 {
144   DM                 dm;
145   PetscSection       s;
146   PetscDraw          draw, popup;
147   DM                 cdm;
148   PetscSection       coordSection;
149   Vec                coordinates;
150   const PetscScalar *coords, *array;
151   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
152   PetscReal          vbound[2], time;
153   PetscBool          isnull, flg;
154   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
155   const char        *name;
156   char               title[PETSC_MAX_PATH_LEN];
157   PetscErrorCode     ierr;
158 
159   PetscFunctionBegin;
160   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
161   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
162   if (isnull) PetscFunctionReturn(0);
163 
164   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
165   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
166   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
167   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
168   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
169   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
170   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
171   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
172   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
173   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
174   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
175 
176   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
177   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
178 
179   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
180   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
181   for (c = 0; c < N; c += dim) {
182     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
183     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
184   }
185   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
186   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
187 
188   /* Could implement something like DMDASelectFields() */
189   for (f = 0; f < Nf; ++f) {
190     DM   fdm = dm;
191     Vec  fv  = v;
192     IS   fis;
193     char prefix[PETSC_MAX_PATH_LEN];
194     const char *fname;
195 
196     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
197     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
198 
199     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
200     else               {prefix[0] = '\0';}
201     if (Nf > 1) {
202       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
203       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
204       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
205       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
206     }
207     for (comp = 0; comp < Nc; ++comp, ++w) {
208       PetscInt nmax = 2;
209 
210       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
211       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
212       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
213       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
214 
215       /* TODO Get max and min only for this component */
216       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
217       if (!flg) {
218         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
219         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
220         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
221       }
222       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
223       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
224       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
225 
226       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
227       for (c = cStart; c < cEnd; ++c) {
228         PetscScalar *coords = NULL, *a = NULL;
229         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
230 
231         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
232         if (a) {
233           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
234           color[1] = color[2] = color[3] = color[0];
235         } else {
236           PetscScalar *vals = NULL;
237           PetscInt     numVals, va;
238 
239           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
240           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
241           switch (numVals/Nc) {
242           case 3: /* P1 Triangle */
243           case 4: /* P1 Quadrangle */
244             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
245             break;
246           case 6: /* P2 Triangle */
247           case 8: /* P2 Quadrangle */
248             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
249             break;
250           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
251           }
252           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
253         }
254         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
255         switch (numCoords) {
256         case 6:
257         case 12: /* Localized triangle */
258           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
259           break;
260         case 8:
261         case 16: /* Localized quadrilateral */
262           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
263           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
264           break;
265         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
266         }
267         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
268       }
269       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
270       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
271       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
272       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
273     }
274     if (Nf > 1) {
275       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
276       ierr = ISDestroy(&fis);CHKERRQ(ierr);
277       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
278     }
279   }
280   PetscFunctionReturn(0);
281 }
282 
283 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
284 {
285   DM                      dm;
286   Vec                     locv;
287   const char              *name;
288   PetscSection            section;
289   PetscInt                pStart, pEnd;
290   PetscInt                numFields;
291   PetscViewerVTKFieldType ft;
292   PetscErrorCode          ierr;
293 
294   PetscFunctionBegin;
295   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
296   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
297   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
298   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
299   ierr = VecCopy(v, locv);CHKERRQ(ierr);
300   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
301   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
302   if (!numFields) {
303     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
304     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
305   } else {
306     PetscInt f;
307 
308     for (f = 0; f < numFields; f++) {
309       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
310       if (ft == PETSC_VTK_INVALID) continue;
311       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
312       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
313     }
314     ierr = VecDestroy(&locv);CHKERRQ(ierr);
315   }
316   PetscFunctionReturn(0);
317 }
318 
319 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
320 {
321   DM             dm;
322   PetscBool      isvtk, ishdf5, isdraw, isglvis;
323   PetscErrorCode ierr;
324 
325   PetscFunctionBegin;
326   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
327   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
328   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
329   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
330   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
331   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
332   if (isvtk || ishdf5 || isdraw || isglvis) {
333     PetscInt    i,numFields;
334     PetscObject fe;
335     PetscBool   fem = PETSC_FALSE;
336     Vec         locv = v;
337     const char  *name;
338     PetscInt    step;
339     PetscReal   time;
340 
341     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
342     for (i=0; i<numFields; i++) {
343       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
344       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
345     }
346     if (fem) {
347       PetscObject isZero;
348 
349       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
350       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
351       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
352       ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
353       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
354       ierr = VecCopy(v, locv);CHKERRQ(ierr);
355       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
356       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
357     }
358     if (isvtk) {
359       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
360     } else if (ishdf5) {
361 #if defined(PETSC_HAVE_HDF5)
362       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
363 #else
364       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
365 #endif
366     } else if (isdraw) {
367       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
368     } else if (isglvis) {
369       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
370       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
371       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
372     }
373     if (fem) {
374       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
375       ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
376     }
377   } else {
378     PetscBool isseq;
379 
380     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
381     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
382     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
383   }
384   PetscFunctionReturn(0);
385 }
386 
387 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
388 {
389   DM             dm;
390   PetscBool      isvtk, ishdf5, isdraw, isglvis, isexodusii;
391   PetscErrorCode ierr;
392 
393   PetscFunctionBegin;
394   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
395   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
396   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
397   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
398   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
399   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
400   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
401   if (isvtk || isdraw || isglvis) {
402     Vec         locv;
403     PetscObject isZero;
404     const char *name;
405 
406     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
407     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
408     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
409     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
410     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
411     ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
412     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
413     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
414     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
415     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
416   } else if (ishdf5) {
417 #if defined(PETSC_HAVE_HDF5)
418     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
419 #else
420     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
421 #endif
422   } else if (isexodusii) {
423 #if defined(PETSC_HAVE_EXODUSII)
424     ierr = VecView_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
425 #else
426     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
427 #endif
428   } else {
429     PetscBool isseq;
430 
431     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
432     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
433     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
434   }
435   PetscFunctionReturn(0);
436 }
437 
438 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
439 {
440   DM                dm;
441   MPI_Comm          comm;
442   PetscViewerFormat format;
443   Vec               v;
444   PetscBool         isvtk, ishdf5;
445   PetscErrorCode    ierr;
446 
447   PetscFunctionBegin;
448   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
449   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
450   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
451   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
452   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
453   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
454   if (format == PETSC_VIEWER_NATIVE) {
455     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
456     /* this need a better fix */
457     if (dm->useNatural) {
458       if (dm->sfNatural) {
459         const char *vecname;
460         PetscInt    n, nroots;
461 
462         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
463         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
464         if (n == nroots) {
465           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
466           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
467           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
468           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
469           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
470         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
471       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
472     } else v = originalv;
473   } else v = originalv;
474 
475   if (ishdf5) {
476 #if defined(PETSC_HAVE_HDF5)
477     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
478 #else
479     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
480 #endif
481   } else if (isvtk) {
482     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
483   } else {
484     PetscBool isseq;
485 
486     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
487     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
488     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
489   }
490   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
491   PetscFunctionReturn(0);
492 }
493 
494 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
495 {
496   DM             dm;
497   PetscBool      ishdf5;
498   PetscErrorCode ierr;
499 
500   PetscFunctionBegin;
501   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
502   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
503   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
504   if (ishdf5) {
505     DM          dmBC;
506     Vec         gv;
507     const char *name;
508 
509     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
510     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
511     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
512     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
513     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
514     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
515     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
516     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
517   } else {
518     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
519   }
520   PetscFunctionReturn(0);
521 }
522 
523 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
524 {
525   DM             dm;
526   PetscBool      ishdf5,isexodusii;
527   PetscErrorCode ierr;
528 
529   PetscFunctionBegin;
530   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
531   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
532   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
533   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
534   if (ishdf5) {
535 #if defined(PETSC_HAVE_HDF5)
536     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
537 #else
538     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
539 #endif
540   } else if (isexodusii) {
541 #if defined(PETSC_HAVE_EXODUSII)
542     ierr = VecLoad_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
543 #else
544     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
545 #endif
546   } else {
547     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
548   }
549   PetscFunctionReturn(0);
550 }
551 
552 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
553 {
554   DM                dm;
555   PetscViewerFormat format;
556   PetscBool         ishdf5;
557   PetscErrorCode    ierr;
558 
559   PetscFunctionBegin;
560   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
561   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
562   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
563   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
564   if (format == PETSC_VIEWER_NATIVE) {
565     if (dm->useNatural) {
566       if (dm->sfNatural) {
567         if (ishdf5) {
568 #if defined(PETSC_HAVE_HDF5)
569           Vec         v;
570           const char *vecname;
571 
572           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
573           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
574           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
575           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
576           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
577           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
578           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
579 #else
580           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
581 #endif
582         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
583       }
584     } else {
585       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
586     }
587   }
588   PetscFunctionReturn(0);
589 }
590 
591 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
592 {
593   PetscSection       coordSection;
594   Vec                coordinates;
595   DMLabel            depthLabel, celltypeLabel;
596   const char        *name[4];
597   const PetscScalar *a;
598   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
599   PetscErrorCode     ierr;
600 
601   PetscFunctionBegin;
602   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
603   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
604   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
605   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
606   ierr = DMPlexGetCellTypeLabel(dm, &celltypeLabel);CHKERRQ(ierr);
607   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
608   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
609   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
610   name[0]     = "vertex";
611   name[1]     = "edge";
612   name[dim-1] = "face";
613   name[dim]   = "cell";
614   for (c = cStart; c < cEnd; ++c) {
615     PetscInt *closure = NULL;
616     PetscInt  closureSize, cl, ct;
617 
618     ierr = DMLabelGetValue(celltypeLabel, c, &ct);CHKERRQ(ierr);
619     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D polytope type %s:\n", c, DMPolytopeTypes[ct]);CHKERRQ(ierr);
620     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
621     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
622     for (cl = 0; cl < closureSize*2; cl += 2) {
623       PetscInt point = closure[cl], depth, dof, off, d, p;
624 
625       if ((point < pStart) || (point >= pEnd)) continue;
626       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
627       if (!dof) continue;
628       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
629       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
630       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
631       for (p = 0; p < dof/dim; ++p) {
632         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
633         for (d = 0; d < dim; ++d) {
634           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
635           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
636         }
637         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
638       }
639       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
640     }
641     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
642     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
643   }
644   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
645   PetscFunctionReturn(0);
646 }
647 
648 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
649 {
650   DM_Plex          *mesh = (DM_Plex*) dm->data;
651   DM                cdm;
652   PetscSection      coordSection;
653   Vec               coordinates;
654   PetscViewerFormat format;
655   PetscErrorCode    ierr;
656 
657   PetscFunctionBegin;
658   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
659   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
660   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
661   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
662   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
663     const char *name;
664     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
665     PetscInt    pStart, pEnd, p, numLabels, l;
666     PetscMPIInt rank, size;
667 
668     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
669     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
670     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
671     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
672     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
673     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
674     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
675     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
676     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
677     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
678     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
679     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
680     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
681     for (p = pStart; p < pEnd; ++p) {
682       PetscInt dof, off, s;
683 
684       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
685       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
686       for (s = off; s < off+dof; ++s) {
687         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
688       }
689     }
690     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
691     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
692     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
693     for (p = pStart; p < pEnd; ++p) {
694       PetscInt dof, off, c;
695 
696       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
697       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
698       for (c = off; c < off+dof; ++c) {
699         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
700       }
701     }
702     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
703     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
704     if (coordSection && coordinates) {
705       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
706     }
707     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
708     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
709     for (l = 0; l < numLabels; ++l) {
710       DMLabel     label;
711       PetscBool   isdepth;
712       const char *name;
713 
714       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
715       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
716       if (isdepth) continue;
717       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
718       ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
719     }
720     if (size > 1) {
721       PetscSF sf;
722 
723       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
724       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
725     }
726     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
727   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
728     const char  *name, *color;
729     const char  *defcolors[3]  = {"gray", "orange", "green"};
730     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
731     char         lname[PETSC_MAX_PATH_LEN];
732     PetscReal    scale         = 2.0;
733     PetscReal    tikzscale     = 1.0;
734     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
735     double       tcoords[3];
736     PetscScalar *coords;
737     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
738     PetscMPIInt  rank, size;
739     char         **names, **colors, **lcolors;
740     PetscBool    plotEdges, flg, lflg;
741     PetscBT      wp = NULL;
742     PetscInt     pEnd, pStart;
743 
744     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
745     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
746     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
747     numLabels  = PetscMax(numLabels, 10);
748     numColors  = 10;
749     numLColors = 10;
750     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
751     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
752     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
753     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
754     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
755     if (!useLabels) numLabels = 0;
756     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
757     if (!useColors) {
758       numColors = 3;
759       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
760     }
761     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
762     if (!useColors) {
763       numLColors = 4;
764       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
765     }
766     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg);CHKERRQ(ierr);
767     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
768     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
769     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
770     if (depth < dim) plotEdges = PETSC_FALSE;
771 
772     /* filter points with labelvalue != labeldefaultvalue */
773     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
774     if (lflg) {
775       DMLabel lbl;
776 
777       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
778       if (lbl) {
779         PetscInt val, defval;
780 
781         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
782         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
783         for (c = pStart;  c < pEnd; c++) {
784           PetscInt *closure = NULL;
785           PetscInt  closureSize;
786 
787           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
788           if (val == defval) continue;
789 
790           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
791           for (p = 0; p < closureSize*2; p += 2) {
792             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
793           }
794           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
795         }
796       }
797     }
798 
799     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
800     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
801     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
802     ierr = PetscViewerASCIIPrintf(viewer, "\
803 \\documentclass[tikz]{standalone}\n\n\
804 \\usepackage{pgflibraryshapes}\n\
805 \\usetikzlibrary{backgrounds}\n\
806 \\usetikzlibrary{arrows}\n\
807 \\begin{document}\n");CHKERRQ(ierr);
808     if (size > 1) {
809       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
810       for (p = 0; p < size; ++p) {
811         if (p > 0 && p == size-1) {
812           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
813         } else if (p > 0) {
814           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
815         }
816         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
817       }
818       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
819     }
820     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
821 
822     /* Plot vertices */
823     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
824     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
825     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
826     for (v = vStart; v < vEnd; ++v) {
827       PetscInt  off, dof, d;
828       PetscBool isLabeled = PETSC_FALSE;
829 
830       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
831       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
832       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
833       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
834       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
835       for (d = 0; d < dof; ++d) {
836         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
837         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
838       }
839       /* Rotate coordinates since PGF makes z point out of the page instead of up */
840       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
841       for (d = 0; d < dof; ++d) {
842         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
843         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
844       }
845       color = colors[rank%numColors];
846       for (l = 0; l < numLabels; ++l) {
847         PetscInt val;
848         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
849         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
850       }
851       if (useNumbers) {
852         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
853       } else {
854         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
855       }
856     }
857     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
858     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
859     /* Plot cells */
860     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
861     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
862     if (dim == 3 || !useNumbers) {
863       for (e = eStart; e < eEnd; ++e) {
864         const PetscInt *cone;
865 
866         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
867         color = colors[rank%numColors];
868         for (l = 0; l < numLabels; ++l) {
869           PetscInt val;
870           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
871           if (val >= 0) {color = lcolors[l%numLColors]; break;}
872         }
873         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
874         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
875       }
876     } else {
877       for (c = cStart; c < cEnd; ++c) {
878         PetscInt *closure = NULL;
879         PetscInt  closureSize, firstPoint = -1;
880 
881         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
882         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
883         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
884         for (p = 0; p < closureSize*2; p += 2) {
885           const PetscInt point = closure[p];
886 
887           if ((point < vStart) || (point >= vEnd)) continue;
888           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
889           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
890           if (firstPoint < 0) firstPoint = point;
891         }
892         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
893         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
894         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
895       }
896     }
897     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
898     for (c = cStart; c < cEnd; ++c) {
899       double    ccoords[3] = {0.0, 0.0, 0.0};
900       PetscBool isLabeled  = PETSC_FALSE;
901       PetscInt *closure    = NULL;
902       PetscInt  closureSize, dof, d, n = 0;
903 
904       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
905       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
906       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
907       for (p = 0; p < closureSize*2; p += 2) {
908         const PetscInt point = closure[p];
909         PetscInt       off;
910 
911         if ((point < vStart) || (point >= vEnd)) continue;
912         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
913         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
914         for (d = 0; d < dof; ++d) {
915           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
916           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
917         }
918         /* Rotate coordinates since PGF makes z point out of the page instead of up */
919         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
920         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
921         ++n;
922       }
923       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
924       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
925       for (d = 0; d < dof; ++d) {
926         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
927         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
928       }
929       color = colors[rank%numColors];
930       for (l = 0; l < numLabels; ++l) {
931         PetscInt val;
932         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
933         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
934       }
935       if (useNumbers) {
936         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
937       } else {
938         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
939       }
940     }
941     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
942     /* Plot edges */
943     if (plotEdges) {
944       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
945       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
946       for (e = eStart; e < eEnd; ++e) {
947         const PetscInt *cone;
948         PetscInt        coneSize, offA, offB, dof, d;
949 
950         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
951         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
952         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
953         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
954         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
955         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
956         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
957         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
958         for (d = 0; d < dof; ++d) {
959           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
960           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
961         }
962         /* Rotate coordinates since PGF makes z point out of the page instead of up */
963         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
964         for (d = 0; d < dof; ++d) {
965           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
966           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
967         }
968         color = colors[rank%numColors];
969         for (l = 0; l < numLabels; ++l) {
970           PetscInt val;
971           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
972           if (val >= 0) {color = lcolors[l%numLColors]; break;}
973         }
974         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
975       }
976       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
977       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
978       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
979     }
980     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
981     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
982     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
983     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
984     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
985     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
986     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
987     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
988     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
989   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
990     Vec                    cown,acown;
991     VecScatter             sct;
992     ISLocalToGlobalMapping g2l;
993     IS                     gid,acis;
994     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
995     MPI_Group              ggroup,ngroup;
996     PetscScalar            *array,nid;
997     const PetscInt         *idxs;
998     PetscInt               *idxs2,*start,*adjacency,*work;
999     PetscInt64             lm[3],gm[3];
1000     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
1001     PetscMPIInt            d1,d2,rank;
1002 
1003     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1004     ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr);
1005 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1006     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRMPI(ierr);
1007 #endif
1008     if (ncomm != MPI_COMM_NULL) {
1009       ierr = MPI_Comm_group(comm,&ggroup);CHKERRMPI(ierr);
1010       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRMPI(ierr);
1011       d1   = 0;
1012       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRMPI(ierr);
1013       nid  = d2;
1014       ierr = MPI_Group_free(&ggroup);CHKERRMPI(ierr);
1015       ierr = MPI_Group_free(&ngroup);CHKERRMPI(ierr);
1016       ierr = MPI_Comm_free(&ncomm);CHKERRMPI(ierr);
1017     } else nid = 0.0;
1018 
1019     /* Get connectivity */
1020     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
1021     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
1022 
1023     /* filter overlapped local cells */
1024     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
1025     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
1026     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
1027     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
1028     for (c = cStart, cum = 0; c < cEnd; c++) {
1029       if (idxs[c-cStart] < 0) continue;
1030       idxs2[cum++] = idxs[c-cStart];
1031     }
1032     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
1033     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
1034     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1035     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
1036 
1037     /* support for node-aware cell locality */
1038     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
1039     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
1040     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
1041     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
1042     for (c = 0; c < numVertices; c++) array[c] = nid;
1043     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
1044     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
1045     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1046     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1047     ierr = ISDestroy(&acis);CHKERRQ(ierr);
1048     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
1049     ierr = VecDestroy(&cown);CHKERRQ(ierr);
1050 
1051     /* compute edgeCut */
1052     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1053     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
1054     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
1055     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
1056     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1057     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1058     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1059       PetscInt totl;
1060 
1061       totl = start[c+1]-start[c];
1062       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1063       for (i = 0; i < totl; i++) {
1064         if (work[i] < 0) {
1065           ect  += 1;
1066           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1067         }
1068       }
1069     }
1070     ierr  = PetscFree(work);CHKERRQ(ierr);
1071     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1072     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1073     lm[1] = -numVertices;
1074     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRMPI(ierr);
1075     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1076     lm[0] = ect; /* edgeCut */
1077     lm[1] = ectn; /* node-aware edgeCut */
1078     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1079     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRMPI(ierr);
1080     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1081 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1082     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1083 #else
1084     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1085 #endif
1086     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1087     ierr  = PetscFree(start);CHKERRQ(ierr);
1088     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1089     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1090   } else {
1091     const char    *name;
1092     PetscInt      *sizes, *hybsizes, *ghostsizes;
1093     PetscInt       locDepth, depth, cellHeight, dim, d;
1094     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1095     PetscInt       numLabels, l;
1096     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1097     MPI_Comm       comm;
1098     PetscMPIInt    size, rank;
1099 
1100     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
1101     ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1102     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1103     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1104     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1105     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1106     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1107     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1108     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1109     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1110     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRMPI(ierr);
1111     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1112     gcNum = gcEnd - gcStart;
1113     ierr = PetscCalloc3(size,&sizes,size,&hybsizes,size,&ghostsizes);CHKERRQ(ierr);
1114     for (d = 0; d <= depth; d++) {
1115       PetscInt Nc[2] = {0, 0}, ict;
1116 
1117       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1118       if (pStart < pEnd) {ierr = DMPlexGetCellType(dm, pStart, &ct0);CHKERRQ(ierr);}
1119       ict  = ct0;
1120       ierr = MPI_Bcast(&ict, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1121       ct0  = (DMPolytopeType) ict;
1122       for (p = pStart; p < pEnd; ++p) {
1123         DMPolytopeType ct;
1124 
1125         ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
1126         if (ct == ct0) ++Nc[0];
1127         else           ++Nc[1];
1128       }
1129       ierr = MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1130       ierr = MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1131       if (d == depth) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);}
1132       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1133       for (p = 0; p < size; ++p) {
1134         if (!rank) {
1135           ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]+hybsizes[p]);CHKERRQ(ierr);
1136           if (hybsizes[p]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", hybsizes[p]);CHKERRQ(ierr);}
1137           if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1138         }
1139       }
1140       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1141     }
1142     ierr = PetscFree3(sizes,hybsizes,ghostsizes);CHKERRQ(ierr);
1143     {
1144       const PetscReal      *maxCell;
1145       const PetscReal      *L;
1146       const DMBoundaryType *bd;
1147       PetscBool             per, localized;
1148 
1149       ierr = DMGetPeriodicity(dm, &per, &maxCell, &L, &bd);CHKERRQ(ierr);
1150       ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
1151       if (per) {
1152         ierr = PetscViewerASCIIPrintf(viewer, "Periodic mesh (");CHKERRQ(ierr);
1153         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1154         for (d = 0; d < dim; ++d) {
1155           if (bd && d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1156           if (bd)    {ierr = PetscViewerASCIIPrintf(viewer, "%s", DMBoundaryTypes[bd[d]]);CHKERRQ(ierr);}
1157         }
1158         ierr = PetscViewerASCIIPrintf(viewer, ") coordinates %s\n", localized ? "localized" : "not localized");CHKERRQ(ierr);
1159         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1160       }
1161     }
1162     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1163     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1164     for (l = 0; l < numLabels; ++l) {
1165       DMLabel         label;
1166       const char     *name;
1167       IS              valueIS;
1168       const PetscInt *values;
1169       PetscInt        numValues, v;
1170 
1171       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1172       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1173       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1174       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1175       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1176       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1177       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1178       for (v = 0; v < numValues; ++v) {
1179         PetscInt size;
1180 
1181         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1182         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1183         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1184       }
1185       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1186       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1187       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1188       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1189     }
1190     /* If no fields are specified, people do not want to see adjacency */
1191     if (dm->Nf) {
1192       PetscInt f;
1193 
1194       for (f = 0; f < dm->Nf; ++f) {
1195         const char *name;
1196 
1197         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1198         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1199         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1200         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1201         if (dm->fields[f].adjacency[0]) {
1202           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1203           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1204         } else {
1205           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1206           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1207         }
1208         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1209       }
1210     }
1211     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1212     if (cdm) {
1213       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1214       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1215       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1216     }
1217   }
1218   PetscFunctionReturn(0);
1219 }
1220 
1221 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1222 {
1223   DMPolytopeType ct;
1224   PetscMPIInt    rank;
1225   PetscErrorCode ierr;
1226 
1227   PetscFunctionBegin;
1228   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1229   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1230   switch (ct) {
1231   case DM_POLYTOPE_TRIANGLE:
1232     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1233                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1234                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1235                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1236     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1237     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1238     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1239     break;
1240   case DM_POLYTOPE_QUADRILATERAL:
1241     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1242                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1243                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1244                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1245     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1246                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1247                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1248                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1249     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1250     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1251     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1252     ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1253     break;
1254   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1255   }
1256   PetscFunctionReturn(0);
1257 }
1258 
1259 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1260 {
1261   DMPolytopeType ct;
1262   PetscReal      centroid[2] = {0., 0.};
1263   PetscMPIInt    rank;
1264   PetscInt       fillColor, v, e, d;
1265   PetscErrorCode ierr;
1266 
1267   PetscFunctionBegin;
1268   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1269   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1270   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2;
1271   switch (ct) {
1272   case DM_POLYTOPE_TRIANGLE:
1273     {
1274       PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1275 
1276       for (v = 0; v < 3; ++v) {centroid[0] += PetscRealPart(coords[v*2+0])/3.;centroid[1] += PetscRealPart(coords[v*2+1])/3.;}
1277       for (e = 0; e < 3; ++e) {
1278         refCoords[0] = refVertices[e*2+0];
1279         refCoords[1] = refVertices[e*2+1];
1280         for (d = 1; d <= edgeDiv; ++d) {
1281           refCoords[d*2+0] = refCoords[0] + (refVertices[(e+1)%3 * 2 + 0] - refCoords[0])*d/edgeDiv;
1282           refCoords[d*2+1] = refCoords[1] + (refVertices[(e+1)%3 * 2 + 1] - refCoords[1])*d/edgeDiv;
1283         }
1284         ierr = DMPlexReferenceToCoordinates(dm, cell, edgeDiv+1, refCoords, edgeCoords);CHKERRQ(ierr);
1285         for (d = 0; d < edgeDiv; ++d) {
1286           ierr = PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], fillColor, fillColor, fillColor);CHKERRQ(ierr);
1287           ierr = PetscDrawLine(draw, edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], PETSC_DRAW_BLACK);CHKERRQ(ierr);
1288         }
1289       }
1290     }
1291     break;
1292   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1293   }
1294   PetscFunctionReturn(0);
1295 }
1296 
1297 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1298 {
1299   PetscDraw          draw;
1300   DM                 cdm;
1301   PetscSection       coordSection;
1302   Vec                coordinates;
1303   const PetscScalar *coords;
1304   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1305   PetscReal         *refCoords, *edgeCoords;
1306   PetscBool          isnull, drawAffine = PETSC_TRUE;
1307   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N, edgeDiv = 4;
1308   PetscErrorCode     ierr;
1309 
1310   PetscFunctionBegin;
1311   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1312   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1313   ierr = PetscOptionsGetBool(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL);CHKERRQ(ierr);
1314   if (!drawAffine) {ierr = PetscMalloc2((edgeDiv+1)*dim, &refCoords, (edgeDiv+1)*dim, &edgeCoords);CHKERRQ(ierr);}
1315   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1316   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1317   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1318   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1319   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1320 
1321   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1322   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1323   if (isnull) PetscFunctionReturn(0);
1324   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1325 
1326   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1327   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1328   for (c = 0; c < N; c += dim) {
1329     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1330     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1331   }
1332   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1333   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1334   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1335   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1336   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1337 
1338   for (c = cStart; c < cEnd; ++c) {
1339     PetscScalar *coords = NULL;
1340     PetscInt     numCoords;
1341 
1342     ierr = DMPlexVecGetClosureAtDepth_Internal(dm, coordSection, coordinates, c, 0, &numCoords, &coords);CHKERRQ(ierr);
1343     if (drawAffine) {
1344       ierr = DMPlexDrawCell(dm, draw, c, coords);CHKERRQ(ierr);
1345     } else {
1346       ierr = DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords);CHKERRQ(ierr);
1347     }
1348     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1349   }
1350   if (!drawAffine) {ierr = PetscFree2(refCoords, edgeCoords);CHKERRQ(ierr);}
1351   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1352   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1353   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1354   PetscFunctionReturn(0);
1355 }
1356 
1357 #if defined(PETSC_HAVE_EXODUSII)
1358 #include <exodusII.h>
1359 #include <petscviewerexodusii.h>
1360 #endif
1361 
1362 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1363 {
1364   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1365   char           name[PETSC_MAX_PATH_LEN];
1366   PetscErrorCode ierr;
1367 
1368   PetscFunctionBegin;
1369   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1370   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii);CHKERRQ(ierr);
1372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
1373   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
1374   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
1375   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
1376   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus);CHKERRQ(ierr);
1377   if (iascii) {
1378     PetscViewerFormat format;
1379     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1380     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1381       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1382     } else {
1383       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1384     }
1385   } else if (ishdf5) {
1386 #if defined(PETSC_HAVE_HDF5)
1387     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1388 #else
1389     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1390 #endif
1391   } else if (isvtk) {
1392     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1393   } else if (isdraw) {
1394     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1395   } else if (isglvis) {
1396     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1397 #if defined(PETSC_HAVE_EXODUSII)
1398   } else if (isexodus) {
1399 /*
1400       exodusII requires that all sets be part of exactly one cell set.
1401       If the dm does not have a "Cell Sets" label defined, we create one
1402       with ID 1, containig all cells.
1403       Note that if the Cell Sets label is defined but does not cover all cells,
1404       we may still have a problem. This should probably be checked here or in the viewer;
1405     */
1406     PetscInt numCS;
1407     ierr = DMGetLabelSize(dm,"Cell Sets",&numCS);CHKERRQ(ierr);
1408     if (!numCS) {
1409       PetscInt cStart, cEnd, c;
1410       ierr = DMCreateLabel(dm, "Cell Sets");CHKERRQ(ierr);
1411       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1412       for (c = cStart; c < cEnd; ++c) {ierr = DMSetLabelValue(dm, "Cell Sets", c, 1);CHKERRQ(ierr);}
1413     }
1414     ierr = DMView_PlexExodusII(dm, viewer);CHKERRQ(ierr);
1415 #endif
1416   } else {
1417     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1418   }
1419   /* Optionally view the partition */
1420   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1421   if (flg) {
1422     Vec ranks;
1423     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1424     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1425     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1426   }
1427   /* Optionally view a label */
1428   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, sizeof(name), &flg);CHKERRQ(ierr);
1429   if (flg) {
1430     DMLabel label;
1431     Vec     val;
1432 
1433     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1434     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1435     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1436     ierr = VecView(val, viewer);CHKERRQ(ierr);
1437     ierr = VecDestroy(&val);CHKERRQ(ierr);
1438   }
1439   PetscFunctionReturn(0);
1440 }
1441 
1442 /*@
1443   DMPlexTopologyView - Saves a DMPlex topology into a file
1444 
1445   Collective on DM
1446 
1447   Input Parameters:
1448 + dm     - The DM whose topology is to be saved
1449 - viewer - The PetscViewer for saving
1450 
1451   Level: advanced
1452 
1453 .seealso: DMView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexTopologyLoad()
1454 @*/
1455 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1456 {
1457   PetscBool      ishdf5;
1458   PetscErrorCode ierr;
1459 
1460   PetscFunctionBegin;
1461   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1462   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1463   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1464   if (ishdf5) {
1465 #if defined(PETSC_HAVE_HDF5)
1466     PetscViewerFormat format;
1467     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1468     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1469       IS globalPointNumbering;
1470 
1471       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1472       ierr = DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1473       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1474     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1475 #else
1476     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1477 #endif
1478   }
1479   PetscFunctionReturn(0);
1480 }
1481 
1482 /*@
1483   DMPlexCoordinatesView - Saves DMPlex coordinates into a file
1484 
1485   Collective on DM
1486 
1487   Input Parameters:
1488 + dm     - The DM whose coordinates are to be saved
1489 - viewer - The PetscViewer for saving
1490 
1491   Level: advanced
1492 
1493 .seealso: DMView(), DMPlexTopologyView(), DMPlexLabelsView(), DMPlexCoordinatesLoad()
1494 @*/
1495 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
1496 {
1497   PetscBool      ishdf5;
1498   PetscErrorCode ierr;
1499 
1500   PetscFunctionBegin;
1501   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1502   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1503   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1504   if (ishdf5) {
1505 #if defined(PETSC_HAVE_HDF5)
1506     PetscViewerFormat format;
1507     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1508     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1509       ierr = DMPlexCoordinatesView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1510     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1511 #else
1512     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1513 #endif
1514   }
1515   PetscFunctionReturn(0);
1516 }
1517 
1518 /*@
1519   DMPlexLabelsView - Saves DMPlex labels into a file
1520 
1521   Collective on DM
1522 
1523   Input Parameters:
1524 + dm     - The DM whose labels are to be saved
1525 - viewer - The PetscViewer for saving
1526 
1527   Level: advanced
1528 
1529 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsLoad()
1530 @*/
1531 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
1532 {
1533   PetscBool      ishdf5;
1534   PetscErrorCode ierr;
1535 
1536   PetscFunctionBegin;
1537   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1538   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1539   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1540   if (ishdf5) {
1541 #if defined(PETSC_HAVE_HDF5)
1542     IS                globalPointNumbering;
1543     PetscViewerFormat format;
1544 
1545     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1546     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1547       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1548       ierr = DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1549       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1550     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1551 #else
1552     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1553 #endif
1554   }
1555   PetscFunctionReturn(0);
1556 }
1557 
1558 /*@
1559   DMPlexSectionView - Saves a section associated with a DMPlex
1560 
1561   Collective on DM
1562 
1563   Input Parameters:
1564 + dm         - The DM that contains the topology on which the section to be saved is defined
1565 . viewer     - The PetscViewer for saving
1566 - sectiondm  - The DM that contains the section to be saved
1567 
1568   Level: advanced
1569 
1570   Notes:
1571   This function is a wrapper around PetscSectionView(); in addition to the raw section, it saves information that associates the section points to the topology (dm) points. When the topology (dm) and the section are later loaded with DMPlexTopologyLoad() and DMPlexSectionLoad(), respectively, this information is used to match section points with topology points.
1572 
1573   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1574 
1575 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexGlobalVectorView(), DMPlexLocalVectorView(), PetscSectionView(), DMPlexSectionLoad()
1576 @*/
1577 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
1578 {
1579   PetscBool      ishdf5;
1580   PetscErrorCode ierr;
1581 
1582   PetscFunctionBegin;
1583   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1584   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1585   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1586   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1587   if (ishdf5) {
1588 #if defined(PETSC_HAVE_HDF5)
1589     ierr = DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm);CHKERRQ(ierr);
1590 #else
1591     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1592 #endif
1593   }
1594   PetscFunctionReturn(0);
1595 }
1596 
1597 /*@
1598   DMPlexGlobalVectorView - Saves a global vector
1599 
1600   Collective on DM
1601 
1602   Input Parameters:
1603 + dm        - The DM that represents the topology
1604 . viewer    - The PetscViewer to save data with
1605 . sectiondm - The DM that contains the global section on which vec is defined
1606 - vec       - The global vector to be saved
1607 
1608   Level: advanced
1609 
1610   Notes:
1611   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1612 
1613   Typical calling sequence
1614 $       DMCreate(PETSC_COMM_WORLD, &dm);
1615 $       DMSetType(dm, DMPLEX);
1616 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1617 $       DMClone(dm, &sectiondm);
1618 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1619 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1620 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1621 $       PetscSectionSetChart(section, pStart, pEnd);
1622 $       PetscSectionSetUp(section);
1623 $       DMSetLocalSection(sectiondm, section);
1624 $       PetscSectionDestroy(&section);
1625 $       DMGetGlobalVector(sectiondm, &vec);
1626 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1627 $       DMPlexTopologyView(dm, viewer);
1628 $       DMPlexSectionView(dm, viewer, sectiondm);
1629 $       DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
1630 $       DMRestoreGlobalVector(sectiondm, &vec);
1631 $       DMDestroy(&sectiondm);
1632 $       DMDestroy(&dm);
1633 
1634 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexLocalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1635 @*/
1636 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1637 {
1638   PetscBool       ishdf5;
1639   PetscErrorCode  ierr;
1640 
1641   PetscFunctionBegin;
1642   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1643   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1644   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1645   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1646   /* Check consistency */
1647   {
1648     PetscSection  section;
1649     PetscBool     includesConstraints;
1650     PetscInt      m, m1;
1651 
1652     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1653     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
1654     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1655     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1656     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1657     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
1658   }
1659   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1660   if (ishdf5) {
1661 #if defined(PETSC_HAVE_HDF5)
1662     ierr = DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1663 #else
1664     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1665 #endif
1666   }
1667   PetscFunctionReturn(0);
1668 }
1669 
1670 /*@
1671   DMPlexLocalVectorView - Saves a local vector
1672 
1673   Collective on DM
1674 
1675   Input Parameters:
1676 + dm        - The DM that represents the topology
1677 . viewer    - The PetscViewer to save data with
1678 . sectiondm - The DM that contains the local section on which vec is defined; may be the same as dm
1679 - vec       - The local vector to be saved
1680 
1681   Level: advanced
1682 
1683   Notes:
1684   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1685 
1686   Typical calling sequence
1687 $       DMCreate(PETSC_COMM_WORLD, &dm);
1688 $       DMSetType(dm, DMPLEX);
1689 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1690 $       DMClone(dm, &sectiondm);
1691 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1692 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1693 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1694 $       PetscSectionSetChart(section, pStart, pEnd);
1695 $       PetscSectionSetUp(section);
1696 $       DMSetLocalSection(sectiondm, section);
1697 $       DMGetLocalVector(sectiondm, &vec);
1698 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1699 $       DMPlexTopologyView(dm, viewer);
1700 $       DMPlexSectionView(dm, viewer, sectiondm);
1701 $       DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
1702 $       DMRestoreLocalVector(sectiondm, &vec);
1703 $       DMDestroy(&sectiondm);
1704 $       DMDestroy(&dm);
1705 
1706 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexGlobalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1707 @*/
1708 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1709 {
1710   PetscBool       ishdf5;
1711   PetscErrorCode  ierr;
1712 
1713   PetscFunctionBegin;
1714   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1715   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1716   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1717   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1718   /* Check consistency */
1719   {
1720     PetscSection  section;
1721     PetscBool     includesConstraints;
1722     PetscInt      m, m1;
1723 
1724     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1725     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
1726     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1727     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1728     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1729     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
1730   }
1731   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1732   if (ishdf5) {
1733 #if defined(PETSC_HAVE_HDF5)
1734     ierr = DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1735 #else
1736     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1737 #endif
1738   }
1739   PetscFunctionReturn(0);
1740 }
1741 
1742 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1743 {
1744   PetscBool      ishdf5;
1745   PetscErrorCode ierr;
1746 
1747   PetscFunctionBegin;
1748   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1749   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1750   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1751   if (ishdf5) {
1752 #if defined(PETSC_HAVE_HDF5)
1753     PetscViewerFormat format;
1754     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1755     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1756       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1757     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1758       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1759     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1760     PetscFunctionReturn(0);
1761 #else
1762     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1763 #endif
1764   } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1765 }
1766 
1767 /*@
1768   DMPlexTopologyLoad - Loads a topology into a DMPlex
1769 
1770   Collective on DM
1771 
1772   Input Parameters:
1773 + dm     - The DM into which the topology is loaded
1774 - viewer - The PetscViewer for the saved topology
1775 
1776   Output Parameters:
1777 . globalToLocalPointSF - The PetscSF that pushes points in [0, N) to the associated points in the loaded plex, where N is the global number of points; NULL if unneeded
1778 
1779   Level: advanced
1780 
1781 .seealso: DMLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1782 @*/
1783 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
1784 {
1785   PetscBool      ishdf5;
1786   PetscErrorCode ierr;
1787 
1788   PetscFunctionBegin;
1789   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1790   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1791   if (globalToLocalPointSF) PetscValidPointer(globalToLocalPointSF, 3);
1792   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1793   if (ishdf5) {
1794 #if defined(PETSC_HAVE_HDF5)
1795     PetscViewerFormat format;
1796     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1797     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1798       ierr = DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
1799     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1800 #else
1801     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1802 #endif
1803   }
1804   PetscFunctionReturn(0);
1805 }
1806 
1807 /*@
1808   DMPlexCoordinatesLoad - Loads coordinates into a DMPlex
1809 
1810   Collective on DM
1811 
1812   Input Parameters:
1813 + dm     - The DM into which the coordinates are loaded
1814 - viewer - The PetscViewer for the saved coordinates
1815 
1816   Level: advanced
1817 
1818 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1819 @*/
1820 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer)
1821 {
1822   PetscBool      ishdf5;
1823   PetscErrorCode ierr;
1824 
1825   PetscFunctionBegin;
1826   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1827   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1828   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1829   if (ishdf5) {
1830 #if defined(PETSC_HAVE_HDF5)
1831     PetscViewerFormat format;
1832     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1833     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1834       ierr = DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1835     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1836 #else
1837     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1838 #endif
1839   }
1840   PetscFunctionReturn(0);
1841 }
1842 
1843 /*@
1844   DMPlexLabelsLoad - Loads labels into a DMPlex
1845 
1846   Collective on DM
1847 
1848   Input Parameters:
1849 + dm     - The DM into which the labels are loaded
1850 - viewer - The PetscViewer for the saved labels
1851 
1852   Level: advanced
1853 
1854 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1855 @*/
1856 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer)
1857 {
1858   PetscBool      ishdf5;
1859   PetscErrorCode ierr;
1860 
1861   PetscFunctionBegin;
1862   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1863   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1864   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1865   if (ishdf5) {
1866 #if defined(PETSC_HAVE_HDF5)
1867     PetscViewerFormat format;
1868 
1869     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1870     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1871       ierr = DMPlexLabelsLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1872     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1873 #else
1874     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1875 #endif
1876   }
1877   PetscFunctionReturn(0);
1878 }
1879 
1880 /*@
1881   DMPlexSectionLoad - Loads section into a DMPlex
1882 
1883   Collective on DM
1884 
1885   Input Parameters:
1886 + dm          - The DM that represents the topology
1887 . viewer      - The PetscViewer that represents the on-disk section (sectionA)
1888 . sectiondm   - The DM into which the on-disk section (sectionA) is migrated
1889 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
1890 
1891   Output Parameters
1892 + globalDofSF - The SF that migrates any on-disk Vec data associated with sectionA into a global Vec associated with the sectiondm's global section (NULL if not needed)
1893 - localDofSF  - The SF that migrates any on-disk Vec data associated with sectionA into a local Vec associated with the sectiondm's local section (NULL if not needed)
1894 
1895   Level: advanced
1896 
1897   Notes:
1898   This function is a wrapper around PetscSectionLoad(); it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in dm. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
1899 
1900   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1901 
1902   The output parameter, globalDofSF (localDofSF), can later be used with DMPlexGlobalVectorLoad() (DMPlexLocalVectorLoad()) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
1903 
1904   Example using 2 processes:
1905 $  NX (number of points on dm): 4
1906 $  sectionA                   : the on-disk section
1907 $  vecA                       : a vector associated with sectionA
1908 $  sectionB                   : sectiondm's local section constructed in this function
1909 $  vecB (local)               : a vector associated with sectiondm's local section
1910 $  vecB (global)              : a vector associated with sectiondm's global section
1911 $
1912 $                                     rank 0    rank 1
1913 $  vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
1914 $  sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
1915 $  sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
1916 $  sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
1917 $  [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
1918 $  sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
1919 $  sectionB->atlasDof             :     1 0 1 | 1 3
1920 $  sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
1921 $  vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
1922 $  vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
1923 $
1924 $  where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
1925 
1926 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad(), PetscSectionLoad(), DMPlexSectionView()
1927 @*/
1928 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
1929 {
1930   PetscBool      ishdf5;
1931   PetscErrorCode ierr;
1932 
1933   PetscFunctionBegin;
1934   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1935   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1936   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1937   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
1938   if (globalDofSF) PetscValidPointer(globalDofSF, 5);
1939   if (localDofSF) PetscValidPointer(localDofSF, 6);
1940   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1941   if (ishdf5) {
1942 #if defined(PETSC_HAVE_HDF5)
1943     ierr = DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF);CHKERRQ(ierr);
1944 #else
1945     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1946 #endif
1947   }
1948   PetscFunctionReturn(0);
1949 }
1950 
1951 /*@
1952   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
1953 
1954   Collective on DM
1955 
1956   Input Parameters:
1957 + dm        - The DM that represents the topology
1958 . viewer    - The PetscViewer that represents the on-disk vector data
1959 . sectiondm - The DM that contains the global section on which vec is defined
1960 . sf        - The SF that migrates the on-disk vector data into vec
1961 - vec       - The global vector to set values of
1962 
1963   Level: advanced
1964 
1965   Notes:
1966   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1967 
1968   Typical calling sequence
1969 $       DMCreate(PETSC_COMM_WORLD, &dm);
1970 $       DMSetType(dm, DMPLEX);
1971 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1972 $       DMPlexTopologyLoad(dm, viewer, &sfX);
1973 $       DMClone(dm, &sectiondm);
1974 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1975 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
1976 $       DMGetGlobalVector(sectiondm, &vec);
1977 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1978 $       DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
1979 $       DMRestoreGlobalVector(sectiondm, &vec);
1980 $       PetscSFDestroy(&gsf);
1981 $       PetscSFDestroy(&sfX);
1982 $       DMDestroy(&sectiondm);
1983 $       DMDestroy(&dm);
1984 
1985 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexLocalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
1986 @*/
1987 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
1988 {
1989   PetscBool       ishdf5;
1990   PetscErrorCode  ierr;
1991 
1992   PetscFunctionBegin;
1993   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1994   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1995   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1996   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
1997   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
1998   /* Check consistency */
1999   {
2000     PetscSection  section;
2001     PetscBool     includesConstraints;
2002     PetscInt      m, m1;
2003 
2004     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2005     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
2006     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2007     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2008     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2009     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
2010   }
2011   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2012   if (ishdf5) {
2013 #if defined(PETSC_HAVE_HDF5)
2014     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2015 #else
2016     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2017 #endif
2018   }
2019   PetscFunctionReturn(0);
2020 }
2021 
2022 /*@
2023   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2024 
2025   Collective on DM
2026 
2027   Input Parameters:
2028 + dm        - The DM that represents the topology
2029 . viewer    - The PetscViewer that represents the on-disk vector data
2030 . sectiondm - The DM that contains the local section on which vec is defined
2031 . sf        - The SF that migrates the on-disk vector data into vec
2032 - vec       - The local vector to set values of
2033 
2034   Level: advanced
2035 
2036   Notes:
2037   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2038 
2039   Typical calling sequence
2040 $       DMCreate(PETSC_COMM_WORLD, &dm);
2041 $       DMSetType(dm, DMPLEX);
2042 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2043 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2044 $       DMClone(dm, &sectiondm);
2045 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2046 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2047 $       DMGetLocalVector(sectiondm, &vec);
2048 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2049 $       DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2050 $       DMRestoreLocalVector(sectiondm, &vec);
2051 $       PetscSFDestroy(&lsf);
2052 $       PetscSFDestroy(&sfX);
2053 $       DMDestroy(&sectiondm);
2054 $       DMDestroy(&dm);
2055 
2056 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexGlobalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2057 @*/
2058 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2059 {
2060   PetscBool       ishdf5;
2061   PetscErrorCode  ierr;
2062 
2063   PetscFunctionBegin;
2064   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2065   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2066   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2067   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2068   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2069   /* Check consistency */
2070   {
2071     PetscSection  section;
2072     PetscBool     includesConstraints;
2073     PetscInt      m, m1;
2074 
2075     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2076     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
2077     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2078     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2079     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2080     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
2081   }
2082   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2083   if (ishdf5) {
2084 #if defined(PETSC_HAVE_HDF5)
2085     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2086 #else
2087     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2088 #endif
2089   }
2090   PetscFunctionReturn(0);
2091 }
2092 
2093 PetscErrorCode DMDestroy_Plex(DM dm)
2094 {
2095   DM_Plex       *mesh = (DM_Plex*) dm->data;
2096   PetscErrorCode ierr;
2097 
2098   PetscFunctionBegin;
2099   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
2100   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
2101   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
2102   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C", NULL);CHKERRQ(ierr);
2103   if (--mesh->refct > 0) PetscFunctionReturn(0);
2104   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
2105   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
2106   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
2107   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
2108   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
2109   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
2110   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
2111   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
2112   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
2113   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
2114   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
2115   ierr = ISDestroy(&mesh->subpointIS);CHKERRQ(ierr);
2116   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
2117   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
2118   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
2119   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
2120   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
2121   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
2122   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
2123   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
2124   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
2125   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
2126   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
2127   ierr = PetscFree(mesh->neighbors);CHKERRQ(ierr);
2128   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2129   ierr = PetscFree(mesh);CHKERRQ(ierr);
2130   PetscFunctionReturn(0);
2131 }
2132 
2133 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2134 {
2135   PetscSection           sectionGlobal;
2136   PetscInt               bs = -1, mbs;
2137   PetscInt               localSize;
2138   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2139   PetscErrorCode         ierr;
2140   MatType                mtype;
2141   ISLocalToGlobalMapping ltog;
2142 
2143   PetscFunctionBegin;
2144   ierr = MatInitializePackage();CHKERRQ(ierr);
2145   mtype = dm->mattype;
2146   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
2147   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
2148   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
2149   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
2150   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
2151   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
2152   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
2153   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
2154   if (mbs > 1) bs = mbs;
2155   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
2156   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
2157   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
2158   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
2159   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
2160   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
2161   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
2162   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
2163   if (!isShell) {
2164     PetscSection subSection;
2165     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2166     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
2167     PetscInt     pStart, pEnd, p, dof, cdof;
2168 
2169     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
2170     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
2171       PetscSection section;
2172       PetscInt     size;
2173 
2174       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
2175       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
2176       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
2177       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
2178     } else {
2179       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
2180     }
2181     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
2182     for (p = pStart, lsize = 0; p < pEnd; ++p) {
2183       PetscInt bdof;
2184 
2185       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
2186       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
2187       dof  = dof < 0 ? -(dof+1) : dof;
2188       bdof = cdof && (dof-cdof) ? 1 : dof;
2189       if (dof) {
2190         if (bs < 0)          {bs = bdof;}
2191         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
2192       }
2193       if (isMatIS) {
2194         PetscInt loff,c,off;
2195         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
2196         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
2197         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
2198       }
2199     }
2200     /* Must have same blocksize on all procs (some might have no points) */
2201     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
2202     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
2203     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
2204     else                            {bs = bsMinMax[0];}
2205     bs = PetscMax(1,bs);
2206     if (isMatIS) { /* Must reduce indices by blocksize */
2207       PetscInt l;
2208 
2209       lsize = lsize/bs;
2210       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
2211       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
2212     }
2213     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
2214     if (isMatIS) {
2215       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
2216     }
2217     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
2218     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
2219     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
2220   }
2221   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
2222   PetscFunctionReturn(0);
2223 }
2224 
2225 /*@
2226   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2227 
2228   Not collective
2229 
2230   Input Parameter:
2231 . mesh - The DMPlex
2232 
2233   Output Parameters:
2234 . subsection - The subdomain section
2235 
2236   Level: developer
2237 
2238 .seealso:
2239 @*/
2240 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2241 {
2242   DM_Plex       *mesh = (DM_Plex*) dm->data;
2243   PetscErrorCode ierr;
2244 
2245   PetscFunctionBegin;
2246   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2247   if (!mesh->subdomainSection) {
2248     PetscSection section;
2249     PetscSF      sf;
2250 
2251     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
2252     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
2253     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
2254     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
2255   }
2256   *subsection = mesh->subdomainSection;
2257   PetscFunctionReturn(0);
2258 }
2259 
2260 /*@
2261   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
2262 
2263   Not collective
2264 
2265   Input Parameter:
2266 . mesh - The DMPlex
2267 
2268   Output Parameters:
2269 + pStart - The first mesh point
2270 - pEnd   - The upper bound for mesh points
2271 
2272   Level: beginner
2273 
2274 .seealso: DMPlexCreate(), DMPlexSetChart()
2275 @*/
2276 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2277 {
2278   DM_Plex       *mesh = (DM_Plex*) dm->data;
2279   PetscErrorCode ierr;
2280 
2281   PetscFunctionBegin;
2282   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2283   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2284   PetscFunctionReturn(0);
2285 }
2286 
2287 /*@
2288   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
2289 
2290   Not collective
2291 
2292   Input Parameters:
2293 + mesh - The DMPlex
2294 . pStart - The first mesh point
2295 - pEnd   - The upper bound for mesh points
2296 
2297   Output Parameters:
2298 
2299   Level: beginner
2300 
2301 .seealso: DMPlexCreate(), DMPlexGetChart()
2302 @*/
2303 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2304 {
2305   DM_Plex       *mesh = (DM_Plex*) dm->data;
2306   PetscErrorCode ierr;
2307 
2308   PetscFunctionBegin;
2309   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2310   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2311   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2312   PetscFunctionReturn(0);
2313 }
2314 
2315 /*@
2316   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2317 
2318   Not collective
2319 
2320   Input Parameters:
2321 + mesh - The DMPlex
2322 - p - The point, which must lie in the chart set with DMPlexSetChart()
2323 
2324   Output Parameter:
2325 . size - The cone size for point p
2326 
2327   Level: beginner
2328 
2329 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2330 @*/
2331 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2332 {
2333   DM_Plex       *mesh = (DM_Plex*) dm->data;
2334   PetscErrorCode ierr;
2335 
2336   PetscFunctionBegin;
2337   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2338   PetscValidPointer(size, 3);
2339   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2340   PetscFunctionReturn(0);
2341 }
2342 
2343 /*@
2344   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2345 
2346   Not collective
2347 
2348   Input Parameters:
2349 + mesh - The DMPlex
2350 . p - The point, which must lie in the chart set with DMPlexSetChart()
2351 - size - The cone size for point p
2352 
2353   Output Parameter:
2354 
2355   Note:
2356   This should be called after DMPlexSetChart().
2357 
2358   Level: beginner
2359 
2360 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
2361 @*/
2362 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
2363 {
2364   DM_Plex       *mesh = (DM_Plex*) dm->data;
2365   PetscErrorCode ierr;
2366 
2367   PetscFunctionBegin;
2368   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2369   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2370 
2371   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
2372   PetscFunctionReturn(0);
2373 }
2374 
2375 /*@
2376   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
2377 
2378   Not collective
2379 
2380   Input Parameters:
2381 + mesh - The DMPlex
2382 . p - The point, which must lie in the chart set with DMPlexSetChart()
2383 - size - The additional cone size for point p
2384 
2385   Output Parameter:
2386 
2387   Note:
2388   This should be called after DMPlexSetChart().
2389 
2390   Level: beginner
2391 
2392 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
2393 @*/
2394 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
2395 {
2396   DM_Plex       *mesh = (DM_Plex*) dm->data;
2397   PetscInt       csize;
2398   PetscErrorCode ierr;
2399 
2400   PetscFunctionBegin;
2401   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2402   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2403   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
2404 
2405   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
2406   PetscFunctionReturn(0);
2407 }
2408 
2409 /*@C
2410   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
2411 
2412   Not collective
2413 
2414   Input Parameters:
2415 + dm - The DMPlex
2416 - p - The point, which must lie in the chart set with DMPlexSetChart()
2417 
2418   Output Parameter:
2419 . cone - An array of points which are on the in-edges for point p
2420 
2421   Level: beginner
2422 
2423   Fortran Notes:
2424   Since it returns an array, this routine is only available in Fortran 90, and you must
2425   include petsc.h90 in your code.
2426   You must also call DMPlexRestoreCone() after you finish using the returned array.
2427   DMPlexRestoreCone() is not needed/available in C.
2428 
2429 .seealso: DMPlexGetConeSize(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
2430 @*/
2431 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
2432 {
2433   DM_Plex       *mesh = (DM_Plex*) dm->data;
2434   PetscInt       off;
2435   PetscErrorCode ierr;
2436 
2437   PetscFunctionBegin;
2438   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2439   PetscValidPointer(cone, 3);
2440   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2441   *cone = &mesh->cones[off];
2442   PetscFunctionReturn(0);
2443 }
2444 
2445 /*@C
2446   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
2447 
2448   Not collective
2449 
2450   Input Parameters:
2451 + dm - The DMPlex
2452 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
2453 
2454   Output Parameter:
2455 + pConesSection - PetscSection describing the layout of pCones
2456 - pCones - An array of points which are on the in-edges for the point set p
2457 
2458   Level: intermediate
2459 
2460 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
2461 @*/
2462 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
2463 {
2464   PetscSection        cs, newcs;
2465   PetscInt            *cones;
2466   PetscInt            *newarr=NULL;
2467   PetscInt            n;
2468   PetscErrorCode      ierr;
2469 
2470   PetscFunctionBegin;
2471   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2472   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
2473   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
2474   if (pConesSection) *pConesSection = newcs;
2475   if (pCones) {
2476     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
2477     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
2478   }
2479   PetscFunctionReturn(0);
2480 }
2481 
2482 /*@
2483   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
2484 
2485   Not collective
2486 
2487   Input Parameters:
2488 + dm - The DMPlex
2489 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2490 
2491   Output Parameter:
2492 . expandedPoints - An array of vertices recursively expanded from input points
2493 
2494   Level: advanced
2495 
2496   Notes:
2497   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
2498   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
2499 
2500 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
2501 @*/
2502 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
2503 {
2504   IS                  *expandedPointsAll;
2505   PetscInt            depth;
2506   PetscErrorCode      ierr;
2507 
2508   PetscFunctionBegin;
2509   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2510   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2511   PetscValidPointer(expandedPoints, 3);
2512   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2513   *expandedPoints = expandedPointsAll[0];
2514   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);CHKERRQ(ierr);
2515   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2516   PetscFunctionReturn(0);
2517 }
2518 
2519 /*@
2520   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
2521 
2522   Not collective
2523 
2524   Input Parameters:
2525 + dm - The DMPlex
2526 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2527 
2528   Output Parameter:
2529 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2530 . expandedPoints - (optional) An array of index sets with recursively expanded cones
2531 - sections - (optional) An array of sections which describe mappings from points to their cone points
2532 
2533   Level: advanced
2534 
2535   Notes:
2536   Like DMPlexGetConeTuple() but recursive.
2537 
2538   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
2539   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
2540 
2541   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
2542   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
2543   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
2544 
2545 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2546 @*/
2547 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2548 {
2549   const PetscInt      *arr0=NULL, *cone=NULL;
2550   PetscInt            *arr=NULL, *newarr=NULL;
2551   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
2552   IS                  *expandedPoints_;
2553   PetscSection        *sections_;
2554   PetscErrorCode      ierr;
2555 
2556   PetscFunctionBegin;
2557   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2558   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2559   if (depth) PetscValidIntPointer(depth, 3);
2560   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
2561   if (sections) PetscValidPointer(sections, 5);
2562   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
2563   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
2564   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2565   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
2566   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
2567   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
2568   for (d=depth_-1; d>=0; d--) {
2569     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
2570     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
2571     for (i=0; i<n; i++) {
2572       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
2573       if (arr[i] >= start && arr[i] < end) {
2574         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
2575         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
2576       } else {
2577         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
2578       }
2579     }
2580     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
2581     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
2582     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
2583     for (i=0; i<n; i++) {
2584       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
2585       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
2586       if (cn > 1) {
2587         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
2588         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
2589       } else {
2590         newarr[co] = arr[i];
2591       }
2592     }
2593     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
2594     arr = newarr;
2595     n = newn;
2596   }
2597   ierr = ISRestoreIndices(points, &arr0);CHKERRQ(ierr);
2598   *depth = depth_;
2599   if (expandedPoints) *expandedPoints = expandedPoints_;
2600   else {
2601     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
2602     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
2603   }
2604   if (sections) *sections = sections_;
2605   else {
2606     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
2607     ierr = PetscFree(sections_);CHKERRQ(ierr);
2608   }
2609   PetscFunctionReturn(0);
2610 }
2611 
2612 /*@
2613   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
2614 
2615   Not collective
2616 
2617   Input Parameters:
2618 + dm - The DMPlex
2619 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2620 
2621   Output Parameter:
2622 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2623 . expandedPoints - (optional) An array of recursively expanded cones
2624 - sections - (optional) An array of sections which describe mappings from points to their cone points
2625 
2626   Level: advanced
2627 
2628   Notes:
2629   See DMPlexGetConeRecursive() for details.
2630 
2631 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2632 @*/
2633 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2634 {
2635   PetscInt            d, depth_;
2636   PetscErrorCode      ierr;
2637 
2638   PetscFunctionBegin;
2639   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2640   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
2641   if (depth) *depth = 0;
2642   if (expandedPoints) {
2643     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
2644     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
2645   }
2646   if (sections)  {
2647     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
2648     ierr = PetscFree(*sections);CHKERRQ(ierr);
2649   }
2650   PetscFunctionReturn(0);
2651 }
2652 
2653 /*@
2654   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
2655 
2656   Not collective
2657 
2658   Input Parameters:
2659 + mesh - The DMPlex
2660 . p - The point, which must lie in the chart set with DMPlexSetChart()
2661 - cone - An array of points which are on the in-edges for point p
2662 
2663   Output Parameter:
2664 
2665   Note:
2666   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2667 
2668   Developer Note: Why not call this DMPlexSetCover()
2669 
2670   Level: beginner
2671 
2672 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
2673 @*/
2674 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
2675 {
2676   DM_Plex       *mesh = (DM_Plex*) dm->data;
2677   PetscInt       pStart, pEnd;
2678   PetscInt       dof, off, c;
2679   PetscErrorCode ierr;
2680 
2681   PetscFunctionBegin;
2682   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2683   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2684   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2685   if (dof) PetscValidPointer(cone, 3);
2686   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2687   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2688   for (c = 0; c < dof; ++c) {
2689     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
2690     mesh->cones[off+c] = cone[c];
2691   }
2692   PetscFunctionReturn(0);
2693 }
2694 
2695 /*@C
2696   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
2697 
2698   Not collective
2699 
2700   Input Parameters:
2701 + mesh - The DMPlex
2702 - p - The point, which must lie in the chart set with DMPlexSetChart()
2703 
2704   Output Parameter:
2705 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2706                     integer giving the prescription for cone traversal. If it is negative, the cone is
2707                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
2708                     the index of the cone point on which to start.
2709 
2710   Level: beginner
2711 
2712   Fortran Notes:
2713   Since it returns an array, this routine is only available in Fortran 90, and you must
2714   include petsc.h90 in your code.
2715   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
2716   DMPlexRestoreConeOrientation() is not needed/available in C.
2717 
2718 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
2719 @*/
2720 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
2721 {
2722   DM_Plex       *mesh = (DM_Plex*) dm->data;
2723   PetscInt       off;
2724   PetscErrorCode ierr;
2725 
2726   PetscFunctionBegin;
2727   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2728   if (PetscDefined(USE_DEBUG)) {
2729     PetscInt dof;
2730     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2731     if (dof) PetscValidPointer(coneOrientation, 3);
2732   }
2733   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2734 
2735   *coneOrientation = &mesh->coneOrientations[off];
2736   PetscFunctionReturn(0);
2737 }
2738 
2739 /*@
2740   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
2741 
2742   Not collective
2743 
2744   Input Parameters:
2745 + mesh - The DMPlex
2746 . p - The point, which must lie in the chart set with DMPlexSetChart()
2747 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2748                     integer giving the prescription for cone traversal. If it is negative, the cone is
2749                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
2750                     the index of the cone point on which to start.
2751 
2752   Output Parameter:
2753 
2754   Note:
2755   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2756 
2757   Level: beginner
2758 
2759 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2760 @*/
2761 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2762 {
2763   DM_Plex       *mesh = (DM_Plex*) dm->data;
2764   PetscInt       pStart, pEnd;
2765   PetscInt       dof, off, c;
2766   PetscErrorCode ierr;
2767 
2768   PetscFunctionBegin;
2769   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2770   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2771   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2772   if (dof) PetscValidPointer(coneOrientation, 3);
2773   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2774   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2775   for (c = 0; c < dof; ++c) {
2776     PetscInt cdof, o = coneOrientation[c];
2777 
2778     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2779     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2780     mesh->coneOrientations[off+c] = o;
2781   }
2782   PetscFunctionReturn(0);
2783 }
2784 
2785 /*@
2786   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2787 
2788   Not collective
2789 
2790   Input Parameters:
2791 + mesh - The DMPlex
2792 . p - The point, which must lie in the chart set with DMPlexSetChart()
2793 . conePos - The local index in the cone where the point should be put
2794 - conePoint - The mesh point to insert
2795 
2796   Level: beginner
2797 
2798 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2799 @*/
2800 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2801 {
2802   DM_Plex       *mesh = (DM_Plex*) dm->data;
2803   PetscInt       pStart, pEnd;
2804   PetscInt       dof, off;
2805   PetscErrorCode ierr;
2806 
2807   PetscFunctionBegin;
2808   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2809   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2810   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2811   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2812   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2813   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2814   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2815   mesh->cones[off+conePos] = conePoint;
2816   PetscFunctionReturn(0);
2817 }
2818 
2819 /*@
2820   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2821 
2822   Not collective
2823 
2824   Input Parameters:
2825 + mesh - The DMPlex
2826 . p - The point, which must lie in the chart set with DMPlexSetChart()
2827 . conePos - The local index in the cone where the point should be put
2828 - coneOrientation - The point orientation to insert
2829 
2830   Level: beginner
2831 
2832 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2833 @*/
2834 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2835 {
2836   DM_Plex       *mesh = (DM_Plex*) dm->data;
2837   PetscInt       pStart, pEnd;
2838   PetscInt       dof, off;
2839   PetscErrorCode ierr;
2840 
2841   PetscFunctionBegin;
2842   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2843   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2844   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2845   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2846   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2847   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2848   mesh->coneOrientations[off+conePos] = coneOrientation;
2849   PetscFunctionReturn(0);
2850 }
2851 
2852 /*@
2853   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2854 
2855   Not collective
2856 
2857   Input Parameters:
2858 + mesh - The DMPlex
2859 - p - The point, which must lie in the chart set with DMPlexSetChart()
2860 
2861   Output Parameter:
2862 . size - The support size for point p
2863 
2864   Level: beginner
2865 
2866 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2867 @*/
2868 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2869 {
2870   DM_Plex       *mesh = (DM_Plex*) dm->data;
2871   PetscErrorCode ierr;
2872 
2873   PetscFunctionBegin;
2874   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2875   PetscValidPointer(size, 3);
2876   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2877   PetscFunctionReturn(0);
2878 }
2879 
2880 /*@
2881   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2882 
2883   Not collective
2884 
2885   Input Parameters:
2886 + mesh - The DMPlex
2887 . p - The point, which must lie in the chart set with DMPlexSetChart()
2888 - size - The support size for point p
2889 
2890   Output Parameter:
2891 
2892   Note:
2893   This should be called after DMPlexSetChart().
2894 
2895   Level: beginner
2896 
2897 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2898 @*/
2899 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2900 {
2901   DM_Plex       *mesh = (DM_Plex*) dm->data;
2902   PetscErrorCode ierr;
2903 
2904   PetscFunctionBegin;
2905   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2906   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2907 
2908   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2909   PetscFunctionReturn(0);
2910 }
2911 
2912 /*@C
2913   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2914 
2915   Not collective
2916 
2917   Input Parameters:
2918 + mesh - The DMPlex
2919 - p - The point, which must lie in the chart set with DMPlexSetChart()
2920 
2921   Output Parameter:
2922 . support - An array of points which are on the out-edges for point p
2923 
2924   Level: beginner
2925 
2926   Fortran Notes:
2927   Since it returns an array, this routine is only available in Fortran 90, and you must
2928   include petsc.h90 in your code.
2929   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2930   DMPlexRestoreSupport() is not needed/available in C.
2931 
2932 .seealso: DMPlexGetSupportSize(), DMPlexSetSupport(), DMPlexGetCone(), DMPlexSetChart()
2933 @*/
2934 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2935 {
2936   DM_Plex       *mesh = (DM_Plex*) dm->data;
2937   PetscInt       off;
2938   PetscErrorCode ierr;
2939 
2940   PetscFunctionBegin;
2941   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2942   PetscValidPointer(support, 3);
2943   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2944   *support = &mesh->supports[off];
2945   PetscFunctionReturn(0);
2946 }
2947 
2948 /*@
2949   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2950 
2951   Not collective
2952 
2953   Input Parameters:
2954 + mesh - The DMPlex
2955 . p - The point, which must lie in the chart set with DMPlexSetChart()
2956 - support - An array of points which are on the out-edges for point p
2957 
2958   Output Parameter:
2959 
2960   Note:
2961   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2962 
2963   Level: beginner
2964 
2965 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2966 @*/
2967 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2968 {
2969   DM_Plex       *mesh = (DM_Plex*) dm->data;
2970   PetscInt       pStart, pEnd;
2971   PetscInt       dof, off, c;
2972   PetscErrorCode ierr;
2973 
2974   PetscFunctionBegin;
2975   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2976   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2977   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2978   if (dof) PetscValidPointer(support, 3);
2979   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2980   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2981   for (c = 0; c < dof; ++c) {
2982     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2983     mesh->supports[off+c] = support[c];
2984   }
2985   PetscFunctionReturn(0);
2986 }
2987 
2988 /*@
2989   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2990 
2991   Not collective
2992 
2993   Input Parameters:
2994 + mesh - The DMPlex
2995 . p - The point, which must lie in the chart set with DMPlexSetChart()
2996 . supportPos - The local index in the cone where the point should be put
2997 - supportPoint - The mesh point to insert
2998 
2999   Level: beginner
3000 
3001 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3002 @*/
3003 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3004 {
3005   DM_Plex       *mesh = (DM_Plex*) dm->data;
3006   PetscInt       pStart, pEnd;
3007   PetscInt       dof, off;
3008   PetscErrorCode ierr;
3009 
3010   PetscFunctionBegin;
3011   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3012   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3013   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3014   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3015   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3016   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
3017   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
3018   mesh->supports[off+supportPos] = supportPoint;
3019   PetscFunctionReturn(0);
3020 }
3021 
3022 /*@C
3023   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
3024 
3025   Not collective
3026 
3027   Input Parameters:
3028 + mesh - The DMPlex
3029 . p - The point, which must lie in the chart set with DMPlexSetChart()
3030 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3031 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
3032 
3033   Output Parameters:
3034 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3035 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3036 
3037   Note:
3038   If using internal storage (points is NULL on input), each call overwrites the last output.
3039 
3040   Fortran Notes:
3041   Since it returns an array, this routine is only available in Fortran 90, and you must
3042   include petsc.h90 in your code.
3043 
3044   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3045 
3046   Level: beginner
3047 
3048 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3049 @*/
3050 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3051 {
3052   DM_Plex        *mesh = (DM_Plex*) dm->data;
3053   PetscInt       *closure, *fifo;
3054   const PetscInt *tmp = NULL, *tmpO = NULL;
3055   PetscInt        tmpSize, t;
3056   PetscInt        depth       = 0, maxSize;
3057   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
3058   PetscErrorCode  ierr;
3059 
3060   PetscFunctionBegin;
3061   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3062   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3063   /* This is only 1-level */
3064   if (useCone) {
3065     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3066     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3067     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3068   } else {
3069     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3070     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3071   }
3072   if (depth == 1) {
3073     if (*points) {
3074       closure = *points;
3075     } else {
3076       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
3077       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3078     }
3079     closure[0] = p; closure[1] = 0;
3080     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
3081       closure[closureSize]   = tmp[t];
3082       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
3083     }
3084     if (numPoints) *numPoints = closureSize/2;
3085     if (points)    *points    = closure;
3086     PetscFunctionReturn(0);
3087   }
3088   {
3089     PetscInt c, coneSeries, s,supportSeries;
3090 
3091     c = mesh->maxConeSize;
3092     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
3093     s = mesh->maxSupportSize;
3094     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
3095     maxSize = 2*PetscMax(coneSeries,supportSeries);
3096   }
3097   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3098   if (*points) {
3099     closure = *points;
3100   } else {
3101     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3102   }
3103   closure[0] = p; closure[1] = 0;
3104   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
3105     const PetscInt cp = tmp[t];
3106     const PetscInt co = tmpO ? tmpO[t] : 0;
3107 
3108     closure[closureSize]   = cp;
3109     closure[closureSize+1] = co;
3110     fifo[fifoSize]         = cp;
3111     fifo[fifoSize+1]       = co;
3112   }
3113   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3114   while (fifoSize - fifoStart) {
3115     const PetscInt q   = fifo[fifoStart];
3116     const PetscInt o   = fifo[fifoStart+1];
3117     const PetscInt rev = o >= 0 ? 0 : 1;
3118     const PetscInt off = rev ? -(o+1) : o;
3119 
3120     if (useCone) {
3121       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3122       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3123       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3124     } else {
3125       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3126       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3127       tmpO = NULL;
3128     }
3129     for (t = 0; t < tmpSize; ++t) {
3130       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
3131       const PetscInt cp = tmp[i];
3132       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
3133       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
3134        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
3135       PetscInt       co = tmpO ? tmpO[i] : 0;
3136       PetscInt       c;
3137 
3138       if (rev) {
3139         PetscInt childSize, coff;
3140         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3141         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
3142         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3143       }
3144       /* Check for duplicate */
3145       for (c = 0; c < closureSize; c += 2) {
3146         if (closure[c] == cp) break;
3147       }
3148       if (c == closureSize) {
3149         closure[closureSize]   = cp;
3150         closure[closureSize+1] = co;
3151         fifo[fifoSize]         = cp;
3152         fifo[fifoSize+1]       = co;
3153         closureSize           += 2;
3154         fifoSize              += 2;
3155       }
3156     }
3157     fifoStart += 2;
3158   }
3159   if (numPoints) *numPoints = closureSize/2;
3160   if (points)    *points    = closure;
3161   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3162   PetscFunctionReturn(0);
3163 }
3164 
3165 /*@C
3166   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
3167 
3168   Not collective
3169 
3170   Input Parameters:
3171 + mesh - The DMPlex
3172 . p - The point, which must lie in the chart set with DMPlexSetChart()
3173 . orientation - The orientation of the point
3174 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3175 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
3176 
3177   Output Parameters:
3178 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3179 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3180 
3181   Note:
3182   If using internal storage (points is NULL on input), each call overwrites the last output.
3183 
3184   Fortran Notes:
3185   Since it returns an array, this routine is only available in Fortran 90, and you must
3186   include petsc.h90 in your code.
3187 
3188   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3189 
3190   Level: beginner
3191 
3192 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3193 @*/
3194 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3195 {
3196   DM_Plex        *mesh = (DM_Plex*) dm->data;
3197   PetscInt       *closure, *fifo;
3198   const PetscInt *tmp = NULL, *tmpO = NULL;
3199   PetscInt        tmpSize, t;
3200   PetscInt        depth       = 0, maxSize;
3201   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
3202   PetscErrorCode  ierr;
3203 
3204   PetscFunctionBegin;
3205   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3206   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3207   /* This is only 1-level */
3208   if (useCone) {
3209     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3210     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3211     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3212   } else {
3213     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3214     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3215   }
3216   if (depth == 1) {
3217     if (*points) {
3218       closure = *points;
3219     } else {
3220       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
3221       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3222     }
3223     closure[0] = p; closure[1] = ornt;
3224     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
3225       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
3226       closure[closureSize]   = tmp[i];
3227       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
3228     }
3229     if (numPoints) *numPoints = closureSize/2;
3230     if (points)    *points    = closure;
3231     PetscFunctionReturn(0);
3232   }
3233   {
3234     PetscInt c, coneSeries, s,supportSeries;
3235 
3236     c = mesh->maxConeSize;
3237     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
3238     s = mesh->maxSupportSize;
3239     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
3240     maxSize = 2*PetscMax(coneSeries,supportSeries);
3241   }
3242   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3243   if (*points) {
3244     closure = *points;
3245   } else {
3246     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3247   }
3248   closure[0] = p; closure[1] = ornt;
3249   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
3250     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
3251     const PetscInt cp = tmp[i];
3252     PetscInt       co = tmpO ? tmpO[i] : 0;
3253 
3254     if (ornt < 0) {
3255       PetscInt childSize, coff;
3256       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3257       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
3258       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3259     }
3260     closure[closureSize]   = cp;
3261     closure[closureSize+1] = co;
3262     fifo[fifoSize]         = cp;
3263     fifo[fifoSize+1]       = co;
3264   }
3265   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3266   while (fifoSize - fifoStart) {
3267     const PetscInt q   = fifo[fifoStart];
3268     const PetscInt o   = fifo[fifoStart+1];
3269     const PetscInt rev = o >= 0 ? 0 : 1;
3270     const PetscInt off = rev ? -(o+1) : o;
3271 
3272     if (useCone) {
3273       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3274       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3275       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3276     } else {
3277       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3278       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3279       tmpO = NULL;
3280     }
3281     for (t = 0; t < tmpSize; ++t) {
3282       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
3283       const PetscInt cp = tmp[i];
3284       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
3285       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
3286        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
3287       PetscInt       co = tmpO ? tmpO[i] : 0;
3288       PetscInt       c;
3289 
3290       if (rev) {
3291         PetscInt childSize, coff;
3292         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3293         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
3294         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3295       }
3296       /* Check for duplicate */
3297       for (c = 0; c < closureSize; c += 2) {
3298         if (closure[c] == cp) break;
3299       }
3300       if (c == closureSize) {
3301         closure[closureSize]   = cp;
3302         closure[closureSize+1] = co;
3303         fifo[fifoSize]         = cp;
3304         fifo[fifoSize+1]       = co;
3305         closureSize           += 2;
3306         fifoSize              += 2;
3307       }
3308     }
3309     fifoStart += 2;
3310   }
3311   if (numPoints) *numPoints = closureSize/2;
3312   if (points)    *points    = closure;
3313   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3314   PetscFunctionReturn(0);
3315 }
3316 
3317 /*@C
3318   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
3319 
3320   Not collective
3321 
3322   Input Parameters:
3323 + mesh - The DMPlex
3324 . p - The point, which must lie in the chart set with DMPlexSetChart()
3325 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3326 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
3327 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
3328 
3329   Note:
3330   If not using internal storage (points is not NULL on input), this call is unnecessary
3331 
3332   Fortran Notes:
3333   Since it returns an array, this routine is only available in Fortran 90, and you must
3334   include petsc.h90 in your code.
3335 
3336   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3337 
3338   Level: beginner
3339 
3340 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3341 @*/
3342 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3343 {
3344   PetscErrorCode ierr;
3345 
3346   PetscFunctionBegin;
3347   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3348   if (numPoints) PetscValidIntPointer(numPoints,4);
3349   if (points) PetscValidPointer(points,5);
3350   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
3351   if (numPoints) *numPoints = 0;
3352   PetscFunctionReturn(0);
3353 }
3354 
3355 /*@
3356   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
3357 
3358   Not collective
3359 
3360   Input Parameter:
3361 . mesh - The DMPlex
3362 
3363   Output Parameters:
3364 + maxConeSize - The maximum number of in-edges
3365 - maxSupportSize - The maximum number of out-edges
3366 
3367   Level: beginner
3368 
3369 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
3370 @*/
3371 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
3372 {
3373   DM_Plex *mesh = (DM_Plex*) dm->data;
3374 
3375   PetscFunctionBegin;
3376   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3377   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
3378   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
3379   PetscFunctionReturn(0);
3380 }
3381 
3382 PetscErrorCode DMSetUp_Plex(DM dm)
3383 {
3384   DM_Plex       *mesh = (DM_Plex*) dm->data;
3385   PetscInt       size;
3386   PetscErrorCode ierr;
3387 
3388   PetscFunctionBegin;
3389   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3390   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
3391   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
3392   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
3393   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
3394   ierr = PetscLogObjectMemory((PetscObject) dm, size*2*sizeof(PetscInt));CHKERRQ(ierr);
3395   if (mesh->maxSupportSize) {
3396     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3397     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
3398     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
3399     ierr = PetscLogObjectMemory((PetscObject) dm, size*sizeof(PetscInt));CHKERRQ(ierr);
3400   }
3401   PetscFunctionReturn(0);
3402 }
3403 
3404 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
3405 {
3406   PetscErrorCode ierr;
3407 
3408   PetscFunctionBegin;
3409   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
3410   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
3411   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
3412   if (dm->useNatural && dm->sfMigration) {
3413     PetscSF        sfMigrationInv,sfNatural;
3414     PetscSection   section, sectionSeq;
3415 
3416     (*subdm)->sfMigration = dm->sfMigration;
3417     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
3418     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
3419     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3420     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
3421     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3422 
3423     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3424     (*subdm)->sfNatural = sfNatural;
3425     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3426     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3427   }
3428   PetscFunctionReturn(0);
3429 }
3430 
3431 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
3432 {
3433   PetscErrorCode ierr;
3434   PetscInt       i = 0;
3435 
3436   PetscFunctionBegin;
3437   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
3438   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
3439   (*superdm)->useNatural = PETSC_FALSE;
3440   for (i = 0; i < len; i++) {
3441     if (dms[i]->useNatural && dms[i]->sfMigration) {
3442       PetscSF        sfMigrationInv,sfNatural;
3443       PetscSection   section, sectionSeq;
3444 
3445       (*superdm)->sfMigration = dms[i]->sfMigration;
3446       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
3447       (*superdm)->useNatural = PETSC_TRUE;
3448       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
3449       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3450       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
3451       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3452 
3453       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3454       (*superdm)->sfNatural = sfNatural;
3455       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3456       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3457       break;
3458     }
3459   }
3460   PetscFunctionReturn(0);
3461 }
3462 
3463 /*@
3464   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
3465 
3466   Not collective
3467 
3468   Input Parameter:
3469 . mesh - The DMPlex
3470 
3471   Output Parameter:
3472 
3473   Note:
3474   This should be called after all calls to DMPlexSetCone()
3475 
3476   Level: beginner
3477 
3478 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
3479 @*/
3480 PetscErrorCode DMPlexSymmetrize(DM dm)
3481 {
3482   DM_Plex       *mesh = (DM_Plex*) dm->data;
3483   PetscInt      *offsets;
3484   PetscInt       supportSize;
3485   PetscInt       pStart, pEnd, p;
3486   PetscErrorCode ierr;
3487 
3488   PetscFunctionBegin;
3489   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3490   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
3491   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3492   /* Calculate support sizes */
3493   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3494   for (p = pStart; p < pEnd; ++p) {
3495     PetscInt dof, off, c;
3496 
3497     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3498     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3499     for (c = off; c < off+dof; ++c) {
3500       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
3501     }
3502   }
3503   for (p = pStart; p < pEnd; ++p) {
3504     PetscInt dof;
3505 
3506     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3507 
3508     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
3509   }
3510   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3511   /* Calculate supports */
3512   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
3513   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
3514   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
3515   for (p = pStart; p < pEnd; ++p) {
3516     PetscInt dof, off, c;
3517 
3518     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3519     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3520     for (c = off; c < off+dof; ++c) {
3521       const PetscInt q = mesh->cones[c];
3522       PetscInt       offS;
3523 
3524       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
3525 
3526       mesh->supports[offS+offsets[q]] = p;
3527       ++offsets[q];
3528     }
3529   }
3530   ierr = PetscFree(offsets);CHKERRQ(ierr);
3531   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3532   PetscFunctionReturn(0);
3533 }
3534 
3535 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
3536 {
3537   IS             stratumIS;
3538   PetscErrorCode ierr;
3539 
3540   PetscFunctionBegin;
3541   if (pStart >= pEnd) PetscFunctionReturn(0);
3542   if (PetscDefined(USE_DEBUG)) {
3543     PetscInt  qStart, qEnd, numLevels, level;
3544     PetscBool overlap = PETSC_FALSE;
3545     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
3546     for (level = 0; level < numLevels; level++) {
3547       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3548       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
3549     }
3550     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
3551   }
3552   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
3553   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
3554   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
3555   PetscFunctionReturn(0);
3556 }
3557 
3558 /*@
3559   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
3560   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
3561   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
3562   the DAG.
3563 
3564   Collective on dm
3565 
3566   Input Parameter:
3567 . mesh - The DMPlex
3568 
3569   Output Parameter:
3570 
3571   Notes:
3572   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
3573   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
3574   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
3575   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
3576   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
3577 
3578   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
3579   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
3580   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
3581   to interpolate only that one (e0), so that
3582 $  cone(c0) = {e0, v2}
3583 $  cone(e0) = {v0, v1}
3584   If DMPlexStratify() is run on this mesh, it will give depths
3585 $  depth 0 = {v0, v1, v2}
3586 $  depth 1 = {e0, c0}
3587   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
3588 
3589   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
3590 
3591   Level: beginner
3592 
3593 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexComputeCellTypes()
3594 @*/
3595 PetscErrorCode DMPlexStratify(DM dm)
3596 {
3597   DM_Plex       *mesh = (DM_Plex*) dm->data;
3598   DMLabel        label;
3599   PetscInt       pStart, pEnd, p;
3600   PetscInt       numRoots = 0, numLeaves = 0;
3601   PetscErrorCode ierr;
3602 
3603   PetscFunctionBegin;
3604   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3605   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3606 
3607   /* Create depth label */
3608   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3609   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
3610   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3611 
3612   {
3613     /* Initialize roots and count leaves */
3614     PetscInt sMin = PETSC_MAX_INT;
3615     PetscInt sMax = PETSC_MIN_INT;
3616     PetscInt coneSize, supportSize;
3617 
3618     for (p = pStart; p < pEnd; ++p) {
3619       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3620       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3621       if (!coneSize && supportSize) {
3622         sMin = PetscMin(p, sMin);
3623         sMax = PetscMax(p, sMax);
3624         ++numRoots;
3625       } else if (!supportSize && coneSize) {
3626         ++numLeaves;
3627       } else if (!supportSize && !coneSize) {
3628         /* Isolated points */
3629         sMin = PetscMin(p, sMin);
3630         sMax = PetscMax(p, sMax);
3631       }
3632     }
3633     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
3634   }
3635 
3636   if (numRoots + numLeaves == (pEnd - pStart)) {
3637     PetscInt sMin = PETSC_MAX_INT;
3638     PetscInt sMax = PETSC_MIN_INT;
3639     PetscInt coneSize, supportSize;
3640 
3641     for (p = pStart; p < pEnd; ++p) {
3642       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3643       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3644       if (!supportSize && coneSize) {
3645         sMin = PetscMin(p, sMin);
3646         sMax = PetscMax(p, sMax);
3647       }
3648     }
3649     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
3650   } else {
3651     PetscInt level = 0;
3652     PetscInt qStart, qEnd, q;
3653 
3654     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3655     while (qEnd > qStart) {
3656       PetscInt sMin = PETSC_MAX_INT;
3657       PetscInt sMax = PETSC_MIN_INT;
3658 
3659       for (q = qStart; q < qEnd; ++q) {
3660         const PetscInt *support;
3661         PetscInt        supportSize, s;
3662 
3663         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
3664         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
3665         for (s = 0; s < supportSize; ++s) {
3666           sMin = PetscMin(support[s], sMin);
3667           sMax = PetscMax(support[s], sMax);
3668         }
3669       }
3670       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
3671       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
3672       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3673     }
3674   }
3675   { /* just in case there is an empty process */
3676     PetscInt numValues, maxValues = 0, v;
3677 
3678     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
3679     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
3680     for (v = numValues; v < maxValues; v++) {
3681       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
3682     }
3683   }
3684   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
3685   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3686   PetscFunctionReturn(0);
3687 }
3688 
3689 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
3690 {
3691   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3692   PetscInt       dim, depth, pheight, coneSize;
3693   PetscErrorCode ierr;
3694 
3695   PetscFunctionBeginHot;
3696   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3697   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3698   ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3699   pheight = depth - pdepth;
3700   if (depth <= 1) {
3701     switch (pdepth) {
3702       case 0: ct = DM_POLYTOPE_POINT;break;
3703       case 1:
3704         switch (coneSize) {
3705           case 2: ct = DM_POLYTOPE_SEGMENT;break;
3706           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3707           case 4:
3708           switch (dim) {
3709             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
3710             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
3711             default: break;
3712           }
3713           break;
3714         case 5: ct = DM_POLYTOPE_PYRAMID;break;
3715         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3716         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
3717         default: break;
3718       }
3719     }
3720   } else {
3721     if (pdepth == 0) {
3722       ct = DM_POLYTOPE_POINT;
3723     } else if (pheight == 0) {
3724       switch (dim) {
3725         case 1:
3726           switch (coneSize) {
3727             case 2: ct = DM_POLYTOPE_SEGMENT;break;
3728             default: break;
3729           }
3730           break;
3731         case 2:
3732           switch (coneSize) {
3733             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3734             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3735             default: break;
3736           }
3737           break;
3738         case 3:
3739           switch (coneSize) {
3740             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
3741             case 5:
3742             {
3743               const PetscInt *cone;
3744               PetscInt        faceConeSize;
3745 
3746               ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
3747               ierr = DMPlexGetConeSize(dm, cone[0], &faceConeSize);CHKERRQ(ierr);
3748               switch (faceConeSize) {
3749                 case 3: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3750                 case 4: ct = DM_POLYTOPE_PYRAMID;break;
3751               }
3752             }
3753             break;
3754             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
3755             default: break;
3756           }
3757           break;
3758         default: break;
3759       }
3760     } else if (pheight > 0) {
3761       switch (coneSize) {
3762         case 2: ct = DM_POLYTOPE_SEGMENT;break;
3763         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3764         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3765         default: break;
3766       }
3767     }
3768   }
3769   *pt = ct;
3770   PetscFunctionReturn(0);
3771 }
3772 
3773 /*@
3774   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
3775 
3776   Collective on dm
3777 
3778   Input Parameter:
3779 . mesh - The DMPlex
3780 
3781   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
3782 
3783   Level: developer
3784 
3785   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
3786   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
3787   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
3788 
3789 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexStratify(), DMGetLabel(), DMCreateLabel()
3790 @*/
3791 PetscErrorCode DMPlexComputeCellTypes(DM dm)
3792 {
3793   DM_Plex       *mesh;
3794   DMLabel        ctLabel;
3795   PetscInt       pStart, pEnd, p;
3796   PetscErrorCode ierr;
3797 
3798   PetscFunctionBegin;
3799   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3800   mesh = (DM_Plex *) dm->data;
3801   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
3802   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
3803   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3804   for (p = pStart; p < pEnd; ++p) {
3805     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3806     PetscInt       pdepth;
3807 
3808     ierr = DMPlexGetPointDepth(dm, p, &pdepth);CHKERRQ(ierr);
3809     ierr = DMPlexComputeCellType_Internal(dm, p, pdepth, &ct);CHKERRQ(ierr);
3810     if (ct == DM_POLYTOPE_UNKNOWN) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %D is screwed up", p);
3811     ierr = DMLabelSetValue(ctLabel, p, ct);CHKERRQ(ierr);
3812   }
3813   ierr = PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState);CHKERRQ(ierr);
3814   ierr = PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view");CHKERRQ(ierr);
3815   PetscFunctionReturn(0);
3816 }
3817 
3818 /*@C
3819   DMPlexGetJoin - Get an array for the join of the set of points
3820 
3821   Not Collective
3822 
3823   Input Parameters:
3824 + dm - The DMPlex object
3825 . numPoints - The number of input points for the join
3826 - points - The input points
3827 
3828   Output Parameters:
3829 + numCoveredPoints - The number of points in the join
3830 - coveredPoints - The points in the join
3831 
3832   Level: intermediate
3833 
3834   Note: Currently, this is restricted to a single level join
3835 
3836   Fortran Notes:
3837   Since it returns an array, this routine is only available in Fortran 90, and you must
3838   include petsc.h90 in your code.
3839 
3840   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3841 
3842 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
3843 @*/
3844 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3845 {
3846   DM_Plex       *mesh = (DM_Plex*) dm->data;
3847   PetscInt      *join[2];
3848   PetscInt       joinSize, i = 0;
3849   PetscInt       dof, off, p, c, m;
3850   PetscErrorCode ierr;
3851 
3852   PetscFunctionBegin;
3853   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3854   PetscValidIntPointer(points, 3);
3855   PetscValidIntPointer(numCoveredPoints, 4);
3856   PetscValidPointer(coveredPoints, 5);
3857   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3858   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3859   /* Copy in support of first point */
3860   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
3861   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
3862   for (joinSize = 0; joinSize < dof; ++joinSize) {
3863     join[i][joinSize] = mesh->supports[off+joinSize];
3864   }
3865   /* Check each successive support */
3866   for (p = 1; p < numPoints; ++p) {
3867     PetscInt newJoinSize = 0;
3868 
3869     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
3870     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
3871     for (c = 0; c < dof; ++c) {
3872       const PetscInt point = mesh->supports[off+c];
3873 
3874       for (m = 0; m < joinSize; ++m) {
3875         if (point == join[i][m]) {
3876           join[1-i][newJoinSize++] = point;
3877           break;
3878         }
3879       }
3880     }
3881     joinSize = newJoinSize;
3882     i        = 1-i;
3883   }
3884   *numCoveredPoints = joinSize;
3885   *coveredPoints    = join[i];
3886   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3887   PetscFunctionReturn(0);
3888 }
3889 
3890 /*@C
3891   DMPlexRestoreJoin - Restore an array for the join of the set of points
3892 
3893   Not Collective
3894 
3895   Input Parameters:
3896 + dm - The DMPlex object
3897 . numPoints - The number of input points for the join
3898 - points - The input points
3899 
3900   Output Parameters:
3901 + numCoveredPoints - The number of points in the join
3902 - coveredPoints - The points in the join
3903 
3904   Fortran Notes:
3905   Since it returns an array, this routine is only available in Fortran 90, and you must
3906   include petsc.h90 in your code.
3907 
3908   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3909 
3910   Level: intermediate
3911 
3912 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3913 @*/
3914 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3915 {
3916   PetscErrorCode ierr;
3917 
3918   PetscFunctionBegin;
3919   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3920   if (points) PetscValidIntPointer(points,3);
3921   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3922   PetscValidPointer(coveredPoints, 5);
3923   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3924   if (numCoveredPoints) *numCoveredPoints = 0;
3925   PetscFunctionReturn(0);
3926 }
3927 
3928 /*@C
3929   DMPlexGetFullJoin - Get an array for the join of the set of points
3930 
3931   Not Collective
3932 
3933   Input Parameters:
3934 + dm - The DMPlex object
3935 . numPoints - The number of input points for the join
3936 - points - The input points
3937 
3938   Output Parameters:
3939 + numCoveredPoints - The number of points in the join
3940 - coveredPoints - The points in the join
3941 
3942   Fortran Notes:
3943   Since it returns an array, this routine is only available in Fortran 90, and you must
3944   include petsc.h90 in your code.
3945 
3946   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3947 
3948   Level: intermediate
3949 
3950 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3951 @*/
3952 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3953 {
3954   DM_Plex       *mesh = (DM_Plex*) dm->data;
3955   PetscInt      *offsets, **closures;
3956   PetscInt      *join[2];
3957   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3958   PetscInt       p, d, c, m, ms;
3959   PetscErrorCode ierr;
3960 
3961   PetscFunctionBegin;
3962   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3963   PetscValidIntPointer(points, 3);
3964   PetscValidIntPointer(numCoveredPoints, 4);
3965   PetscValidPointer(coveredPoints, 5);
3966 
3967   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3968   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3969   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3970   ms      = mesh->maxSupportSize;
3971   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3972   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3973   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3974 
3975   for (p = 0; p < numPoints; ++p) {
3976     PetscInt closureSize;
3977 
3978     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3979 
3980     offsets[p*(depth+2)+0] = 0;
3981     for (d = 0; d < depth+1; ++d) {
3982       PetscInt pStart, pEnd, i;
3983 
3984       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
3985       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
3986         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3987           offsets[p*(depth+2)+d+1] = i;
3988           break;
3989         }
3990       }
3991       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
3992     }
3993     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
3994   }
3995   for (d = 0; d < depth+1; ++d) {
3996     PetscInt dof;
3997 
3998     /* Copy in support of first point */
3999     dof = offsets[d+1] - offsets[d];
4000     for (joinSize = 0; joinSize < dof; ++joinSize) {
4001       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
4002     }
4003     /* Check each successive cone */
4004     for (p = 1; p < numPoints && joinSize; ++p) {
4005       PetscInt newJoinSize = 0;
4006 
4007       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
4008       for (c = 0; c < dof; ++c) {
4009         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
4010 
4011         for (m = 0; m < joinSize; ++m) {
4012           if (point == join[i][m]) {
4013             join[1-i][newJoinSize++] = point;
4014             break;
4015           }
4016         }
4017       }
4018       joinSize = newJoinSize;
4019       i        = 1-i;
4020     }
4021     if (joinSize) break;
4022   }
4023   *numCoveredPoints = joinSize;
4024   *coveredPoints    = join[i];
4025   for (p = 0; p < numPoints; ++p) {
4026     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
4027   }
4028   ierr = PetscFree(closures);CHKERRQ(ierr);
4029   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4030   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4031   PetscFunctionReturn(0);
4032 }
4033 
4034 /*@C
4035   DMPlexGetMeet - Get an array for the meet of the set of points
4036 
4037   Not Collective
4038 
4039   Input Parameters:
4040 + dm - The DMPlex object
4041 . numPoints - The number of input points for the meet
4042 - points - The input points
4043 
4044   Output Parameters:
4045 + numCoveredPoints - The number of points in the meet
4046 - coveredPoints - The points in the meet
4047 
4048   Level: intermediate
4049 
4050   Note: Currently, this is restricted to a single level meet
4051 
4052   Fortran Notes:
4053   Since it returns an array, this routine is only available in Fortran 90, and you must
4054   include petsc.h90 in your code.
4055 
4056   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4057 
4058 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
4059 @*/
4060 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
4061 {
4062   DM_Plex       *mesh = (DM_Plex*) dm->data;
4063   PetscInt      *meet[2];
4064   PetscInt       meetSize, i = 0;
4065   PetscInt       dof, off, p, c, m;
4066   PetscErrorCode ierr;
4067 
4068   PetscFunctionBegin;
4069   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4070   PetscValidPointer(points, 3);
4071   PetscValidPointer(numCoveringPoints, 4);
4072   PetscValidPointer(coveringPoints, 5);
4073   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4074   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4075   /* Copy in cone of first point */
4076   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
4077   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
4078   for (meetSize = 0; meetSize < dof; ++meetSize) {
4079     meet[i][meetSize] = mesh->cones[off+meetSize];
4080   }
4081   /* Check each successive cone */
4082   for (p = 1; p < numPoints; ++p) {
4083     PetscInt newMeetSize = 0;
4084 
4085     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
4086     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
4087     for (c = 0; c < dof; ++c) {
4088       const PetscInt point = mesh->cones[off+c];
4089 
4090       for (m = 0; m < meetSize; ++m) {
4091         if (point == meet[i][m]) {
4092           meet[1-i][newMeetSize++] = point;
4093           break;
4094         }
4095       }
4096     }
4097     meetSize = newMeetSize;
4098     i        = 1-i;
4099   }
4100   *numCoveringPoints = meetSize;
4101   *coveringPoints    = meet[i];
4102   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4103   PetscFunctionReturn(0);
4104 }
4105 
4106 /*@C
4107   DMPlexRestoreMeet - Restore an array for the meet of the set of points
4108 
4109   Not Collective
4110 
4111   Input Parameters:
4112 + dm - The DMPlex object
4113 . numPoints - The number of input points for the meet
4114 - points - The input points
4115 
4116   Output Parameters:
4117 + numCoveredPoints - The number of points in the meet
4118 - coveredPoints - The points in the meet
4119 
4120   Level: intermediate
4121 
4122   Fortran Notes:
4123   Since it returns an array, this routine is only available in Fortran 90, and you must
4124   include petsc.h90 in your code.
4125 
4126   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4127 
4128 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
4129 @*/
4130 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4131 {
4132   PetscErrorCode ierr;
4133 
4134   PetscFunctionBegin;
4135   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4136   if (points) PetscValidIntPointer(points,3);
4137   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4138   PetscValidPointer(coveredPoints,5);
4139   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4140   if (numCoveredPoints) *numCoveredPoints = 0;
4141   PetscFunctionReturn(0);
4142 }
4143 
4144 /*@C
4145   DMPlexGetFullMeet - Get an array for the meet of the set of points
4146 
4147   Not Collective
4148 
4149   Input Parameters:
4150 + dm - The DMPlex object
4151 . numPoints - The number of input points for the meet
4152 - points - The input points
4153 
4154   Output Parameters:
4155 + numCoveredPoints - The number of points in the meet
4156 - coveredPoints - The points in the meet
4157 
4158   Level: intermediate
4159 
4160   Fortran Notes:
4161   Since it returns an array, this routine is only available in Fortran 90, and you must
4162   include petsc.h90 in your code.
4163 
4164   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4165 
4166 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
4167 @*/
4168 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4169 {
4170   DM_Plex       *mesh = (DM_Plex*) dm->data;
4171   PetscInt      *offsets, **closures;
4172   PetscInt      *meet[2];
4173   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
4174   PetscInt       p, h, c, m, mc;
4175   PetscErrorCode ierr;
4176 
4177   PetscFunctionBegin;
4178   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4179   PetscValidPointer(points, 3);
4180   PetscValidPointer(numCoveredPoints, 4);
4181   PetscValidPointer(coveredPoints, 5);
4182 
4183   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
4184   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
4185   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4186   mc      = mesh->maxConeSize;
4187   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
4188   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4189   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4190 
4191   for (p = 0; p < numPoints; ++p) {
4192     PetscInt closureSize;
4193 
4194     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
4195 
4196     offsets[p*(height+2)+0] = 0;
4197     for (h = 0; h < height+1; ++h) {
4198       PetscInt pStart, pEnd, i;
4199 
4200       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
4201       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
4202         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4203           offsets[p*(height+2)+h+1] = i;
4204           break;
4205         }
4206       }
4207       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
4208     }
4209     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
4210   }
4211   for (h = 0; h < height+1; ++h) {
4212     PetscInt dof;
4213 
4214     /* Copy in cone of first point */
4215     dof = offsets[h+1] - offsets[h];
4216     for (meetSize = 0; meetSize < dof; ++meetSize) {
4217       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
4218     }
4219     /* Check each successive cone */
4220     for (p = 1; p < numPoints && meetSize; ++p) {
4221       PetscInt newMeetSize = 0;
4222 
4223       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
4224       for (c = 0; c < dof; ++c) {
4225         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
4226 
4227         for (m = 0; m < meetSize; ++m) {
4228           if (point == meet[i][m]) {
4229             meet[1-i][newMeetSize++] = point;
4230             break;
4231           }
4232         }
4233       }
4234       meetSize = newMeetSize;
4235       i        = 1-i;
4236     }
4237     if (meetSize) break;
4238   }
4239   *numCoveredPoints = meetSize;
4240   *coveredPoints    = meet[i];
4241   for (p = 0; p < numPoints; ++p) {
4242     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
4243   }
4244   ierr = PetscFree(closures);CHKERRQ(ierr);
4245   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4246   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4247   PetscFunctionReturn(0);
4248 }
4249 
4250 /*@C
4251   DMPlexEqual - Determine if two DMs have the same topology
4252 
4253   Not Collective
4254 
4255   Input Parameters:
4256 + dmA - A DMPlex object
4257 - dmB - A DMPlex object
4258 
4259   Output Parameters:
4260 . equal - PETSC_TRUE if the topologies are identical
4261 
4262   Level: intermediate
4263 
4264   Notes:
4265   We are not solving graph isomorphism, so we do not permutation.
4266 
4267 .seealso: DMPlexGetCone()
4268 @*/
4269 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
4270 {
4271   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
4272   PetscErrorCode ierr;
4273 
4274   PetscFunctionBegin;
4275   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4276   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4277   PetscValidPointer(equal, 3);
4278 
4279   *equal = PETSC_FALSE;
4280   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
4281   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
4282   if (depth != depthB) PetscFunctionReturn(0);
4283   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
4284   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
4285   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
4286   for (p = pStart; p < pEnd; ++p) {
4287     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
4288     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
4289 
4290     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
4291     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
4292     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
4293     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
4294     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
4295     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
4296     if (coneSize != coneSizeB) PetscFunctionReturn(0);
4297     for (c = 0; c < coneSize; ++c) {
4298       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
4299       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
4300     }
4301     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
4302     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
4303     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
4304     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
4305     if (supportSize != supportSizeB) PetscFunctionReturn(0);
4306     for (s = 0; s < supportSize; ++s) {
4307       if (support[s] != supportB[s]) PetscFunctionReturn(0);
4308     }
4309   }
4310   *equal = PETSC_TRUE;
4311   PetscFunctionReturn(0);
4312 }
4313 
4314 /*@C
4315   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
4316 
4317   Not Collective
4318 
4319   Input Parameters:
4320 + dm         - The DMPlex
4321 . cellDim    - The cell dimension
4322 - numCorners - The number of vertices on a cell
4323 
4324   Output Parameters:
4325 . numFaceVertices - The number of vertices on a face
4326 
4327   Level: developer
4328 
4329   Notes:
4330   Of course this can only work for a restricted set of symmetric shapes
4331 
4332 .seealso: DMPlexGetCone()
4333 @*/
4334 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
4335 {
4336   MPI_Comm       comm;
4337   PetscErrorCode ierr;
4338 
4339   PetscFunctionBegin;
4340   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
4341   PetscValidPointer(numFaceVertices,4);
4342   switch (cellDim) {
4343   case 0:
4344     *numFaceVertices = 0;
4345     break;
4346   case 1:
4347     *numFaceVertices = 1;
4348     break;
4349   case 2:
4350     switch (numCorners) {
4351     case 3: /* triangle */
4352       *numFaceVertices = 2; /* Edge has 2 vertices */
4353       break;
4354     case 4: /* quadrilateral */
4355       *numFaceVertices = 2; /* Edge has 2 vertices */
4356       break;
4357     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
4358       *numFaceVertices = 3; /* Edge has 3 vertices */
4359       break;
4360     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
4361       *numFaceVertices = 3; /* Edge has 3 vertices */
4362       break;
4363     default:
4364       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4365     }
4366     break;
4367   case 3:
4368     switch (numCorners) {
4369     case 4: /* tetradehdron */
4370       *numFaceVertices = 3; /* Face has 3 vertices */
4371       break;
4372     case 6: /* tet cohesive cells */
4373       *numFaceVertices = 4; /* Face has 4 vertices */
4374       break;
4375     case 8: /* hexahedron */
4376       *numFaceVertices = 4; /* Face has 4 vertices */
4377       break;
4378     case 9: /* tet cohesive Lagrange cells */
4379       *numFaceVertices = 6; /* Face has 6 vertices */
4380       break;
4381     case 10: /* quadratic tetrahedron */
4382       *numFaceVertices = 6; /* Face has 6 vertices */
4383       break;
4384     case 12: /* hex cohesive Lagrange cells */
4385       *numFaceVertices = 6; /* Face has 6 vertices */
4386       break;
4387     case 18: /* quadratic tet cohesive Lagrange cells */
4388       *numFaceVertices = 6; /* Face has 6 vertices */
4389       break;
4390     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
4391       *numFaceVertices = 9; /* Face has 9 vertices */
4392       break;
4393     default:
4394       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4395     }
4396     break;
4397   default:
4398     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
4399   }
4400   PetscFunctionReturn(0);
4401 }
4402 
4403 /*@
4404   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4405 
4406   Not Collective
4407 
4408   Input Parameter:
4409 . dm    - The DMPlex object
4410 
4411   Output Parameter:
4412 . depthLabel - The DMLabel recording point depth
4413 
4414   Level: developer
4415 
4416 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(),
4417 @*/
4418 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4419 {
4420   PetscFunctionBegin;
4421   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4422   PetscValidPointer(depthLabel, 2);
4423   *depthLabel = dm->depthLabel;
4424   PetscFunctionReturn(0);
4425 }
4426 
4427 /*@
4428   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4429 
4430   Not Collective
4431 
4432   Input Parameter:
4433 . dm    - The DMPlex object
4434 
4435   Output Parameter:
4436 . depth - The number of strata (breadth first levels) in the DAG
4437 
4438   Level: developer
4439 
4440   Notes:
4441   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
4442   The point depth is described more in detail in DMPlexGetDepthStratum().
4443   An empty mesh gives -1.
4444 
4445 .seealso: DMPlexGetDepthLabel(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(), DMPlexSymmetrize()
4446 @*/
4447 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4448 {
4449   DMLabel        label;
4450   PetscInt       d = 0;
4451   PetscErrorCode ierr;
4452 
4453   PetscFunctionBegin;
4454   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4455   PetscValidPointer(depth, 2);
4456   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4457   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4458   *depth = d-1;
4459   PetscFunctionReturn(0);
4460 }
4461 
4462 /*@
4463   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4464 
4465   Not Collective
4466 
4467   Input Parameters:
4468 + dm           - The DMPlex object
4469 - stratumValue - The requested depth
4470 
4471   Output Parameters:
4472 + start - The first point at this depth
4473 - end   - One beyond the last point at this depth
4474 
4475   Notes:
4476   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
4477   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
4478   higher dimension, e.g., "edges".
4479 
4480   Level: developer
4481 
4482 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth(), DMPlexGetDepthLabel(), DMPlexGetPointDepth(), DMPlexSymmetrize(), DMPlexInterpolate()
4483 @*/
4484 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4485 {
4486   DMLabel        label;
4487   PetscInt       pStart, pEnd;
4488   PetscErrorCode ierr;
4489 
4490   PetscFunctionBegin;
4491   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4492   if (start) {PetscValidPointer(start, 3); *start = 0;}
4493   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4494   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4495   if (pStart == pEnd) PetscFunctionReturn(0);
4496   if (stratumValue < 0) {
4497     if (start) *start = pStart;
4498     if (end)   *end   = pEnd;
4499     PetscFunctionReturn(0);
4500   }
4501   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4502   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4503   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4504   PetscFunctionReturn(0);
4505 }
4506 
4507 /*@
4508   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4509 
4510   Not Collective
4511 
4512   Input Parameters:
4513 + dm           - The DMPlex object
4514 - stratumValue - The requested height
4515 
4516   Output Parameters:
4517 + start - The first point at this height
4518 - end   - One beyond the last point at this height
4519 
4520   Notes:
4521   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
4522   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
4523   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
4524 
4525   Level: developer
4526 
4527 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth(), DMPlexGetPointHeight()
4528 @*/
4529 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4530 {
4531   DMLabel        label;
4532   PetscInt       depth, pStart, pEnd;
4533   PetscErrorCode ierr;
4534 
4535   PetscFunctionBegin;
4536   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4537   if (start) {PetscValidPointer(start, 3); *start = 0;}
4538   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4539   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4540   if (pStart == pEnd) PetscFunctionReturn(0);
4541   if (stratumValue < 0) {
4542     if (start) *start = pStart;
4543     if (end)   *end   = pEnd;
4544     PetscFunctionReturn(0);
4545   }
4546   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4547   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4548   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4549   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4550   PetscFunctionReturn(0);
4551 }
4552 
4553 /*@
4554   DMPlexGetPointDepth - Get the depth of a given point
4555 
4556   Not Collective
4557 
4558   Input Parameter:
4559 + dm    - The DMPlex object
4560 - point - The point
4561 
4562   Output Parameter:
4563 . depth - The depth of the point
4564 
4565   Level: intermediate
4566 
4567 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointHeight()
4568 @*/
4569 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
4570 {
4571   PetscErrorCode ierr;
4572 
4573   PetscFunctionBegin;
4574   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4575   PetscValidIntPointer(depth, 3);
4576   ierr = DMLabelGetValue(dm->depthLabel, point, depth);CHKERRQ(ierr);
4577   PetscFunctionReturn(0);
4578 }
4579 
4580 /*@
4581   DMPlexGetPointHeight - Get the height of a given point
4582 
4583   Not Collective
4584 
4585   Input Parameter:
4586 + dm    - The DMPlex object
4587 - point - The point
4588 
4589   Output Parameter:
4590 . height - The height of the point
4591 
4592   Level: intermediate
4593 
4594 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointDepth()
4595 @*/
4596 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
4597 {
4598   PetscInt       n, pDepth;
4599   PetscErrorCode ierr;
4600 
4601   PetscFunctionBegin;
4602   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4603   PetscValidIntPointer(height, 3);
4604   ierr = DMLabelGetNumValues(dm->depthLabel, &n);CHKERRQ(ierr);
4605   ierr = DMLabelGetValue(dm->depthLabel, point, &pDepth);CHKERRQ(ierr);
4606   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
4607   PetscFunctionReturn(0);
4608 }
4609 
4610 /*@
4611   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
4612 
4613   Not Collective
4614 
4615   Input Parameter:
4616 . dm - The DMPlex object
4617 
4618   Output Parameter:
4619 . celltypeLabel - The DMLabel recording cell polytope type
4620 
4621   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
4622   DMCreateLabel(dm, "celltype") beforehand.
4623 
4624   Level: developer
4625 
4626 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMCreateLabel()
4627 @*/
4628 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
4629 {
4630   PetscErrorCode ierr;
4631 
4632   PetscFunctionBegin;
4633   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4634   PetscValidPointer(celltypeLabel, 2);
4635   if (!dm->celltypeLabel) {ierr = DMPlexComputeCellTypes(dm);CHKERRQ(ierr);}
4636   *celltypeLabel = dm->celltypeLabel;
4637   PetscFunctionReturn(0);
4638 }
4639 
4640 /*@
4641   DMPlexGetCellType - Get the polytope type of a given cell
4642 
4643   Not Collective
4644 
4645   Input Parameter:
4646 + dm   - The DMPlex object
4647 - cell - The cell
4648 
4649   Output Parameter:
4650 . celltype - The polytope type of the cell
4651 
4652   Level: intermediate
4653 
4654 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth()
4655 @*/
4656 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
4657 {
4658   DMLabel        label;
4659   PetscInt       ct;
4660   PetscErrorCode ierr;
4661 
4662   PetscFunctionBegin;
4663   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4664   PetscValidPointer(celltype, 3);
4665   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4666   ierr = DMLabelGetValue(label, cell, &ct);CHKERRQ(ierr);
4667   if (ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D has not been assigned a cell type", cell);
4668   *celltype = (DMPolytopeType) ct;
4669   PetscFunctionReturn(0);
4670 }
4671 
4672 /*@
4673   DMPlexSetCellType - Set the polytope type of a given cell
4674 
4675   Not Collective
4676 
4677   Input Parameters:
4678 + dm   - The DMPlex object
4679 . cell - The cell
4680 - celltype - The polytope type of the cell
4681 
4682   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
4683   is executed. This function will override the computed type. However, if automatic classification will not succeed
4684   and a user wants to manually specify all types, the classification must be disabled by calling
4685   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
4686 
4687   Level: advanced
4688 
4689 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexComputeCellTypes(), DMCreateLabel()
4690 @*/
4691 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
4692 {
4693   DMLabel        label;
4694   PetscErrorCode ierr;
4695 
4696   PetscFunctionBegin;
4697   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4698   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4699   ierr = DMLabelSetValue(label, cell, celltype);CHKERRQ(ierr);
4700   PetscFunctionReturn(0);
4701 }
4702 
4703 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4704 {
4705   PetscSection   section, s;
4706   Mat            m;
4707   PetscInt       maxHeight;
4708   PetscErrorCode ierr;
4709 
4710   PetscFunctionBegin;
4711   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4712   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
4713   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
4714   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4715   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
4716   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4717   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
4718   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
4719   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
4720   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
4721   ierr = MatDestroy(&m);CHKERRQ(ierr);
4722 
4723   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
4724   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
4725   PetscFunctionReturn(0);
4726 }
4727 
4728 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
4729 {
4730   Vec            coordsLocal;
4731   DM             coordsDM;
4732   PetscErrorCode ierr;
4733 
4734   PetscFunctionBegin;
4735   *field = NULL;
4736   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
4737   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
4738   if (coordsLocal && coordsDM) {
4739     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
4740   }
4741   PetscFunctionReturn(0);
4742 }
4743 
4744 /*@C
4745   DMPlexGetConeSection - Return a section which describes the layout of cone data
4746 
4747   Not Collective
4748 
4749   Input Parameters:
4750 . dm        - The DMPlex object
4751 
4752   Output Parameter:
4753 . section - The PetscSection object
4754 
4755   Level: developer
4756 
4757 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
4758 @*/
4759 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4760 {
4761   DM_Plex *mesh = (DM_Plex*) dm->data;
4762 
4763   PetscFunctionBegin;
4764   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4765   if (section) *section = mesh->coneSection;
4766   PetscFunctionReturn(0);
4767 }
4768 
4769 /*@C
4770   DMPlexGetSupportSection - Return a section which describes the layout of support data
4771 
4772   Not Collective
4773 
4774   Input Parameters:
4775 . dm        - The DMPlex object
4776 
4777   Output Parameter:
4778 . section - The PetscSection object
4779 
4780   Level: developer
4781 
4782 .seealso: DMPlexGetConeSection()
4783 @*/
4784 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4785 {
4786   DM_Plex *mesh = (DM_Plex*) dm->data;
4787 
4788   PetscFunctionBegin;
4789   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4790   if (section) *section = mesh->supportSection;
4791   PetscFunctionReturn(0);
4792 }
4793 
4794 /*@C
4795   DMPlexGetCones - Return cone data
4796 
4797   Not Collective
4798 
4799   Input Parameters:
4800 . dm        - The DMPlex object
4801 
4802   Output Parameter:
4803 . cones - The cone for each point
4804 
4805   Level: developer
4806 
4807 .seealso: DMPlexGetConeSection()
4808 @*/
4809 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4810 {
4811   DM_Plex *mesh = (DM_Plex*) dm->data;
4812 
4813   PetscFunctionBegin;
4814   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4815   if (cones) *cones = mesh->cones;
4816   PetscFunctionReturn(0);
4817 }
4818 
4819 /*@C
4820   DMPlexGetConeOrientations - Return cone orientation data
4821 
4822   Not Collective
4823 
4824   Input Parameters:
4825 . dm        - The DMPlex object
4826 
4827   Output Parameter:
4828 . coneOrientations - The cone orientation for each point
4829 
4830   Level: developer
4831 
4832 .seealso: DMPlexGetConeSection()
4833 @*/
4834 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4835 {
4836   DM_Plex *mesh = (DM_Plex*) dm->data;
4837 
4838   PetscFunctionBegin;
4839   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4840   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4841   PetscFunctionReturn(0);
4842 }
4843 
4844 /******************************** FEM Support **********************************/
4845 
4846 /*
4847  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
4848  representing a line in the section.
4849 */
4850 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
4851 {
4852   PetscErrorCode ierr;
4853 
4854   PetscFunctionBeginHot;
4855   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
4856   if (line < 0) {
4857     *k = 0;
4858     *Nc = 0;
4859   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
4860     *k = 1;
4861   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
4862     /* An order k SEM disc has k-1 dofs on an edge */
4863     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
4864     *k = *k / *Nc + 1;
4865   }
4866   PetscFunctionReturn(0);
4867 }
4868 
4869 /*@
4870 
4871   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
4872   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
4873   section provided (or the section of the DM).
4874 
4875   Input Parameters:
4876 + dm      - The DM
4877 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
4878 - section - The PetscSection to reorder, or NULL for the default section
4879 
4880   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
4881   degree of the basis.
4882 
4883   Example:
4884   A typical interpolated single-quad mesh might order points as
4885 .vb
4886   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
4887 
4888   v4 -- e6 -- v3
4889   |           |
4890   e7    c0    e8
4891   |           |
4892   v1 -- e5 -- v2
4893 .ve
4894 
4895   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
4896   dofs in the order of points, e.g.,
4897 .vb
4898     c0 -> [0,1,2,3]
4899     v1 -> [4]
4900     ...
4901     e5 -> [8, 9]
4902 .ve
4903 
4904   which corresponds to the dofs
4905 .vb
4906     6   10  11  7
4907     13  2   3   15
4908     12  0   1   14
4909     4   8   9   5
4910 .ve
4911 
4912   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
4913 .vb
4914   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
4915 .ve
4916 
4917   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
4918 .vb
4919    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
4920 .ve
4921 
4922   Level: developer
4923 
4924 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
4925 @*/
4926 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
4927 {
4928   DMLabel        label;
4929   PetscInt       dim, depth = -1, eStart = -1, Nf;
4930   PetscBool      vertexchart;
4931   PetscErrorCode ierr;
4932 
4933   PetscFunctionBegin;
4934   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4935   if (dim < 1) PetscFunctionReturn(0);
4936   if (point < 0) {
4937     PetscInt sStart,sEnd;
4938 
4939     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
4940     point = sEnd-sStart ? sStart : point;
4941   }
4942   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4943   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
4944   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4945   if (depth == 1) {eStart = point;}
4946   else if  (depth == dim) {
4947     const PetscInt *cone;
4948 
4949     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4950     if (dim == 2) eStart = cone[0];
4951     else if (dim == 3) {
4952       const PetscInt *cone2;
4953       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
4954       eStart = cone2[0];
4955     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
4956   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
4957   {                             /* Determine whether the chart covers all points or just vertices. */
4958     PetscInt pStart,pEnd,cStart,cEnd;
4959     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
4960     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
4961     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
4962     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
4963   }
4964   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
4965   for (PetscInt d=1; d<=dim; d++) {
4966     PetscInt k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
4967     PetscInt *perm;
4968 
4969     for (f = 0; f < Nf; ++f) {
4970       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4971       size += PetscPowInt(k+1, d)*Nc;
4972     }
4973     ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
4974     for (f = 0; f < Nf; ++f) {
4975       switch (d) {
4976       case 1:
4977         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4978         /*
4979          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
4980          We want              [ vtx0; edge of length k-1; vtx1 ]
4981          */
4982         for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
4983         for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
4984         for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
4985         foffset = offset;
4986         break;
4987       case 2:
4988         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
4989         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4990         /* The SEM order is
4991 
4992          v_lb, {e_b}, v_rb,
4993          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
4994          v_lt, reverse {e_t}, v_rt
4995          */
4996         {
4997           const PetscInt of   = 0;
4998           const PetscInt oeb  = of   + PetscSqr(k-1);
4999           const PetscInt oer  = oeb  + (k-1);
5000           const PetscInt oet  = oer  + (k-1);
5001           const PetscInt oel  = oet  + (k-1);
5002           const PetscInt ovlb = oel  + (k-1);
5003           const PetscInt ovrb = ovlb + 1;
5004           const PetscInt ovrt = ovrb + 1;
5005           const PetscInt ovlt = ovrt + 1;
5006           PetscInt       o;
5007 
5008           /* bottom */
5009           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
5010           for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5011           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
5012           /* middle */
5013           for (i = 0; i < k-1; ++i) {
5014             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
5015             for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5016             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
5017           }
5018           /* top */
5019           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
5020           for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5021           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
5022           foffset = offset;
5023         }
5024         break;
5025       case 3:
5026         /* The original hex closure is
5027 
5028          {c,
5029          f_b, f_t, f_f, f_b, f_r, f_l,
5030          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5031          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5032          */
5033         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5034         /* The SEM order is
5035          Bottom Slice
5036          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5037          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5038          v_blb, {e_bb}, v_brb,
5039 
5040          Middle Slice (j)
5041          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5042          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5043          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5044 
5045          Top Slice
5046          v_tlf, {e_tf}, v_trf,
5047          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5048          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5049          */
5050         {
5051           const PetscInt oc    = 0;
5052           const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
5053           const PetscInt oft   = ofb   + PetscSqr(k-1);
5054           const PetscInt off   = oft   + PetscSqr(k-1);
5055           const PetscInt ofk   = off   + PetscSqr(k-1);
5056           const PetscInt ofr   = ofk   + PetscSqr(k-1);
5057           const PetscInt ofl   = ofr   + PetscSqr(k-1);
5058           const PetscInt oebl  = ofl   + PetscSqr(k-1);
5059           const PetscInt oebb  = oebl  + (k-1);
5060           const PetscInt oebr  = oebb  + (k-1);
5061           const PetscInt oebf  = oebr  + (k-1);
5062           const PetscInt oetf  = oebf  + (k-1);
5063           const PetscInt oetr  = oetf  + (k-1);
5064           const PetscInt oetb  = oetr  + (k-1);
5065           const PetscInt oetl  = oetb  + (k-1);
5066           const PetscInt oerf  = oetl  + (k-1);
5067           const PetscInt oelf  = oerf  + (k-1);
5068           const PetscInt oelb  = oelf  + (k-1);
5069           const PetscInt oerb  = oelb  + (k-1);
5070           const PetscInt ovblf = oerb  + (k-1);
5071           const PetscInt ovblb = ovblf + 1;
5072           const PetscInt ovbrb = ovblb + 1;
5073           const PetscInt ovbrf = ovbrb + 1;
5074           const PetscInt ovtlf = ovbrf + 1;
5075           const PetscInt ovtrf = ovtlf + 1;
5076           const PetscInt ovtrb = ovtrf + 1;
5077           const PetscInt ovtlb = ovtrb + 1;
5078           PetscInt       o, n;
5079 
5080           /* Bottom Slice */
5081           /*   bottom */
5082           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
5083           for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5084           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
5085           /*   middle */
5086           for (i = 0; i < k-1; ++i) {
5087             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
5088             for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
5089             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
5090           }
5091           /*   top */
5092           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
5093           for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5094           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
5095 
5096           /* Middle Slice */
5097           for (j = 0; j < k-1; ++j) {
5098             /*   bottom */
5099             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
5100             for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5101             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
5102             /*   middle */
5103             for (i = 0; i < k-1; ++i) {
5104               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
5105               for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
5106               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
5107             }
5108             /*   top */
5109             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
5110             for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5111             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
5112           }
5113 
5114           /* Top Slice */
5115           /*   bottom */
5116           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
5117           for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5118           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
5119           /*   middle */
5120           for (i = 0; i < k-1; ++i) {
5121             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
5122             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
5123             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
5124           }
5125           /*   top */
5126           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
5127           for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5128           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
5129 
5130           foffset = offset;
5131         }
5132         break;
5133       default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", d);
5134       }
5135     }
5136     if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
5137     /* Check permutation */
5138     {
5139       PetscInt *check;
5140 
5141       ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
5142       for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
5143       for (i = 0; i < size; ++i) check[perm[i]] = i;
5144       for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
5145       ierr = PetscFree(check);CHKERRQ(ierr);
5146     }
5147     ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
5148   }
5149   PetscFunctionReturn(0);
5150 }
5151 
5152 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
5153 {
5154   PetscDS        prob;
5155   PetscInt       depth, Nf, h;
5156   DMLabel        label;
5157   PetscErrorCode ierr;
5158 
5159   PetscFunctionBeginHot;
5160   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
5161   Nf      = prob->Nf;
5162   label   = dm->depthLabel;
5163   *dspace = NULL;
5164   if (field < Nf) {
5165     PetscObject disc = prob->disc[field];
5166 
5167     if (disc->classid == PETSCFE_CLASSID) {
5168       PetscDualSpace dsp;
5169 
5170       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
5171       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
5172       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
5173       h    = depth - 1 - h;
5174       if (h) {
5175         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
5176       } else {
5177         *dspace = dsp;
5178       }
5179     }
5180   }
5181   PetscFunctionReturn(0);
5182 }
5183 
5184 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5185 {
5186   PetscScalar    *array, *vArray;
5187   const PetscInt *cone, *coneO;
5188   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
5189   PetscErrorCode  ierr;
5190 
5191   PetscFunctionBeginHot;
5192   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5193   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5194   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5195   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5196   if (!values || !*values) {
5197     if ((point >= pStart) && (point < pEnd)) {
5198       PetscInt dof;
5199 
5200       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5201       size += dof;
5202     }
5203     for (p = 0; p < numPoints; ++p) {
5204       const PetscInt cp = cone[p];
5205       PetscInt       dof;
5206 
5207       if ((cp < pStart) || (cp >= pEnd)) continue;
5208       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5209       size += dof;
5210     }
5211     if (!values) {
5212       if (csize) *csize = size;
5213       PetscFunctionReturn(0);
5214     }
5215     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
5216   } else {
5217     array = *values;
5218   }
5219   size = 0;
5220   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5221   if ((point >= pStart) && (point < pEnd)) {
5222     PetscInt     dof, off, d;
5223     PetscScalar *varr;
5224 
5225     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5226     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5227     varr = &vArray[off];
5228     for (d = 0; d < dof; ++d, ++offset) {
5229       array[offset] = varr[d];
5230     }
5231     size += dof;
5232   }
5233   for (p = 0; p < numPoints; ++p) {
5234     const PetscInt cp = cone[p];
5235     PetscInt       o  = coneO[p];
5236     PetscInt       dof, off, d;
5237     PetscScalar   *varr;
5238 
5239     if ((cp < pStart) || (cp >= pEnd)) continue;
5240     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5241     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
5242     varr = &vArray[off];
5243     if (o >= 0) {
5244       for (d = 0; d < dof; ++d, ++offset) {
5245         array[offset] = varr[d];
5246       }
5247     } else {
5248       for (d = dof-1; d >= 0; --d, ++offset) {
5249         array[offset] = varr[d];
5250       }
5251     }
5252     size += dof;
5253   }
5254   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5255   if (!*values) {
5256     if (csize) *csize = size;
5257     *values = array;
5258   } else {
5259     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5260     *csize = size;
5261   }
5262   PetscFunctionReturn(0);
5263 }
5264 
5265 /* Compress out points not in the section */
5266 PETSC_STATIC_INLINE PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
5267 {
5268   const PetscInt np = *numPoints;
5269   PetscInt       pStart, pEnd, p, q;
5270   PetscErrorCode ierr;
5271 
5272   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5273   for (p = 0, q = 0; p < np; ++p) {
5274     const PetscInt r = points[p*2];
5275     if ((r >= pStart) && (r < pEnd)) {
5276       points[q*2]   = r;
5277       points[q*2+1] = points[p*2+1];
5278       ++q;
5279     }
5280   }
5281   *numPoints = q;
5282   return 0;
5283 }
5284 
5285 static PetscErrorCode DMPlexTransitiveClosure_Hybrid_Internal(DM dm, PetscInt point, PetscInt np, PetscInt *numPoints, PetscInt **points)
5286 {
5287   const PetscInt *cone, *ornt;
5288   PetscInt       *pts,  *closure = NULL;
5289   PetscInt        dim, coneSize, c, d, clSize, cl;
5290   PetscErrorCode  ierr;
5291 
5292   PetscFunctionBeginHot;
5293   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5294   ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
5295   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5296   ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
5297   ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5298   ierr = DMGetWorkArray(dm, np*2, MPIU_INT, &pts);CHKERRQ(ierr);
5299   c    = 0;
5300   pts[c*2+0] = point;
5301   pts[c*2+1] = 0;
5302   ++c;
5303   for (cl = 0; cl < clSize*2; cl += 2, ++c) {pts[c*2+0] = closure[cl]; pts[c*2+1] = closure[cl+1];}
5304   ierr = DMPlexGetTransitiveClosure(dm, cone[1], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5305   for (cl = 0; cl < clSize*2; cl += 2, ++c) {pts[c*2+0] = closure[cl]; pts[c*2+1] = closure[cl+1];}
5306   ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5307   if (dim >= 2) {
5308     for (d = 2; d < coneSize; ++d, ++c) {pts[c*2+0] = cone[d]; pts[c*2+1] = ornt[d];}
5309   }
5310   if (dim >= 3) {
5311     for (d = 2; d < coneSize; ++d) {
5312       const PetscInt  fpoint = cone[d];
5313       const PetscInt *fcone;
5314       PetscInt        fconeSize, fc, i;
5315 
5316       ierr = DMPlexGetConeSize(dm, fpoint, &fconeSize);CHKERRQ(ierr);
5317       ierr = DMPlexGetCone(dm, fpoint, &fcone);CHKERRQ(ierr);
5318       for (fc = 0; fc < fconeSize; ++fc) {
5319         for (i = 0; i < c; ++i) if (pts[i*2] == fcone[fc]) break;
5320         if (i == c) {pts[c*2+0] = fcone[fc]; pts[c*2+1] = 0; ++c;}
5321       }
5322     }
5323   }
5324   if (c != np) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid closure for hybrid point %D, size %D != %D", point, c, np);
5325   *numPoints = np;
5326   *points    = pts;
5327   PetscFunctionReturn(0);
5328 }
5329 
5330 /* Compressed closure does not apply closure permutation */
5331 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5332 {
5333   const PetscInt *cla = NULL;
5334   PetscInt       np, *pts = NULL;
5335   PetscErrorCode ierr;
5336 
5337   PetscFunctionBeginHot;
5338   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
5339   if (*clPoints) {
5340     PetscInt dof, off;
5341 
5342     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
5343     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
5344     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
5345     np   = dof/2;
5346     pts  = (PetscInt *) &cla[off];
5347   } else {
5348     DMPolytopeType ct;
5349 
5350     /* Do not make the label if it does not exist */
5351     if (!dm->celltypeLabel) {ct = DM_POLYTOPE_POINT;}
5352     else                    {ierr = DMPlexGetCellType(dm, point, &ct);CHKERRQ(ierr);}
5353     switch (ct) {
5354       case DM_POLYTOPE_SEG_PRISM_TENSOR:
5355         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 9, &np, &pts);CHKERRQ(ierr);
5356         break;
5357       case DM_POLYTOPE_TRI_PRISM_TENSOR:
5358         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 21, &np, &pts);CHKERRQ(ierr);
5359         break;
5360       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
5361         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 27, &np, &pts);CHKERRQ(ierr);
5362         break;
5363       default:
5364         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
5365     }
5366     ierr = CompressPoints_Private(section, &np, pts);CHKERRQ(ierr);
5367   }
5368   *numPoints = np;
5369   *points    = pts;
5370   *clp       = cla;
5371   PetscFunctionReturn(0);
5372 }
5373 
5374 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5375 {
5376   PetscErrorCode ierr;
5377 
5378   PetscFunctionBeginHot;
5379   if (!*clPoints) {
5380     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
5381   } else {
5382     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
5383   }
5384   *numPoints = 0;
5385   *points    = NULL;
5386   *clSec     = NULL;
5387   *clPoints  = NULL;
5388   *clp       = NULL;
5389   PetscFunctionReturn(0);
5390 }
5391 
5392 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5393 {
5394   PetscInt          offset = 0, p;
5395   const PetscInt    **perms = NULL;
5396   const PetscScalar **flips = NULL;
5397   PetscErrorCode    ierr;
5398 
5399   PetscFunctionBeginHot;
5400   *size = 0;
5401   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5402   for (p = 0; p < numPoints; p++) {
5403     const PetscInt    point = points[2*p];
5404     const PetscInt    *perm = perms ? perms[p] : NULL;
5405     const PetscScalar *flip = flips ? flips[p] : NULL;
5406     PetscInt          dof, off, d;
5407     const PetscScalar *varr;
5408 
5409     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5410     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5411     varr = &vArray[off];
5412     if (clperm) {
5413       if (perm) {
5414         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
5415       } else {
5416         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
5417       }
5418       if (flip) {
5419         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
5420       }
5421     } else {
5422       if (perm) {
5423         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
5424       } else {
5425         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
5426       }
5427       if (flip) {
5428         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
5429       }
5430     }
5431     offset += dof;
5432   }
5433   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5434   *size = offset;
5435   PetscFunctionReturn(0);
5436 }
5437 
5438 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5439 {
5440   PetscInt          offset = 0, f;
5441   PetscErrorCode    ierr;
5442 
5443   PetscFunctionBeginHot;
5444   *size = 0;
5445   for (f = 0; f < numFields; ++f) {
5446     PetscInt          p;
5447     const PetscInt    **perms = NULL;
5448     const PetscScalar **flips = NULL;
5449 
5450     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5451     for (p = 0; p < numPoints; p++) {
5452       const PetscInt    point = points[2*p];
5453       PetscInt          fdof, foff, b;
5454       const PetscScalar *varr;
5455       const PetscInt    *perm = perms ? perms[p] : NULL;
5456       const PetscScalar *flip = flips ? flips[p] : NULL;
5457 
5458       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5459       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5460       varr = &vArray[foff];
5461       if (clperm) {
5462         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
5463         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
5464         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
5465       } else {
5466         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
5467         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
5468         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
5469       }
5470       offset += fdof;
5471     }
5472     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5473   }
5474   *size = offset;
5475   PetscFunctionReturn(0);
5476 }
5477 
5478 /*@C
5479   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
5480 
5481   Not collective
5482 
5483   Input Parameters:
5484 + dm - The DM
5485 . section - The section describing the layout in v, or NULL to use the default section
5486 . v - The local vector
5487 . point - The point in the DM
5488 . csize - The size of the input values array, or NULL
5489 - values - An array to use for the values, or NULL to have it allocated automatically
5490 
5491   Output Parameters:
5492 + csize - The number of values in the closure
5493 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
5494 
5495 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
5496 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
5497 $ assembly function, and a user may already have allocated storage for this operation.
5498 $
5499 $ A typical use could be
5500 $
5501 $  values = NULL;
5502 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5503 $  for (cl = 0; cl < clSize; ++cl) {
5504 $    <Compute on closure>
5505 $  }
5506 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5507 $
5508 $ or
5509 $
5510 $  PetscMalloc1(clMaxSize, &values);
5511 $  for (p = pStart; p < pEnd; ++p) {
5512 $    clSize = clMaxSize;
5513 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5514 $    for (cl = 0; cl < clSize; ++cl) {
5515 $      <Compute on closure>
5516 $    }
5517 $  }
5518 $  PetscFree(values);
5519 
5520   Fortran Notes:
5521   Since it returns an array, this routine is only available in Fortran 90, and you must
5522   include petsc.h90 in your code.
5523 
5524   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5525 
5526   Level: intermediate
5527 
5528 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5529 @*/
5530 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5531 {
5532   PetscSection       clSection;
5533   IS                 clPoints;
5534   PetscInt          *points = NULL;
5535   const PetscInt    *clp, *perm;
5536   PetscInt           depth, numFields, numPoints, asize;
5537   PetscErrorCode     ierr;
5538 
5539   PetscFunctionBeginHot;
5540   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5541   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5542   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5543   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5544   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5545   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5546   if (depth == 1 && numFields < 2) {
5547     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5548     PetscFunctionReturn(0);
5549   }
5550   /* Get points */
5551   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5552   /* Get sizes */
5553   asize = 0;
5554   for (PetscInt p = 0; p < numPoints*2; p += 2) {
5555     PetscInt dof;
5556     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5557     asize += dof;
5558   }
5559   if (values) {
5560     const PetscScalar *vArray;
5561     PetscInt          size;
5562 
5563     if (*values) {
5564       if (PetscUnlikely(*csize < asize)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %D not sufficient to hold closure size %D", *csize, asize);
5565     } else {ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, values);CHKERRQ(ierr);}
5566     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, asize, &perm);CHKERRQ(ierr);
5567     ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5568     /* Get values */
5569     if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values);CHKERRQ(ierr);}
5570     else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values);CHKERRQ(ierr);}
5571     if (PetscUnlikely(asize != size)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %D does not match Vec closure size %D", asize, size);
5572     /* Cleanup array */
5573     ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5574   }
5575   if (csize) *csize = asize;
5576   /* Cleanup points */
5577   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5578   PetscFunctionReturn(0);
5579 }
5580 
5581 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
5582 {
5583   DMLabel            depthLabel;
5584   PetscSection       clSection;
5585   IS                 clPoints;
5586   PetscScalar       *array;
5587   const PetscScalar *vArray;
5588   PetscInt          *points = NULL;
5589   const PetscInt    *clp, *perm = NULL;
5590   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
5591   PetscErrorCode     ierr;
5592 
5593   PetscFunctionBeginHot;
5594   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5595   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5596   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5597   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5598   ierr = DMPlexGetDepth(dm, &mdepth);CHKERRQ(ierr);
5599   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
5600   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5601   if (mdepth == 1 && numFields < 2) {
5602     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5603     PetscFunctionReturn(0);
5604   }
5605   /* Get points */
5606   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5607   for (clsize=0,p=0; p<Np; p++) {
5608     PetscInt dof;
5609     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
5610     clsize += dof;
5611   }
5612   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &perm);CHKERRQ(ierr);
5613   /* Filter points */
5614   for (p = 0; p < numPoints*2; p += 2) {
5615     PetscInt dep;
5616 
5617     ierr = DMLabelGetValue(depthLabel, points[p], &dep);CHKERRQ(ierr);
5618     if (dep != depth) continue;
5619     points[Np*2+0] = points[p];
5620     points[Np*2+1] = points[p+1];
5621     ++Np;
5622   }
5623   /* Get array */
5624   if (!values || !*values) {
5625     PetscInt asize = 0, dof;
5626 
5627     for (p = 0; p < Np*2; p += 2) {
5628       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5629       asize += dof;
5630     }
5631     if (!values) {
5632       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5633       if (csize) *csize = asize;
5634       PetscFunctionReturn(0);
5635     }
5636     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
5637   } else {
5638     array = *values;
5639   }
5640   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5641   /* Get values */
5642   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
5643   else               {ierr = DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array);CHKERRQ(ierr);}
5644   /* Cleanup points */
5645   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5646   /* Cleanup array */
5647   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5648   if (!*values) {
5649     if (csize) *csize = size;
5650     *values = array;
5651   } else {
5652     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5653     *csize = size;
5654   }
5655   PetscFunctionReturn(0);
5656 }
5657 
5658 /*@C
5659   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5660 
5661   Not collective
5662 
5663   Input Parameters:
5664 + dm - The DM
5665 . section - The section describing the layout in v, or NULL to use the default section
5666 . v - The local vector
5667 . point - The point in the DM
5668 . csize - The number of values in the closure, or NULL
5669 - values - The array of values, which is a borrowed array and should not be freed
5670 
5671   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
5672 
5673   Fortran Notes:
5674   Since it returns an array, this routine is only available in Fortran 90, and you must
5675   include petsc.h90 in your code.
5676 
5677   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5678 
5679   Level: intermediate
5680 
5681 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5682 @*/
5683 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5684 {
5685   PetscInt       size = 0;
5686   PetscErrorCode ierr;
5687 
5688   PetscFunctionBegin;
5689   /* Should work without recalculating size */
5690   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
5691   *values = NULL;
5692   PetscFunctionReturn(0);
5693 }
5694 
5695 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5696 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5697 
5698 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5699 {
5700   PetscInt        cdof;   /* The number of constraints on this point */
5701   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5702   PetscScalar    *a;
5703   PetscInt        off, cind = 0, k;
5704   PetscErrorCode  ierr;
5705 
5706   PetscFunctionBegin;
5707   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5708   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5709   a    = &array[off];
5710   if (!cdof || setBC) {
5711     if (clperm) {
5712       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
5713       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
5714     } else {
5715       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
5716       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
5717     }
5718   } else {
5719     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5720     if (clperm) {
5721       if (perm) {for (k = 0; k < dof; ++k) {
5722           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5723           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5724         }
5725       } else {
5726         for (k = 0; k < dof; ++k) {
5727           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5728           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5729         }
5730       }
5731     } else {
5732       if (perm) {
5733         for (k = 0; k < dof; ++k) {
5734           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5735           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5736         }
5737       } else {
5738         for (k = 0; k < dof; ++k) {
5739           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5740           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5741         }
5742       }
5743     }
5744   }
5745   PetscFunctionReturn(0);
5746 }
5747 
5748 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5749 {
5750   PetscInt        cdof;   /* The number of constraints on this point */
5751   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5752   PetscScalar    *a;
5753   PetscInt        off, cind = 0, k;
5754   PetscErrorCode  ierr;
5755 
5756   PetscFunctionBegin;
5757   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5758   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5759   a    = &array[off];
5760   if (cdof) {
5761     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5762     if (clperm) {
5763       if (perm) {
5764         for (k = 0; k < dof; ++k) {
5765           if ((cind < cdof) && (k == cdofs[cind])) {
5766             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5767             cind++;
5768           }
5769         }
5770       } else {
5771         for (k = 0; k < dof; ++k) {
5772           if ((cind < cdof) && (k == cdofs[cind])) {
5773             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5774             cind++;
5775           }
5776         }
5777       }
5778     } else {
5779       if (perm) {
5780         for (k = 0; k < dof; ++k) {
5781           if ((cind < cdof) && (k == cdofs[cind])) {
5782             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5783             cind++;
5784           }
5785         }
5786       } else {
5787         for (k = 0; k < dof; ++k) {
5788           if ((cind < cdof) && (k == cdofs[cind])) {
5789             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5790             cind++;
5791           }
5792         }
5793       }
5794     }
5795   }
5796   PetscFunctionReturn(0);
5797 }
5798 
5799 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5800 {
5801   PetscScalar    *a;
5802   PetscInt        fdof, foff, fcdof, foffset = *offset;
5803   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5804   PetscInt        cind = 0, b;
5805   PetscErrorCode  ierr;
5806 
5807   PetscFunctionBegin;
5808   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5809   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5810   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5811   a    = &array[foff];
5812   if (!fcdof || setBC) {
5813     if (clperm) {
5814       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
5815       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
5816     } else {
5817       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
5818       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
5819     }
5820   } else {
5821     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5822     if (clperm) {
5823       if (perm) {
5824         for (b = 0; b < fdof; b++) {
5825           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5826           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5827         }
5828       } else {
5829         for (b = 0; b < fdof; b++) {
5830           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5831           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
5832         }
5833       }
5834     } else {
5835       if (perm) {
5836         for (b = 0; b < fdof; b++) {
5837           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5838           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
5839         }
5840       } else {
5841         for (b = 0; b < fdof; b++) {
5842           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5843           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
5844         }
5845       }
5846     }
5847   }
5848   *offset += fdof;
5849   PetscFunctionReturn(0);
5850 }
5851 
5852 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5853 {
5854   PetscScalar    *a;
5855   PetscInt        fdof, foff, fcdof, foffset = *offset;
5856   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5857   PetscInt        Nc, cind = 0, ncind = 0, b;
5858   PetscBool       ncSet, fcSet;
5859   PetscErrorCode  ierr;
5860 
5861   PetscFunctionBegin;
5862   ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
5863   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5864   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5865   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5866   a    = &array[foff];
5867   if (fcdof) {
5868     /* We just override fcdof and fcdofs with Ncc and comps */
5869     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5870     if (clperm) {
5871       if (perm) {
5872         if (comps) {
5873           for (b = 0; b < fdof; b++) {
5874             ncSet = fcSet = PETSC_FALSE;
5875             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5876             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5877             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
5878           }
5879         } else {
5880           for (b = 0; b < fdof; b++) {
5881             if ((cind < fcdof) && (b == fcdofs[cind])) {
5882               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5883               ++cind;
5884             }
5885           }
5886         }
5887       } else {
5888         if (comps) {
5889           for (b = 0; b < fdof; b++) {
5890             ncSet = fcSet = PETSC_FALSE;
5891             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5892             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5893             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
5894           }
5895         } else {
5896           for (b = 0; b < fdof; b++) {
5897             if ((cind < fcdof) && (b == fcdofs[cind])) {
5898               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
5899               ++cind;
5900             }
5901           }
5902         }
5903       }
5904     } else {
5905       if (perm) {
5906         if (comps) {
5907           for (b = 0; b < fdof; b++) {
5908             ncSet = fcSet = PETSC_FALSE;
5909             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5910             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5911             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
5912           }
5913         } else {
5914           for (b = 0; b < fdof; b++) {
5915             if ((cind < fcdof) && (b == fcdofs[cind])) {
5916               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
5917               ++cind;
5918             }
5919           }
5920         }
5921       } else {
5922         if (comps) {
5923           for (b = 0; b < fdof; b++) {
5924             ncSet = fcSet = PETSC_FALSE;
5925             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5926             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5927             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
5928           }
5929         } else {
5930           for (b = 0; b < fdof; b++) {
5931             if ((cind < fcdof) && (b == fcdofs[cind])) {
5932               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
5933               ++cind;
5934             }
5935           }
5936         }
5937       }
5938     }
5939   }
5940   *offset += fdof;
5941   PetscFunctionReturn(0);
5942 }
5943 
5944 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5945 {
5946   PetscScalar    *array;
5947   const PetscInt *cone, *coneO;
5948   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5949   PetscErrorCode  ierr;
5950 
5951   PetscFunctionBeginHot;
5952   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5953   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5954   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5955   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5956   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5957   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5958     const PetscInt cp = !p ? point : cone[p-1];
5959     const PetscInt o  = !p ? 0     : coneO[p-1];
5960 
5961     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5962     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5963     /* ADD_VALUES */
5964     {
5965       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5966       PetscScalar    *a;
5967       PetscInt        cdof, coff, cind = 0, k;
5968 
5969       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5970       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5971       a    = &array[coff];
5972       if (!cdof) {
5973         if (o >= 0) {
5974           for (k = 0; k < dof; ++k) {
5975             a[k] += values[off+k];
5976           }
5977         } else {
5978           for (k = 0; k < dof; ++k) {
5979             a[k] += values[off+dof-k-1];
5980           }
5981         }
5982       } else {
5983         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
5984         if (o >= 0) {
5985           for (k = 0; k < dof; ++k) {
5986             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5987             a[k] += values[off+k];
5988           }
5989         } else {
5990           for (k = 0; k < dof; ++k) {
5991             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5992             a[k] += values[off+dof-k-1];
5993           }
5994         }
5995       }
5996     }
5997   }
5998   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5999   PetscFunctionReturn(0);
6000 }
6001 
6002 /*@C
6003   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6004 
6005   Not collective
6006 
6007   Input Parameters:
6008 + dm - The DM
6009 . section - The section describing the layout in v, or NULL to use the default section
6010 . v - The local vector
6011 . point - The point in the DM
6012 . values - The array of values
6013 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
6014          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
6015 
6016   Fortran Notes:
6017   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6018 
6019   Level: intermediate
6020 
6021 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6022 @*/
6023 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6024 {
6025   PetscSection    clSection;
6026   IS              clPoints;
6027   PetscScalar    *array;
6028   PetscInt       *points = NULL;
6029   const PetscInt *clp, *clperm = NULL;
6030   PetscInt        depth, numFields, numPoints, p, clsize;
6031   PetscErrorCode  ierr;
6032 
6033   PetscFunctionBeginHot;
6034   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6035   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6036   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6037   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6038   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6039   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6040   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6041     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
6042     PetscFunctionReturn(0);
6043   }
6044   /* Get points */
6045   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6046   for (clsize=0,p=0; p<numPoints; p++) {
6047     PetscInt dof;
6048     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
6049     clsize += dof;
6050   }
6051   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
6052   /* Get array */
6053   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6054   /* Get values */
6055   if (numFields > 0) {
6056     PetscInt offset = 0, f;
6057     for (f = 0; f < numFields; ++f) {
6058       const PetscInt    **perms = NULL;
6059       const PetscScalar **flips = NULL;
6060 
6061       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6062       switch (mode) {
6063       case INSERT_VALUES:
6064         for (p = 0; p < numPoints; p++) {
6065           const PetscInt    point = points[2*p];
6066           const PetscInt    *perm = perms ? perms[p] : NULL;
6067           const PetscScalar *flip = flips ? flips[p] : NULL;
6068           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
6069         } break;
6070       case INSERT_ALL_VALUES:
6071         for (p = 0; p < numPoints; p++) {
6072           const PetscInt    point = points[2*p];
6073           const PetscInt    *perm = perms ? perms[p] : NULL;
6074           const PetscScalar *flip = flips ? flips[p] : NULL;
6075           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
6076         } break;
6077       case INSERT_BC_VALUES:
6078         for (p = 0; p < numPoints; p++) {
6079           const PetscInt    point = points[2*p];
6080           const PetscInt    *perm = perms ? perms[p] : NULL;
6081           const PetscScalar *flip = flips ? flips[p] : NULL;
6082           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
6083         } break;
6084       case ADD_VALUES:
6085         for (p = 0; p < numPoints; p++) {
6086           const PetscInt    point = points[2*p];
6087           const PetscInt    *perm = perms ? perms[p] : NULL;
6088           const PetscScalar *flip = flips ? flips[p] : NULL;
6089           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
6090         } break;
6091       case ADD_ALL_VALUES:
6092         for (p = 0; p < numPoints; p++) {
6093           const PetscInt    point = points[2*p];
6094           const PetscInt    *perm = perms ? perms[p] : NULL;
6095           const PetscScalar *flip = flips ? flips[p] : NULL;
6096           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
6097         } break;
6098       case ADD_BC_VALUES:
6099         for (p = 0; p < numPoints; p++) {
6100           const PetscInt    point = points[2*p];
6101           const PetscInt    *perm = perms ? perms[p] : NULL;
6102           const PetscScalar *flip = flips ? flips[p] : NULL;
6103           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
6104         } break;
6105       default:
6106         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6107       }
6108       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6109     }
6110   } else {
6111     PetscInt dof, off;
6112     const PetscInt    **perms = NULL;
6113     const PetscScalar **flips = NULL;
6114 
6115     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6116     switch (mode) {
6117     case INSERT_VALUES:
6118       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6119         const PetscInt    point = points[2*p];
6120         const PetscInt    *perm = perms ? perms[p] : NULL;
6121         const PetscScalar *flip = flips ? flips[p] : NULL;
6122         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6123         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
6124       } break;
6125     case INSERT_ALL_VALUES:
6126       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6127         const PetscInt    point = points[2*p];
6128         const PetscInt    *perm = perms ? perms[p] : NULL;
6129         const PetscScalar *flip = flips ? flips[p] : NULL;
6130         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6131         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
6132       } break;
6133     case INSERT_BC_VALUES:
6134       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6135         const PetscInt    point = points[2*p];
6136         const PetscInt    *perm = perms ? perms[p] : NULL;
6137         const PetscScalar *flip = flips ? flips[p] : NULL;
6138         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6139         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
6140       } break;
6141     case ADD_VALUES:
6142       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6143         const PetscInt    point = points[2*p];
6144         const PetscInt    *perm = perms ? perms[p] : NULL;
6145         const PetscScalar *flip = flips ? flips[p] : NULL;
6146         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6147         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
6148       } break;
6149     case ADD_ALL_VALUES:
6150       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6151         const PetscInt    point = points[2*p];
6152         const PetscInt    *perm = perms ? perms[p] : NULL;
6153         const PetscScalar *flip = flips ? flips[p] : NULL;
6154         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6155         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
6156       } break;
6157     case ADD_BC_VALUES:
6158       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6159         const PetscInt    point = points[2*p];
6160         const PetscInt    *perm = perms ? perms[p] : NULL;
6161         const PetscScalar *flip = flips ? flips[p] : NULL;
6162         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6163         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
6164       } break;
6165     default:
6166       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6167     }
6168     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6169   }
6170   /* Cleanup points */
6171   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6172   /* Cleanup array */
6173   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6174   PetscFunctionReturn(0);
6175 }
6176 
6177 /* Check whether the given point is in the label. If not, update the offset to skip this point */
6178 PETSC_STATIC_INLINE PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset)
6179 {
6180   PetscFunctionBegin;
6181   if (label) {
6182     PetscInt       val, fdof;
6183     PetscErrorCode ierr;
6184 
6185     /* There is a problem with this:
6186          Suppose we have two label values, defining surfaces, interecting along a line in 3D. When we add cells to the label, the cells that
6187        touch both surfaces must pick a label value. Thus we miss setting values for the surface with that other value intersecting that cell.
6188        Thus I am only going to check val != -1, not val != labelId
6189     */
6190     ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
6191     if (val < 0) {
6192       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6193       *offset += fdof;
6194       PetscFunctionReturn(1);
6195     }
6196   }
6197   PetscFunctionReturn(0);
6198 }
6199 
6200 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
6201 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
6202 {
6203   PetscSection      clSection;
6204   IS                clPoints;
6205   PetscScalar       *array;
6206   PetscInt          *points = NULL;
6207   const PetscInt    *clp;
6208   PetscInt          numFields, numPoints, p;
6209   PetscInt          offset = 0, f;
6210   PetscErrorCode    ierr;
6211 
6212   PetscFunctionBeginHot;
6213   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6214   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6215   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6216   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6217   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6218   /* Get points */
6219   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6220   /* Get array */
6221   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6222   /* Get values */
6223   for (f = 0; f < numFields; ++f) {
6224     const PetscInt    **perms = NULL;
6225     const PetscScalar **flips = NULL;
6226 
6227     if (!fieldActive[f]) {
6228       for (p = 0; p < numPoints*2; p += 2) {
6229         PetscInt fdof;
6230         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6231         offset += fdof;
6232       }
6233       continue;
6234     }
6235     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6236     switch (mode) {
6237     case INSERT_VALUES:
6238       for (p = 0; p < numPoints; p++) {
6239         const PetscInt    point = points[2*p];
6240         const PetscInt    *perm = perms ? perms[p] : NULL;
6241         const PetscScalar *flip = flips ? flips[p] : NULL;
6242         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6243         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array);
6244       } break;
6245     case INSERT_ALL_VALUES:
6246       for (p = 0; p < numPoints; p++) {
6247         const PetscInt    point = points[2*p];
6248         const PetscInt    *perm = perms ? perms[p] : NULL;
6249         const PetscScalar *flip = flips ? flips[p] : NULL;
6250         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6251         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array);
6252       } break;
6253     case INSERT_BC_VALUES:
6254       for (p = 0; p < numPoints; p++) {
6255         const PetscInt    point = points[2*p];
6256         const PetscInt    *perm = perms ? perms[p] : NULL;
6257         const PetscScalar *flip = flips ? flips[p] : NULL;
6258         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6259         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array);
6260       } break;
6261     case ADD_VALUES:
6262       for (p = 0; p < numPoints; p++) {
6263         const PetscInt    point = points[2*p];
6264         const PetscInt    *perm = perms ? perms[p] : NULL;
6265         const PetscScalar *flip = flips ? flips[p] : NULL;
6266         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6267         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array);
6268       } break;
6269     case ADD_ALL_VALUES:
6270       for (p = 0; p < numPoints; p++) {
6271         const PetscInt    point = points[2*p];
6272         const PetscInt    *perm = perms ? perms[p] : NULL;
6273         const PetscScalar *flip = flips ? flips[p] : NULL;
6274         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6275         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array);
6276       } break;
6277     default:
6278       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6279     }
6280     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6281   }
6282   /* Cleanup points */
6283   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6284   /* Cleanup array */
6285   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6286   PetscFunctionReturn(0);
6287 }
6288 
6289 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
6290 {
6291   PetscMPIInt    rank;
6292   PetscInt       i, j;
6293   PetscErrorCode ierr;
6294 
6295   PetscFunctionBegin;
6296   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr);
6297   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
6298   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
6299   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
6300   numCIndices = numCIndices ? numCIndices : numRIndices;
6301   if (!values) PetscFunctionReturn(0);
6302   for (i = 0; i < numRIndices; i++) {
6303     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
6304     for (j = 0; j < numCIndices; j++) {
6305 #if defined(PETSC_USE_COMPLEX)
6306       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
6307 #else
6308       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
6309 #endif
6310     }
6311     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6312   }
6313   PetscFunctionReturn(0);
6314 }
6315 
6316 /*
6317   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
6318 
6319   Input Parameters:
6320 + section - The section for this data layout
6321 . islocal - Is the section (and thus indices being requested) local or global?
6322 . point   - The point contributing dofs with these indices
6323 . off     - The global offset of this point
6324 . loff    - The local offset of each field
6325 . setBC   - The flag determining whether to include indices of bounsary values
6326 . perm    - A permutation of the dofs on this point, or NULL
6327 - indperm - A permutation of the entire indices array, or NULL
6328 
6329   Output Parameter:
6330 . indices - Indices for dofs on this point
6331 
6332   Level: developer
6333 
6334   Note: The indices could be local or global, depending on the value of 'off'.
6335 */
6336 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
6337 {
6338   PetscInt        dof;   /* The number of unknowns on this point */
6339   PetscInt        cdof;  /* The number of constraints on this point */
6340   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6341   PetscInt        cind = 0, k;
6342   PetscErrorCode  ierr;
6343 
6344   PetscFunctionBegin;
6345   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6346   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6347   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6348   if (!cdof || setBC) {
6349     for (k = 0; k < dof; ++k) {
6350       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6351       const PetscInt ind    = indperm ? indperm[preind] : preind;
6352 
6353       indices[ind] = off + k;
6354     }
6355   } else {
6356     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6357     for (k = 0; k < dof; ++k) {
6358       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6359       const PetscInt ind    = indperm ? indperm[preind] : preind;
6360 
6361       if ((cind < cdof) && (k == cdofs[cind])) {
6362         /* Insert check for returning constrained indices */
6363         indices[ind] = -(off+k+1);
6364         ++cind;
6365       } else {
6366         indices[ind] = off + k - (islocal ? 0 : cind);
6367       }
6368     }
6369   }
6370   *loff += dof;
6371   PetscFunctionReturn(0);
6372 }
6373 
6374 /*
6375  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
6376 
6377  Input Parameters:
6378 + section - a section (global or local)
6379 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
6380 . point - point within section
6381 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
6382 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
6383 . setBC - identify constrained (boundary condition) points via involution.
6384 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
6385 . permsoff - offset
6386 - indperm - index permutation
6387 
6388  Output Parameter:
6389 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
6390 . indices - array to hold indices (as defined by section) of each dof associated with point
6391 
6392  Notes:
6393  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
6394  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
6395  in the local vector.
6396 
6397  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
6398  significant).  It is invalid to call with a global section and setBC=true.
6399 
6400  Developer Note:
6401  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
6402  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
6403  offset could be obtained from the section instead of passing it explicitly as we do now.
6404 
6405  Example:
6406  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
6407  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
6408  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
6409  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
6410 
6411  Level: developer
6412 */
6413 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6414 {
6415   PetscInt       numFields, foff, f;
6416   PetscErrorCode ierr;
6417 
6418   PetscFunctionBegin;
6419   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6420   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6421   for (f = 0, foff = 0; f < numFields; ++f) {
6422     PetscInt        fdof, cfdof;
6423     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6424     PetscInt        cind = 0, b;
6425     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6426 
6427     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6428     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6429     if (!cfdof || setBC) {
6430       for (b = 0; b < fdof; ++b) {
6431         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6432         const PetscInt ind    = indperm ? indperm[preind] : preind;
6433 
6434         indices[ind] = off+foff+b;
6435       }
6436     } else {
6437       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6438       for (b = 0; b < fdof; ++b) {
6439         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6440         const PetscInt ind    = indperm ? indperm[preind] : preind;
6441 
6442         if ((cind < cfdof) && (b == fcdofs[cind])) {
6443           indices[ind] = -(off+foff+b+1);
6444           ++cind;
6445         } else {
6446           indices[ind] = off + foff + b - (islocal ? 0 : cind);
6447         }
6448       }
6449     }
6450     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
6451     foffs[f] += fdof;
6452   }
6453   PetscFunctionReturn(0);
6454 }
6455 
6456 /*
6457   This version believes the globalSection offsets for each field, rather than just the point offset
6458 
6459  . foffs - The offset into 'indices' for each field, since it is segregated by field
6460 
6461  Notes:
6462  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
6463  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
6464 */
6465 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6466 {
6467   PetscInt       numFields, foff, f;
6468   PetscErrorCode ierr;
6469 
6470   PetscFunctionBegin;
6471   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6472   for (f = 0; f < numFields; ++f) {
6473     PetscInt        fdof, cfdof;
6474     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6475     PetscInt        cind = 0, b;
6476     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6477 
6478     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6479     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6480     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
6481     if (!cfdof) {
6482       for (b = 0; b < fdof; ++b) {
6483         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6484         const PetscInt ind    = indperm ? indperm[preind] : preind;
6485 
6486         indices[ind] = foff+b;
6487       }
6488     } else {
6489       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6490       for (b = 0; b < fdof; ++b) {
6491         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6492         const PetscInt ind    = indperm ? indperm[preind] : preind;
6493 
6494         if ((cind < cfdof) && (b == fcdofs[cind])) {
6495           indices[ind] = -(foff+b+1);
6496           ++cind;
6497         } else {
6498           indices[ind] = foff+b-cind;
6499         }
6500       }
6501     }
6502     foffs[f] += fdof;
6503   }
6504   PetscFunctionReturn(0);
6505 }
6506 
6507 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
6508 {
6509   Mat             cMat;
6510   PetscSection    aSec, cSec;
6511   IS              aIS;
6512   PetscInt        aStart = -1, aEnd = -1;
6513   const PetscInt  *anchors;
6514   PetscInt        numFields, f, p, q, newP = 0;
6515   PetscInt        newNumPoints = 0, newNumIndices = 0;
6516   PetscInt        *newPoints, *indices, *newIndices;
6517   PetscInt        maxAnchor, maxDof;
6518   PetscInt        newOffsets[32];
6519   PetscInt        *pointMatOffsets[32];
6520   PetscInt        *newPointOffsets[32];
6521   PetscScalar     *pointMat[32];
6522   PetscScalar     *newValues=NULL,*tmpValues;
6523   PetscBool       anyConstrained = PETSC_FALSE;
6524   PetscErrorCode  ierr;
6525 
6526   PetscFunctionBegin;
6527   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6528   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6529   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6530 
6531   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
6532   /* if there are point-to-point constraints */
6533   if (aSec) {
6534     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
6535     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
6536     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
6537     /* figure out how many points are going to be in the new element matrix
6538      * (we allow double counting, because it's all just going to be summed
6539      * into the global matrix anyway) */
6540     for (p = 0; p < 2*numPoints; p+=2) {
6541       PetscInt b    = points[p];
6542       PetscInt bDof = 0, bSecDof;
6543 
6544       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6545       if (!bSecDof) {
6546         continue;
6547       }
6548       if (b >= aStart && b < aEnd) {
6549         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
6550       }
6551       if (bDof) {
6552         /* this point is constrained */
6553         /* it is going to be replaced by its anchors */
6554         PetscInt bOff, q;
6555 
6556         anyConstrained = PETSC_TRUE;
6557         newNumPoints  += bDof;
6558         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
6559         for (q = 0; q < bDof; q++) {
6560           PetscInt a = anchors[bOff + q];
6561           PetscInt aDof;
6562 
6563           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
6564           newNumIndices += aDof;
6565           for (f = 0; f < numFields; ++f) {
6566             PetscInt fDof;
6567 
6568             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
6569             newOffsets[f+1] += fDof;
6570           }
6571         }
6572       }
6573       else {
6574         /* this point is not constrained */
6575         newNumPoints++;
6576         newNumIndices += bSecDof;
6577         for (f = 0; f < numFields; ++f) {
6578           PetscInt fDof;
6579 
6580           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6581           newOffsets[f+1] += fDof;
6582         }
6583       }
6584     }
6585   }
6586   if (!anyConstrained) {
6587     if (outNumPoints)  *outNumPoints  = 0;
6588     if (outNumIndices) *outNumIndices = 0;
6589     if (outPoints)     *outPoints     = NULL;
6590     if (outValues)     *outValues     = NULL;
6591     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6592     PetscFunctionReturn(0);
6593   }
6594 
6595   if (outNumPoints)  *outNumPoints  = newNumPoints;
6596   if (outNumIndices) *outNumIndices = newNumIndices;
6597 
6598   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
6599 
6600   if (!outPoints && !outValues) {
6601     if (offsets) {
6602       for (f = 0; f <= numFields; f++) {
6603         offsets[f] = newOffsets[f];
6604       }
6605     }
6606     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6607     PetscFunctionReturn(0);
6608   }
6609 
6610   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
6611 
6612   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
6613 
6614   /* workspaces */
6615   if (numFields) {
6616     for (f = 0; f < numFields; f++) {
6617       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
6618       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
6619     }
6620   }
6621   else {
6622     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
6623     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
6624   }
6625 
6626   /* get workspaces for the point-to-point matrices */
6627   if (numFields) {
6628     PetscInt totalOffset, totalMatOffset;
6629 
6630     for (p = 0; p < numPoints; p++) {
6631       PetscInt b    = points[2*p];
6632       PetscInt bDof = 0, bSecDof;
6633 
6634       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6635       if (!bSecDof) {
6636         for (f = 0; f < numFields; f++) {
6637           newPointOffsets[f][p + 1] = 0;
6638           pointMatOffsets[f][p + 1] = 0;
6639         }
6640         continue;
6641       }
6642       if (b >= aStart && b < aEnd) {
6643         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6644       }
6645       if (bDof) {
6646         for (f = 0; f < numFields; f++) {
6647           PetscInt fDof, q, bOff, allFDof = 0;
6648 
6649           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6650           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6651           for (q = 0; q < bDof; q++) {
6652             PetscInt a = anchors[bOff + q];
6653             PetscInt aFDof;
6654 
6655             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
6656             allFDof += aFDof;
6657           }
6658           newPointOffsets[f][p+1] = allFDof;
6659           pointMatOffsets[f][p+1] = fDof * allFDof;
6660         }
6661       }
6662       else {
6663         for (f = 0; f < numFields; f++) {
6664           PetscInt fDof;
6665 
6666           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6667           newPointOffsets[f][p+1] = fDof;
6668           pointMatOffsets[f][p+1] = 0;
6669         }
6670       }
6671     }
6672     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
6673       newPointOffsets[f][0] = totalOffset;
6674       pointMatOffsets[f][0] = totalMatOffset;
6675       for (p = 0; p < numPoints; p++) {
6676         newPointOffsets[f][p+1] += newPointOffsets[f][p];
6677         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
6678       }
6679       totalOffset    = newPointOffsets[f][numPoints];
6680       totalMatOffset = pointMatOffsets[f][numPoints];
6681       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
6682     }
6683   }
6684   else {
6685     for (p = 0; p < numPoints; p++) {
6686       PetscInt b    = points[2*p];
6687       PetscInt bDof = 0, bSecDof;
6688 
6689       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6690       if (!bSecDof) {
6691         newPointOffsets[0][p + 1] = 0;
6692         pointMatOffsets[0][p + 1] = 0;
6693         continue;
6694       }
6695       if (b >= aStart && b < aEnd) {
6696         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6697       }
6698       if (bDof) {
6699         PetscInt bOff, q, allDof = 0;
6700 
6701         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6702         for (q = 0; q < bDof; q++) {
6703           PetscInt a = anchors[bOff + q], aDof;
6704 
6705           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
6706           allDof += aDof;
6707         }
6708         newPointOffsets[0][p+1] = allDof;
6709         pointMatOffsets[0][p+1] = bSecDof * allDof;
6710       }
6711       else {
6712         newPointOffsets[0][p+1] = bSecDof;
6713         pointMatOffsets[0][p+1] = 0;
6714       }
6715     }
6716     newPointOffsets[0][0] = 0;
6717     pointMatOffsets[0][0] = 0;
6718     for (p = 0; p < numPoints; p++) {
6719       newPointOffsets[0][p+1] += newPointOffsets[0][p];
6720       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
6721     }
6722     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
6723   }
6724 
6725   /* output arrays */
6726   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6727 
6728   /* get the point-to-point matrices; construct newPoints */
6729   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
6730   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6731   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6732   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
6733   if (numFields) {
6734     for (p = 0, newP = 0; p < numPoints; p++) {
6735       PetscInt b    = points[2*p];
6736       PetscInt o    = points[2*p+1];
6737       PetscInt bDof = 0, bSecDof;
6738 
6739       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6740       if (!bSecDof) {
6741         continue;
6742       }
6743       if (b >= aStart && b < aEnd) {
6744         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6745       }
6746       if (bDof) {
6747         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
6748 
6749         fStart[0] = 0;
6750         fEnd[0]   = 0;
6751         for (f = 0; f < numFields; f++) {
6752           PetscInt fDof;
6753 
6754           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
6755           fStart[f+1] = fStart[f] + fDof;
6756           fEnd[f+1]   = fStart[f+1];
6757         }
6758         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6759         ierr = DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
6760 
6761         fAnchorStart[0] = 0;
6762         fAnchorEnd[0]   = 0;
6763         for (f = 0; f < numFields; f++) {
6764           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
6765 
6766           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
6767           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
6768         }
6769         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6770         for (q = 0; q < bDof; q++) {
6771           PetscInt a = anchors[bOff + q], aOff;
6772 
6773           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6774           newPoints[2*(newP + q)]     = a;
6775           newPoints[2*(newP + q) + 1] = 0;
6776           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6777           ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
6778         }
6779         newP += bDof;
6780 
6781         if (outValues) {
6782           /* get the point-to-point submatrix */
6783           for (f = 0; f < numFields; f++) {
6784             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
6785           }
6786         }
6787       }
6788       else {
6789         newPoints[2 * newP]     = b;
6790         newPoints[2 * newP + 1] = o;
6791         newP++;
6792       }
6793     }
6794   } else {
6795     for (p = 0; p < numPoints; p++) {
6796       PetscInt b    = points[2*p];
6797       PetscInt o    = points[2*p+1];
6798       PetscInt bDof = 0, bSecDof;
6799 
6800       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6801       if (!bSecDof) {
6802         continue;
6803       }
6804       if (b >= aStart && b < aEnd) {
6805         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6806       }
6807       if (bDof) {
6808         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
6809 
6810         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6811         ierr = DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
6812 
6813         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
6814         for (q = 0; q < bDof; q++) {
6815           PetscInt a = anchors[bOff + q], aOff;
6816 
6817           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6818 
6819           newPoints[2*(newP + q)]     = a;
6820           newPoints[2*(newP + q) + 1] = 0;
6821           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6822           ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
6823         }
6824         newP += bDof;
6825 
6826         /* get the point-to-point submatrix */
6827         if (outValues) {
6828           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
6829         }
6830       }
6831       else {
6832         newPoints[2 * newP]     = b;
6833         newPoints[2 * newP + 1] = o;
6834         newP++;
6835       }
6836     }
6837   }
6838 
6839   if (outValues) {
6840     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
6841     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
6842     /* multiply constraints on the right */
6843     if (numFields) {
6844       for (f = 0; f < numFields; f++) {
6845         PetscInt oldOff = offsets[f];
6846 
6847         for (p = 0; p < numPoints; p++) {
6848           PetscInt cStart = newPointOffsets[f][p];
6849           PetscInt b      = points[2 * p];
6850           PetscInt c, r, k;
6851           PetscInt dof;
6852 
6853           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
6854           if (!dof) {
6855             continue;
6856           }
6857           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
6858             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
6859             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
6860 
6861             for (r = 0; r < numIndices; r++) {
6862               for (c = 0; c < nCols; c++) {
6863                 for (k = 0; k < dof; k++) {
6864                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
6865                 }
6866               }
6867             }
6868           }
6869           else {
6870             /* copy this column as is */
6871             for (r = 0; r < numIndices; r++) {
6872               for (c = 0; c < dof; c++) {
6873                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
6874               }
6875             }
6876           }
6877           oldOff += dof;
6878         }
6879       }
6880     }
6881     else {
6882       PetscInt oldOff = 0;
6883       for (p = 0; p < numPoints; p++) {
6884         PetscInt cStart = newPointOffsets[0][p];
6885         PetscInt b      = points[2 * p];
6886         PetscInt c, r, k;
6887         PetscInt dof;
6888 
6889         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
6890         if (!dof) {
6891           continue;
6892         }
6893         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
6894           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
6895           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
6896 
6897           for (r = 0; r < numIndices; r++) {
6898             for (c = 0; c < nCols; c++) {
6899               for (k = 0; k < dof; k++) {
6900                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
6901               }
6902             }
6903           }
6904         }
6905         else {
6906           /* copy this column as is */
6907           for (r = 0; r < numIndices; r++) {
6908             for (c = 0; c < dof; c++) {
6909               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
6910             }
6911           }
6912         }
6913         oldOff += dof;
6914       }
6915     }
6916 
6917     if (multiplyLeft) {
6918       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6919       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
6920       /* multiply constraints transpose on the left */
6921       if (numFields) {
6922         for (f = 0; f < numFields; f++) {
6923           PetscInt oldOff = offsets[f];
6924 
6925           for (p = 0; p < numPoints; p++) {
6926             PetscInt rStart = newPointOffsets[f][p];
6927             PetscInt b      = points[2 * p];
6928             PetscInt c, r, k;
6929             PetscInt dof;
6930 
6931             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
6932             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
6933               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
6934               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
6935 
6936               for (r = 0; r < nRows; r++) {
6937                 for (c = 0; c < newNumIndices; c++) {
6938                   for (k = 0; k < dof; k++) {
6939                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
6940                   }
6941                 }
6942               }
6943             }
6944             else {
6945               /* copy this row as is */
6946               for (r = 0; r < dof; r++) {
6947                 for (c = 0; c < newNumIndices; c++) {
6948                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
6949                 }
6950               }
6951             }
6952             oldOff += dof;
6953           }
6954         }
6955       }
6956       else {
6957         PetscInt oldOff = 0;
6958 
6959         for (p = 0; p < numPoints; p++) {
6960           PetscInt rStart = newPointOffsets[0][p];
6961           PetscInt b      = points[2 * p];
6962           PetscInt c, r, k;
6963           PetscInt dof;
6964 
6965           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
6966           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
6967             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
6968             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
6969 
6970             for (r = 0; r < nRows; r++) {
6971               for (c = 0; c < newNumIndices; c++) {
6972                 for (k = 0; k < dof; k++) {
6973                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
6974                 }
6975               }
6976             }
6977           }
6978           else {
6979             /* copy this row as is */
6980             for (r = 0; r < dof; r++) {
6981               for (c = 0; c < newNumIndices; c++) {
6982                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
6983               }
6984             }
6985           }
6986           oldOff += dof;
6987         }
6988       }
6989 
6990       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
6991     }
6992     else {
6993       newValues = tmpValues;
6994     }
6995   }
6996 
6997   /* clean up */
6998   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6999   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
7000 
7001   if (numFields) {
7002     for (f = 0; f < numFields; f++) {
7003       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
7004       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
7005       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
7006     }
7007   }
7008   else {
7009     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
7010     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
7011     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
7012   }
7013   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7014 
7015   /* output */
7016   if (outPoints) {
7017     *outPoints = newPoints;
7018   }
7019   else {
7020     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
7021   }
7022   if (outValues) {
7023     *outValues = newValues;
7024   }
7025   for (f = 0; f <= numFields; f++) {
7026     offsets[f] = newOffsets[f];
7027   }
7028   PetscFunctionReturn(0);
7029 }
7030 
7031 /*@C
7032   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
7033 
7034   Not collective
7035 
7036   Input Parameters:
7037 + dm         - The DM
7038 . section    - The PetscSection describing the points (a local section)
7039 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7040 . point      - The point defining the closure
7041 - useClPerm  - Use the closure point permutation if available
7042 
7043   Output Parameters:
7044 + numIndices - The number of dof indices in the closure of point with the input sections
7045 . indices    - The dof indices
7046 . outOffsets - Array to write the field offsets into, or NULL
7047 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7048 
7049   Notes:
7050   Must call DMPlexRestoreClosureIndices() to free allocated memory
7051 
7052   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7053   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7054   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7055   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7056   indices (with the above semantics) are implied.
7057 
7058   Level: advanced
7059 
7060 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7061 @*/
7062 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7063                                        PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7064 {
7065   /* Closure ordering */
7066   PetscSection        clSection;
7067   IS                  clPoints;
7068   const PetscInt     *clp;
7069   PetscInt           *points;
7070   const PetscInt     *clperm = NULL;
7071   /* Dof permutation and sign flips */
7072   const PetscInt    **perms[32] = {NULL};
7073   const PetscScalar **flips[32] = {NULL};
7074   PetscScalar        *valCopy   = NULL;
7075   /* Hanging node constraints */
7076   PetscInt           *pointsC = NULL;
7077   PetscScalar        *valuesC = NULL;
7078   PetscInt            NclC, NiC;
7079 
7080   PetscInt           *idx;
7081   PetscInt            Nf, Ncl, Ni = 0, offsets[32], p, f;
7082   PetscBool           isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7083   PetscErrorCode      ierr;
7084 
7085   PetscFunctionBeginHot;
7086   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7087   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7088   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7089   if (numIndices) PetscValidPointer(numIndices, 6);
7090   if (indices)    PetscValidPointer(indices, 7);
7091   if (outOffsets) PetscValidPointer(outOffsets, 8);
7092   if (values)     PetscValidPointer(values, 9);
7093   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7094   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
7095   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
7096   /* 1) Get points in closure */
7097   ierr = DMPlexGetCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7098   if (useClPerm) {
7099     PetscInt depth, clsize;
7100     ierr = DMPlexGetPointDepth(dm, point, &depth);CHKERRQ(ierr);
7101     for (clsize=0,p=0; p<Ncl; p++) {
7102       PetscInt dof;
7103       ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
7104       clsize += dof;
7105     }
7106     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
7107   }
7108   /* 2) Get number of indices on these points and field offsets from section */
7109   for (p = 0; p < Ncl*2; p += 2) {
7110     PetscInt dof, fdof;
7111 
7112     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7113     for (f = 0; f < Nf; ++f) {
7114       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7115       offsets[f+1] += fdof;
7116     }
7117     Ni += dof;
7118   }
7119   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
7120   if (Nf && offsets[Nf] != Ni) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Ni);
7121   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7122   for (f = 0; f < PetscMax(1, Nf); ++f) {
7123     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7124     else    {ierr = PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7125     /* may need to apply sign changes to the element matrix */
7126     if (values && flips[f]) {
7127       PetscInt foffset = offsets[f];
7128 
7129       for (p = 0; p < Ncl; ++p) {
7130         PetscInt           pnt  = points[2*p], fdof;
7131         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7132 
7133         if (!Nf) {ierr = PetscSectionGetDof(section, pnt, &fdof);CHKERRQ(ierr);}
7134         else     {ierr = PetscSectionGetFieldDof(section, pnt, f, &fdof);CHKERRQ(ierr);}
7135         if (flip) {
7136           PetscInt i, j, k;
7137 
7138           if (!valCopy) {
7139             ierr = DMGetWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);
7140             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7141             *values = valCopy;
7142           }
7143           for (i = 0; i < fdof; ++i) {
7144             PetscScalar fval = flip[i];
7145 
7146             for (k = 0; k < Ni; ++k) {
7147               valCopy[Ni * (foffset + i) + k] *= fval;
7148               valCopy[Ni * k + (foffset + i)] *= fval;
7149             }
7150           }
7151         }
7152         foffset += fdof;
7153       }
7154     }
7155   }
7156   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7157   ierr = DMPlexAnchorsModifyMat(dm, section, Ncl, Ni, points, perms, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
7158   if (NclC) {
7159     if (valCopy) {ierr = DMRestoreWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);}
7160     for (f = 0; f < PetscMax(1, Nf); ++f) {
7161       if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7162       else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7163     }
7164     for (f = 0; f < PetscMax(1, Nf); ++f) {
7165       if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7166       else    {ierr = PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7167     }
7168     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7169     Ncl     = NclC;
7170     Ni      = NiC;
7171     points  = pointsC;
7172     if (values) *values = valuesC;
7173   }
7174   /* 5) Calculate indices */
7175   ierr = DMGetWorkArray(dm, Ni, MPIU_INT, &idx);CHKERRQ(ierr);
7176   if (Nf) {
7177     PetscInt  idxOff;
7178     PetscBool useFieldOffsets;
7179 
7180     if (outOffsets) {for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];}
7181     ierr = PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets);CHKERRQ(ierr);
7182     if (useFieldOffsets) {
7183       for (p = 0; p < Ncl; ++p) {
7184         const PetscInt pnt = points[p*2];
7185 
7186         ierr = DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx);CHKERRQ(ierr);
7187       }
7188     } else {
7189       for (p = 0; p < Ncl; ++p) {
7190         const PetscInt pnt = points[p*2];
7191 
7192         ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7193         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7194          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
7195          * global section. */
7196         ierr = DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx);CHKERRQ(ierr);
7197       }
7198     }
7199   } else {
7200     PetscInt off = 0, idxOff;
7201 
7202     for (p = 0; p < Ncl; ++p) {
7203       const PetscInt  pnt  = points[p*2];
7204       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
7205 
7206       ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7207       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7208        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
7209       ierr = DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx);CHKERRQ(ierr);
7210     }
7211   }
7212   /* 6) Cleanup */
7213   for (f = 0; f < PetscMax(1, Nf); ++f) {
7214     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7215     else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7216   }
7217   if (NclC) {
7218     ierr = DMRestoreWorkArray(dm, NclC*2, MPIU_INT, &pointsC);CHKERRQ(ierr);
7219   } else {
7220     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7221   }
7222 
7223   if (numIndices) *numIndices = Ni;
7224   if (indices)    *indices    = idx;
7225   PetscFunctionReturn(0);
7226 }
7227 
7228 /*@C
7229   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
7230 
7231   Not collective
7232 
7233   Input Parameters:
7234 + dm         - The DM
7235 . section    - The PetscSection describing the points (a local section)
7236 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7237 . point      - The point defining the closure
7238 - useClPerm  - Use the closure point permutation if available
7239 
7240   Output Parameters:
7241 + numIndices - The number of dof indices in the closure of point with the input sections
7242 . indices    - The dof indices
7243 . outOffsets - Array to write the field offsets into, or NULL
7244 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7245 
7246   Notes:
7247   If values were modified, the user is responsible for calling DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values).
7248 
7249   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7250   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7251   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7252   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7253   indices (with the above semantics) are implied.
7254 
7255   Level: advanced
7256 
7257 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7258 @*/
7259 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7260                                            PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7261 {
7262   PetscErrorCode ierr;
7263 
7264   PetscFunctionBegin;
7265   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7266   PetscValidPointer(indices, 7);
7267   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
7268   PetscFunctionReturn(0);
7269 }
7270 
7271 /*@C
7272   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7273 
7274   Not collective
7275 
7276   Input Parameters:
7277 + dm - The DM
7278 . section - The section describing the layout in v, or NULL to use the default section
7279 . globalSection - The section describing the layout in v, or NULL to use the default global section
7280 . A - The matrix
7281 . point - The point in the DM
7282 . values - The array of values
7283 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7284 
7285   Fortran Notes:
7286   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7287 
7288   Level: intermediate
7289 
7290 .seealso DMPlexMatSetClosureGeneral(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7291 @*/
7292 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7293 {
7294   DM_Plex           *mesh = (DM_Plex*) dm->data;
7295   PetscInt          *indices;
7296   PetscInt           numIndices;
7297   const PetscScalar *valuesOrig = values;
7298   PetscErrorCode     ierr;
7299 
7300   PetscFunctionBegin;
7301   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7302   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
7303   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7304   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
7305   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7306   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7307 
7308   ierr = DMPlexGetClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7309 
7310   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
7311   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7312   if (ierr) {
7313     PetscMPIInt    rank;
7314     PetscErrorCode ierr2;
7315 
7316     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7317     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7318     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
7319     ierr2 = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7320     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7321     CHKERRQ(ierr);
7322   }
7323   if (mesh->printFEM > 1) {
7324     PetscInt i;
7325     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
7326     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
7327     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7328   }
7329 
7330   ierr = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7331   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7332   PetscFunctionReturn(0);
7333 }
7334 
7335 /*@C
7336   DMPlexMatSetClosure - Set an array of the values on the closure of 'point' using a different row and column section
7337 
7338   Not collective
7339 
7340   Input Parameters:
7341 + dmRow - The DM for the row fields
7342 . sectionRow - The section describing the layout, or NULL to use the default section in dmRow
7343 . globalSectionRow - The section describing the layout, or NULL to use the default global section in dmRow
7344 . dmCol - The DM for the column fields
7345 . sectionCol - The section describing the layout, or NULL to use the default section in dmCol
7346 . globalSectionCol - The section describing the layout, or NULL to use the default global section in dmCol
7347 . A - The matrix
7348 . point - The point in the DMs
7349 . values - The array of values
7350 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7351 
7352   Level: intermediate
7353 
7354 .seealso DMPlexMatSetClosure(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7355 @*/
7356 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7357 {
7358   DM_Plex           *mesh = (DM_Plex*) dmRow->data;
7359   PetscInt          *indicesRow, *indicesCol;
7360   PetscInt           numIndicesRow, numIndicesCol;
7361   const PetscScalar *valuesOrig = values;
7362   PetscErrorCode     ierr;
7363 
7364   PetscFunctionBegin;
7365   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
7366   if (!sectionRow) {ierr = DMGetLocalSection(dmRow, &sectionRow);CHKERRQ(ierr);}
7367   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
7368   if (!globalSectionRow) {ierr = DMGetGlobalSection(dmRow, &globalSectionRow);CHKERRQ(ierr);}
7369   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
7370   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 4);
7371   if (!sectionCol) {ierr = DMGetLocalSection(dmCol, &sectionCol);CHKERRQ(ierr);}
7372   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 5);
7373   if (!globalSectionCol) {ierr = DMGetGlobalSection(dmCol, &globalSectionCol);CHKERRQ(ierr);}
7374   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 6);
7375   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7376 
7377   ierr = DMPlexGetClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7378   ierr = DMPlexGetClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7379 
7380   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr);}
7381   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values, mode);
7382   if (ierr) {
7383     PetscMPIInt    rank;
7384     PetscErrorCode ierr2;
7385 
7386     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7387     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7388     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr2);
7389     ierr2 = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7390     ierr2 = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7391     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7392     CHKERRQ(ierr);
7393   }
7394 
7395   ierr = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7396   ierr = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7397   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7398   PetscFunctionReturn(0);
7399 }
7400 
7401 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7402 {
7403   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
7404   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
7405   PetscInt       *cpoints = NULL;
7406   PetscInt       *findices, *cindices;
7407   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7408   PetscInt        foffsets[32], coffsets[32];
7409   DMPolytopeType  ct;
7410   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7411   PetscErrorCode  ierr;
7412 
7413   PetscFunctionBegin;
7414   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7415   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7416   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7417   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7418   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7419   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7420   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7421   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7422   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7423   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7424   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7425   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7426   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7427   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7428   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7429   /* Column indices */
7430   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7431   maxFPoints = numCPoints;
7432   /* Compress out points not in the section */
7433   /*   TODO: Squeeze out points with 0 dof as well */
7434   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7435   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7436     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7437       cpoints[q*2]   = cpoints[p];
7438       cpoints[q*2+1] = cpoints[p+1];
7439       ++q;
7440     }
7441   }
7442   numCPoints = q;
7443   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7444     PetscInt fdof;
7445 
7446     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7447     if (!dof) continue;
7448     for (f = 0; f < numFields; ++f) {
7449       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7450       coffsets[f+1] += fdof;
7451     }
7452     numCIndices += dof;
7453   }
7454   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7455   /* Row indices */
7456   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7457   {
7458     DMPlexCellRefiner cr;
7459     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7460     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7461     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7462   }
7463   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7464   for (r = 0, q = 0; r < numSubcells; ++r) {
7465     /* TODO Map from coarse to fine cells */
7466     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7467     /* Compress out points not in the section */
7468     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7469     for (p = 0; p < numFPoints*2; p += 2) {
7470       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7471         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7472         if (!dof) continue;
7473         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7474         if (s < q) continue;
7475         ftotpoints[q*2]   = fpoints[p];
7476         ftotpoints[q*2+1] = fpoints[p+1];
7477         ++q;
7478       }
7479     }
7480     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7481   }
7482   numFPoints = q;
7483   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7484     PetscInt fdof;
7485 
7486     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7487     if (!dof) continue;
7488     for (f = 0; f < numFields; ++f) {
7489       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7490       foffsets[f+1] += fdof;
7491     }
7492     numFIndices += dof;
7493   }
7494   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7495 
7496   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7497   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7498   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7499   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7500   if (numFields) {
7501     const PetscInt **permsF[32] = {NULL};
7502     const PetscInt **permsC[32] = {NULL};
7503 
7504     for (f = 0; f < numFields; f++) {
7505       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7506       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7507     }
7508     for (p = 0; p < numFPoints; p++) {
7509       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7510       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7511     }
7512     for (p = 0; p < numCPoints; p++) {
7513       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7514       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7515     }
7516     for (f = 0; f < numFields; f++) {
7517       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7518       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7519     }
7520   } else {
7521     const PetscInt **permsF = NULL;
7522     const PetscInt **permsC = NULL;
7523 
7524     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7525     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7526     for (p = 0, off = 0; p < numFPoints; p++) {
7527       const PetscInt *perm = permsF ? permsF[p] : NULL;
7528 
7529       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7530       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7531     }
7532     for (p = 0, off = 0; p < numCPoints; p++) {
7533       const PetscInt *perm = permsC ? permsC[p] : NULL;
7534 
7535       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7536       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7537     }
7538     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7539     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7540   }
7541   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
7542   /* TODO: flips */
7543   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
7544   if (ierr) {
7545     PetscMPIInt    rank;
7546     PetscErrorCode ierr2;
7547 
7548     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7549     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7550     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
7551     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
7552     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
7553     CHKERRQ(ierr);
7554   }
7555   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7556   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7557   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7558   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7559   PetscFunctionReturn(0);
7560 }
7561 
7562 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
7563 {
7564   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
7565   PetscInt      *cpoints = NULL;
7566   PetscInt       foffsets[32], coffsets[32];
7567   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7568   DMPolytopeType ct;
7569   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7570   PetscErrorCode ierr;
7571 
7572   PetscFunctionBegin;
7573   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7574   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7575   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7576   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7577   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7578   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7579   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7580   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7581   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7582   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7583   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7584   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7585   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7586   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7587   /* Column indices */
7588   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7589   maxFPoints = numCPoints;
7590   /* Compress out points not in the section */
7591   /*   TODO: Squeeze out points with 0 dof as well */
7592   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7593   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7594     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7595       cpoints[q*2]   = cpoints[p];
7596       cpoints[q*2+1] = cpoints[p+1];
7597       ++q;
7598     }
7599   }
7600   numCPoints = q;
7601   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7602     PetscInt fdof;
7603 
7604     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7605     if (!dof) continue;
7606     for (f = 0; f < numFields; ++f) {
7607       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7608       coffsets[f+1] += fdof;
7609     }
7610     numCIndices += dof;
7611   }
7612   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7613   /* Row indices */
7614   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7615   {
7616     DMPlexCellRefiner cr;
7617     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7618     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7619     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7620   }
7621   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7622   for (r = 0, q = 0; r < numSubcells; ++r) {
7623     /* TODO Map from coarse to fine cells */
7624     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7625     /* Compress out points not in the section */
7626     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7627     for (p = 0; p < numFPoints*2; p += 2) {
7628       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7629         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7630         if (!dof) continue;
7631         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7632         if (s < q) continue;
7633         ftotpoints[q*2]   = fpoints[p];
7634         ftotpoints[q*2+1] = fpoints[p+1];
7635         ++q;
7636       }
7637     }
7638     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7639   }
7640   numFPoints = q;
7641   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7642     PetscInt fdof;
7643 
7644     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7645     if (!dof) continue;
7646     for (f = 0; f < numFields; ++f) {
7647       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7648       foffsets[f+1] += fdof;
7649     }
7650     numFIndices += dof;
7651   }
7652   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7653 
7654   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7655   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7656   if (numFields) {
7657     const PetscInt **permsF[32] = {NULL};
7658     const PetscInt **permsC[32] = {NULL};
7659 
7660     for (f = 0; f < numFields; f++) {
7661       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7662       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7663     }
7664     for (p = 0; p < numFPoints; p++) {
7665       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7666       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7667     }
7668     for (p = 0; p < numCPoints; p++) {
7669       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7670       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7671     }
7672     for (f = 0; f < numFields; f++) {
7673       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7674       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7675     }
7676   } else {
7677     const PetscInt **permsF = NULL;
7678     const PetscInt **permsC = NULL;
7679 
7680     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7681     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7682     for (p = 0, off = 0; p < numFPoints; p++) {
7683       const PetscInt *perm = permsF ? permsF[p] : NULL;
7684 
7685       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7686       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7687     }
7688     for (p = 0, off = 0; p < numCPoints; p++) {
7689       const PetscInt *perm = permsC ? permsC[p] : NULL;
7690 
7691       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7692       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7693     }
7694     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7695     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7696   }
7697   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7698   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7699   PetscFunctionReturn(0);
7700 }
7701 
7702 /*@C
7703   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
7704 
7705   Input Parameter:
7706 . dm   - The DMPlex object
7707 
7708   Output Parameter:
7709 . cellHeight - The height of a cell
7710 
7711   Level: developer
7712 
7713 .seealso DMPlexSetVTKCellHeight()
7714 @*/
7715 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7716 {
7717   DM_Plex *mesh = (DM_Plex*) dm->data;
7718 
7719   PetscFunctionBegin;
7720   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7721   PetscValidPointer(cellHeight, 2);
7722   *cellHeight = mesh->vtkCellHeight;
7723   PetscFunctionReturn(0);
7724 }
7725 
7726 /*@C
7727   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
7728 
7729   Input Parameters:
7730 + dm   - The DMPlex object
7731 - cellHeight - The height of a cell
7732 
7733   Level: developer
7734 
7735 .seealso DMPlexGetVTKCellHeight()
7736 @*/
7737 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7738 {
7739   DM_Plex *mesh = (DM_Plex*) dm->data;
7740 
7741   PetscFunctionBegin;
7742   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7743   mesh->vtkCellHeight = cellHeight;
7744   PetscFunctionReturn(0);
7745 }
7746 
7747 /*@
7748   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
7749 
7750   Input Parameter:
7751 . dm - The DMPlex object
7752 
7753   Output Parameters:
7754 + gcStart - The first ghost cell, or NULL
7755 - gcEnd   - The upper bound on ghost cells, or NULL
7756 
7757   Level: advanced
7758 
7759 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
7760 @*/
7761 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
7762 {
7763   DMLabel        ctLabel;
7764   PetscErrorCode ierr;
7765 
7766   PetscFunctionBegin;
7767   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7768   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
7769   ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd);CHKERRQ(ierr);
7770   PetscFunctionReturn(0);
7771 }
7772 
7773 /* We can easily have a form that takes an IS instead */
7774 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
7775 {
7776   PetscSection   section, globalSection;
7777   PetscInt      *numbers, p;
7778   PetscErrorCode ierr;
7779 
7780   PetscFunctionBegin;
7781   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
7782   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
7783   for (p = pStart; p < pEnd; ++p) {
7784     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
7785   }
7786   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
7787   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
7788   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
7789   for (p = pStart; p < pEnd; ++p) {
7790     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
7791     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
7792     else                       numbers[p-pStart] += shift;
7793   }
7794   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
7795   if (globalSize) {
7796     PetscLayout layout;
7797     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
7798     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
7799     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
7800   }
7801   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
7802   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
7803   PetscFunctionReturn(0);
7804 }
7805 
7806 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
7807 {
7808   PetscInt       cellHeight, cStart, cEnd;
7809   PetscErrorCode ierr;
7810 
7811   PetscFunctionBegin;
7812   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
7813   if (includeHybrid) {ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7814   else               {ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7815   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
7816   PetscFunctionReturn(0);
7817 }
7818 
7819 /*@
7820   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
7821 
7822   Input Parameter:
7823 . dm   - The DMPlex object
7824 
7825   Output Parameter:
7826 . globalCellNumbers - Global cell numbers for all cells on this process
7827 
7828   Level: developer
7829 
7830 .seealso DMPlexGetVertexNumbering()
7831 @*/
7832 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
7833 {
7834   DM_Plex       *mesh = (DM_Plex*) dm->data;
7835   PetscErrorCode ierr;
7836 
7837   PetscFunctionBegin;
7838   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7839   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
7840   *globalCellNumbers = mesh->globalCellNumbers;
7841   PetscFunctionReturn(0);
7842 }
7843 
7844 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
7845 {
7846   PetscInt       vStart, vEnd;
7847   PetscErrorCode ierr;
7848 
7849   PetscFunctionBegin;
7850   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7851   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7852   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
7853   PetscFunctionReturn(0);
7854 }
7855 
7856 /*@
7857   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
7858 
7859   Input Parameter:
7860 . dm   - The DMPlex object
7861 
7862   Output Parameter:
7863 . globalVertexNumbers - Global vertex numbers for all vertices on this process
7864 
7865   Level: developer
7866 
7867 .seealso DMPlexGetCellNumbering()
7868 @*/
7869 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
7870 {
7871   DM_Plex       *mesh = (DM_Plex*) dm->data;
7872   PetscErrorCode ierr;
7873 
7874   PetscFunctionBegin;
7875   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7876   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
7877   *globalVertexNumbers = mesh->globalVertexNumbers;
7878   PetscFunctionReturn(0);
7879 }
7880 
7881 /*@
7882   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
7883 
7884   Input Parameter:
7885 . dm   - The DMPlex object
7886 
7887   Output Parameter:
7888 . globalPointNumbers - Global numbers for all points on this process
7889 
7890   Level: developer
7891 
7892 .seealso DMPlexGetCellNumbering()
7893 @*/
7894 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
7895 {
7896   IS             nums[4];
7897   PetscInt       depths[4], gdepths[4], starts[4];
7898   PetscInt       depth, d, shift = 0;
7899   PetscErrorCode ierr;
7900 
7901   PetscFunctionBegin;
7902   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7903   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7904   /* For unstratified meshes use dim instead of depth */
7905   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
7906   for (d = 0; d <= depth; ++d) {
7907     PetscInt end;
7908 
7909     depths[d] = depth-d;
7910     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
7911     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
7912   }
7913   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
7914   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
7915   for (d = 0; d <= depth; ++d) {
7916     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
7917   }
7918   for (d = 0; d <= depth; ++d) {
7919     PetscInt pStart, pEnd, gsize;
7920 
7921     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
7922     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
7923     shift += gsize;
7924   }
7925   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
7926   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
7927   PetscFunctionReturn(0);
7928 }
7929 
7930 /*@
7931   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
7932 
7933   Input Parameter:
7934 . dm - The DMPlex object
7935 
7936   Output Parameter:
7937 . ranks - The rank field
7938 
7939   Options Database Keys:
7940 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
7941 
7942   Level: intermediate
7943 
7944 .seealso: DMView()
7945 @*/
7946 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
7947 {
7948   DM             rdm;
7949   PetscFE        fe;
7950   PetscScalar   *r;
7951   PetscMPIInt    rank;
7952   DMPolytopeType ct;
7953   PetscInt       dim, cStart, cEnd, c;
7954   PetscBool      simplex;
7955   PetscErrorCode ierr;
7956 
7957   PetscFunctionBeginUser;
7958   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7959   PetscValidPointer(ranks, 2);
7960   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
7961   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
7962   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
7963   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7964   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
7965   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
7966   ierr = PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
7967   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
7968   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
7969   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
7970   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
7971   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
7972   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
7973   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
7974   for (c = cStart; c < cEnd; ++c) {
7975     PetscScalar *lr;
7976 
7977     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
7978     if (lr) *lr = rank;
7979   }
7980   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
7981   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
7982   PetscFunctionReturn(0);
7983 }
7984 
7985 /*@
7986   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
7987 
7988   Input Parameters:
7989 + dm    - The DMPlex
7990 - label - The DMLabel
7991 
7992   Output Parameter:
7993 . val - The label value field
7994 
7995   Options Database Keys:
7996 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
7997 
7998   Level: intermediate
7999 
8000 .seealso: DMView()
8001 @*/
8002 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
8003 {
8004   DM             rdm;
8005   PetscFE        fe;
8006   PetscScalar   *v;
8007   PetscInt       dim, cStart, cEnd, c;
8008   PetscErrorCode ierr;
8009 
8010   PetscFunctionBeginUser;
8011   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8012   PetscValidPointer(label, 2);
8013   PetscValidPointer(val, 3);
8014   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8015   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8016   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
8017   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
8018   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8019   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8020   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8021   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8022   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
8023   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
8024   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
8025   for (c = cStart; c < cEnd; ++c) {
8026     PetscScalar *lv;
8027     PetscInt     cval;
8028 
8029     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
8030     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
8031     *lv = cval;
8032   }
8033   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
8034   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8035   PetscFunctionReturn(0);
8036 }
8037 
8038 /*@
8039   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
8040 
8041   Input Parameter:
8042 . dm - The DMPlex object
8043 
8044   Notes:
8045   This is a useful diagnostic when creating meshes programmatically.
8046 
8047   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8048 
8049   Level: developer
8050 
8051 .seealso: DMCreate(), DMSetFromOptions()
8052 @*/
8053 PetscErrorCode DMPlexCheckSymmetry(DM dm)
8054 {
8055   PetscSection    coneSection, supportSection;
8056   const PetscInt *cone, *support;
8057   PetscInt        coneSize, c, supportSize, s;
8058   PetscInt        pStart, pEnd, p, pp, csize, ssize;
8059   PetscBool       storagecheck = PETSC_TRUE;
8060   PetscErrorCode  ierr;
8061 
8062   PetscFunctionBegin;
8063   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8064   ierr = DMViewFromOptions(dm, NULL, "-sym_dm_view");CHKERRQ(ierr);
8065   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
8066   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
8067   /* Check that point p is found in the support of its cone points, and vice versa */
8068   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
8069   for (p = pStart; p < pEnd; ++p) {
8070     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
8071     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
8072     for (c = 0; c < coneSize; ++c) {
8073       PetscBool dup = PETSC_FALSE;
8074       PetscInt  d;
8075       for (d = c-1; d >= 0; --d) {
8076         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
8077       }
8078       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
8079       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
8080       for (s = 0; s < supportSize; ++s) {
8081         if (support[s] == p) break;
8082       }
8083       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
8084         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
8085         for (s = 0; s < coneSize; ++s) {
8086           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
8087         }
8088         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8089         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
8090         for (s = 0; s < supportSize; ++s) {
8091           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
8092         }
8093         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8094         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
8095         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
8096       }
8097     }
8098     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
8099     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
8100     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
8101     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
8102     for (s = 0; s < supportSize; ++s) {
8103       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
8104       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
8105       for (c = 0; c < coneSize; ++c) {
8106         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
8107         if (cone[c] != pp) { c = 0; break; }
8108         if (cone[c] == p) break;
8109       }
8110       if (c >= coneSize) {
8111         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
8112         for (c = 0; c < supportSize; ++c) {
8113           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
8114         }
8115         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8116         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
8117         for (c = 0; c < coneSize; ++c) {
8118           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
8119         }
8120         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8121         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
8122       }
8123     }
8124   }
8125   if (storagecheck) {
8126     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
8127     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
8128     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
8129   }
8130   PetscFunctionReturn(0);
8131 }
8132 
8133 /*
8134   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
8135 */
8136 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
8137 {
8138   DMPolytopeType  cct;
8139   PetscInt        ptpoints[4];
8140   const PetscInt *cone, *ccone, *ptcone;
8141   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
8142   PetscErrorCode  ierr;
8143 
8144   PetscFunctionBegin;
8145   *unsplit = 0;
8146   switch (ct) {
8147     case DM_POLYTOPE_SEG_PRISM_TENSOR:
8148       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8149       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8150       for (cp = 0; cp < coneSize; ++cp) {
8151         ierr = DMPlexGetCellType(dm, cone[cp], &cct);CHKERRQ(ierr);
8152         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
8153       }
8154       break;
8155     case DM_POLYTOPE_TRI_PRISM_TENSOR:
8156     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8157       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8158       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8159       for (cp = 0; cp < coneSize; ++cp) {
8160         ierr = DMPlexGetCone(dm, cone[cp], &ccone);CHKERRQ(ierr);
8161         ierr = DMPlexGetConeSize(dm, cone[cp], &cconeSize);CHKERRQ(ierr);
8162         for (ccp = 0; ccp < cconeSize; ++ccp) {
8163           ierr = DMPlexGetCellType(dm, ccone[ccp], &cct);CHKERRQ(ierr);
8164           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
8165             PetscInt p;
8166             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
8167             if (p == npt) ptpoints[npt++] = ccone[ccp];
8168           }
8169         }
8170       }
8171       break;
8172     default: break;
8173   }
8174   for (pt = 0; pt < npt; ++pt) {
8175     ierr = DMPlexGetCone(dm, ptpoints[pt], &ptcone);CHKERRQ(ierr);
8176     if (ptcone[0] == ptcone[1]) ++(*unsplit);
8177   }
8178   PetscFunctionReturn(0);
8179 }
8180 
8181 /*@
8182   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
8183 
8184   Input Parameters:
8185 + dm - The DMPlex object
8186 - cellHeight - Normally 0
8187 
8188   Notes:
8189   This is a useful diagnostic when creating meshes programmatically.
8190   Currently applicable only to homogeneous simplex or tensor meshes.
8191 
8192   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8193 
8194   Level: developer
8195 
8196 .seealso: DMCreate(), DMSetFromOptions()
8197 @*/
8198 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
8199 {
8200   DMPlexInterpolatedFlag interp;
8201   DMPolytopeType         ct;
8202   PetscInt               vStart, vEnd, cStart, cEnd, c;
8203   PetscErrorCode         ierr;
8204 
8205   PetscFunctionBegin;
8206   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8207   ierr = DMPlexIsInterpolated(dm, &interp);CHKERRQ(ierr);
8208   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8209   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8210   for (c = cStart; c < cEnd; ++c) {
8211     PetscInt *closure = NULL;
8212     PetscInt  coneSize, closureSize, cl, Nv = 0;
8213 
8214     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8215     if ((PetscInt) ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has no cell type", c);
8216     if (ct == DM_POLYTOPE_UNKNOWN) continue;
8217     if (interp == DMPLEX_INTERPOLATED_FULL) {
8218       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8219       if (coneSize != DMPolytopeTypeGetConeSize(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has cone size %D != %D", c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
8220     }
8221     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8222     for (cl = 0; cl < closureSize*2; cl += 2) {
8223       const PetscInt p = closure[cl];
8224       if ((p >= vStart) && (p < vEnd)) ++Nv;
8225     }
8226     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8227     /* Special Case: Tensor faces with identified vertices */
8228     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
8229       PetscInt unsplit;
8230 
8231       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8232       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
8233     }
8234     if (Nv != DMPolytopeTypeGetNumVertices(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D vertices != %D", c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
8235   }
8236   PetscFunctionReturn(0);
8237 }
8238 
8239 /*@
8240   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
8241 
8242   Not Collective
8243 
8244   Input Parameters:
8245 + dm - The DMPlex object
8246 - cellHeight - Normally 0
8247 
8248   Notes:
8249   This is a useful diagnostic when creating meshes programmatically.
8250   This routine is only relevant for meshes that are fully interpolated across all ranks.
8251   It will error out if a partially interpolated mesh is given on some rank.
8252   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
8253 
8254   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8255 
8256   Level: developer
8257 
8258 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
8259 @*/
8260 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
8261 {
8262   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
8263   PetscErrorCode ierr;
8264   DMPlexInterpolatedFlag interpEnum;
8265 
8266   PetscFunctionBegin;
8267   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8268   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
8269   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
8270   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
8271     PetscMPIInt rank;
8272     MPI_Comm    comm;
8273 
8274     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8275     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8276     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
8277   }
8278 
8279   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8280   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8281   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8282   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
8283     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
8284     for (c = cStart; c < cEnd; ++c) {
8285       const PetscInt      *cone, *ornt, *faceSizes, *faces;
8286       const DMPolytopeType *faceTypes;
8287       DMPolytopeType        ct;
8288       PetscInt              numFaces, coneSize, f;
8289       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
8290 
8291       ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8292       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8293       if (unsplit) continue;
8294       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8295       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8296       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
8297       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8298       for (cl = 0; cl < closureSize*2; cl += 2) {
8299         const PetscInt p = closure[cl];
8300         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
8301       }
8302       ierr = DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8303       if (coneSize != numFaces) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D faces but should have %D", c, DMPolytopeTypes[ct], coneSize, numFaces);
8304       for (f = 0; f < numFaces; ++f) {
8305         DMPolytopeType fct;
8306         PetscInt       *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
8307 
8308         ierr = DMPlexGetCellType(dm, cone[f], &fct);CHKERRQ(ierr);
8309         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8310         for (cl = 0; cl < fclosureSize*2; cl += 2) {
8311           const PetscInt p = fclosure[cl];
8312           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
8313         }
8314         if (fnumCorners != faceSizes[f]) SETERRQ7(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %D) of cell %D of type %s has %D vertices but should have %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
8315         for (v = 0; v < fnumCorners; ++v) {
8316           if (fclosure[v] != faces[fOff+v]) SETERRQ8(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %d) of cell %D of type %s vertex %D, %D != %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff+v]);
8317         }
8318         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8319         fOff += faceSizes[f];
8320       }
8321       ierr = DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8322       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8323     }
8324   }
8325   PetscFunctionReturn(0);
8326 }
8327 
8328 /*@
8329   DMPlexCheckGeometry - Check the geometry of mesh cells
8330 
8331   Input Parameter:
8332 . dm - The DMPlex object
8333 
8334   Notes:
8335   This is a useful diagnostic when creating meshes programmatically.
8336 
8337   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8338 
8339   Level: developer
8340 
8341 .seealso: DMCreate(), DMSetFromOptions()
8342 @*/
8343 PetscErrorCode DMPlexCheckGeometry(DM dm)
8344 {
8345   Vec            coordinates;
8346   PetscReal      detJ, J[9], refVol = 1.0;
8347   PetscReal      vol;
8348   PetscBool      periodic;
8349   PetscInt       dim, depth, dE, d, cStart, cEnd, c;
8350   PetscErrorCode ierr;
8351 
8352   PetscFunctionBegin;
8353   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8354   ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
8355   if (dim != dE) PetscFunctionReturn(0);
8356   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8357   ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
8358   for (d = 0; d < dim; ++d) refVol *= 2.0;
8359   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8360   /* Make sure local coordinates are created, because that step is collective */
8361   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
8362   for (c = cStart; c < cEnd; ++c) {
8363     DMPolytopeType ct;
8364     PetscInt       unsplit;
8365     PetscBool      ignoreZeroVol = PETSC_FALSE;
8366 
8367     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8368     switch (ct) {
8369       case DM_POLYTOPE_SEG_PRISM_TENSOR:
8370       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8371       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8372         ignoreZeroVol = PETSC_TRUE; break;
8373       default: break;
8374     }
8375     switch (ct) {
8376       case DM_POLYTOPE_TRI_PRISM:
8377       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8378       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8379       case DM_POLYTOPE_PYRAMID:
8380         continue;
8381       default: break;
8382     }
8383     ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8384     if (unsplit) continue;
8385     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
8386     if (detJ < -PETSC_SMALL || (detJ <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double) detJ);
8387     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
8388     if (depth > 1 && !periodic) {
8389       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
8390       if (vol < -PETSC_SMALL || (vol <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double) vol);
8391       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
8392     }
8393   }
8394   PetscFunctionReturn(0);
8395 }
8396 
8397 /*@
8398   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
8399 
8400   Input Parameters:
8401 . dm - The DMPlex object
8402 
8403   Notes:
8404   This is mainly intended for debugging/testing purposes.
8405   It currently checks only meshes with no partition overlapping.
8406 
8407   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8408 
8409   Level: developer
8410 
8411 .seealso: DMGetPointSF(), DMSetFromOptions()
8412 @*/
8413 PetscErrorCode DMPlexCheckPointSF(DM dm)
8414 {
8415   PetscSF         pointSF;
8416   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
8417   const PetscInt *locals, *rootdegree;
8418   PetscBool       distributed;
8419   PetscErrorCode  ierr;
8420 
8421   PetscFunctionBegin;
8422   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8423   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
8424   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
8425   if (!distributed) PetscFunctionReturn(0);
8426   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
8427   if (overlap) {
8428     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");CHKERRQ(ierr);
8429     PetscFunctionReturn(0);
8430   }
8431   if (!pointSF) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
8432   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
8433   if (nroots < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
8434   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
8435   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
8436 
8437   /* 1) check there are no faces in 2D, cells in 3D, in interface */
8438   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8439   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8440   for (l = 0; l < nleaves; ++l) {
8441     const PetscInt point = locals[l];
8442 
8443     if (point >= cStart && point < cEnd) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
8444   }
8445 
8446   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
8447   for (l = 0; l < nleaves; ++l) {
8448     const PetscInt  point = locals[l];
8449     const PetscInt *cone;
8450     PetscInt        coneSize, c, idx;
8451 
8452     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
8453     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
8454     for (c = 0; c < coneSize; ++c) {
8455       if (!rootdegree[cone[c]]) {
8456         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
8457         if (idx < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
8458       }
8459     }
8460   }
8461   PetscFunctionReturn(0);
8462 }
8463 
8464 typedef struct cell_stats
8465 {
8466   PetscReal min, max, sum, squaresum;
8467   PetscInt  count;
8468 } cell_stats_t;
8469 
8470 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
8471 {
8472   PetscInt i, N = *len;
8473 
8474   for (i = 0; i < N; i++) {
8475     cell_stats_t *A = (cell_stats_t *) a;
8476     cell_stats_t *B = (cell_stats_t *) b;
8477 
8478     B->min = PetscMin(A->min,B->min);
8479     B->max = PetscMax(A->max,B->max);
8480     B->sum += A->sum;
8481     B->squaresum += A->squaresum;
8482     B->count += A->count;
8483   }
8484 }
8485 
8486 /*@
8487   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
8488 
8489   Collective on dm
8490 
8491   Input Parameters:
8492 + dm        - The DMPlex object
8493 . output    - If true, statistics will be displayed on stdout
8494 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
8495 
8496   Notes:
8497   This is mainly intended for debugging/testing purposes.
8498 
8499   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8500 
8501   Level: developer
8502 
8503 .seealso: DMSetFromOptions(), DMPlexComputeOrthogonalQuality()
8504 @*/
8505 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
8506 {
8507   DM             dmCoarse;
8508   cell_stats_t   stats, globalStats;
8509   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
8510   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
8511   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
8512   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
8513   PetscMPIInt    rank,size;
8514   PetscErrorCode ierr;
8515 
8516   PetscFunctionBegin;
8517   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8518   stats.min   = PETSC_MAX_REAL;
8519   stats.max   = PETSC_MIN_REAL;
8520   stats.sum   = stats.squaresum = 0.;
8521   stats.count = 0;
8522 
8523   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
8524   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8525   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
8526   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
8527   ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
8528   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
8529   for (c = cStart; c < cEnd; c++) {
8530     PetscInt  i;
8531     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
8532 
8533     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
8534     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
8535     for (i = 0; i < PetscSqr(cdim); ++i) {
8536       frobJ    += J[i] * J[i];
8537       frobInvJ += invJ[i] * invJ[i];
8538     }
8539     cond2 = frobJ * frobInvJ;
8540     cond  = PetscSqrtReal(cond2);
8541 
8542     stats.min        = PetscMin(stats.min,cond);
8543     stats.max        = PetscMax(stats.max,cond);
8544     stats.sum       += cond;
8545     stats.squaresum += cond2;
8546     stats.count++;
8547     if (output && cond > limit) {
8548       PetscSection coordSection;
8549       Vec          coordsLocal;
8550       PetscScalar *coords = NULL;
8551       PetscInt     Nv, d, clSize, cl, *closure = NULL;
8552 
8553       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
8554       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
8555       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8556       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
8557       for (i = 0; i < Nv/cdim; ++i) {
8558         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
8559         for (d = 0; d < cdim; ++d) {
8560           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
8561           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
8562         }
8563         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
8564       }
8565       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8566       for (cl = 0; cl < clSize*2; cl += 2) {
8567         const PetscInt edge = closure[cl];
8568 
8569         if ((edge >= eStart) && (edge < eEnd)) {
8570           PetscReal len;
8571 
8572           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
8573           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
8574         }
8575       }
8576       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8577       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8578     }
8579   }
8580   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
8581 
8582   if (size > 1) {
8583     PetscMPIInt   blockLengths[2] = {4,1};
8584     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
8585     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
8586     MPI_Op        statReduce;
8587 
8588     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRMPI(ierr);
8589     ierr = MPI_Type_commit(&statType);CHKERRMPI(ierr);
8590     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRMPI(ierr);
8591     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRMPI(ierr);
8592     ierr = MPI_Op_free(&statReduce);CHKERRMPI(ierr);
8593     ierr = MPI_Type_free(&statType);CHKERRMPI(ierr);
8594   } else {
8595     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
8596   }
8597   if (!rank) {
8598     count = globalStats.count;
8599     min   = globalStats.min;
8600     max   = globalStats.max;
8601     mean  = globalStats.sum / globalStats.count;
8602     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
8603   }
8604 
8605   if (output) {
8606     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
8607   }
8608   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
8609 
8610   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
8611   if (dmCoarse) {
8612     PetscBool isplex;
8613 
8614     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
8615     if (isplex) {
8616       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
8617     }
8618   }
8619   PetscFunctionReturn(0);
8620 }
8621 
8622 /*@
8623   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
8624   orthogonal quality below given tolerance.
8625 
8626   Collective on dm
8627 
8628   Input Parameters:
8629 + dm   - The DMPlex object
8630 . fv   - Optional PetscFV object for pre-computed cell/face centroid information
8631 - atol - [0, 1] Absolute tolerance for tagging cells.
8632 
8633   Output Parameters:
8634 + OrthQual      - Vec containing orthogonal quality per cell
8635 - OrthQualLabel - DMLabel tagging cells below atol with DM_ADAPT_REFINE
8636 
8637   Options Database Keys:
8638 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only PETSCVIEWERASCII is
8639 supported.
8640 - -dm_plex_orthogonal_quality_vec_view - view OrthQual vector.
8641 
8642   Notes:
8643   Orthogonal quality is given by the following formula:
8644 
8645   \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]
8646 
8647   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
8648   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
8649   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
8650   calculating the cosine of the angle between these vectors.
8651 
8652   Orthogonal quality ranges from 1 (best) to 0 (worst).
8653 
8654   This routine is mainly useful for FVM, however is not restricted to only FVM. The PetscFV object is optionally used to check for
8655   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
8656 
8657   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
8658 
8659   Level: intermediate
8660 
8661 .seealso: DMPlexCheckCellShape(), DMCreateLabel()
8662 @*/
8663 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
8664 {
8665   PetscInt                nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
8666   PetscInt                *idx;
8667   PetscScalar             *oqVals;
8668   const PetscScalar       *cellGeomArr, *faceGeomArr;
8669   PetscReal               *ci, *fi, *Ai;
8670   MPI_Comm                comm;
8671   Vec                     cellgeom, facegeom;
8672   DM                      dmFace, dmCell;
8673   IS                      glob;
8674   ISLocalToGlobalMapping  ltog;
8675   PetscViewer             vwr;
8676   PetscErrorCode          ierr;
8677 
8678   PetscFunctionBegin;
8679   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8680   if (fv) {PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);}
8681   PetscValidPointer(OrthQual, 4);
8682   if (PetscUnlikelyDebug(atol < 0.0 || atol > 1.0)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Absolute tolerance %g not in [0,1]",(double)atol);
8683   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8684   ierr = DMGetDimension(dm, &nc);CHKERRQ(ierr);
8685   if (nc < 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %D)", nc);
8686   {
8687     DMPlexInterpolatedFlag interpFlag;
8688 
8689     ierr = DMPlexIsInterpolated(dm, &interpFlag);CHKERRQ(ierr);
8690     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
8691       PetscMPIInt rank;
8692 
8693       ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8694       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
8695     }
8696   }
8697   if (OrthQualLabel) {
8698     PetscValidPointer(OrthQualLabel, 5);
8699     ierr = DMCreateLabel(dm, "Orthogonal_Quality");CHKERRQ(ierr);
8700     ierr = DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel);CHKERRQ(ierr);
8701   } else {*OrthQualLabel = NULL;}
8702   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8703   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8704   ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_TRUE, &glob);CHKERRQ(ierr);
8705   ierr = ISLocalToGlobalMappingCreateIS(glob, &ltog);CHKERRQ(ierr);
8706   ierr = ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
8707   ierr = VecCreate(comm, OrthQual);CHKERRQ(ierr);
8708   ierr = VecSetType(*OrthQual, VECSTANDARD);CHKERRQ(ierr);
8709   ierr = VecSetSizes(*OrthQual, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
8710   ierr = VecSetLocalToGlobalMapping(*OrthQual, ltog);CHKERRQ(ierr);
8711   ierr = VecSetUp(*OrthQual);CHKERRQ(ierr);
8712   ierr = ISDestroy(&glob);CHKERRQ(ierr);
8713   ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
8714   ierr = DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL);CHKERRQ(ierr);
8715   ierr = VecGetArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8716   ierr = VecGetArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8717   ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
8718   ierr = VecGetDM(facegeom, &dmFace);CHKERRQ(ierr);
8719   ierr = PetscMalloc5(cEnd-cStart, &idx, cEnd-cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai);CHKERRQ(ierr);
8720   for (cell = cStart; cell < cEnd; cellIter++,cell++) {
8721     PetscInt           cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
8722     PetscInt           cellarr[2], *adj = NULL;
8723     PetscScalar        *cArr, *fArr;
8724     PetscReal          minvalc = 1.0, minvalf = 1.0;
8725     PetscFVCellGeom    *cg;
8726 
8727     idx[cellIter] = cell-cStart;
8728     cellarr[0] = cell;
8729     /* Make indexing into cellGeom easier */
8730     ierr = DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg);CHKERRQ(ierr);
8731     ierr = DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
8732     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
8733     ierr = PetscCalloc2(adjSize, &cArr, adjSize, &fArr);CHKERRQ(ierr);
8734     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++,cellneigh++) {
8735       PetscInt         i;
8736       const PetscInt   neigh = adj[cellneigh];
8737       PetscReal        normci = 0, normfi = 0, normai = 0;
8738       PetscFVCellGeom  *cgneigh;
8739       PetscFVFaceGeom  *fg;
8740 
8741       /* Don't count ourselves in the neighbor list */
8742       if (neigh == cell) continue;
8743       ierr = DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh);CHKERRQ(ierr);
8744       cellarr[1] = neigh;
8745       {
8746         PetscInt       numcovpts;
8747         const PetscInt *covpts;
8748 
8749         ierr = DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8750         ierr = DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg);CHKERRQ(ierr);
8751         ierr = DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8752       }
8753 
8754       /* Compute c_i, f_i and their norms */
8755       for (i = 0; i < nc; i++) {
8756         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
8757         fi[i] = fg->centroid[i] - cg->centroid[i];
8758         Ai[i] = fg->normal[i];
8759         normci += PetscPowReal(ci[i], 2);
8760         normfi += PetscPowReal(fi[i], 2);
8761         normai += PetscPowReal(Ai[i], 2);
8762       }
8763       normci = PetscSqrtReal(normci);
8764       normfi = PetscSqrtReal(normfi);
8765       normai = PetscSqrtReal(normai);
8766 
8767       /* Normalize and compute for each face-cell-normal pair */
8768       for (i = 0; i < nc; i++) {
8769         ci[i] = ci[i]/normci;
8770         fi[i] = fi[i]/normfi;
8771         Ai[i] = Ai[i]/normai;
8772         /* PetscAbs because I don't know if normals are guaranteed to point out */
8773         cArr[cellneighiter] += PetscAbs(Ai[i]*ci[i]);
8774         fArr[cellneighiter] += PetscAbs(Ai[i]*fi[i]);
8775       }
8776       if (PetscRealPart(cArr[cellneighiter]) < minvalc) {
8777         minvalc = PetscRealPart(cArr[cellneighiter]);
8778       }
8779       if (PetscRealPart(fArr[cellneighiter]) < minvalf) {
8780         minvalf = PetscRealPart(fArr[cellneighiter]);
8781       }
8782     }
8783     ierr = PetscFree(adj);CHKERRQ(ierr);
8784     ierr = PetscFree2(cArr, fArr);CHKERRQ(ierr);
8785     /* Defer to cell if they're equal */
8786     oqVals[cellIter] = PetscMin(minvalf, minvalc);
8787     if (OrthQualLabel) {
8788       if (PetscRealPart(oqVals[cellIter]) <= atol) {ierr = DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE);CHKERRQ(ierr);}
8789     }
8790   }
8791   ierr = VecSetValuesLocal(*OrthQual, cEnd-cStart, idx, oqVals, INSERT_VALUES);CHKERRQ(ierr);
8792   ierr = VecAssemblyBegin(*OrthQual);CHKERRQ(ierr);
8793   ierr = VecAssemblyEnd(*OrthQual);CHKERRQ(ierr);
8794   ierr = VecRestoreArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8795   ierr = VecRestoreArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8796   ierr = PetscOptionsGetViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL);CHKERRQ(ierr);
8797   if (OrthQualLabel) {
8798     if (vwr) {ierr = DMLabelView(*OrthQualLabel, vwr);CHKERRQ(ierr);}
8799   }
8800   ierr = PetscFree5(idx, oqVals, ci, fi, Ai);CHKERRQ(ierr);
8801   ierr = PetscViewerDestroy(&vwr);CHKERRQ(ierr);
8802   ierr = VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view");CHKERRQ(ierr);
8803   PetscFunctionReturn(0);
8804 }
8805 
8806 /* this is here insead of DMGetOutputDM because output DM still has constraints in the local indices that affect
8807  * interpolator construction */
8808 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
8809 {
8810   PetscSection   section, newSection, gsection;
8811   PetscSF        sf;
8812   PetscBool      hasConstraints, ghasConstraints;
8813   PetscErrorCode ierr;
8814 
8815   PetscFunctionBegin;
8816   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
8817   PetscValidPointer(odm,2);
8818   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
8819   ierr = PetscSectionHasConstraints(section, &hasConstraints);CHKERRQ(ierr);
8820   ierr = MPI_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8821   if (!ghasConstraints) {
8822     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
8823     *odm = dm;
8824     PetscFunctionReturn(0);
8825   }
8826   ierr = DMClone(dm, odm);CHKERRQ(ierr);
8827   ierr = DMCopyFields(dm, *odm);CHKERRQ(ierr);
8828   ierr = DMGetLocalSection(*odm, &newSection);CHKERRQ(ierr);
8829   ierr = DMGetPointSF(*odm, &sf);CHKERRQ(ierr);
8830   ierr = PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_FALSE, &gsection);CHKERRQ(ierr);
8831   ierr = DMSetGlobalSection(*odm, gsection);CHKERRQ(ierr);
8832   ierr = PetscSectionDestroy(&gsection);CHKERRQ(ierr);
8833   PetscFunctionReturn(0);
8834 }
8835 
8836 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
8837 {
8838   DM             dmco, dmfo;
8839   Mat            interpo;
8840   Vec            rscale;
8841   Vec            cglobalo, clocal;
8842   Vec            fglobal, fglobalo, flocal;
8843   PetscBool      regular;
8844   PetscErrorCode ierr;
8845 
8846   PetscFunctionBegin;
8847   ierr = DMGetFullDM(dmc, &dmco);CHKERRQ(ierr);
8848   ierr = DMGetFullDM(dmf, &dmfo);CHKERRQ(ierr);
8849   ierr = DMSetCoarseDM(dmfo, dmco);CHKERRQ(ierr);
8850   ierr = DMPlexGetRegularRefinement(dmf, &regular);CHKERRQ(ierr);
8851   ierr = DMPlexSetRegularRefinement(dmfo, regular);CHKERRQ(ierr);
8852   ierr = DMCreateInterpolation(dmco, dmfo, &interpo, &rscale);CHKERRQ(ierr);
8853   ierr = DMCreateGlobalVector(dmco, &cglobalo);CHKERRQ(ierr);
8854   ierr = DMCreateLocalVector(dmc, &clocal);CHKERRQ(ierr);
8855   ierr = VecSet(cglobalo, 0.);CHKERRQ(ierr);
8856   ierr = VecSet(clocal, 0.);CHKERRQ(ierr);
8857   ierr = DMCreateGlobalVector(dmf, &fglobal);CHKERRQ(ierr);
8858   ierr = DMCreateGlobalVector(dmfo, &fglobalo);CHKERRQ(ierr);
8859   ierr = DMCreateLocalVector(dmf, &flocal);CHKERRQ(ierr);
8860   ierr = VecSet(fglobal, 0.);CHKERRQ(ierr);
8861   ierr = VecSet(fglobalo, 0.);CHKERRQ(ierr);
8862   ierr = VecSet(flocal, 0.);CHKERRQ(ierr);
8863   ierr = DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL);CHKERRQ(ierr);
8864   ierr = DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8865   ierr = DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8866   ierr = MatMult(interpo, cglobalo, fglobalo);CHKERRQ(ierr);
8867   ierr = DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8868   ierr = DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8869   ierr = DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8870   ierr = DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8871   *shift = fglobal;
8872   ierr = VecDestroy(&flocal);CHKERRQ(ierr);
8873   ierr = VecDestroy(&fglobalo);CHKERRQ(ierr);
8874   ierr = VecDestroy(&clocal);CHKERRQ(ierr);
8875   ierr = VecDestroy(&cglobalo);CHKERRQ(ierr);
8876   ierr = VecDestroy(&rscale);CHKERRQ(ierr);
8877   ierr = MatDestroy(&interpo);CHKERRQ(ierr);
8878   ierr = DMDestroy(&dmfo);CHKERRQ(ierr);
8879   ierr = DMDestroy(&dmco);CHKERRQ(ierr);
8880   PetscFunctionReturn(0);
8881 }
8882 
8883 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
8884 {
8885   PetscObject    shifto;
8886   Vec            shift;
8887 
8888   PetscErrorCode ierr;
8889 
8890   PetscFunctionBegin;
8891   if (!interp) {
8892     Vec rscale;
8893 
8894     ierr = DMCreateInterpolation(coarse, fine, &interp, &rscale);CHKERRQ(ierr);
8895     ierr = VecDestroy(&rscale);CHKERRQ(ierr);
8896   } else {
8897     ierr = PetscObjectReference((PetscObject)interp);CHKERRQ(ierr);
8898   }
8899   ierr = PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto);CHKERRQ(ierr);
8900   if (!shifto) {
8901     ierr = DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift);CHKERRQ(ierr);
8902     ierr = PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject) shift);CHKERRQ(ierr);
8903     shifto = (PetscObject) shift;
8904     ierr = VecDestroy(&shift);CHKERRQ(ierr);
8905   }
8906   shift = (Vec) shifto;
8907   ierr = MatInterpolate(interp, coarseSol, fineSol);CHKERRQ(ierr);
8908   ierr = VecAXPY(fineSol, 1.0, shift);CHKERRQ(ierr);
8909   ierr = MatDestroy(&interp);CHKERRQ(ierr);
8910   PetscFunctionReturn(0);
8911 }
8912 
8913 /* Pointwise interpolation
8914      Just code FEM for now
8915      u^f = I u^c
8916      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
8917      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
8918      I_{ij} = psi^f_i phi^c_j
8919 */
8920 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
8921 {
8922   PetscSection   gsc, gsf;
8923   PetscInt       m, n;
8924   void          *ctx;
8925   DM             cdm;
8926   PetscBool      regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
8927   PetscErrorCode ierr;
8928 
8929   PetscFunctionBegin;
8930   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
8931   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
8932   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
8933   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
8934 
8935   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
8936   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
8937   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
8938   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
8939   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
8940 
8941   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
8942   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
8943   if (!isRefined || (regular && cdm == dmCoarse)) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx);CHKERRQ(ierr);}
8944   else                                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
8945   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
8946   if (scaling) {
8947     /* Use naive scaling */
8948     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
8949   }
8950   PetscFunctionReturn(0);
8951 }
8952 
8953 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
8954 {
8955   PetscErrorCode ierr;
8956   VecScatter     ctx;
8957 
8958   PetscFunctionBegin;
8959   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
8960   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
8961   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
8962   PetscFunctionReturn(0);
8963 }
8964 
8965 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux,
8966                                 const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
8967                                 const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
8968                                 PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
8969 {
8970   g0[0] = 1.0;
8971 }
8972 
8973 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
8974 {
8975   PetscSection   gsc, gsf;
8976   PetscInt       m, n;
8977   void          *ctx;
8978   DM             cdm;
8979   PetscBool      regular;
8980   PetscErrorCode ierr;
8981 
8982   PetscFunctionBegin;
8983   if (dmFine == dmCoarse) {
8984     DM       dmc;
8985     PetscDS  ds;
8986     Vec      u;
8987     IS       cellIS;
8988     PetscFormKey key;
8989     PetscInt depth;
8990 
8991     ierr = DMClone(dmFine, &dmc);CHKERRQ(ierr);
8992     ierr = DMCopyDisc(dmFine, dmc);CHKERRQ(ierr);
8993     ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
8994     ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
8995     ierr = DMCreateMatrix(dmc, mass);CHKERRQ(ierr);
8996     ierr = DMGetGlobalVector(dmc, &u);CHKERRQ(ierr);
8997     ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
8998     ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
8999     ierr = MatZeroEntries(*mass);CHKERRQ(ierr);
9000     key.label = NULL;
9001     key.value = 0;
9002     key.field = 0;
9003     key.part  = 0;
9004     ierr = DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL);CHKERRQ(ierr);
9005     ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9006     ierr = DMRestoreGlobalVector(dmc, &u);CHKERRQ(ierr);
9007     ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9008   } else {
9009     ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9010     ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9011     ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9012     ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9013 
9014     ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
9015     ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9016     ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
9017     ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9018 
9019     ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9020     ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9021     if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9022     else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9023   }
9024   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
9025   PetscFunctionReturn(0);
9026 }
9027 
9028 /*@
9029   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9030 
9031   Input Parameter:
9032 . dm - The DMPlex object
9033 
9034   Output Parameter:
9035 . regular - The flag
9036 
9037   Level: intermediate
9038 
9039 .seealso: DMPlexSetRegularRefinement()
9040 @*/
9041 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
9042 {
9043   PetscFunctionBegin;
9044   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9045   PetscValidPointer(regular, 2);
9046   *regular = ((DM_Plex *) dm->data)->regularRefinement;
9047   PetscFunctionReturn(0);
9048 }
9049 
9050 /*@
9051   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9052 
9053   Input Parameters:
9054 + dm - The DMPlex object
9055 - regular - The flag
9056 
9057   Level: intermediate
9058 
9059 .seealso: DMPlexGetRegularRefinement()
9060 @*/
9061 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
9062 {
9063   PetscFunctionBegin;
9064   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9065   ((DM_Plex *) dm->data)->regularRefinement = regular;
9066   PetscFunctionReturn(0);
9067 }
9068 
9069 /*@
9070   DMPlexGetCellRefinerType - Get the strategy for refining a cell
9071 
9072   Input Parameter:
9073 . dm - The DMPlex object
9074 
9075   Output Parameter:
9076 . cr - The strategy number
9077 
9078   Level: intermediate
9079 
9080 .seealso: DMPlexSetCellRefinerType(), DMPlexSetRegularRefinement()
9081 @*/
9082 PetscErrorCode DMPlexGetCellRefinerType(DM dm, DMPlexCellRefinerType *cr)
9083 {
9084   PetscFunctionBegin;
9085   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9086   PetscValidPointer(cr, 2);
9087   *cr = ((DM_Plex *) dm->data)->cellRefiner;
9088   PetscFunctionReturn(0);
9089 }
9090 
9091 /*@
9092   DMPlexSetCellRefinerType - Set the strategy for refining a cell
9093 
9094   Input Parameters:
9095 + dm - The DMPlex object
9096 - cr - The strategy number
9097 
9098   Level: intermediate
9099 
9100 .seealso: DMPlexGetCellRefinerType(), DMPlexGetRegularRefinement()
9101 @*/
9102 PetscErrorCode DMPlexSetCellRefinerType(DM dm, DMPlexCellRefinerType cr)
9103 {
9104   PetscFunctionBegin;
9105   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9106   ((DM_Plex *) dm->data)->cellRefiner = cr;
9107   PetscFunctionReturn(0);
9108 }
9109 
9110 /* anchors */
9111 /*@
9112   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
9113   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
9114 
9115   not collective
9116 
9117   Input Parameters:
9118 . dm - The DMPlex object
9119 
9120   Output Parameters:
9121 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
9122 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
9123 
9124   Level: intermediate
9125 
9126 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
9127 @*/
9128 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
9129 {
9130   DM_Plex *plex = (DM_Plex *)dm->data;
9131   PetscErrorCode ierr;
9132 
9133   PetscFunctionBegin;
9134   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9135   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
9136   if (anchorSection) *anchorSection = plex->anchorSection;
9137   if (anchorIS) *anchorIS = plex->anchorIS;
9138   PetscFunctionReturn(0);
9139 }
9140 
9141 /*@
9142   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
9143   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
9144   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
9145 
9146   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
9147   DMGetConstraints() and filling in the entries in the constraint matrix.
9148 
9149   collective on dm
9150 
9151   Input Parameters:
9152 + dm - The DMPlex object
9153 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9154 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9155 
9156   The reference counts of anchorSection and anchorIS are incremented.
9157 
9158   Level: intermediate
9159 
9160 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
9161 @*/
9162 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
9163 {
9164   DM_Plex        *plex = (DM_Plex *)dm->data;
9165   PetscMPIInt    result;
9166   PetscErrorCode ierr;
9167 
9168   PetscFunctionBegin;
9169   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9170   if (anchorSection) {
9171     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
9172     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRMPI(ierr);
9173     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
9174   }
9175   if (anchorIS) {
9176     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
9177     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRMPI(ierr);
9178     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
9179   }
9180 
9181   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
9182   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
9183   plex->anchorSection = anchorSection;
9184 
9185   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
9186   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
9187   plex->anchorIS = anchorIS;
9188 
9189   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
9190     PetscInt size, a, pStart, pEnd;
9191     const PetscInt *anchors;
9192 
9193     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9194     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
9195     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
9196     for (a = 0; a < size; a++) {
9197       PetscInt p;
9198 
9199       p = anchors[a];
9200       if (p >= pStart && p < pEnd) {
9201         PetscInt dof;
9202 
9203         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9204         if (dof) {
9205           PetscErrorCode ierr2;
9206 
9207           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
9208           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
9209         }
9210       }
9211     }
9212     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
9213   }
9214   /* reset the generic constraints */
9215   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
9216   PetscFunctionReturn(0);
9217 }
9218 
9219 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
9220 {
9221   PetscSection anchorSection;
9222   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
9223   PetscErrorCode ierr;
9224 
9225   PetscFunctionBegin;
9226   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9227   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9228   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
9229   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9230   if (numFields) {
9231     PetscInt f;
9232     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
9233 
9234     for (f = 0; f < numFields; f++) {
9235       PetscInt numComp;
9236 
9237       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
9238       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
9239     }
9240   }
9241   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9242   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9243   pStart = PetscMax(pStart,sStart);
9244   pEnd   = PetscMin(pEnd,sEnd);
9245   pEnd   = PetscMax(pStart,pEnd);
9246   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
9247   for (p = pStart; p < pEnd; p++) {
9248     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9249     if (dof) {
9250       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
9251       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
9252       for (f = 0; f < numFields; f++) {
9253         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
9254         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
9255       }
9256     }
9257   }
9258   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
9259   ierr = PetscObjectSetName((PetscObject) *cSec, "Constraint Section");CHKERRQ(ierr);
9260   PetscFunctionReturn(0);
9261 }
9262 
9263 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
9264 {
9265   PetscSection   aSec;
9266   PetscInt       pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
9267   const PetscInt *anchors;
9268   PetscInt       numFields, f;
9269   IS             aIS;
9270   PetscErrorCode ierr;
9271   MatType        mtype;
9272   PetscBool      iscuda,iskokkos;
9273 
9274   PetscFunctionBegin;
9275   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9276   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
9277   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
9278   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
9279   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
9280   ierr = PetscStrcmp(dm->mattype,MATSEQAIJCUSPARSE,&iscuda);CHKERRQ(ierr);
9281   if (!iscuda) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJCUSPARSE,&iscuda);CHKERRQ(ierr); }
9282   ierr = PetscStrcmp(dm->mattype,MATSEQAIJKOKKOS,&iskokkos);CHKERRQ(ierr);
9283   if (!iskokkos) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJKOKKOS,&iskokkos);CHKERRQ(ierr); }
9284   if (iscuda) mtype = MATSEQAIJCUSPARSE;
9285   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
9286   else mtype = MATSEQAIJ;
9287   ierr = MatSetType(*cMat,mtype);CHKERRQ(ierr);
9288   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
9289   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
9290   /* cSec will be a subset of aSec and section */
9291   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
9292   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9293   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
9294   i[0] = 0;
9295   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9296   for (p = pStart; p < pEnd; p++) {
9297     PetscInt rDof, rOff, r;
9298 
9299     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9300     if (!rDof) continue;
9301     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9302     if (numFields) {
9303       for (f = 0; f < numFields; f++) {
9304         annz = 0;
9305         for (r = 0; r < rDof; r++) {
9306           a = anchors[rOff + r];
9307           if (a < sStart || a >= sEnd) continue;
9308           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9309           annz += aDof;
9310         }
9311         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9312         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
9313         for (q = 0; q < dof; q++) {
9314           i[off + q + 1] = i[off + q] + annz;
9315         }
9316       }
9317     }
9318     else {
9319       annz = 0;
9320       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9321       for (q = 0; q < dof; q++) {
9322         a = anchors[rOff + q];
9323         if (a < sStart || a >= sEnd) continue;
9324         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9325         annz += aDof;
9326       }
9327       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9328       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
9329       for (q = 0; q < dof; q++) {
9330         i[off + q + 1] = i[off + q] + annz;
9331       }
9332     }
9333   }
9334   nnz = i[m];
9335   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
9336   offset = 0;
9337   for (p = pStart; p < pEnd; p++) {
9338     if (numFields) {
9339       for (f = 0; f < numFields; f++) {
9340         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9341         for (q = 0; q < dof; q++) {
9342           PetscInt rDof, rOff, r;
9343           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9344           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9345           for (r = 0; r < rDof; r++) {
9346             PetscInt s;
9347 
9348             a = anchors[rOff + r];
9349             if (a < sStart || a >= sEnd) continue;
9350             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9351             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
9352             for (s = 0; s < aDof; s++) {
9353               j[offset++] = aOff + s;
9354             }
9355           }
9356         }
9357       }
9358     }
9359     else {
9360       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9361       for (q = 0; q < dof; q++) {
9362         PetscInt rDof, rOff, r;
9363         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9364         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9365         for (r = 0; r < rDof; r++) {
9366           PetscInt s;
9367 
9368           a = anchors[rOff + r];
9369           if (a < sStart || a >= sEnd) continue;
9370           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9371           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
9372           for (s = 0; s < aDof; s++) {
9373             j[offset++] = aOff + s;
9374           }
9375         }
9376       }
9377     }
9378   }
9379   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
9380   ierr = PetscFree(i);CHKERRQ(ierr);
9381   ierr = PetscFree(j);CHKERRQ(ierr);
9382   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
9383   PetscFunctionReturn(0);
9384 }
9385 
9386 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
9387 {
9388   DM_Plex        *plex = (DM_Plex *)dm->data;
9389   PetscSection   anchorSection, section, cSec;
9390   Mat            cMat;
9391   PetscErrorCode ierr;
9392 
9393   PetscFunctionBegin;
9394   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9395   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9396   if (anchorSection) {
9397     PetscInt Nf;
9398 
9399     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
9400     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
9401     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
9402     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
9403     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
9404     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
9405     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
9406     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
9407   }
9408   PetscFunctionReturn(0);
9409 }
9410 
9411 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
9412 {
9413   IS             subis;
9414   PetscSection   section, subsection;
9415   PetscErrorCode ierr;
9416 
9417   PetscFunctionBegin;
9418   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9419   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
9420   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
9421   /* Create subdomain */
9422   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
9423   /* Create submodel */
9424   ierr = DMPlexGetSubpointIS(*subdm, &subis);CHKERRQ(ierr);
9425   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
9426   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
9427   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
9428   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
9429   /* Create map from submodel to global model */
9430   if (is) {
9431     PetscSection    sectionGlobal, subsectionGlobal;
9432     IS              spIS;
9433     const PetscInt *spmap;
9434     PetscInt       *subIndices;
9435     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
9436     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
9437 
9438     ierr = DMPlexGetSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
9439     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
9440     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
9441     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
9442     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
9443     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
9444     for (p = pStart; p < pEnd; ++p) {
9445       PetscInt gdof, pSubSize  = 0;
9446 
9447       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
9448       if (gdof > 0) {
9449         for (f = 0; f < Nf; ++f) {
9450           PetscInt fdof, fcdof;
9451 
9452           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
9453           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
9454           pSubSize += fdof-fcdof;
9455         }
9456         subSize += pSubSize;
9457         if (pSubSize) {
9458           if (bs < 0) {
9459             bs = pSubSize;
9460           } else if (bs != pSubSize) {
9461             /* Layout does not admit a pointwise block size */
9462             bs = 1;
9463           }
9464         }
9465       }
9466     }
9467     /* Must have same blocksize on all procs (some might have no points) */
9468     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
9469     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
9470     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
9471     else                            {bs = bsMinMax[0];}
9472     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
9473     for (p = pStart; p < pEnd; ++p) {
9474       PetscInt gdof, goff;
9475 
9476       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
9477       if (gdof > 0) {
9478         const PetscInt point = spmap[p];
9479 
9480         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
9481         for (f = 0; f < Nf; ++f) {
9482           PetscInt fdof, fcdof, fc, f2, poff = 0;
9483 
9484           /* Can get rid of this loop by storing field information in the global section */
9485           for (f2 = 0; f2 < f; ++f2) {
9486             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
9487             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
9488             poff += fdof-fcdof;
9489           }
9490           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
9491           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
9492           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
9493             subIndices[subOff] = goff+poff+fc;
9494           }
9495         }
9496       }
9497     }
9498     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
9499     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
9500     if (bs > 1) {
9501       /* We need to check that the block size does not come from non-contiguous fields */
9502       PetscInt i, j, set = 1;
9503       for (i = 0; i < subSize; i += bs) {
9504         for (j = 0; j < bs; ++j) {
9505           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
9506         }
9507       }
9508       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
9509     }
9510     /* Attach nullspace */
9511     for (f = 0; f < Nf; ++f) {
9512       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
9513       if ((*subdm)->nullspaceConstructors[f]) break;
9514     }
9515     if (f < Nf) {
9516       MatNullSpace nullSpace;
9517       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace);CHKERRQ(ierr);
9518 
9519       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
9520       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
9521     }
9522   }
9523   PetscFunctionReturn(0);
9524 }
9525 
9526 /*@
9527   DMPlexMonitorThroughput - Report the cell throughput of FE integration
9528 
9529   Input Parameter:
9530 - dm - The DM
9531 
9532   Level: developer
9533 
9534   Options Database Keys:
9535 . -dm_plex_monitor_throughput - Activate the monitor
9536 
9537 .seealso: DMSetFromOptions(), DMPlexCreate()
9538 @*/
9539 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
9540 {
9541 #if defined(PETSC_USE_LOG)
9542   PetscStageLog      stageLog;
9543   PetscLogEvent      event;
9544   PetscLogStage      stage;
9545   PetscEventPerfInfo eventInfo;
9546   PetscReal          cellRate, flopRate;
9547   PetscInt           cStart, cEnd, Nf, N;
9548   const char        *name;
9549   PetscErrorCode     ierr;
9550 #endif
9551 
9552   PetscFunctionBegin;
9553   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9554 #if defined(PETSC_USE_LOG)
9555   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
9556   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
9557   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
9558   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
9559   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
9560   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
9561   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
9562   N        = (cEnd - cStart)*Nf*eventInfo.count;
9563   flopRate = eventInfo.flops/eventInfo.time;
9564   cellRate = N/eventInfo.time;
9565   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6));CHKERRQ(ierr);
9566 #else
9567   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
9568 #endif
9569   PetscFunctionReturn(0);
9570 }
9571