xref: /petsc/src/dm/impls/plex/plex.c (revision b7f6ffafa12eb2d9bbc843a6262863d3adbb8bbb) 
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF,DMPLEX_LocatePoints;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
17 
18   Input Parameter:
19 . dm      - The DMPlex object
20 
21   Output Parameter:
22 . simplex - Flag checking for a simplex
23 
24   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
25   If the mesh has no cells, this returns PETSC_FALSE.
26 
27   Level: intermediate
28 
29 .seealso DMPlexGetSimplexOrBoxCells(), DMPlexGetCellType(), DMPlexGetHeightStratum(), DMPolytopeTypeGetNumVertices()
30 @*/
31 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
32 {
33   DMPolytopeType ct;
34   PetscInt       cStart, cEnd;
35   PetscErrorCode ierr;
36 
37   PetscFunctionBegin;
38   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
39   if (cEnd <= cStart) {*simplex = PETSC_FALSE; PetscFunctionReturn(0);}
40   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
41   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
42   PetscFunctionReturn(0);
43 }
44 
45 /*@
46   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
47 
48   Input Parameter:
49 + dm     - The DMPlex object
50 - height - The cell height in the Plex, 0 is the default
51 
52   Output Parameters:
53 + cStart - The first "normal" cell
54 - cEnd   - The upper bound on "normal"" cells
55 
56   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
57 
58   Level: developer
59 
60 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
61 @*/
62 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
63 {
64   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
65   PetscInt       cS, cE, c;
66   PetscErrorCode ierr;
67 
68   PetscFunctionBegin;
69   ierr = DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE);CHKERRQ(ierr);
70   for (c = cS; c < cE; ++c) {
71     DMPolytopeType cct;
72 
73     ierr = DMPlexGetCellType(dm, c, &cct);CHKERRQ(ierr);
74     if ((PetscInt) cct < 0) break;
75     switch (cct) {
76       case DM_POLYTOPE_POINT:
77       case DM_POLYTOPE_SEGMENT:
78       case DM_POLYTOPE_TRIANGLE:
79       case DM_POLYTOPE_QUADRILATERAL:
80       case DM_POLYTOPE_TETRAHEDRON:
81       case DM_POLYTOPE_HEXAHEDRON:
82         ct = cct;
83         break;
84       default: break;
85     }
86     if (ct != DM_POLYTOPE_UNKNOWN) break;
87   }
88   if (ct != DM_POLYTOPE_UNKNOWN) {
89     DMLabel ctLabel;
90 
91     ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
92     ierr = DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE);CHKERRQ(ierr);
93   }
94   if (cStart) *cStart = cS;
95   if (cEnd)   *cEnd   = cE;
96   PetscFunctionReturn(0);
97 }
98 
99 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
100 {
101   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
102   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
103   PetscErrorCode ierr;
104 
105   PetscFunctionBegin;
106   *ft  = PETSC_VTK_INVALID;
107   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
108   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
109   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
110   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
111   if (field >= 0) {
112     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
113     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
114   } else {
115     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
116     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
117   }
118   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
119   if (globalvcdof[0]) {
120     *sStart = vStart;
121     *sEnd   = vEnd;
122     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
123     else                        *ft = PETSC_VTK_POINT_FIELD;
124   } else if (globalvcdof[1]) {
125     *sStart = cStart;
126     *sEnd   = cEnd;
127     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
128     else                        *ft = PETSC_VTK_CELL_FIELD;
129   } else {
130     if (field >= 0) {
131       const char *fieldname;
132 
133       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
134       ierr = PetscInfo2((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
135     } else {
136       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section\"%s\"\n");CHKERRQ(ierr);
137     }
138   }
139   PetscFunctionReturn(0);
140 }
141 
142 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
143 {
144   DM                 dm;
145   PetscSection       s;
146   PetscDraw          draw, popup;
147   DM                 cdm;
148   PetscSection       coordSection;
149   Vec                coordinates;
150   const PetscScalar *coords, *array;
151   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
152   PetscReal          vbound[2], time;
153   PetscBool          isnull, flg;
154   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
155   const char        *name;
156   char               title[PETSC_MAX_PATH_LEN];
157   PetscErrorCode     ierr;
158 
159   PetscFunctionBegin;
160   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
161   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
162   if (isnull) PetscFunctionReturn(0);
163 
164   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
165   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
166   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
167   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
168   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
169   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
170   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
171   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
172   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
173   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
174   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
175 
176   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
177   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
178 
179   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
180   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
181   for (c = 0; c < N; c += dim) {
182     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
183     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
184   }
185   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
186   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
187 
188   /* Could implement something like DMDASelectFields() */
189   for (f = 0; f < Nf; ++f) {
190     DM   fdm = dm;
191     Vec  fv  = v;
192     IS   fis;
193     char prefix[PETSC_MAX_PATH_LEN];
194     const char *fname;
195 
196     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
197     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
198 
199     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
200     else               {prefix[0] = '\0';}
201     if (Nf > 1) {
202       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
203       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
204       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
205       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
206     }
207     for (comp = 0; comp < Nc; ++comp, ++w) {
208       PetscInt nmax = 2;
209 
210       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
211       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
212       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
213       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
214 
215       /* TODO Get max and min only for this component */
216       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
217       if (!flg) {
218         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
219         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
220         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
221       }
222       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
223       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
224       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
225 
226       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
227       for (c = cStart; c < cEnd; ++c) {
228         PetscScalar *coords = NULL, *a = NULL;
229         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
230 
231         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
232         if (a) {
233           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
234           color[1] = color[2] = color[3] = color[0];
235         } else {
236           PetscScalar *vals = NULL;
237           PetscInt     numVals, va;
238 
239           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
240           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
241           switch (numVals/Nc) {
242           case 3: /* P1 Triangle */
243           case 4: /* P1 Quadrangle */
244             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
245             break;
246           case 6: /* P2 Triangle */
247           case 8: /* P2 Quadrangle */
248             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
249             break;
250           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
251           }
252           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
253         }
254         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
255         switch (numCoords) {
256         case 6:
257         case 12: /* Localized triangle */
258           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
259           break;
260         case 8:
261         case 16: /* Localized quadrilateral */
262           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
263           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
264           break;
265         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
266         }
267         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
268       }
269       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
270       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
271       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
272       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
273     }
274     if (Nf > 1) {
275       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
276       ierr = ISDestroy(&fis);CHKERRQ(ierr);
277       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
278     }
279   }
280   PetscFunctionReturn(0);
281 }
282 
283 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
284 {
285   DM                      dm;
286   Vec                     locv;
287   const char              *name;
288   PetscSection            section;
289   PetscInt                pStart, pEnd;
290   PetscInt                numFields;
291   PetscViewerVTKFieldType ft;
292   PetscErrorCode          ierr;
293 
294   PetscFunctionBegin;
295   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
296   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
297   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
298   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
299   ierr = VecCopy(v, locv);CHKERRQ(ierr);
300   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
301   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
302   if (!numFields) {
303     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
304     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
305   } else {
306     PetscInt f;
307 
308     for (f = 0; f < numFields; f++) {
309       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
310       if (ft == PETSC_VTK_INVALID) continue;
311       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
312       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
313     }
314     ierr = VecDestroy(&locv);CHKERRQ(ierr);
315   }
316   PetscFunctionReturn(0);
317 }
318 
319 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
320 {
321   DM             dm;
322   PetscBool      isvtk, ishdf5, isdraw, isglvis;
323   PetscErrorCode ierr;
324 
325   PetscFunctionBegin;
326   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
327   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
328   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
329   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
330   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
331   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
332   if (isvtk || ishdf5 || isdraw || isglvis) {
333     PetscInt    i,numFields;
334     PetscObject fe;
335     PetscBool   fem = PETSC_FALSE;
336     Vec         locv = v;
337     const char  *name;
338     PetscInt    step;
339     PetscReal   time;
340 
341     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
342     for (i=0; i<numFields; i++) {
343       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
344       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
345     }
346     if (fem) {
347       PetscObject isZero;
348 
349       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
350       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
351       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
352       ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
353       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
354       ierr = VecCopy(v, locv);CHKERRQ(ierr);
355       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
356       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
357     }
358     if (isvtk) {
359       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
360     } else if (ishdf5) {
361 #if defined(PETSC_HAVE_HDF5)
362       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
363 #else
364       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
365 #endif
366     } else if (isdraw) {
367       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
368     } else if (isglvis) {
369       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
370       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
371       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
372     }
373     if (fem) {
374       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
375       ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
376     }
377   } else {
378     PetscBool isseq;
379 
380     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
381     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
382     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
383   }
384   PetscFunctionReturn(0);
385 }
386 
387 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
388 {
389   DM             dm;
390   PetscBool      isvtk, ishdf5, isdraw, isglvis, isexodusii;
391   PetscErrorCode ierr;
392 
393   PetscFunctionBegin;
394   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
395   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
396   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
397   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
398   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
399   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
400   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
401   if (isvtk || isdraw || isglvis) {
402     Vec         locv;
403     PetscObject isZero;
404     const char *name;
405 
406     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
407     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
408     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
409     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
410     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
411     ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
412     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
413     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
414     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
415     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
416   } else if (ishdf5) {
417 #if defined(PETSC_HAVE_HDF5)
418     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
419 #else
420     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
421 #endif
422   } else if (isexodusii) {
423 #if defined(PETSC_HAVE_EXODUSII)
424     ierr = VecView_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
425 #else
426     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
427 #endif
428   } else {
429     PetscBool isseq;
430 
431     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
432     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
433     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
434   }
435   PetscFunctionReturn(0);
436 }
437 
438 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
439 {
440   DM                dm;
441   MPI_Comm          comm;
442   PetscViewerFormat format;
443   Vec               v;
444   PetscBool         isvtk, ishdf5;
445   PetscErrorCode    ierr;
446 
447   PetscFunctionBegin;
448   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
449   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
450   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
451   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
452   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
453   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
454   if (format == PETSC_VIEWER_NATIVE) {
455     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
456     /* this need a better fix */
457     if (dm->useNatural) {
458       if (dm->sfNatural) {
459         const char *vecname;
460         PetscInt    n, nroots;
461 
462         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
463         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
464         if (n == nroots) {
465           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
466           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
467           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
468           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
469           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
470         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
471       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
472     } else v = originalv;
473   } else v = originalv;
474 
475   if (ishdf5) {
476 #if defined(PETSC_HAVE_HDF5)
477     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
478 #else
479     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
480 #endif
481   } else if (isvtk) {
482     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
483   } else {
484     PetscBool isseq;
485 
486     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
487     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
488     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
489   }
490   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
491   PetscFunctionReturn(0);
492 }
493 
494 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
495 {
496   DM             dm;
497   PetscBool      ishdf5;
498   PetscErrorCode ierr;
499 
500   PetscFunctionBegin;
501   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
502   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
503   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
504   if (ishdf5) {
505     DM          dmBC;
506     Vec         gv;
507     const char *name;
508 
509     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
510     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
511     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
512     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
513     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
514     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
515     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
516     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
517   } else {
518     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
519   }
520   PetscFunctionReturn(0);
521 }
522 
523 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
524 {
525   DM             dm;
526   PetscBool      ishdf5,isexodusii;
527   PetscErrorCode ierr;
528 
529   PetscFunctionBegin;
530   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
531   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
532   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
533   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
534   if (ishdf5) {
535 #if defined(PETSC_HAVE_HDF5)
536     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
537 #else
538     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
539 #endif
540   } else if (isexodusii) {
541 #if defined(PETSC_HAVE_EXODUSII)
542     ierr = VecLoad_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
543 #else
544     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
545 #endif
546   } else {
547     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
548   }
549   PetscFunctionReturn(0);
550 }
551 
552 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
553 {
554   DM                dm;
555   PetscViewerFormat format;
556   PetscBool         ishdf5;
557   PetscErrorCode    ierr;
558 
559   PetscFunctionBegin;
560   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
561   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
562   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
563   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
564   if (format == PETSC_VIEWER_NATIVE) {
565     if (dm->useNatural) {
566       if (dm->sfNatural) {
567         if (ishdf5) {
568 #if defined(PETSC_HAVE_HDF5)
569           Vec         v;
570           const char *vecname;
571 
572           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
573           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
574           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
575           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
576           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
577           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
578           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
579 #else
580           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
581 #endif
582         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
583       }
584     } else {
585       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
586     }
587   }
588   PetscFunctionReturn(0);
589 }
590 
591 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
592 {
593   PetscSection       coordSection;
594   Vec                coordinates;
595   DMLabel            depthLabel, celltypeLabel;
596   const char        *name[4];
597   const PetscScalar *a;
598   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
599   PetscErrorCode     ierr;
600 
601   PetscFunctionBegin;
602   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
603   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
604   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
605   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
606   ierr = DMPlexGetCellTypeLabel(dm, &celltypeLabel);CHKERRQ(ierr);
607   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
608   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
609   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
610   name[0]     = "vertex";
611   name[1]     = "edge";
612   name[dim-1] = "face";
613   name[dim]   = "cell";
614   for (c = cStart; c < cEnd; ++c) {
615     PetscInt *closure = NULL;
616     PetscInt  closureSize, cl, ct;
617 
618     ierr = DMLabelGetValue(celltypeLabel, c, &ct);CHKERRQ(ierr);
619     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D polytope type %s:\n", c, DMPolytopeTypes[ct]);CHKERRQ(ierr);
620     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
621     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
622     for (cl = 0; cl < closureSize*2; cl += 2) {
623       PetscInt point = closure[cl], depth, dof, off, d, p;
624 
625       if ((point < pStart) || (point >= pEnd)) continue;
626       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
627       if (!dof) continue;
628       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
629       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
630       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
631       for (p = 0; p < dof/dim; ++p) {
632         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
633         for (d = 0; d < dim; ++d) {
634           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
635           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
636         }
637         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
638       }
639       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
640     }
641     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
642     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
643   }
644   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
645   PetscFunctionReturn(0);
646 }
647 
648 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
649 {
650   DM_Plex          *mesh = (DM_Plex*) dm->data;
651   DM                cdm;
652   PetscSection      coordSection;
653   Vec               coordinates;
654   PetscViewerFormat format;
655   PetscErrorCode    ierr;
656 
657   PetscFunctionBegin;
658   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
659   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
660   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
661   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
662   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
663     const char *name;
664     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
665     PetscInt    pStart, pEnd, p, numLabels, l;
666     PetscMPIInt rank, size;
667 
668     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
669     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
670     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
671     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
672     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
673     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
674     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
675     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
676     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
677     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
678     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
679     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
680     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
681     for (p = pStart; p < pEnd; ++p) {
682       PetscInt dof, off, s;
683 
684       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
685       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
686       for (s = off; s < off+dof; ++s) {
687         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
688       }
689     }
690     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
691     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
692     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
693     for (p = pStart; p < pEnd; ++p) {
694       PetscInt dof, off, c;
695 
696       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
697       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
698       for (c = off; c < off+dof; ++c) {
699         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
700       }
701     }
702     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
703     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
704     if (coordSection && coordinates) {
705       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
706     }
707     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
708     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
709     for (l = 0; l < numLabels; ++l) {
710       DMLabel     label;
711       PetscBool   isdepth;
712       const char *name;
713 
714       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
715       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
716       if (isdepth) continue;
717       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
718       ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
719     }
720     if (size > 1) {
721       PetscSF sf;
722 
723       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
724       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
725     }
726     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
727   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
728     const char  *name, *color;
729     const char  *defcolors[3]  = {"gray", "orange", "green"};
730     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
731     char         lname[PETSC_MAX_PATH_LEN];
732     PetscReal    scale         = 2.0;
733     PetscReal    tikzscale     = 1.0;
734     PetscBool    useNumbers    = PETSC_TRUE, drawNumbers[4], drawColors[4], useLabels, useColors, plotEdges, drawHasse = PETSC_FALSE;
735     double       tcoords[3];
736     PetscScalar *coords;
737     PetscInt     numLabels, l, numColors, numLColors, dim, d, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p, n;
738     PetscMPIInt  rank, size;
739     char         **names, **colors, **lcolors;
740     PetscBool    flg, lflg;
741     PetscBT      wp = NULL;
742     PetscInt     pEnd, pStart;
743 
744     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
745     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
746     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
747     numLabels  = PetscMax(numLabels, 10);
748     numColors  = 10;
749     numLColors = 10;
750     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
751     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
752     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
753     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
754     for (d = 0; d < 4; ++d) drawNumbers[d] = useNumbers;
755     for (d = 0; d < 4; ++d) drawColors[d]  = PETSC_TRUE;
756     n = 4;
757     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers_depth", drawNumbers, &n, &flg);CHKERRQ(ierr);
758     if (flg && n != dim+1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
759     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors_depth", drawColors, &n, &flg);CHKERRQ(ierr);
760     if (flg && n != dim+1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
761     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
762     if (!useLabels) numLabels = 0;
763     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
764     if (!useColors) {
765       numColors = 3;
766       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
767     }
768     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
769     if (!useColors) {
770       numLColors = 4;
771       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
772     }
773     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg);CHKERRQ(ierr);
774     plotEdges = (PetscBool)(depth > 1 && drawNumbers[1] && dim < 3);
775     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
776     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
777     if (depth < dim) plotEdges = PETSC_FALSE;
778     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_hasse", &drawHasse, NULL);CHKERRQ(ierr);
779 
780     /* filter points with labelvalue != labeldefaultvalue */
781     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
782     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
783     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
784     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
785     if (lflg) {
786       DMLabel lbl;
787 
788       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
789       if (lbl) {
790         PetscInt val, defval;
791 
792         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
793         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
794         for (c = pStart;  c < pEnd; c++) {
795           PetscInt *closure = NULL;
796           PetscInt  closureSize;
797 
798           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
799           if (val == defval) continue;
800 
801           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
802           for (p = 0; p < closureSize*2; p += 2) {
803             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
804           }
805           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
806         }
807       }
808     }
809 
810     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
811     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
812     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
813     ierr = PetscViewerASCIIPrintf(viewer, "\
814 \\documentclass[tikz]{standalone}\n\n\
815 \\usepackage{pgflibraryshapes}\n\
816 \\usetikzlibrary{backgrounds}\n\
817 \\usetikzlibrary{arrows}\n\
818 \\begin{document}\n");CHKERRQ(ierr);
819     if (size > 1) {
820       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
821       for (p = 0; p < size; ++p) {
822         if (p > 0 && p == size-1) {
823           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
824         } else if (p > 0) {
825           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
826         }
827         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
828       }
829       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
830     }
831     if (drawHasse) {
832       PetscInt maxStratum = PetscMax(vEnd-vStart, PetscMax(eEnd-eStart, cEnd-cStart));
833 
834       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vStart}{%D}\n", vStart);CHKERRQ(ierr);
835       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vEnd}{%D}\n", vEnd-1);CHKERRQ(ierr);
836       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numVertices}{%D}\n", vEnd-vStart);CHKERRQ(ierr);
837       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vShift}{%.2f}\n", 3 + (maxStratum-(vEnd-vStart))/2.);CHKERRQ(ierr);
838       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eStart}{%D}\n", eStart);CHKERRQ(ierr);
839       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eEnd}{%D}\n", eEnd-1);CHKERRQ(ierr);
840       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eShift}{%.2f}\n", 3 + (maxStratum-(eEnd-eStart))/2.);CHKERRQ(ierr);
841       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numEdges}{%D}\n", eEnd-eStart);CHKERRQ(ierr);
842       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cStart}{%D}\n", cStart);CHKERRQ(ierr);
843       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cEnd}{%D}\n", cEnd-1);CHKERRQ(ierr);
844       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numCells}{%D}\n", cEnd-cStart);CHKERRQ(ierr);
845       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cShift}{%.2f}\n", 3 + (maxStratum-(cEnd-cStart))/2.);CHKERRQ(ierr);
846     }
847     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
848 
849     /* Plot vertices */
850     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
851     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
852     for (v = vStart; v < vEnd; ++v) {
853       PetscInt  off, dof, d;
854       PetscBool isLabeled = PETSC_FALSE;
855 
856       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
857       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
858       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
859       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
860       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
861       for (d = 0; d < dof; ++d) {
862         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
863         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
864       }
865       /* Rotate coordinates since PGF makes z point out of the page instead of up */
866       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
867       for (d = 0; d < dof; ++d) {
868         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
869         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
870       }
871       if (drawHasse) color = colors[0%numColors];
872       else           color = colors[rank%numColors];
873       for (l = 0; l < numLabels; ++l) {
874         PetscInt val;
875         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
876         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
877       }
878       if (drawNumbers[0]) {
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
880       } else if (drawColors[0]) {
881         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
882       } else {
883         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", v, rank);CHKERRQ(ierr);
884       }
885     }
886     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
887     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
888     /* Plot edges */
889     if (plotEdges) {
890       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
891       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
892       for (e = eStart; e < eEnd; ++e) {
893         const PetscInt *cone;
894         PetscInt        coneSize, offA, offB, dof, d;
895 
896         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
897         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
898         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
899         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
900         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
901         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
902         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
903         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
904         for (d = 0; d < dof; ++d) {
905           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
906           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
907         }
908         /* Rotate coordinates since PGF makes z point out of the page instead of up */
909         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
910         for (d = 0; d < dof; ++d) {
911           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
912           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
913         }
914         if (drawHasse) color = colors[1%numColors];
915         else           color = colors[rank%numColors];
916         for (l = 0; l < numLabels; ++l) {
917           PetscInt val;
918           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
919           if (val >= 0) {color = lcolors[l%numLColors]; break;}
920         }
921         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
922       }
923       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
924       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
925       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
926     }
927     /* Plot cells */
928     if (dim == 3 || !drawNumbers[1]) {
929       for (e = eStart; e < eEnd; ++e) {
930         const PetscInt *cone;
931 
932         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
933         color = colors[rank%numColors];
934         for (l = 0; l < numLabels; ++l) {
935           PetscInt val;
936           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
937           if (val >= 0) {color = lcolors[l%numLColors]; break;}
938         }
939         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
940         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
941       }
942     } else {
943        DMPolytopeType ct;
944 
945       /* Drawing a 2D polygon */
946       for (c = cStart; c < cEnd; ++c) {
947         if (wp && !PetscBTLookup(wp, c - pStart)) continue;
948         ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
949         if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR ||
950             ct == DM_POLYTOPE_TRI_PRISM_TENSOR ||
951             ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
952           const PetscInt *cone;
953           PetscInt        coneSize, e;
954 
955           ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
956           ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
957           for (e = 0; e < coneSize; ++e) {
958             const PetscInt *econe;
959 
960             ierr = DMPlexGetCone(dm, cone[e], &econe);CHKERRQ(ierr);
961             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d) -- (%D_%d);\n", colors[rank%numColors], econe[0], rank, cone[e], rank, econe[1], rank);CHKERRQ(ierr);
962           }
963         } else {
964           PetscInt *closure = NULL;
965           PetscInt  closureSize, Nv = 0, v;
966 
967           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
968           for (p = 0; p < closureSize*2; p += 2) {
969             const PetscInt point = closure[p];
970 
971             if ((point >= vStart) && (point < vEnd)) closure[Nv++] = point;
972           }
973           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
974           for (v = 0; v <= Nv; ++v) {
975             const PetscInt vertex = closure[v%Nv];
976 
977             if (v > 0) {
978               if (plotEdges) {
979                 const PetscInt *edge;
980                 PetscInt        endpoints[2], ne;
981 
982                 endpoints[0] = closure[v-1]; endpoints[1] = vertex;
983                 ierr = DMPlexGetJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
984                 if (ne != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find edge for vertices %D, %D", endpoints[0], endpoints[1]);
985                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d) -- ", edge[0], rank);CHKERRQ(ierr);
986                 ierr = DMPlexRestoreJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
987               } else {
988                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);
989               }
990             }
991             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", vertex, rank);CHKERRQ(ierr);
992           }
993           ierr = PetscViewerASCIISynchronizedPrintf(viewer, ";\n");CHKERRQ(ierr);
994           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
995         }
996       }
997     }
998     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
999     for (c = cStart; c < cEnd; ++c) {
1000       double    ccoords[3] = {0.0, 0.0, 0.0};
1001       PetscBool isLabeled  = PETSC_FALSE;
1002       PetscInt *closure    = NULL;
1003       PetscInt  closureSize, dof, d, n = 0;
1004 
1005       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
1006       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1007       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
1008       for (p = 0; p < closureSize*2; p += 2) {
1009         const PetscInt point = closure[p];
1010         PetscInt       off;
1011 
1012         if ((point < vStart) || (point >= vEnd)) continue;
1013         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
1014         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
1015         for (d = 0; d < dof; ++d) {
1016           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
1017           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1018         }
1019         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1020         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1021         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
1022         ++n;
1023       }
1024       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
1025       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1026       for (d = 0; d < dof; ++d) {
1027         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
1028         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
1029       }
1030       if (drawHasse) color = colors[depth%numColors];
1031       else           color = colors[rank%numColors];
1032       for (l = 0; l < numLabels; ++l) {
1033         PetscInt val;
1034         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
1035         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
1036       }
1037       if (drawNumbers[dim]) {
1038         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
1039       } else if (drawColors[dim]) {
1040         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
1041       } else {
1042         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", c, rank);CHKERRQ(ierr);
1043       }
1044     }
1045     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1046     if (drawHasse) {
1047       color = colors[depth%numColors];
1048       ierr = PetscViewerASCIIPrintf(viewer, "%% Cells\n");CHKERRQ(ierr);
1049       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\c in {\\cStart,...,\\cEnd}\n");CHKERRQ(ierr);
1050       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1051       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\c_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\cShift+\\c-\\cStart,0) {\\c};\n", rank, color);CHKERRQ(ierr);
1052       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1053 
1054       color = colors[1%numColors];
1055       ierr = PetscViewerASCIIPrintf(viewer, "%% Edges\n");CHKERRQ(ierr);
1056       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\e in {\\eStart,...,\\eEnd}\n");CHKERRQ(ierr);
1057       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1058       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\e_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\eShift+\\e-\\eStart,1) {\\e};\n", rank, color);CHKERRQ(ierr);
1059       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1060 
1061       color = colors[0%numColors];
1062       ierr = PetscViewerASCIIPrintf(viewer, "%% Vertices\n");CHKERRQ(ierr);
1063       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\v in {\\vStart,...,\\vEnd}\n");CHKERRQ(ierr);
1064       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1065       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\v_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\vShift+\\v-\\vStart,2) {\\v};\n", rank, color);CHKERRQ(ierr);
1066       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1067 
1068       for (p = pStart; p < pEnd; ++p) {
1069         const PetscInt *cone;
1070         PetscInt        coneSize, cp;
1071 
1072         ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1073         ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1074         for (cp = 0; cp < coneSize; ++cp) {
1075           ierr = PetscViewerASCIIPrintf(viewer, "\\draw[->, shorten >=1pt] (%D_%d) -- (%D_%d);\n", cone[cp], rank, p, rank);CHKERRQ(ierr);
1076         }
1077       }
1078     }
1079     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
1080     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
1081     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
1082     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
1083     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
1084     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
1085     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
1086     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
1087     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
1088   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
1089     Vec                    cown,acown;
1090     VecScatter             sct;
1091     ISLocalToGlobalMapping g2l;
1092     IS                     gid,acis;
1093     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
1094     MPI_Group              ggroup,ngroup;
1095     PetscScalar            *array,nid;
1096     const PetscInt         *idxs;
1097     PetscInt               *idxs2,*start,*adjacency,*work;
1098     PetscInt64             lm[3],gm[3];
1099     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
1100     PetscMPIInt            d1,d2,rank;
1101 
1102     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1103     ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr);
1104 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1105     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRMPI(ierr);
1106 #endif
1107     if (ncomm != MPI_COMM_NULL) {
1108       ierr = MPI_Comm_group(comm,&ggroup);CHKERRMPI(ierr);
1109       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRMPI(ierr);
1110       d1   = 0;
1111       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRMPI(ierr);
1112       nid  = d2;
1113       ierr = MPI_Group_free(&ggroup);CHKERRMPI(ierr);
1114       ierr = MPI_Group_free(&ngroup);CHKERRMPI(ierr);
1115       ierr = MPI_Comm_free(&ncomm);CHKERRMPI(ierr);
1116     } else nid = 0.0;
1117 
1118     /* Get connectivity */
1119     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
1120     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
1121 
1122     /* filter overlapped local cells */
1123     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
1124     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
1125     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
1126     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
1127     for (c = cStart, cum = 0; c < cEnd; c++) {
1128       if (idxs[c-cStart] < 0) continue;
1129       idxs2[cum++] = idxs[c-cStart];
1130     }
1131     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
1132     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
1133     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1134     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
1135 
1136     /* support for node-aware cell locality */
1137     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
1138     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
1139     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
1140     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
1141     for (c = 0; c < numVertices; c++) array[c] = nid;
1142     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
1143     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
1144     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1145     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1146     ierr = ISDestroy(&acis);CHKERRQ(ierr);
1147     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
1148     ierr = VecDestroy(&cown);CHKERRQ(ierr);
1149 
1150     /* compute edgeCut */
1151     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1152     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
1153     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
1154     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
1155     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1156     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1157     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1158       PetscInt totl;
1159 
1160       totl = start[c+1]-start[c];
1161       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1162       for (i = 0; i < totl; i++) {
1163         if (work[i] < 0) {
1164           ect  += 1;
1165           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1166         }
1167       }
1168     }
1169     ierr  = PetscFree(work);CHKERRQ(ierr);
1170     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1171     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1172     lm[1] = -numVertices;
1173     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRMPI(ierr);
1174     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1175     lm[0] = ect; /* edgeCut */
1176     lm[1] = ectn; /* node-aware edgeCut */
1177     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1178     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRMPI(ierr);
1179     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1180 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1181     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1182 #else
1183     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1184 #endif
1185     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1186     ierr  = PetscFree(start);CHKERRQ(ierr);
1187     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1188     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1189   } else {
1190     const char    *name;
1191     PetscInt      *sizes, *hybsizes, *ghostsizes;
1192     PetscInt       locDepth, depth, cellHeight, dim, d;
1193     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1194     PetscInt       numLabels, l;
1195     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1196     MPI_Comm       comm;
1197     PetscMPIInt    size, rank;
1198 
1199     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
1200     ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1201     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1202     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1203     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1204     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1205     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1206     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1207     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1208     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1209     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRMPI(ierr);
1210     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1211     gcNum = gcEnd - gcStart;
1212     ierr = PetscCalloc3(size,&sizes,size,&hybsizes,size,&ghostsizes);CHKERRQ(ierr);
1213     for (d = 0; d <= depth; d++) {
1214       PetscInt Nc[2] = {0, 0}, ict;
1215 
1216       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1217       if (pStart < pEnd) {ierr = DMPlexGetCellType(dm, pStart, &ct0);CHKERRQ(ierr);}
1218       ict  = ct0;
1219       ierr = MPI_Bcast(&ict, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1220       ct0  = (DMPolytopeType) ict;
1221       for (p = pStart; p < pEnd; ++p) {
1222         DMPolytopeType ct;
1223 
1224         ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
1225         if (ct == ct0) ++Nc[0];
1226         else           ++Nc[1];
1227       }
1228       ierr = MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1229       ierr = MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1230       if (d == depth) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);}
1231       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1232       for (p = 0; p < size; ++p) {
1233         if (!rank) {
1234           ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]+hybsizes[p]);CHKERRQ(ierr);
1235           if (hybsizes[p]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", hybsizes[p]);CHKERRQ(ierr);}
1236           if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1237         }
1238       }
1239       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1240     }
1241     ierr = PetscFree3(sizes,hybsizes,ghostsizes);CHKERRQ(ierr);
1242     {
1243       const PetscReal      *maxCell;
1244       const PetscReal      *L;
1245       const DMBoundaryType *bd;
1246       PetscBool             per, localized;
1247 
1248       ierr = DMGetPeriodicity(dm, &per, &maxCell, &L, &bd);CHKERRQ(ierr);
1249       ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
1250       if (per) {
1251         ierr = PetscViewerASCIIPrintf(viewer, "Periodic mesh (");CHKERRQ(ierr);
1252         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1253         for (d = 0; d < dim; ++d) {
1254           if (bd && d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1255           if (bd)    {ierr = PetscViewerASCIIPrintf(viewer, "%s", DMBoundaryTypes[bd[d]]);CHKERRQ(ierr);}
1256         }
1257         ierr = PetscViewerASCIIPrintf(viewer, ") coordinates %s\n", localized ? "localized" : "not localized");CHKERRQ(ierr);
1258         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1259       }
1260     }
1261     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1262     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1263     for (l = 0; l < numLabels; ++l) {
1264       DMLabel         label;
1265       const char     *name;
1266       IS              valueIS;
1267       const PetscInt *values;
1268       PetscInt        numValues, v;
1269 
1270       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1271       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1272       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1273       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1274       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1275       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1276       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1277       for (v = 0; v < numValues; ++v) {
1278         PetscInt size;
1279 
1280         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1281         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1282         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1283       }
1284       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1285       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1286       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1287       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1288     }
1289     {
1290       char    **labelNames;
1291       PetscInt  Nl = numLabels;
1292       PetscBool flg;
1293 
1294       ierr = PetscMalloc1(Nl, &labelNames);CHKERRQ(ierr);
1295       ierr = PetscOptionsGetStringArray(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_plex_view_labels", labelNames, &Nl, &flg);CHKERRQ(ierr);
1296       for (l = 0; l < Nl; ++l) {
1297         DMLabel label;
1298 
1299         ierr = DMHasLabel(dm, labelNames[l], &flg);CHKERRQ(ierr);
1300         if (flg) {
1301           ierr = DMGetLabel(dm, labelNames[l], &label);CHKERRQ(ierr);
1302           ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
1303         }
1304         ierr = PetscFree(labelNames[l]);CHKERRQ(ierr);
1305       }
1306       ierr = PetscFree(labelNames);CHKERRQ(ierr);
1307     }
1308     /* If no fields are specified, people do not want to see adjacency */
1309     if (dm->Nf) {
1310       PetscInt f;
1311 
1312       for (f = 0; f < dm->Nf; ++f) {
1313         const char *name;
1314 
1315         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1316         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1317         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1318         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1319         if (dm->fields[f].adjacency[0]) {
1320           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1321           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1322         } else {
1323           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1324           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1325         }
1326         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1327       }
1328     }
1329     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1330     if (cdm) {
1331       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1332       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1333       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1334     }
1335   }
1336   PetscFunctionReturn(0);
1337 }
1338 
1339 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1340 {
1341   DMPolytopeType ct;
1342   PetscMPIInt    rank;
1343   PetscErrorCode ierr;
1344 
1345   PetscFunctionBegin;
1346   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1347   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1348   switch (ct) {
1349   case DM_POLYTOPE_TRIANGLE:
1350     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1351                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1352                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1353                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1354     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1355     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1356     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1357     break;
1358   case DM_POLYTOPE_QUADRILATERAL:
1359     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1360                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1361                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1362                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1363     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1364                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1365                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1366                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1367     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1368     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1369     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1370     ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1371     break;
1372   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1373   }
1374   PetscFunctionReturn(0);
1375 }
1376 
1377 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1378 {
1379   DMPolytopeType ct;
1380   PetscReal      centroid[2] = {0., 0.};
1381   PetscMPIInt    rank;
1382   PetscInt       fillColor, v, e, d;
1383   PetscErrorCode ierr;
1384 
1385   PetscFunctionBegin;
1386   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1387   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1388   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2;
1389   switch (ct) {
1390   case DM_POLYTOPE_TRIANGLE:
1391     {
1392       PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1393 
1394       for (v = 0; v < 3; ++v) {centroid[0] += PetscRealPart(coords[v*2+0])/3.;centroid[1] += PetscRealPart(coords[v*2+1])/3.;}
1395       for (e = 0; e < 3; ++e) {
1396         refCoords[0] = refVertices[e*2+0];
1397         refCoords[1] = refVertices[e*2+1];
1398         for (d = 1; d <= edgeDiv; ++d) {
1399           refCoords[d*2+0] = refCoords[0] + (refVertices[(e+1)%3 * 2 + 0] - refCoords[0])*d/edgeDiv;
1400           refCoords[d*2+1] = refCoords[1] + (refVertices[(e+1)%3 * 2 + 1] - refCoords[1])*d/edgeDiv;
1401         }
1402         ierr = DMPlexReferenceToCoordinates(dm, cell, edgeDiv+1, refCoords, edgeCoords);CHKERRQ(ierr);
1403         for (d = 0; d < edgeDiv; ++d) {
1404           ierr = PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], fillColor, fillColor, fillColor);CHKERRQ(ierr);
1405           ierr = PetscDrawLine(draw, edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], PETSC_DRAW_BLACK);CHKERRQ(ierr);
1406         }
1407       }
1408     }
1409     break;
1410   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1411   }
1412   PetscFunctionReturn(0);
1413 }
1414 
1415 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1416 {
1417   PetscDraw          draw;
1418   DM                 cdm;
1419   PetscSection       coordSection;
1420   Vec                coordinates;
1421   const PetscScalar *coords;
1422   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1423   PetscReal         *refCoords, *edgeCoords;
1424   PetscBool          isnull, drawAffine = PETSC_TRUE;
1425   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N, edgeDiv = 4;
1426   PetscErrorCode     ierr;
1427 
1428   PetscFunctionBegin;
1429   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1430   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1431   ierr = PetscOptionsGetBool(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL);CHKERRQ(ierr);
1432   if (!drawAffine) {ierr = PetscMalloc2((edgeDiv+1)*dim, &refCoords, (edgeDiv+1)*dim, &edgeCoords);CHKERRQ(ierr);}
1433   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1434   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1435   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1436   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1437   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1438 
1439   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1440   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1441   if (isnull) PetscFunctionReturn(0);
1442   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1443 
1444   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1445   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1446   for (c = 0; c < N; c += dim) {
1447     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1448     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1449   }
1450   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1451   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1452   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1453   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1454   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1455 
1456   for (c = cStart; c < cEnd; ++c) {
1457     PetscScalar *coords = NULL;
1458     PetscInt     numCoords;
1459 
1460     ierr = DMPlexVecGetClosureAtDepth_Internal(dm, coordSection, coordinates, c, 0, &numCoords, &coords);CHKERRQ(ierr);
1461     if (drawAffine) {
1462       ierr = DMPlexDrawCell(dm, draw, c, coords);CHKERRQ(ierr);
1463     } else {
1464       ierr = DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords);CHKERRQ(ierr);
1465     }
1466     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1467   }
1468   if (!drawAffine) {ierr = PetscFree2(refCoords, edgeCoords);CHKERRQ(ierr);}
1469   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1470   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1471   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1472   PetscFunctionReturn(0);
1473 }
1474 
1475 #if defined(PETSC_HAVE_EXODUSII)
1476 #include <exodusII.h>
1477 #include <petscviewerexodusii.h>
1478 #endif
1479 
1480 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1481 {
1482   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1483   char           name[PETSC_MAX_PATH_LEN];
1484   PetscErrorCode ierr;
1485 
1486   PetscFunctionBegin;
1487   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1488   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1489   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii);CHKERRQ(ierr);
1490   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
1491   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
1492   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
1493   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
1494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus);CHKERRQ(ierr);
1495   if (iascii) {
1496     PetscViewerFormat format;
1497     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1498     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1499       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1500     } else {
1501       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1502     }
1503   } else if (ishdf5) {
1504 #if defined(PETSC_HAVE_HDF5)
1505     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1506 #else
1507     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1508 #endif
1509   } else if (isvtk) {
1510     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1511   } else if (isdraw) {
1512     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1513   } else if (isglvis) {
1514     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1515 #if defined(PETSC_HAVE_EXODUSII)
1516   } else if (isexodus) {
1517 /*
1518       exodusII requires that all sets be part of exactly one cell set.
1519       If the dm does not have a "Cell Sets" label defined, we create one
1520       with ID 1, containig all cells.
1521       Note that if the Cell Sets label is defined but does not cover all cells,
1522       we may still have a problem. This should probably be checked here or in the viewer;
1523     */
1524     PetscInt numCS;
1525     ierr = DMGetLabelSize(dm,"Cell Sets",&numCS);CHKERRQ(ierr);
1526     if (!numCS) {
1527       PetscInt cStart, cEnd, c;
1528       ierr = DMCreateLabel(dm, "Cell Sets");CHKERRQ(ierr);
1529       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1530       for (c = cStart; c < cEnd; ++c) {ierr = DMSetLabelValue(dm, "Cell Sets", c, 1);CHKERRQ(ierr);}
1531     }
1532     ierr = DMView_PlexExodusII(dm, viewer);CHKERRQ(ierr);
1533 #endif
1534   } else {
1535     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1536   }
1537   /* Optionally view the partition */
1538   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1539   if (flg) {
1540     Vec ranks;
1541     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1542     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1543     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1544   }
1545   /* Optionally view a label */
1546   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, sizeof(name), &flg);CHKERRQ(ierr);
1547   if (flg) {
1548     DMLabel label;
1549     Vec     val;
1550 
1551     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1552     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1553     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1554     ierr = VecView(val, viewer);CHKERRQ(ierr);
1555     ierr = VecDestroy(&val);CHKERRQ(ierr);
1556   }
1557   PetscFunctionReturn(0);
1558 }
1559 
1560 /*@
1561   DMPlexTopologyView - Saves a DMPlex topology into a file
1562 
1563   Collective on DM
1564 
1565   Input Parameters:
1566 + dm     - The DM whose topology is to be saved
1567 - viewer - The PetscViewer for saving
1568 
1569   Level: advanced
1570 
1571 .seealso: DMView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexTopologyLoad()
1572 @*/
1573 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1574 {
1575   PetscBool      ishdf5;
1576   PetscErrorCode ierr;
1577 
1578   PetscFunctionBegin;
1579   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1580   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1581   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1582   if (ishdf5) {
1583 #if defined(PETSC_HAVE_HDF5)
1584     PetscViewerFormat format;
1585     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1586     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1587       IS globalPointNumbering;
1588 
1589       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1590       ierr = DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1591       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1592     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1593 #else
1594     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1595 #endif
1596   }
1597   PetscFunctionReturn(0);
1598 }
1599 
1600 /*@
1601   DMPlexCoordinatesView - Saves DMPlex coordinates into a file
1602 
1603   Collective on DM
1604 
1605   Input Parameters:
1606 + dm     - The DM whose coordinates are to be saved
1607 - viewer - The PetscViewer for saving
1608 
1609   Level: advanced
1610 
1611 .seealso: DMView(), DMPlexTopologyView(), DMPlexLabelsView(), DMPlexCoordinatesLoad()
1612 @*/
1613 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
1614 {
1615   PetscBool      ishdf5;
1616   PetscErrorCode ierr;
1617 
1618   PetscFunctionBegin;
1619   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1620   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1621   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1622   if (ishdf5) {
1623 #if defined(PETSC_HAVE_HDF5)
1624     PetscViewerFormat format;
1625     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1626     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1627       ierr = DMPlexCoordinatesView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1628     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1629 #else
1630     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1631 #endif
1632   }
1633   PetscFunctionReturn(0);
1634 }
1635 
1636 /*@
1637   DMPlexLabelsView - Saves DMPlex labels into a file
1638 
1639   Collective on DM
1640 
1641   Input Parameters:
1642 + dm     - The DM whose labels are to be saved
1643 - viewer - The PetscViewer for saving
1644 
1645   Level: advanced
1646 
1647 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsLoad()
1648 @*/
1649 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
1650 {
1651   PetscBool      ishdf5;
1652   PetscErrorCode ierr;
1653 
1654   PetscFunctionBegin;
1655   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1656   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1657   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1658   if (ishdf5) {
1659 #if defined(PETSC_HAVE_HDF5)
1660     IS                globalPointNumbering;
1661     PetscViewerFormat format;
1662 
1663     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1664     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1665       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1666       ierr = DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1667       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1668     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1669 #else
1670     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1671 #endif
1672   }
1673   PetscFunctionReturn(0);
1674 }
1675 
1676 /*@
1677   DMPlexSectionView - Saves a section associated with a DMPlex
1678 
1679   Collective on DM
1680 
1681   Input Parameters:
1682 + dm         - The DM that contains the topology on which the section to be saved is defined
1683 . viewer     - The PetscViewer for saving
1684 - sectiondm  - The DM that contains the section to be saved
1685 
1686   Level: advanced
1687 
1688   Notes:
1689   This function is a wrapper around PetscSectionView(); in addition to the raw section, it saves information that associates the section points to the topology (dm) points. When the topology (dm) and the section are later loaded with DMPlexTopologyLoad() and DMPlexSectionLoad(), respectively, this information is used to match section points with topology points.
1690 
1691   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1692 
1693 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexGlobalVectorView(), DMPlexLocalVectorView(), PetscSectionView(), DMPlexSectionLoad()
1694 @*/
1695 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
1696 {
1697   PetscBool      ishdf5;
1698   PetscErrorCode ierr;
1699 
1700   PetscFunctionBegin;
1701   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1702   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1703   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1704   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1705   if (ishdf5) {
1706 #if defined(PETSC_HAVE_HDF5)
1707     ierr = DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm);CHKERRQ(ierr);
1708 #else
1709     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1710 #endif
1711   }
1712   PetscFunctionReturn(0);
1713 }
1714 
1715 /*@
1716   DMPlexGlobalVectorView - Saves a global vector
1717 
1718   Collective on DM
1719 
1720   Input Parameters:
1721 + dm        - The DM that represents the topology
1722 . viewer    - The PetscViewer to save data with
1723 . sectiondm - The DM that contains the global section on which vec is defined
1724 - vec       - The global vector to be saved
1725 
1726   Level: advanced
1727 
1728   Notes:
1729   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1730 
1731   Typical calling sequence
1732 $       DMCreate(PETSC_COMM_WORLD, &dm);
1733 $       DMSetType(dm, DMPLEX);
1734 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1735 $       DMClone(dm, &sectiondm);
1736 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1737 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1738 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1739 $       PetscSectionSetChart(section, pStart, pEnd);
1740 $       PetscSectionSetUp(section);
1741 $       DMSetLocalSection(sectiondm, section);
1742 $       PetscSectionDestroy(&section);
1743 $       DMGetGlobalVector(sectiondm, &vec);
1744 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1745 $       DMPlexTopologyView(dm, viewer);
1746 $       DMPlexSectionView(dm, viewer, sectiondm);
1747 $       DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
1748 $       DMRestoreGlobalVector(sectiondm, &vec);
1749 $       DMDestroy(&sectiondm);
1750 $       DMDestroy(&dm);
1751 
1752 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexLocalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1753 @*/
1754 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1755 {
1756   PetscBool       ishdf5;
1757   PetscErrorCode  ierr;
1758 
1759   PetscFunctionBegin;
1760   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1761   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1762   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1763   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1764   /* Check consistency */
1765   {
1766     PetscSection  section;
1767     PetscBool     includesConstraints;
1768     PetscInt      m, m1;
1769 
1770     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1771     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
1772     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1773     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1774     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1775     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
1776   }
1777   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1778   if (ishdf5) {
1779 #if defined(PETSC_HAVE_HDF5)
1780     ierr = DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1781 #else
1782     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1783 #endif
1784   }
1785   PetscFunctionReturn(0);
1786 }
1787 
1788 /*@
1789   DMPlexLocalVectorView - Saves a local vector
1790 
1791   Collective on DM
1792 
1793   Input Parameters:
1794 + dm        - The DM that represents the topology
1795 . viewer    - The PetscViewer to save data with
1796 . sectiondm - The DM that contains the local section on which vec is defined; may be the same as dm
1797 - vec       - The local vector to be saved
1798 
1799   Level: advanced
1800 
1801   Notes:
1802   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1803 
1804   Typical calling sequence
1805 $       DMCreate(PETSC_COMM_WORLD, &dm);
1806 $       DMSetType(dm, DMPLEX);
1807 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1808 $       DMClone(dm, &sectiondm);
1809 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1810 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1811 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1812 $       PetscSectionSetChart(section, pStart, pEnd);
1813 $       PetscSectionSetUp(section);
1814 $       DMSetLocalSection(sectiondm, section);
1815 $       DMGetLocalVector(sectiondm, &vec);
1816 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1817 $       DMPlexTopologyView(dm, viewer);
1818 $       DMPlexSectionView(dm, viewer, sectiondm);
1819 $       DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
1820 $       DMRestoreLocalVector(sectiondm, &vec);
1821 $       DMDestroy(&sectiondm);
1822 $       DMDestroy(&dm);
1823 
1824 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexGlobalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1825 @*/
1826 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1827 {
1828   PetscBool       ishdf5;
1829   PetscErrorCode  ierr;
1830 
1831   PetscFunctionBegin;
1832   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1833   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1834   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1835   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1836   /* Check consistency */
1837   {
1838     PetscSection  section;
1839     PetscBool     includesConstraints;
1840     PetscInt      m, m1;
1841 
1842     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1843     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
1844     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1845     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1846     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1847     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
1848   }
1849   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1850   if (ishdf5) {
1851 #if defined(PETSC_HAVE_HDF5)
1852     ierr = DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1853 #else
1854     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1855 #endif
1856   }
1857   PetscFunctionReturn(0);
1858 }
1859 
1860 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1861 {
1862   PetscBool      ishdf5;
1863   PetscErrorCode ierr;
1864 
1865   PetscFunctionBegin;
1866   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1867   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1868   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1869   if (ishdf5) {
1870 #if defined(PETSC_HAVE_HDF5)
1871     PetscViewerFormat format;
1872     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1873     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1874       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1875     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1876       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1877     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1878     PetscFunctionReturn(0);
1879 #else
1880     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1881 #endif
1882   } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1883 }
1884 
1885 /*@
1886   DMPlexTopologyLoad - Loads a topology into a DMPlex
1887 
1888   Collective on DM
1889 
1890   Input Parameters:
1891 + dm     - The DM into which the topology is loaded
1892 - viewer - The PetscViewer for the saved topology
1893 
1894   Output Parameters:
1895 . globalToLocalPointSF - The PetscSF that pushes points in [0, N) to the associated points in the loaded plex, where N is the global number of points; NULL if unneeded
1896 
1897   Level: advanced
1898 
1899 .seealso: DMLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1900 @*/
1901 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
1902 {
1903   PetscBool      ishdf5;
1904   PetscErrorCode ierr;
1905 
1906   PetscFunctionBegin;
1907   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1908   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1909   if (globalToLocalPointSF) PetscValidPointer(globalToLocalPointSF, 3);
1910   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1911   if (ishdf5) {
1912 #if defined(PETSC_HAVE_HDF5)
1913     PetscViewerFormat format;
1914     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1915     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1916       ierr = DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
1917     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1918 #else
1919     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1920 #endif
1921   }
1922   PetscFunctionReturn(0);
1923 }
1924 
1925 /*@
1926   DMPlexCoordinatesLoad - Loads coordinates into a DMPlex
1927 
1928   Collective on DM
1929 
1930   Input Parameters:
1931 + dm     - The DM into which the coordinates are loaded
1932 - viewer - The PetscViewer for the saved coordinates
1933 
1934   Level: advanced
1935 
1936 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1937 @*/
1938 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer)
1939 {
1940   PetscBool      ishdf5;
1941   PetscErrorCode ierr;
1942 
1943   PetscFunctionBegin;
1944   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1945   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1946   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1947   if (ishdf5) {
1948 #if defined(PETSC_HAVE_HDF5)
1949     PetscViewerFormat format;
1950     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1951     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1952       ierr = DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1953     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1954 #else
1955     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1956 #endif
1957   }
1958   PetscFunctionReturn(0);
1959 }
1960 
1961 /*@
1962   DMPlexLabelsLoad - Loads labels into a DMPlex
1963 
1964   Collective on DM
1965 
1966   Input Parameters:
1967 + dm     - The DM into which the labels are loaded
1968 - viewer - The PetscViewer for the saved labels
1969 
1970   Level: advanced
1971 
1972 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1973 @*/
1974 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer)
1975 {
1976   PetscBool      ishdf5;
1977   PetscErrorCode ierr;
1978 
1979   PetscFunctionBegin;
1980   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1981   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1982   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1983   if (ishdf5) {
1984 #if defined(PETSC_HAVE_HDF5)
1985     PetscViewerFormat format;
1986 
1987     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1988     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1989       ierr = DMPlexLabelsLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1990     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1991 #else
1992     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1993 #endif
1994   }
1995   PetscFunctionReturn(0);
1996 }
1997 
1998 /*@
1999   DMPlexSectionLoad - Loads section into a DMPlex
2000 
2001   Collective on DM
2002 
2003   Input Parameters:
2004 + dm          - The DM that represents the topology
2005 . viewer      - The PetscViewer that represents the on-disk section (sectionA)
2006 . sectiondm   - The DM into which the on-disk section (sectionA) is migrated
2007 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2008 
2009   Output Parameters
2010 + globalDofSF - The SF that migrates any on-disk Vec data associated with sectionA into a global Vec associated with the sectiondm's global section (NULL if not needed)
2011 - localDofSF  - The SF that migrates any on-disk Vec data associated with sectionA into a local Vec associated with the sectiondm's local section (NULL if not needed)
2012 
2013   Level: advanced
2014 
2015   Notes:
2016   This function is a wrapper around PetscSectionLoad(); it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in dm. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
2017 
2018   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2019 
2020   The output parameter, globalDofSF (localDofSF), can later be used with DMPlexGlobalVectorLoad() (DMPlexLocalVectorLoad()) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
2021 
2022   Example using 2 processes:
2023 $  NX (number of points on dm): 4
2024 $  sectionA                   : the on-disk section
2025 $  vecA                       : a vector associated with sectionA
2026 $  sectionB                   : sectiondm's local section constructed in this function
2027 $  vecB (local)               : a vector associated with sectiondm's local section
2028 $  vecB (global)              : a vector associated with sectiondm's global section
2029 $
2030 $                                     rank 0    rank 1
2031 $  vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
2032 $  sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
2033 $  sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
2034 $  sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
2035 $  [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
2036 $  sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
2037 $  sectionB->atlasDof             :     1 0 1 | 1 3
2038 $  sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
2039 $  vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
2040 $  vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
2041 $
2042 $  where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
2043 
2044 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad(), PetscSectionLoad(), DMPlexSectionView()
2045 @*/
2046 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
2047 {
2048   PetscBool      ishdf5;
2049   PetscErrorCode ierr;
2050 
2051   PetscFunctionBegin;
2052   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2053   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2054   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2055   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
2056   if (globalDofSF) PetscValidPointer(globalDofSF, 5);
2057   if (localDofSF) PetscValidPointer(localDofSF, 6);
2058   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2059   if (ishdf5) {
2060 #if defined(PETSC_HAVE_HDF5)
2061     ierr = DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF);CHKERRQ(ierr);
2062 #else
2063     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2064 #endif
2065   }
2066   PetscFunctionReturn(0);
2067 }
2068 
2069 /*@
2070   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
2071 
2072   Collective on DM
2073 
2074   Input Parameters:
2075 + dm        - The DM that represents the topology
2076 . viewer    - The PetscViewer that represents the on-disk vector data
2077 . sectiondm - The DM that contains the global section on which vec is defined
2078 . sf        - The SF that migrates the on-disk vector data into vec
2079 - vec       - The global vector to set values of
2080 
2081   Level: advanced
2082 
2083   Notes:
2084   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2085 
2086   Typical calling sequence
2087 $       DMCreate(PETSC_COMM_WORLD, &dm);
2088 $       DMSetType(dm, DMPLEX);
2089 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2090 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2091 $       DMClone(dm, &sectiondm);
2092 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2093 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
2094 $       DMGetGlobalVector(sectiondm, &vec);
2095 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2096 $       DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
2097 $       DMRestoreGlobalVector(sectiondm, &vec);
2098 $       PetscSFDestroy(&gsf);
2099 $       PetscSFDestroy(&sfX);
2100 $       DMDestroy(&sectiondm);
2101 $       DMDestroy(&dm);
2102 
2103 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexLocalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2104 @*/
2105 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2106 {
2107   PetscBool       ishdf5;
2108   PetscErrorCode  ierr;
2109 
2110   PetscFunctionBegin;
2111   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2112   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2113   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2114   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2115   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2116   /* Check consistency */
2117   {
2118     PetscSection  section;
2119     PetscBool     includesConstraints;
2120     PetscInt      m, m1;
2121 
2122     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2123     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
2124     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2125     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2126     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2127     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
2128   }
2129   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2130   if (ishdf5) {
2131 #if defined(PETSC_HAVE_HDF5)
2132     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2133 #else
2134     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2135 #endif
2136   }
2137   PetscFunctionReturn(0);
2138 }
2139 
2140 /*@
2141   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2142 
2143   Collective on DM
2144 
2145   Input Parameters:
2146 + dm        - The DM that represents the topology
2147 . viewer    - The PetscViewer that represents the on-disk vector data
2148 . sectiondm - The DM that contains the local section on which vec is defined
2149 . sf        - The SF that migrates the on-disk vector data into vec
2150 - vec       - The local vector to set values of
2151 
2152   Level: advanced
2153 
2154   Notes:
2155   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2156 
2157   Typical calling sequence
2158 $       DMCreate(PETSC_COMM_WORLD, &dm);
2159 $       DMSetType(dm, DMPLEX);
2160 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2161 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2162 $       DMClone(dm, &sectiondm);
2163 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2164 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2165 $       DMGetLocalVector(sectiondm, &vec);
2166 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2167 $       DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2168 $       DMRestoreLocalVector(sectiondm, &vec);
2169 $       PetscSFDestroy(&lsf);
2170 $       PetscSFDestroy(&sfX);
2171 $       DMDestroy(&sectiondm);
2172 $       DMDestroy(&dm);
2173 
2174 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexGlobalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2175 @*/
2176 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2177 {
2178   PetscBool       ishdf5;
2179   PetscErrorCode  ierr;
2180 
2181   PetscFunctionBegin;
2182   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2183   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2184   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2185   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2186   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2187   /* Check consistency */
2188   {
2189     PetscSection  section;
2190     PetscBool     includesConstraints;
2191     PetscInt      m, m1;
2192 
2193     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2194     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
2195     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2196     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2197     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2198     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
2199   }
2200   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2201   if (ishdf5) {
2202 #if defined(PETSC_HAVE_HDF5)
2203     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2204 #else
2205     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2206 #endif
2207   }
2208   PetscFunctionReturn(0);
2209 }
2210 
2211 PetscErrorCode DMDestroy_Plex(DM dm)
2212 {
2213   DM_Plex       *mesh = (DM_Plex*) dm->data;
2214   PetscErrorCode ierr;
2215 
2216   PetscFunctionBegin;
2217   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
2218   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
2219   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
2220   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C", NULL);CHKERRQ(ierr);
2221   if (--mesh->refct > 0) PetscFunctionReturn(0);
2222   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
2223   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
2224   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
2225   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
2226   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
2227   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
2228   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
2229   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
2230   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
2231   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
2232   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
2233   ierr = ISDestroy(&mesh->subpointIS);CHKERRQ(ierr);
2234   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
2235   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
2236   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
2237   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
2238   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
2239   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
2240   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
2241   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
2242   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
2243   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
2244   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
2245   ierr = PetscFree(mesh->neighbors);CHKERRQ(ierr);
2246   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2247   ierr = PetscFree(mesh);CHKERRQ(ierr);
2248   PetscFunctionReturn(0);
2249 }
2250 
2251 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2252 {
2253   PetscSection           sectionGlobal;
2254   PetscInt               bs = -1, mbs;
2255   PetscInt               localSize;
2256   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2257   PetscErrorCode         ierr;
2258   MatType                mtype;
2259   ISLocalToGlobalMapping ltog;
2260 
2261   PetscFunctionBegin;
2262   ierr = MatInitializePackage();CHKERRQ(ierr);
2263   mtype = dm->mattype;
2264   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
2265   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
2266   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
2267   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
2268   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
2269   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
2270   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
2271   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
2272   if (mbs > 1) bs = mbs;
2273   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
2274   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
2275   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
2276   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
2277   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
2278   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
2279   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
2280   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
2281   if (!isShell) {
2282     PetscSection subSection;
2283     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2284     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
2285     PetscInt     pStart, pEnd, p, dof, cdof;
2286 
2287     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
2288     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
2289       PetscSection section;
2290       PetscInt     size;
2291 
2292       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
2293       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
2294       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
2295       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
2296     } else {
2297       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
2298     }
2299     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
2300     for (p = pStart, lsize = 0; p < pEnd; ++p) {
2301       PetscInt bdof;
2302 
2303       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
2304       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
2305       dof  = dof < 0 ? -(dof+1) : dof;
2306       bdof = cdof && (dof-cdof) ? 1 : dof;
2307       if (dof) {
2308         if (bs < 0)          {bs = bdof;}
2309         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
2310       }
2311       if (isMatIS) {
2312         PetscInt loff,c,off;
2313         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
2314         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
2315         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
2316       }
2317     }
2318     /* Must have same blocksize on all procs (some might have no points) */
2319     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
2320     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
2321     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
2322     else                            {bs = bsMinMax[0];}
2323     bs = PetscMax(1,bs);
2324     if (isMatIS) { /* Must reduce indices by blocksize */
2325       PetscInt l;
2326 
2327       lsize = lsize/bs;
2328       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
2329       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
2330     }
2331     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
2332     if (isMatIS) {
2333       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
2334     }
2335     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
2336     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
2337     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
2338   }
2339   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
2340   PetscFunctionReturn(0);
2341 }
2342 
2343 /*@
2344   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2345 
2346   Not collective
2347 
2348   Input Parameter:
2349 . mesh - The DMPlex
2350 
2351   Output Parameters:
2352 . subsection - The subdomain section
2353 
2354   Level: developer
2355 
2356 .seealso:
2357 @*/
2358 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2359 {
2360   DM_Plex       *mesh = (DM_Plex*) dm->data;
2361   PetscErrorCode ierr;
2362 
2363   PetscFunctionBegin;
2364   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2365   if (!mesh->subdomainSection) {
2366     PetscSection section;
2367     PetscSF      sf;
2368 
2369     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
2370     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
2371     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
2372     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
2373   }
2374   *subsection = mesh->subdomainSection;
2375   PetscFunctionReturn(0);
2376 }
2377 
2378 /*@
2379   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
2380 
2381   Not collective
2382 
2383   Input Parameter:
2384 . mesh - The DMPlex
2385 
2386   Output Parameters:
2387 + pStart - The first mesh point
2388 - pEnd   - The upper bound for mesh points
2389 
2390   Level: beginner
2391 
2392 .seealso: DMPlexCreate(), DMPlexSetChart()
2393 @*/
2394 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2395 {
2396   DM_Plex       *mesh = (DM_Plex*) dm->data;
2397   PetscErrorCode ierr;
2398 
2399   PetscFunctionBegin;
2400   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2401   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2402   PetscFunctionReturn(0);
2403 }
2404 
2405 /*@
2406   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
2407 
2408   Not collective
2409 
2410   Input Parameters:
2411 + mesh - The DMPlex
2412 . pStart - The first mesh point
2413 - pEnd   - The upper bound for mesh points
2414 
2415   Output Parameters:
2416 
2417   Level: beginner
2418 
2419 .seealso: DMPlexCreate(), DMPlexGetChart()
2420 @*/
2421 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2422 {
2423   DM_Plex       *mesh = (DM_Plex*) dm->data;
2424   PetscErrorCode ierr;
2425 
2426   PetscFunctionBegin;
2427   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2428   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2429   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2430   PetscFunctionReturn(0);
2431 }
2432 
2433 /*@
2434   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2435 
2436   Not collective
2437 
2438   Input Parameters:
2439 + mesh - The DMPlex
2440 - p - The point, which must lie in the chart set with DMPlexSetChart()
2441 
2442   Output Parameter:
2443 . size - The cone size for point p
2444 
2445   Level: beginner
2446 
2447 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2448 @*/
2449 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2450 {
2451   DM_Plex       *mesh = (DM_Plex*) dm->data;
2452   PetscErrorCode ierr;
2453 
2454   PetscFunctionBegin;
2455   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2456   PetscValidPointer(size, 3);
2457   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2458   PetscFunctionReturn(0);
2459 }
2460 
2461 /*@
2462   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2463 
2464   Not collective
2465 
2466   Input Parameters:
2467 + mesh - The DMPlex
2468 . p - The point, which must lie in the chart set with DMPlexSetChart()
2469 - size - The cone size for point p
2470 
2471   Output Parameter:
2472 
2473   Note:
2474   This should be called after DMPlexSetChart().
2475 
2476   Level: beginner
2477 
2478 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
2479 @*/
2480 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
2481 {
2482   DM_Plex       *mesh = (DM_Plex*) dm->data;
2483   PetscErrorCode ierr;
2484 
2485   PetscFunctionBegin;
2486   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2487   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2488 
2489   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
2490   PetscFunctionReturn(0);
2491 }
2492 
2493 /*@
2494   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
2495 
2496   Not collective
2497 
2498   Input Parameters:
2499 + mesh - The DMPlex
2500 . p - The point, which must lie in the chart set with DMPlexSetChart()
2501 - size - The additional cone size for point p
2502 
2503   Output Parameter:
2504 
2505   Note:
2506   This should be called after DMPlexSetChart().
2507 
2508   Level: beginner
2509 
2510 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
2511 @*/
2512 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
2513 {
2514   DM_Plex       *mesh = (DM_Plex*) dm->data;
2515   PetscInt       csize;
2516   PetscErrorCode ierr;
2517 
2518   PetscFunctionBegin;
2519   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2520   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2521   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
2522 
2523   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
2524   PetscFunctionReturn(0);
2525 }
2526 
2527 /*@C
2528   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
2529 
2530   Not collective
2531 
2532   Input Parameters:
2533 + dm - The DMPlex
2534 - p - The point, which must lie in the chart set with DMPlexSetChart()
2535 
2536   Output Parameter:
2537 . cone - An array of points which are on the in-edges for point p
2538 
2539   Level: beginner
2540 
2541   Fortran Notes:
2542   Since it returns an array, this routine is only available in Fortran 90, and you must
2543   include petsc.h90 in your code.
2544   You must also call DMPlexRestoreCone() after you finish using the returned array.
2545   DMPlexRestoreCone() is not needed/available in C.
2546 
2547 .seealso: DMPlexGetConeSize(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
2548 @*/
2549 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
2550 {
2551   DM_Plex       *mesh = (DM_Plex*) dm->data;
2552   PetscInt       off;
2553   PetscErrorCode ierr;
2554 
2555   PetscFunctionBegin;
2556   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2557   PetscValidPointer(cone, 3);
2558   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2559   *cone = &mesh->cones[off];
2560   PetscFunctionReturn(0);
2561 }
2562 
2563 /*@C
2564   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
2565 
2566   Not collective
2567 
2568   Input Parameters:
2569 + dm - The DMPlex
2570 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
2571 
2572   Output Parameter:
2573 + pConesSection - PetscSection describing the layout of pCones
2574 - pCones - An array of points which are on the in-edges for the point set p
2575 
2576   Level: intermediate
2577 
2578 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
2579 @*/
2580 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
2581 {
2582   PetscSection        cs, newcs;
2583   PetscInt            *cones;
2584   PetscInt            *newarr=NULL;
2585   PetscInt            n;
2586   PetscErrorCode      ierr;
2587 
2588   PetscFunctionBegin;
2589   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2590   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
2591   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
2592   if (pConesSection) *pConesSection = newcs;
2593   if (pCones) {
2594     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
2595     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
2596   }
2597   PetscFunctionReturn(0);
2598 }
2599 
2600 /*@
2601   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
2602 
2603   Not collective
2604 
2605   Input Parameters:
2606 + dm - The DMPlex
2607 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2608 
2609   Output Parameter:
2610 . expandedPoints - An array of vertices recursively expanded from input points
2611 
2612   Level: advanced
2613 
2614   Notes:
2615   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
2616   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
2617 
2618 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
2619 @*/
2620 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
2621 {
2622   IS                  *expandedPointsAll;
2623   PetscInt            depth;
2624   PetscErrorCode      ierr;
2625 
2626   PetscFunctionBegin;
2627   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2628   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2629   PetscValidPointer(expandedPoints, 3);
2630   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2631   *expandedPoints = expandedPointsAll[0];
2632   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);CHKERRQ(ierr);
2633   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2634   PetscFunctionReturn(0);
2635 }
2636 
2637 /*@
2638   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
2639 
2640   Not collective
2641 
2642   Input Parameters:
2643 + dm - The DMPlex
2644 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2645 
2646   Output Parameter:
2647 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2648 . expandedPoints - (optional) An array of index sets with recursively expanded cones
2649 - sections - (optional) An array of sections which describe mappings from points to their cone points
2650 
2651   Level: advanced
2652 
2653   Notes:
2654   Like DMPlexGetConeTuple() but recursive.
2655 
2656   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
2657   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
2658 
2659   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
2660   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
2661   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
2662 
2663 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2664 @*/
2665 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2666 {
2667   const PetscInt      *arr0=NULL, *cone=NULL;
2668   PetscInt            *arr=NULL, *newarr=NULL;
2669   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
2670   IS                  *expandedPoints_;
2671   PetscSection        *sections_;
2672   PetscErrorCode      ierr;
2673 
2674   PetscFunctionBegin;
2675   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2676   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2677   if (depth) PetscValidIntPointer(depth, 3);
2678   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
2679   if (sections) PetscValidPointer(sections, 5);
2680   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
2681   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
2682   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2683   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
2684   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
2685   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
2686   for (d=depth_-1; d>=0; d--) {
2687     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
2688     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
2689     for (i=0; i<n; i++) {
2690       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
2691       if (arr[i] >= start && arr[i] < end) {
2692         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
2693         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
2694       } else {
2695         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
2696       }
2697     }
2698     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
2699     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
2700     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
2701     for (i=0; i<n; i++) {
2702       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
2703       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
2704       if (cn > 1) {
2705         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
2706         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
2707       } else {
2708         newarr[co] = arr[i];
2709       }
2710     }
2711     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
2712     arr = newarr;
2713     n = newn;
2714   }
2715   ierr = ISRestoreIndices(points, &arr0);CHKERRQ(ierr);
2716   *depth = depth_;
2717   if (expandedPoints) *expandedPoints = expandedPoints_;
2718   else {
2719     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
2720     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
2721   }
2722   if (sections) *sections = sections_;
2723   else {
2724     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
2725     ierr = PetscFree(sections_);CHKERRQ(ierr);
2726   }
2727   PetscFunctionReturn(0);
2728 }
2729 
2730 /*@
2731   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
2732 
2733   Not collective
2734 
2735   Input Parameters:
2736 + dm - The DMPlex
2737 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2738 
2739   Output Parameter:
2740 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2741 . expandedPoints - (optional) An array of recursively expanded cones
2742 - sections - (optional) An array of sections which describe mappings from points to their cone points
2743 
2744   Level: advanced
2745 
2746   Notes:
2747   See DMPlexGetConeRecursive() for details.
2748 
2749 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2750 @*/
2751 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2752 {
2753   PetscInt            d, depth_;
2754   PetscErrorCode      ierr;
2755 
2756   PetscFunctionBegin;
2757   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2758   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
2759   if (depth) *depth = 0;
2760   if (expandedPoints) {
2761     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
2762     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
2763   }
2764   if (sections)  {
2765     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
2766     ierr = PetscFree(*sections);CHKERRQ(ierr);
2767   }
2768   PetscFunctionReturn(0);
2769 }
2770 
2771 /*@
2772   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
2773 
2774   Not collective
2775 
2776   Input Parameters:
2777 + mesh - The DMPlex
2778 . p - The point, which must lie in the chart set with DMPlexSetChart()
2779 - cone - An array of points which are on the in-edges for point p
2780 
2781   Output Parameter:
2782 
2783   Note:
2784   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2785 
2786   Level: beginner
2787 
2788 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
2789 @*/
2790 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
2791 {
2792   DM_Plex       *mesh = (DM_Plex*) dm->data;
2793   PetscInt       pStart, pEnd;
2794   PetscInt       dof, off, c;
2795   PetscErrorCode ierr;
2796 
2797   PetscFunctionBegin;
2798   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2799   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2800   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2801   if (dof) PetscValidPointer(cone, 3);
2802   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2803   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2804   for (c = 0; c < dof; ++c) {
2805     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
2806     mesh->cones[off+c] = cone[c];
2807   }
2808   PetscFunctionReturn(0);
2809 }
2810 
2811 /*@C
2812   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
2813 
2814   Not collective
2815 
2816   Input Parameters:
2817 + mesh - The DMPlex
2818 - p - The point, which must lie in the chart set with DMPlexSetChart()
2819 
2820   Output Parameter:
2821 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2822                     integer giving the prescription for cone traversal. If it is negative, the cone is
2823                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
2824                     the index of the cone point on which to start.
2825 
2826   Level: beginner
2827 
2828   Fortran Notes:
2829   Since it returns an array, this routine is only available in Fortran 90, and you must
2830   include petsc.h90 in your code.
2831   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
2832   DMPlexRestoreConeOrientation() is not needed/available in C.
2833 
2834 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
2835 @*/
2836 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
2837 {
2838   DM_Plex       *mesh = (DM_Plex*) dm->data;
2839   PetscInt       off;
2840   PetscErrorCode ierr;
2841 
2842   PetscFunctionBegin;
2843   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2844   if (PetscDefined(USE_DEBUG)) {
2845     PetscInt dof;
2846     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2847     if (dof) PetscValidPointer(coneOrientation, 3);
2848   }
2849   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2850 
2851   *coneOrientation = &mesh->coneOrientations[off];
2852   PetscFunctionReturn(0);
2853 }
2854 
2855 /*@
2856   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
2857 
2858   Not collective
2859 
2860   Input Parameters:
2861 + mesh - The DMPlex
2862 . p - The point, which must lie in the chart set with DMPlexSetChart()
2863 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2864                     integer giving the prescription for cone traversal. If it is negative, the cone is
2865                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
2866                     the index of the cone point on which to start.
2867 
2868   Output Parameter:
2869 
2870   Note:
2871   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2872 
2873   Level: beginner
2874 
2875 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2876 @*/
2877 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2878 {
2879   DM_Plex       *mesh = (DM_Plex*) dm->data;
2880   PetscInt       pStart, pEnd;
2881   PetscInt       dof, off, c;
2882   PetscErrorCode ierr;
2883 
2884   PetscFunctionBegin;
2885   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2886   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2887   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2888   if (dof) PetscValidPointer(coneOrientation, 3);
2889   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2890   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2891   for (c = 0; c < dof; ++c) {
2892     PetscInt cdof, o = coneOrientation[c];
2893 
2894     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2895     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2896     mesh->coneOrientations[off+c] = o;
2897   }
2898   PetscFunctionReturn(0);
2899 }
2900 
2901 /*@
2902   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2903 
2904   Not collective
2905 
2906   Input Parameters:
2907 + mesh - The DMPlex
2908 . p - The point, which must lie in the chart set with DMPlexSetChart()
2909 . conePos - The local index in the cone where the point should be put
2910 - conePoint - The mesh point to insert
2911 
2912   Level: beginner
2913 
2914 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2915 @*/
2916 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2917 {
2918   DM_Plex       *mesh = (DM_Plex*) dm->data;
2919   PetscInt       pStart, pEnd;
2920   PetscInt       dof, off;
2921   PetscErrorCode ierr;
2922 
2923   PetscFunctionBegin;
2924   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2925   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2926   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2927   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2928   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2929   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2930   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2931   mesh->cones[off+conePos] = conePoint;
2932   PetscFunctionReturn(0);
2933 }
2934 
2935 /*@
2936   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2937 
2938   Not collective
2939 
2940   Input Parameters:
2941 + mesh - The DMPlex
2942 . p - The point, which must lie in the chart set with DMPlexSetChart()
2943 . conePos - The local index in the cone where the point should be put
2944 - coneOrientation - The point orientation to insert
2945 
2946   Level: beginner
2947 
2948 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2949 @*/
2950 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2951 {
2952   DM_Plex       *mesh = (DM_Plex*) dm->data;
2953   PetscInt       pStart, pEnd;
2954   PetscInt       dof, off;
2955   PetscErrorCode ierr;
2956 
2957   PetscFunctionBegin;
2958   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2959   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2960   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2961   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2962   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2963   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2964   mesh->coneOrientations[off+conePos] = coneOrientation;
2965   PetscFunctionReturn(0);
2966 }
2967 
2968 /*@
2969   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2970 
2971   Not collective
2972 
2973   Input Parameters:
2974 + mesh - The DMPlex
2975 - p - The point, which must lie in the chart set with DMPlexSetChart()
2976 
2977   Output Parameter:
2978 . size - The support size for point p
2979 
2980   Level: beginner
2981 
2982 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2983 @*/
2984 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2985 {
2986   DM_Plex       *mesh = (DM_Plex*) dm->data;
2987   PetscErrorCode ierr;
2988 
2989   PetscFunctionBegin;
2990   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2991   PetscValidPointer(size, 3);
2992   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2993   PetscFunctionReturn(0);
2994 }
2995 
2996 /*@
2997   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2998 
2999   Not collective
3000 
3001   Input Parameters:
3002 + mesh - The DMPlex
3003 . p - The point, which must lie in the chart set with DMPlexSetChart()
3004 - size - The support size for point p
3005 
3006   Output Parameter:
3007 
3008   Note:
3009   This should be called after DMPlexSetChart().
3010 
3011   Level: beginner
3012 
3013 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
3014 @*/
3015 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
3016 {
3017   DM_Plex       *mesh = (DM_Plex*) dm->data;
3018   PetscErrorCode ierr;
3019 
3020   PetscFunctionBegin;
3021   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3022   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
3023 
3024   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
3025   PetscFunctionReturn(0);
3026 }
3027 
3028 /*@C
3029   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
3030 
3031   Not collective
3032 
3033   Input Parameters:
3034 + mesh - The DMPlex
3035 - p - The point, which must lie in the chart set with DMPlexSetChart()
3036 
3037   Output Parameter:
3038 . support - An array of points which are on the out-edges for point p
3039 
3040   Level: beginner
3041 
3042   Fortran Notes:
3043   Since it returns an array, this routine is only available in Fortran 90, and you must
3044   include petsc.h90 in your code.
3045   You must also call DMPlexRestoreSupport() after you finish using the returned array.
3046   DMPlexRestoreSupport() is not needed/available in C.
3047 
3048 .seealso: DMPlexGetSupportSize(), DMPlexSetSupport(), DMPlexGetCone(), DMPlexSetChart()
3049 @*/
3050 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
3051 {
3052   DM_Plex       *mesh = (DM_Plex*) dm->data;
3053   PetscInt       off;
3054   PetscErrorCode ierr;
3055 
3056   PetscFunctionBegin;
3057   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3058   PetscValidPointer(support, 3);
3059   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3060   *support = &mesh->supports[off];
3061   PetscFunctionReturn(0);
3062 }
3063 
3064 /*@
3065   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
3066 
3067   Not collective
3068 
3069   Input Parameters:
3070 + mesh - The DMPlex
3071 . p - The point, which must lie in the chart set with DMPlexSetChart()
3072 - support - An array of points which are on the out-edges for point p
3073 
3074   Output Parameter:
3075 
3076   Note:
3077   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
3078 
3079   Level: beginner
3080 
3081 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
3082 @*/
3083 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
3084 {
3085   DM_Plex       *mesh = (DM_Plex*) dm->data;
3086   PetscInt       pStart, pEnd;
3087   PetscInt       dof, off, c;
3088   PetscErrorCode ierr;
3089 
3090   PetscFunctionBegin;
3091   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3092   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3093   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3094   if (dof) PetscValidPointer(support, 3);
3095   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3096   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3097   for (c = 0; c < dof; ++c) {
3098     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
3099     mesh->supports[off+c] = support[c];
3100   }
3101   PetscFunctionReturn(0);
3102 }
3103 
3104 /*@
3105   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
3106 
3107   Not collective
3108 
3109   Input Parameters:
3110 + mesh - The DMPlex
3111 . p - The point, which must lie in the chart set with DMPlexSetChart()
3112 . supportPos - The local index in the cone where the point should be put
3113 - supportPoint - The mesh point to insert
3114 
3115   Level: beginner
3116 
3117 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3118 @*/
3119 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3120 {
3121   DM_Plex       *mesh = (DM_Plex*) dm->data;
3122   PetscInt       pStart, pEnd;
3123   PetscInt       dof, off;
3124   PetscErrorCode ierr;
3125 
3126   PetscFunctionBegin;
3127   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3128   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3129   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3130   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3131   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3132   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
3133   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
3134   mesh->supports[off+supportPos] = supportPoint;
3135   PetscFunctionReturn(0);
3136 }
3137 
3138 /*@C
3139   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
3140 
3141   Not collective
3142 
3143   Input Parameters:
3144 + mesh - The DMPlex
3145 . p - The point, which must lie in the chart set with DMPlexSetChart()
3146 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3147 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
3148 
3149   Output Parameters:
3150 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3151 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3152 
3153   Note:
3154   If using internal storage (points is NULL on input), each call overwrites the last output.
3155 
3156   Fortran Notes:
3157   Since it returns an array, this routine is only available in Fortran 90, and you must
3158   include petsc.h90 in your code.
3159 
3160   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3161 
3162   Level: beginner
3163 
3164 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3165 @*/
3166 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3167 {
3168   DM_Plex        *mesh = (DM_Plex*) dm->data;
3169   PetscInt       *closure, *fifo;
3170   const PetscInt *tmp = NULL, *tmpO = NULL;
3171   PetscInt        tmpSize, t;
3172   PetscInt        depth       = 0, maxSize;
3173   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
3174   PetscErrorCode  ierr;
3175 
3176   PetscFunctionBegin;
3177   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3178   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3179   /* This is only 1-level */
3180   if (useCone) {
3181     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3182     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3183     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3184   } else {
3185     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3186     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3187   }
3188   if (depth == 1) {
3189     if (*points) {
3190       closure = *points;
3191     } else {
3192       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
3193       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3194     }
3195     closure[0] = p; closure[1] = 0;
3196     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
3197       closure[closureSize]   = tmp[t];
3198       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
3199     }
3200     if (numPoints) *numPoints = closureSize/2;
3201     if (points)    *points    = closure;
3202     PetscFunctionReturn(0);
3203   }
3204   {
3205     PetscInt c, coneSeries, s,supportSeries;
3206 
3207     c = mesh->maxConeSize;
3208     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
3209     s = mesh->maxSupportSize;
3210     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
3211     maxSize = 2*PetscMax(coneSeries,supportSeries);
3212   }
3213   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3214   if (*points) {
3215     closure = *points;
3216   } else {
3217     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3218   }
3219   closure[0] = p; closure[1] = 0;
3220   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
3221     const PetscInt cp = tmp[t];
3222     const PetscInt co = tmpO ? tmpO[t] : 0;
3223 
3224     closure[closureSize]   = cp;
3225     closure[closureSize+1] = co;
3226     fifo[fifoSize]         = cp;
3227     fifo[fifoSize+1]       = co;
3228   }
3229   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3230   while (fifoSize - fifoStart) {
3231     const PetscInt q   = fifo[fifoStart];
3232     const PetscInt o   = fifo[fifoStart+1];
3233     const PetscInt rev = o >= 0 ? 0 : 1;
3234     const PetscInt off = rev ? -(o+1) : o;
3235 
3236     if (useCone) {
3237       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3238       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3239       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3240     } else {
3241       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3242       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3243       tmpO = NULL;
3244     }
3245     for (t = 0; t < tmpSize; ++t) {
3246       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
3247       const PetscInt cp = tmp[i];
3248       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
3249       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
3250        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
3251       PetscInt       co = tmpO ? tmpO[i] : 0;
3252       PetscInt       c;
3253 
3254       if (rev) {
3255         PetscInt childSize, coff;
3256         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3257         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
3258         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3259       }
3260       /* Check for duplicate */
3261       for (c = 0; c < closureSize; c += 2) {
3262         if (closure[c] == cp) break;
3263       }
3264       if (c == closureSize) {
3265         closure[closureSize]   = cp;
3266         closure[closureSize+1] = co;
3267         fifo[fifoSize]         = cp;
3268         fifo[fifoSize+1]       = co;
3269         closureSize           += 2;
3270         fifoSize              += 2;
3271       }
3272     }
3273     fifoStart += 2;
3274   }
3275   if (numPoints) *numPoints = closureSize/2;
3276   if (points)    *points    = closure;
3277   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3278   PetscFunctionReturn(0);
3279 }
3280 
3281 /*@C
3282   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
3283 
3284   Not collective
3285 
3286   Input Parameters:
3287 + mesh - The DMPlex
3288 . p - The point, which must lie in the chart set with DMPlexSetChart()
3289 . orientation - The orientation of the point
3290 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3291 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
3292 
3293   Output Parameters:
3294 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3295 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3296 
3297   Note:
3298   If using internal storage (points is NULL on input), each call overwrites the last output.
3299 
3300   Fortran Notes:
3301   Since it returns an array, this routine is only available in Fortran 90, and you must
3302   include petsc.h90 in your code.
3303 
3304   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3305 
3306   Level: beginner
3307 
3308 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3309 @*/
3310 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3311 {
3312   DM_Plex        *mesh = (DM_Plex*) dm->data;
3313   PetscInt       *closure, *fifo;
3314   const PetscInt *tmp = NULL, *tmpO = NULL;
3315   PetscInt        tmpSize, t;
3316   PetscInt        depth       = 0, maxSize;
3317   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
3318   PetscErrorCode  ierr;
3319 
3320   PetscFunctionBegin;
3321   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3322   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3323   /* This is only 1-level */
3324   if (useCone) {
3325     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3326     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3327     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3328   } else {
3329     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3330     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3331   }
3332   if (depth == 1) {
3333     if (*points) {
3334       closure = *points;
3335     } else {
3336       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
3337       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3338     }
3339     closure[0] = p; closure[1] = ornt;
3340     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
3341       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
3342       closure[closureSize]   = tmp[i];
3343       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
3344     }
3345     if (numPoints) *numPoints = closureSize/2;
3346     if (points)    *points    = closure;
3347     PetscFunctionReturn(0);
3348   }
3349   {
3350     PetscInt c, coneSeries, s,supportSeries;
3351 
3352     c = mesh->maxConeSize;
3353     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
3354     s = mesh->maxSupportSize;
3355     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
3356     maxSize = 2*PetscMax(coneSeries,supportSeries);
3357   }
3358   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3359   if (*points) {
3360     closure = *points;
3361   } else {
3362     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3363   }
3364   closure[0] = p; closure[1] = ornt;
3365   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
3366     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
3367     const PetscInt cp = tmp[i];
3368     PetscInt       co = tmpO ? tmpO[i] : 0;
3369 
3370     if (ornt < 0) {
3371       PetscInt childSize, coff;
3372       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3373       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
3374       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3375     }
3376     closure[closureSize]   = cp;
3377     closure[closureSize+1] = co;
3378     fifo[fifoSize]         = cp;
3379     fifo[fifoSize+1]       = co;
3380   }
3381   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3382   while (fifoSize - fifoStart) {
3383     const PetscInt q   = fifo[fifoStart];
3384     const PetscInt o   = fifo[fifoStart+1];
3385     const PetscInt rev = o >= 0 ? 0 : 1;
3386     const PetscInt off = rev ? -(o+1) : o;
3387 
3388     if (useCone) {
3389       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3390       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3391       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3392     } else {
3393       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3394       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3395       tmpO = NULL;
3396     }
3397     for (t = 0; t < tmpSize; ++t) {
3398       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
3399       const PetscInt cp = tmp[i];
3400       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
3401       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
3402        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
3403       PetscInt       co = tmpO ? tmpO[i] : 0;
3404       PetscInt       c;
3405 
3406       if (rev) {
3407         PetscInt childSize, coff;
3408         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3409         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
3410         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3411       }
3412       /* Check for duplicate */
3413       for (c = 0; c < closureSize; c += 2) {
3414         if (closure[c] == cp) break;
3415       }
3416       if (c == closureSize) {
3417         closure[closureSize]   = cp;
3418         closure[closureSize+1] = co;
3419         fifo[fifoSize]         = cp;
3420         fifo[fifoSize+1]       = co;
3421         closureSize           += 2;
3422         fifoSize              += 2;
3423       }
3424     }
3425     fifoStart += 2;
3426   }
3427   if (numPoints) *numPoints = closureSize/2;
3428   if (points)    *points    = closure;
3429   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3430   PetscFunctionReturn(0);
3431 }
3432 
3433 /*@C
3434   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
3435 
3436   Not collective
3437 
3438   Input Parameters:
3439 + mesh - The DMPlex
3440 . p - The point, which must lie in the chart set with DMPlexSetChart()
3441 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3442 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
3443 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
3444 
3445   Note:
3446   If not using internal storage (points is not NULL on input), this call is unnecessary
3447 
3448   Fortran Notes:
3449   Since it returns an array, this routine is only available in Fortran 90, and you must
3450   include petsc.h90 in your code.
3451 
3452   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3453 
3454   Level: beginner
3455 
3456 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3457 @*/
3458 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3459 {
3460   PetscErrorCode ierr;
3461 
3462   PetscFunctionBegin;
3463   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3464   if (numPoints) PetscValidIntPointer(numPoints,4);
3465   if (points) PetscValidPointer(points,5);
3466   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
3467   if (numPoints) *numPoints = 0;
3468   PetscFunctionReturn(0);
3469 }
3470 
3471 /*@
3472   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
3473 
3474   Not collective
3475 
3476   Input Parameter:
3477 . mesh - The DMPlex
3478 
3479   Output Parameters:
3480 + maxConeSize - The maximum number of in-edges
3481 - maxSupportSize - The maximum number of out-edges
3482 
3483   Level: beginner
3484 
3485 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
3486 @*/
3487 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
3488 {
3489   DM_Plex *mesh = (DM_Plex*) dm->data;
3490 
3491   PetscFunctionBegin;
3492   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3493   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
3494   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
3495   PetscFunctionReturn(0);
3496 }
3497 
3498 PetscErrorCode DMSetUp_Plex(DM dm)
3499 {
3500   DM_Plex       *mesh = (DM_Plex*) dm->data;
3501   PetscInt       size;
3502   PetscErrorCode ierr;
3503 
3504   PetscFunctionBegin;
3505   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3506   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
3507   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
3508   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
3509   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
3510   ierr = PetscLogObjectMemory((PetscObject) dm, size*2*sizeof(PetscInt));CHKERRQ(ierr);
3511   if (mesh->maxSupportSize) {
3512     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3513     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
3514     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
3515     ierr = PetscLogObjectMemory((PetscObject) dm, size*sizeof(PetscInt));CHKERRQ(ierr);
3516   }
3517   PetscFunctionReturn(0);
3518 }
3519 
3520 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
3521 {
3522   PetscErrorCode ierr;
3523 
3524   PetscFunctionBegin;
3525   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
3526   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
3527   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
3528   if (dm->useNatural && dm->sfMigration) {
3529     PetscSF        sfMigrationInv,sfNatural;
3530     PetscSection   section, sectionSeq;
3531 
3532     (*subdm)->sfMigration = dm->sfMigration;
3533     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
3534     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
3535     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3536     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
3537     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3538 
3539     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3540     (*subdm)->sfNatural = sfNatural;
3541     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3542     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3543   }
3544   PetscFunctionReturn(0);
3545 }
3546 
3547 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
3548 {
3549   PetscErrorCode ierr;
3550   PetscInt       i = 0;
3551 
3552   PetscFunctionBegin;
3553   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
3554   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
3555   (*superdm)->useNatural = PETSC_FALSE;
3556   for (i = 0; i < len; i++) {
3557     if (dms[i]->useNatural && dms[i]->sfMigration) {
3558       PetscSF        sfMigrationInv,sfNatural;
3559       PetscSection   section, sectionSeq;
3560 
3561       (*superdm)->sfMigration = dms[i]->sfMigration;
3562       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
3563       (*superdm)->useNatural = PETSC_TRUE;
3564       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
3565       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3566       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
3567       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3568 
3569       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3570       (*superdm)->sfNatural = sfNatural;
3571       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3572       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3573       break;
3574     }
3575   }
3576   PetscFunctionReturn(0);
3577 }
3578 
3579 /*@
3580   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
3581 
3582   Not collective
3583 
3584   Input Parameter:
3585 . mesh - The DMPlex
3586 
3587   Output Parameter:
3588 
3589   Note:
3590   This should be called after all calls to DMPlexSetCone()
3591 
3592   Level: beginner
3593 
3594 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
3595 @*/
3596 PetscErrorCode DMPlexSymmetrize(DM dm)
3597 {
3598   DM_Plex       *mesh = (DM_Plex*) dm->data;
3599   PetscInt      *offsets;
3600   PetscInt       supportSize;
3601   PetscInt       pStart, pEnd, p;
3602   PetscErrorCode ierr;
3603 
3604   PetscFunctionBegin;
3605   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3606   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
3607   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3608   /* Calculate support sizes */
3609   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3610   for (p = pStart; p < pEnd; ++p) {
3611     PetscInt dof, off, c;
3612 
3613     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3614     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3615     for (c = off; c < off+dof; ++c) {
3616       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
3617     }
3618   }
3619   for (p = pStart; p < pEnd; ++p) {
3620     PetscInt dof;
3621 
3622     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3623 
3624     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
3625   }
3626   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3627   /* Calculate supports */
3628   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
3629   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
3630   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
3631   for (p = pStart; p < pEnd; ++p) {
3632     PetscInt dof, off, c;
3633 
3634     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3635     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3636     for (c = off; c < off+dof; ++c) {
3637       const PetscInt q = mesh->cones[c];
3638       PetscInt       offS;
3639 
3640       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
3641 
3642       mesh->supports[offS+offsets[q]] = p;
3643       ++offsets[q];
3644     }
3645   }
3646   ierr = PetscFree(offsets);CHKERRQ(ierr);
3647   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3648   PetscFunctionReturn(0);
3649 }
3650 
3651 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
3652 {
3653   IS             stratumIS;
3654   PetscErrorCode ierr;
3655 
3656   PetscFunctionBegin;
3657   if (pStart >= pEnd) PetscFunctionReturn(0);
3658   if (PetscDefined(USE_DEBUG)) {
3659     PetscInt  qStart, qEnd, numLevels, level;
3660     PetscBool overlap = PETSC_FALSE;
3661     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
3662     for (level = 0; level < numLevels; level++) {
3663       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3664       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
3665     }
3666     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
3667   }
3668   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
3669   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
3670   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
3671   PetscFunctionReturn(0);
3672 }
3673 
3674 /*@
3675   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
3676   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
3677   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
3678   the DAG.
3679 
3680   Collective on dm
3681 
3682   Input Parameter:
3683 . mesh - The DMPlex
3684 
3685   Output Parameter:
3686 
3687   Notes:
3688   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
3689   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
3690   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
3691   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
3692   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
3693 
3694   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
3695   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
3696   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
3697   to interpolate only that one (e0), so that
3698 $  cone(c0) = {e0, v2}
3699 $  cone(e0) = {v0, v1}
3700   If DMPlexStratify() is run on this mesh, it will give depths
3701 $  depth 0 = {v0, v1, v2}
3702 $  depth 1 = {e0, c0}
3703   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
3704 
3705   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
3706 
3707   Level: beginner
3708 
3709 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexComputeCellTypes()
3710 @*/
3711 PetscErrorCode DMPlexStratify(DM dm)
3712 {
3713   DM_Plex       *mesh = (DM_Plex*) dm->data;
3714   DMLabel        label;
3715   PetscInt       pStart, pEnd, p;
3716   PetscInt       numRoots = 0, numLeaves = 0;
3717   PetscErrorCode ierr;
3718 
3719   PetscFunctionBegin;
3720   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3721   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3722 
3723   /* Create depth label */
3724   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3725   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
3726   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3727 
3728   {
3729     /* Initialize roots and count leaves */
3730     PetscInt sMin = PETSC_MAX_INT;
3731     PetscInt sMax = PETSC_MIN_INT;
3732     PetscInt coneSize, supportSize;
3733 
3734     for (p = pStart; p < pEnd; ++p) {
3735       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3736       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3737       if (!coneSize && supportSize) {
3738         sMin = PetscMin(p, sMin);
3739         sMax = PetscMax(p, sMax);
3740         ++numRoots;
3741       } else if (!supportSize && coneSize) {
3742         ++numLeaves;
3743       } else if (!supportSize && !coneSize) {
3744         /* Isolated points */
3745         sMin = PetscMin(p, sMin);
3746         sMax = PetscMax(p, sMax);
3747       }
3748     }
3749     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
3750   }
3751 
3752   if (numRoots + numLeaves == (pEnd - pStart)) {
3753     PetscInt sMin = PETSC_MAX_INT;
3754     PetscInt sMax = PETSC_MIN_INT;
3755     PetscInt coneSize, supportSize;
3756 
3757     for (p = pStart; p < pEnd; ++p) {
3758       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3759       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3760       if (!supportSize && coneSize) {
3761         sMin = PetscMin(p, sMin);
3762         sMax = PetscMax(p, sMax);
3763       }
3764     }
3765     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
3766   } else {
3767     PetscInt level = 0;
3768     PetscInt qStart, qEnd, q;
3769 
3770     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3771     while (qEnd > qStart) {
3772       PetscInt sMin = PETSC_MAX_INT;
3773       PetscInt sMax = PETSC_MIN_INT;
3774 
3775       for (q = qStart; q < qEnd; ++q) {
3776         const PetscInt *support;
3777         PetscInt        supportSize, s;
3778 
3779         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
3780         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
3781         for (s = 0; s < supportSize; ++s) {
3782           sMin = PetscMin(support[s], sMin);
3783           sMax = PetscMax(support[s], sMax);
3784         }
3785       }
3786       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
3787       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
3788       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3789     }
3790   }
3791   { /* just in case there is an empty process */
3792     PetscInt numValues, maxValues = 0, v;
3793 
3794     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
3795     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
3796     for (v = numValues; v < maxValues; v++) {
3797       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
3798     }
3799   }
3800   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
3801   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3802   PetscFunctionReturn(0);
3803 }
3804 
3805 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
3806 {
3807   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3808   PetscInt       dim, depth, pheight, coneSize;
3809   PetscErrorCode ierr;
3810 
3811   PetscFunctionBeginHot;
3812   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3813   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3814   ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3815   pheight = depth - pdepth;
3816   if (depth <= 1) {
3817     switch (pdepth) {
3818       case 0: ct = DM_POLYTOPE_POINT;break;
3819       case 1:
3820         switch (coneSize) {
3821           case 2: ct = DM_POLYTOPE_SEGMENT;break;
3822           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3823           case 4:
3824           switch (dim) {
3825             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
3826             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
3827             default: break;
3828           }
3829           break;
3830         case 5: ct = DM_POLYTOPE_PYRAMID;break;
3831         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3832         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
3833         default: break;
3834       }
3835     }
3836   } else {
3837     if (pdepth == 0) {
3838       ct = DM_POLYTOPE_POINT;
3839     } else if (pheight == 0) {
3840       switch (dim) {
3841         case 1:
3842           switch (coneSize) {
3843             case 2: ct = DM_POLYTOPE_SEGMENT;break;
3844             default: break;
3845           }
3846           break;
3847         case 2:
3848           switch (coneSize) {
3849             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3850             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3851             default: break;
3852           }
3853           break;
3854         case 3:
3855           switch (coneSize) {
3856             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
3857             case 5:
3858             {
3859               const PetscInt *cone;
3860               PetscInt        faceConeSize;
3861 
3862               ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
3863               ierr = DMPlexGetConeSize(dm, cone[0], &faceConeSize);CHKERRQ(ierr);
3864               switch (faceConeSize) {
3865                 case 3: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3866                 case 4: ct = DM_POLYTOPE_PYRAMID;break;
3867               }
3868             }
3869             break;
3870             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
3871             default: break;
3872           }
3873           break;
3874         default: break;
3875       }
3876     } else if (pheight > 0) {
3877       switch (coneSize) {
3878         case 2: ct = DM_POLYTOPE_SEGMENT;break;
3879         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3880         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3881         default: break;
3882       }
3883     }
3884   }
3885   *pt = ct;
3886   PetscFunctionReturn(0);
3887 }
3888 
3889 /*@
3890   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
3891 
3892   Collective on dm
3893 
3894   Input Parameter:
3895 . mesh - The DMPlex
3896 
3897   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
3898 
3899   Level: developer
3900 
3901   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
3902   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
3903   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
3904 
3905 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexStratify(), DMGetLabel(), DMCreateLabel()
3906 @*/
3907 PetscErrorCode DMPlexComputeCellTypes(DM dm)
3908 {
3909   DM_Plex       *mesh;
3910   DMLabel        ctLabel;
3911   PetscInt       pStart, pEnd, p;
3912   PetscErrorCode ierr;
3913 
3914   PetscFunctionBegin;
3915   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3916   mesh = (DM_Plex *) dm->data;
3917   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
3918   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
3919   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3920   for (p = pStart; p < pEnd; ++p) {
3921     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3922     PetscInt       pdepth;
3923 
3924     ierr = DMPlexGetPointDepth(dm, p, &pdepth);CHKERRQ(ierr);
3925     ierr = DMPlexComputeCellType_Internal(dm, p, pdepth, &ct);CHKERRQ(ierr);
3926     if (ct == DM_POLYTOPE_UNKNOWN) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %D is screwed up", p);
3927     ierr = DMLabelSetValue(ctLabel, p, ct);CHKERRQ(ierr);
3928   }
3929   ierr = PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState);CHKERRQ(ierr);
3930   ierr = PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view");CHKERRQ(ierr);
3931   PetscFunctionReturn(0);
3932 }
3933 
3934 /*@C
3935   DMPlexGetJoin - Get an array for the join of the set of points
3936 
3937   Not Collective
3938 
3939   Input Parameters:
3940 + dm - The DMPlex object
3941 . numPoints - The number of input points for the join
3942 - points - The input points
3943 
3944   Output Parameters:
3945 + numCoveredPoints - The number of points in the join
3946 - coveredPoints - The points in the join
3947 
3948   Level: intermediate
3949 
3950   Note: Currently, this is restricted to a single level join
3951 
3952   Fortran Notes:
3953   Since it returns an array, this routine is only available in Fortran 90, and you must
3954   include petsc.h90 in your code.
3955 
3956   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3957 
3958 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
3959 @*/
3960 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3961 {
3962   DM_Plex       *mesh = (DM_Plex*) dm->data;
3963   PetscInt      *join[2];
3964   PetscInt       joinSize, i = 0;
3965   PetscInt       dof, off, p, c, m;
3966   PetscErrorCode ierr;
3967 
3968   PetscFunctionBegin;
3969   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3970   PetscValidIntPointer(points, 3);
3971   PetscValidIntPointer(numCoveredPoints, 4);
3972   PetscValidPointer(coveredPoints, 5);
3973   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3974   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3975   /* Copy in support of first point */
3976   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
3977   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
3978   for (joinSize = 0; joinSize < dof; ++joinSize) {
3979     join[i][joinSize] = mesh->supports[off+joinSize];
3980   }
3981   /* Check each successive support */
3982   for (p = 1; p < numPoints; ++p) {
3983     PetscInt newJoinSize = 0;
3984 
3985     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
3986     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
3987     for (c = 0; c < dof; ++c) {
3988       const PetscInt point = mesh->supports[off+c];
3989 
3990       for (m = 0; m < joinSize; ++m) {
3991         if (point == join[i][m]) {
3992           join[1-i][newJoinSize++] = point;
3993           break;
3994         }
3995       }
3996     }
3997     joinSize = newJoinSize;
3998     i        = 1-i;
3999   }
4000   *numCoveredPoints = joinSize;
4001   *coveredPoints    = join[i];
4002   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4003   PetscFunctionReturn(0);
4004 }
4005 
4006 /*@C
4007   DMPlexRestoreJoin - Restore an array for the join of the set of points
4008 
4009   Not Collective
4010 
4011   Input Parameters:
4012 + dm - The DMPlex object
4013 . numPoints - The number of input points for the join
4014 - points - The input points
4015 
4016   Output Parameters:
4017 + numCoveredPoints - The number of points in the join
4018 - coveredPoints - The points in the join
4019 
4020   Fortran Notes:
4021   Since it returns an array, this routine is only available in Fortran 90, and you must
4022   include petsc.h90 in your code.
4023 
4024   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4025 
4026   Level: intermediate
4027 
4028 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
4029 @*/
4030 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4031 {
4032   PetscErrorCode ierr;
4033 
4034   PetscFunctionBegin;
4035   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4036   if (points) PetscValidIntPointer(points,3);
4037   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4038   PetscValidPointer(coveredPoints, 5);
4039   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4040   if (numCoveredPoints) *numCoveredPoints = 0;
4041   PetscFunctionReturn(0);
4042 }
4043 
4044 /*@C
4045   DMPlexGetFullJoin - Get an array for the join of the set of points
4046 
4047   Not Collective
4048 
4049   Input Parameters:
4050 + dm - The DMPlex object
4051 . numPoints - The number of input points for the join
4052 - points - The input points
4053 
4054   Output Parameters:
4055 + numCoveredPoints - The number of points in the join
4056 - coveredPoints - The points in the join
4057 
4058   Fortran Notes:
4059   Since it returns an array, this routine is only available in Fortran 90, and you must
4060   include petsc.h90 in your code.
4061 
4062   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4063 
4064   Level: intermediate
4065 
4066 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
4067 @*/
4068 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4069 {
4070   DM_Plex       *mesh = (DM_Plex*) dm->data;
4071   PetscInt      *offsets, **closures;
4072   PetscInt      *join[2];
4073   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
4074   PetscInt       p, d, c, m, ms;
4075   PetscErrorCode ierr;
4076 
4077   PetscFunctionBegin;
4078   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4079   PetscValidIntPointer(points, 3);
4080   PetscValidIntPointer(numCoveredPoints, 4);
4081   PetscValidPointer(coveredPoints, 5);
4082 
4083   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4084   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
4085   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4086   ms      = mesh->maxSupportSize;
4087   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
4088   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
4089   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
4090 
4091   for (p = 0; p < numPoints; ++p) {
4092     PetscInt closureSize;
4093 
4094     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
4095 
4096     offsets[p*(depth+2)+0] = 0;
4097     for (d = 0; d < depth+1; ++d) {
4098       PetscInt pStart, pEnd, i;
4099 
4100       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
4101       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
4102         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4103           offsets[p*(depth+2)+d+1] = i;
4104           break;
4105         }
4106       }
4107       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
4108     }
4109     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
4110   }
4111   for (d = 0; d < depth+1; ++d) {
4112     PetscInt dof;
4113 
4114     /* Copy in support of first point */
4115     dof = offsets[d+1] - offsets[d];
4116     for (joinSize = 0; joinSize < dof; ++joinSize) {
4117       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
4118     }
4119     /* Check each successive cone */
4120     for (p = 1; p < numPoints && joinSize; ++p) {
4121       PetscInt newJoinSize = 0;
4122 
4123       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
4124       for (c = 0; c < dof; ++c) {
4125         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
4126 
4127         for (m = 0; m < joinSize; ++m) {
4128           if (point == join[i][m]) {
4129             join[1-i][newJoinSize++] = point;
4130             break;
4131           }
4132         }
4133       }
4134       joinSize = newJoinSize;
4135       i        = 1-i;
4136     }
4137     if (joinSize) break;
4138   }
4139   *numCoveredPoints = joinSize;
4140   *coveredPoints    = join[i];
4141   for (p = 0; p < numPoints; ++p) {
4142     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
4143   }
4144   ierr = PetscFree(closures);CHKERRQ(ierr);
4145   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4146   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4147   PetscFunctionReturn(0);
4148 }
4149 
4150 /*@C
4151   DMPlexGetMeet - Get an array for the meet of the set of points
4152 
4153   Not Collective
4154 
4155   Input Parameters:
4156 + dm - The DMPlex object
4157 . numPoints - The number of input points for the meet
4158 - points - The input points
4159 
4160   Output Parameters:
4161 + numCoveredPoints - The number of points in the meet
4162 - coveredPoints - The points in the meet
4163 
4164   Level: intermediate
4165 
4166   Note: Currently, this is restricted to a single level meet
4167 
4168   Fortran Notes:
4169   Since it returns an array, this routine is only available in Fortran 90, and you must
4170   include petsc.h90 in your code.
4171 
4172   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4173 
4174 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
4175 @*/
4176 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
4177 {
4178   DM_Plex       *mesh = (DM_Plex*) dm->data;
4179   PetscInt      *meet[2];
4180   PetscInt       meetSize, i = 0;
4181   PetscInt       dof, off, p, c, m;
4182   PetscErrorCode ierr;
4183 
4184   PetscFunctionBegin;
4185   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4186   PetscValidPointer(points, 3);
4187   PetscValidPointer(numCoveringPoints, 4);
4188   PetscValidPointer(coveringPoints, 5);
4189   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4190   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4191   /* Copy in cone of first point */
4192   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
4193   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
4194   for (meetSize = 0; meetSize < dof; ++meetSize) {
4195     meet[i][meetSize] = mesh->cones[off+meetSize];
4196   }
4197   /* Check each successive cone */
4198   for (p = 1; p < numPoints; ++p) {
4199     PetscInt newMeetSize = 0;
4200 
4201     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
4202     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
4203     for (c = 0; c < dof; ++c) {
4204       const PetscInt point = mesh->cones[off+c];
4205 
4206       for (m = 0; m < meetSize; ++m) {
4207         if (point == meet[i][m]) {
4208           meet[1-i][newMeetSize++] = point;
4209           break;
4210         }
4211       }
4212     }
4213     meetSize = newMeetSize;
4214     i        = 1-i;
4215   }
4216   *numCoveringPoints = meetSize;
4217   *coveringPoints    = meet[i];
4218   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4219   PetscFunctionReturn(0);
4220 }
4221 
4222 /*@C
4223   DMPlexRestoreMeet - Restore an array for the meet of the set of points
4224 
4225   Not Collective
4226 
4227   Input Parameters:
4228 + dm - The DMPlex object
4229 . numPoints - The number of input points for the meet
4230 - points - The input points
4231 
4232   Output Parameters:
4233 + numCoveredPoints - The number of points in the meet
4234 - coveredPoints - The points in the meet
4235 
4236   Level: intermediate
4237 
4238   Fortran Notes:
4239   Since it returns an array, this routine is only available in Fortran 90, and you must
4240   include petsc.h90 in your code.
4241 
4242   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4243 
4244 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
4245 @*/
4246 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4247 {
4248   PetscErrorCode ierr;
4249 
4250   PetscFunctionBegin;
4251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4252   if (points) PetscValidIntPointer(points,3);
4253   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4254   PetscValidPointer(coveredPoints,5);
4255   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4256   if (numCoveredPoints) *numCoveredPoints = 0;
4257   PetscFunctionReturn(0);
4258 }
4259 
4260 /*@C
4261   DMPlexGetFullMeet - Get an array for the meet of the set of points
4262 
4263   Not Collective
4264 
4265   Input Parameters:
4266 + dm - The DMPlex object
4267 . numPoints - The number of input points for the meet
4268 - points - The input points
4269 
4270   Output Parameters:
4271 + numCoveredPoints - The number of points in the meet
4272 - coveredPoints - The points in the meet
4273 
4274   Level: intermediate
4275 
4276   Fortran Notes:
4277   Since it returns an array, this routine is only available in Fortran 90, and you must
4278   include petsc.h90 in your code.
4279 
4280   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4281 
4282 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
4283 @*/
4284 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4285 {
4286   DM_Plex       *mesh = (DM_Plex*) dm->data;
4287   PetscInt      *offsets, **closures;
4288   PetscInt      *meet[2];
4289   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
4290   PetscInt       p, h, c, m, mc;
4291   PetscErrorCode ierr;
4292 
4293   PetscFunctionBegin;
4294   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4295   PetscValidPointer(points, 3);
4296   PetscValidPointer(numCoveredPoints, 4);
4297   PetscValidPointer(coveredPoints, 5);
4298 
4299   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
4300   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
4301   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4302   mc      = mesh->maxConeSize;
4303   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
4304   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4305   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4306 
4307   for (p = 0; p < numPoints; ++p) {
4308     PetscInt closureSize;
4309 
4310     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
4311 
4312     offsets[p*(height+2)+0] = 0;
4313     for (h = 0; h < height+1; ++h) {
4314       PetscInt pStart, pEnd, i;
4315 
4316       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
4317       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
4318         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4319           offsets[p*(height+2)+h+1] = i;
4320           break;
4321         }
4322       }
4323       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
4324     }
4325     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
4326   }
4327   for (h = 0; h < height+1; ++h) {
4328     PetscInt dof;
4329 
4330     /* Copy in cone of first point */
4331     dof = offsets[h+1] - offsets[h];
4332     for (meetSize = 0; meetSize < dof; ++meetSize) {
4333       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
4334     }
4335     /* Check each successive cone */
4336     for (p = 1; p < numPoints && meetSize; ++p) {
4337       PetscInt newMeetSize = 0;
4338 
4339       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
4340       for (c = 0; c < dof; ++c) {
4341         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
4342 
4343         for (m = 0; m < meetSize; ++m) {
4344           if (point == meet[i][m]) {
4345             meet[1-i][newMeetSize++] = point;
4346             break;
4347           }
4348         }
4349       }
4350       meetSize = newMeetSize;
4351       i        = 1-i;
4352     }
4353     if (meetSize) break;
4354   }
4355   *numCoveredPoints = meetSize;
4356   *coveredPoints    = meet[i];
4357   for (p = 0; p < numPoints; ++p) {
4358     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
4359   }
4360   ierr = PetscFree(closures);CHKERRQ(ierr);
4361   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4362   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4363   PetscFunctionReturn(0);
4364 }
4365 
4366 /*@C
4367   DMPlexEqual - Determine if two DMs have the same topology
4368 
4369   Not Collective
4370 
4371   Input Parameters:
4372 + dmA - A DMPlex object
4373 - dmB - A DMPlex object
4374 
4375   Output Parameters:
4376 . equal - PETSC_TRUE if the topologies are identical
4377 
4378   Level: intermediate
4379 
4380   Notes:
4381   We are not solving graph isomorphism, so we do not permutation.
4382 
4383 .seealso: DMPlexGetCone()
4384 @*/
4385 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
4386 {
4387   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
4388   PetscErrorCode ierr;
4389 
4390   PetscFunctionBegin;
4391   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4392   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4393   PetscValidPointer(equal, 3);
4394 
4395   *equal = PETSC_FALSE;
4396   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
4397   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
4398   if (depth != depthB) PetscFunctionReturn(0);
4399   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
4400   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
4401   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
4402   for (p = pStart; p < pEnd; ++p) {
4403     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
4404     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
4405 
4406     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
4407     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
4408     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
4409     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
4410     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
4411     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
4412     if (coneSize != coneSizeB) PetscFunctionReturn(0);
4413     for (c = 0; c < coneSize; ++c) {
4414       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
4415       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
4416     }
4417     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
4418     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
4419     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
4420     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
4421     if (supportSize != supportSizeB) PetscFunctionReturn(0);
4422     for (s = 0; s < supportSize; ++s) {
4423       if (support[s] != supportB[s]) PetscFunctionReturn(0);
4424     }
4425   }
4426   *equal = PETSC_TRUE;
4427   PetscFunctionReturn(0);
4428 }
4429 
4430 /*@C
4431   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
4432 
4433   Not Collective
4434 
4435   Input Parameters:
4436 + dm         - The DMPlex
4437 . cellDim    - The cell dimension
4438 - numCorners - The number of vertices on a cell
4439 
4440   Output Parameters:
4441 . numFaceVertices - The number of vertices on a face
4442 
4443   Level: developer
4444 
4445   Notes:
4446   Of course this can only work for a restricted set of symmetric shapes
4447 
4448 .seealso: DMPlexGetCone()
4449 @*/
4450 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
4451 {
4452   MPI_Comm       comm;
4453   PetscErrorCode ierr;
4454 
4455   PetscFunctionBegin;
4456   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
4457   PetscValidPointer(numFaceVertices,4);
4458   switch (cellDim) {
4459   case 0:
4460     *numFaceVertices = 0;
4461     break;
4462   case 1:
4463     *numFaceVertices = 1;
4464     break;
4465   case 2:
4466     switch (numCorners) {
4467     case 3: /* triangle */
4468       *numFaceVertices = 2; /* Edge has 2 vertices */
4469       break;
4470     case 4: /* quadrilateral */
4471       *numFaceVertices = 2; /* Edge has 2 vertices */
4472       break;
4473     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
4474       *numFaceVertices = 3; /* Edge has 3 vertices */
4475       break;
4476     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
4477       *numFaceVertices = 3; /* Edge has 3 vertices */
4478       break;
4479     default:
4480       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4481     }
4482     break;
4483   case 3:
4484     switch (numCorners) {
4485     case 4: /* tetradehdron */
4486       *numFaceVertices = 3; /* Face has 3 vertices */
4487       break;
4488     case 6: /* tet cohesive cells */
4489       *numFaceVertices = 4; /* Face has 4 vertices */
4490       break;
4491     case 8: /* hexahedron */
4492       *numFaceVertices = 4; /* Face has 4 vertices */
4493       break;
4494     case 9: /* tet cohesive Lagrange cells */
4495       *numFaceVertices = 6; /* Face has 6 vertices */
4496       break;
4497     case 10: /* quadratic tetrahedron */
4498       *numFaceVertices = 6; /* Face has 6 vertices */
4499       break;
4500     case 12: /* hex cohesive Lagrange cells */
4501       *numFaceVertices = 6; /* Face has 6 vertices */
4502       break;
4503     case 18: /* quadratic tet cohesive Lagrange cells */
4504       *numFaceVertices = 6; /* Face has 6 vertices */
4505       break;
4506     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
4507       *numFaceVertices = 9; /* Face has 9 vertices */
4508       break;
4509     default:
4510       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4511     }
4512     break;
4513   default:
4514     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
4515   }
4516   PetscFunctionReturn(0);
4517 }
4518 
4519 /*@
4520   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4521 
4522   Not Collective
4523 
4524   Input Parameter:
4525 . dm    - The DMPlex object
4526 
4527   Output Parameter:
4528 . depthLabel - The DMLabel recording point depth
4529 
4530   Level: developer
4531 
4532 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(),
4533 @*/
4534 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4535 {
4536   PetscFunctionBegin;
4537   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4538   PetscValidPointer(depthLabel, 2);
4539   *depthLabel = dm->depthLabel;
4540   PetscFunctionReturn(0);
4541 }
4542 
4543 /*@
4544   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4545 
4546   Not Collective
4547 
4548   Input Parameter:
4549 . dm    - The DMPlex object
4550 
4551   Output Parameter:
4552 . depth - The number of strata (breadth first levels) in the DAG
4553 
4554   Level: developer
4555 
4556   Notes:
4557   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
4558   The point depth is described more in detail in DMPlexGetDepthStratum().
4559   An empty mesh gives -1.
4560 
4561 .seealso: DMPlexGetDepthLabel(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(), DMPlexSymmetrize()
4562 @*/
4563 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4564 {
4565   DMLabel        label;
4566   PetscInt       d = 0;
4567   PetscErrorCode ierr;
4568 
4569   PetscFunctionBegin;
4570   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4571   PetscValidPointer(depth, 2);
4572   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4573   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4574   *depth = d-1;
4575   PetscFunctionReturn(0);
4576 }
4577 
4578 /*@
4579   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4580 
4581   Not Collective
4582 
4583   Input Parameters:
4584 + dm           - The DMPlex object
4585 - stratumValue - The requested depth
4586 
4587   Output Parameters:
4588 + start - The first point at this depth
4589 - end   - One beyond the last point at this depth
4590 
4591   Notes:
4592   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
4593   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
4594   higher dimension, e.g., "edges".
4595 
4596   Level: developer
4597 
4598 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth(), DMPlexGetDepthLabel(), DMPlexGetPointDepth(), DMPlexSymmetrize(), DMPlexInterpolate()
4599 @*/
4600 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4601 {
4602   DMLabel        label;
4603   PetscInt       pStart, pEnd;
4604   PetscErrorCode ierr;
4605 
4606   PetscFunctionBegin;
4607   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4608   if (start) {PetscValidPointer(start, 3); *start = 0;}
4609   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4610   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4611   if (pStart == pEnd) PetscFunctionReturn(0);
4612   if (stratumValue < 0) {
4613     if (start) *start = pStart;
4614     if (end)   *end   = pEnd;
4615     PetscFunctionReturn(0);
4616   }
4617   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4618   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4619   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4620   PetscFunctionReturn(0);
4621 }
4622 
4623 /*@
4624   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4625 
4626   Not Collective
4627 
4628   Input Parameters:
4629 + dm           - The DMPlex object
4630 - stratumValue - The requested height
4631 
4632   Output Parameters:
4633 + start - The first point at this height
4634 - end   - One beyond the last point at this height
4635 
4636   Notes:
4637   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
4638   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
4639   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
4640 
4641   Level: developer
4642 
4643 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth(), DMPlexGetPointHeight()
4644 @*/
4645 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4646 {
4647   DMLabel        label;
4648   PetscInt       depth, pStart, pEnd;
4649   PetscErrorCode ierr;
4650 
4651   PetscFunctionBegin;
4652   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4653   if (start) {PetscValidPointer(start, 3); *start = 0;}
4654   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4655   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4656   if (pStart == pEnd) PetscFunctionReturn(0);
4657   if (stratumValue < 0) {
4658     if (start) *start = pStart;
4659     if (end)   *end   = pEnd;
4660     PetscFunctionReturn(0);
4661   }
4662   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4663   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4664   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4665   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4666   PetscFunctionReturn(0);
4667 }
4668 
4669 /*@
4670   DMPlexGetPointDepth - Get the depth of a given point
4671 
4672   Not Collective
4673 
4674   Input Parameter:
4675 + dm    - The DMPlex object
4676 - point - The point
4677 
4678   Output Parameter:
4679 . depth - The depth of the point
4680 
4681   Level: intermediate
4682 
4683 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointHeight()
4684 @*/
4685 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
4686 {
4687   PetscErrorCode ierr;
4688 
4689   PetscFunctionBegin;
4690   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4691   PetscValidIntPointer(depth, 3);
4692   ierr = DMLabelGetValue(dm->depthLabel, point, depth);CHKERRQ(ierr);
4693   PetscFunctionReturn(0);
4694 }
4695 
4696 /*@
4697   DMPlexGetPointHeight - Get the height of a given point
4698 
4699   Not Collective
4700 
4701   Input Parameter:
4702 + dm    - The DMPlex object
4703 - point - The point
4704 
4705   Output Parameter:
4706 . height - The height of the point
4707 
4708   Level: intermediate
4709 
4710 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointDepth()
4711 @*/
4712 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
4713 {
4714   PetscInt       n, pDepth;
4715   PetscErrorCode ierr;
4716 
4717   PetscFunctionBegin;
4718   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4719   PetscValidIntPointer(height, 3);
4720   ierr = DMLabelGetNumValues(dm->depthLabel, &n);CHKERRQ(ierr);
4721   ierr = DMLabelGetValue(dm->depthLabel, point, &pDepth);CHKERRQ(ierr);
4722   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
4723   PetscFunctionReturn(0);
4724 }
4725 
4726 /*@
4727   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
4728 
4729   Not Collective
4730 
4731   Input Parameter:
4732 . dm - The DMPlex object
4733 
4734   Output Parameter:
4735 . celltypeLabel - The DMLabel recording cell polytope type
4736 
4737   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
4738   DMCreateLabel(dm, "celltype") beforehand.
4739 
4740   Level: developer
4741 
4742 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMCreateLabel()
4743 @*/
4744 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
4745 {
4746   PetscErrorCode ierr;
4747 
4748   PetscFunctionBegin;
4749   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4750   PetscValidPointer(celltypeLabel, 2);
4751   if (!dm->celltypeLabel) {ierr = DMPlexComputeCellTypes(dm);CHKERRQ(ierr);}
4752   *celltypeLabel = dm->celltypeLabel;
4753   PetscFunctionReturn(0);
4754 }
4755 
4756 /*@
4757   DMPlexGetCellType - Get the polytope type of a given cell
4758 
4759   Not Collective
4760 
4761   Input Parameter:
4762 + dm   - The DMPlex object
4763 - cell - The cell
4764 
4765   Output Parameter:
4766 . celltype - The polytope type of the cell
4767 
4768   Level: intermediate
4769 
4770 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth()
4771 @*/
4772 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
4773 {
4774   DMLabel        label;
4775   PetscInt       ct;
4776   PetscErrorCode ierr;
4777 
4778   PetscFunctionBegin;
4779   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4780   PetscValidPointer(celltype, 3);
4781   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4782   ierr = DMLabelGetValue(label, cell, &ct);CHKERRQ(ierr);
4783   if (ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D has not been assigned a cell type", cell);
4784   *celltype = (DMPolytopeType) ct;
4785   PetscFunctionReturn(0);
4786 }
4787 
4788 /*@
4789   DMPlexSetCellType - Set the polytope type of a given cell
4790 
4791   Not Collective
4792 
4793   Input Parameters:
4794 + dm   - The DMPlex object
4795 . cell - The cell
4796 - celltype - The polytope type of the cell
4797 
4798   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
4799   is executed. This function will override the computed type. However, if automatic classification will not succeed
4800   and a user wants to manually specify all types, the classification must be disabled by calling
4801   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
4802 
4803   Level: advanced
4804 
4805 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexComputeCellTypes(), DMCreateLabel()
4806 @*/
4807 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
4808 {
4809   DMLabel        label;
4810   PetscErrorCode ierr;
4811 
4812   PetscFunctionBegin;
4813   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4814   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4815   ierr = DMLabelSetValue(label, cell, celltype);CHKERRQ(ierr);
4816   PetscFunctionReturn(0);
4817 }
4818 
4819 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4820 {
4821   PetscSection   section, s;
4822   Mat            m;
4823   PetscInt       maxHeight;
4824   PetscErrorCode ierr;
4825 
4826   PetscFunctionBegin;
4827   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4828   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
4829   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
4830   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4831   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
4832   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4833   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
4834   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
4835   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
4836   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
4837   ierr = MatDestroy(&m);CHKERRQ(ierr);
4838 
4839   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
4840   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
4841   PetscFunctionReturn(0);
4842 }
4843 
4844 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
4845 {
4846   Vec            coordsLocal;
4847   DM             coordsDM;
4848   PetscErrorCode ierr;
4849 
4850   PetscFunctionBegin;
4851   *field = NULL;
4852   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
4853   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
4854   if (coordsLocal && coordsDM) {
4855     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
4856   }
4857   PetscFunctionReturn(0);
4858 }
4859 
4860 /*@C
4861   DMPlexGetConeSection - Return a section which describes the layout of cone data
4862 
4863   Not Collective
4864 
4865   Input Parameters:
4866 . dm        - The DMPlex object
4867 
4868   Output Parameter:
4869 . section - The PetscSection object
4870 
4871   Level: developer
4872 
4873 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
4874 @*/
4875 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4876 {
4877   DM_Plex *mesh = (DM_Plex*) dm->data;
4878 
4879   PetscFunctionBegin;
4880   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4881   if (section) *section = mesh->coneSection;
4882   PetscFunctionReturn(0);
4883 }
4884 
4885 /*@C
4886   DMPlexGetSupportSection - Return a section which describes the layout of support data
4887 
4888   Not Collective
4889 
4890   Input Parameters:
4891 . dm        - The DMPlex object
4892 
4893   Output Parameter:
4894 . section - The PetscSection object
4895 
4896   Level: developer
4897 
4898 .seealso: DMPlexGetConeSection()
4899 @*/
4900 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4901 {
4902   DM_Plex *mesh = (DM_Plex*) dm->data;
4903 
4904   PetscFunctionBegin;
4905   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4906   if (section) *section = mesh->supportSection;
4907   PetscFunctionReturn(0);
4908 }
4909 
4910 /*@C
4911   DMPlexGetCones - Return cone data
4912 
4913   Not Collective
4914 
4915   Input Parameters:
4916 . dm        - The DMPlex object
4917 
4918   Output Parameter:
4919 . cones - The cone for each point
4920 
4921   Level: developer
4922 
4923 .seealso: DMPlexGetConeSection()
4924 @*/
4925 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4926 {
4927   DM_Plex *mesh = (DM_Plex*) dm->data;
4928 
4929   PetscFunctionBegin;
4930   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4931   if (cones) *cones = mesh->cones;
4932   PetscFunctionReturn(0);
4933 }
4934 
4935 /*@C
4936   DMPlexGetConeOrientations - Return cone orientation data
4937 
4938   Not Collective
4939 
4940   Input Parameters:
4941 . dm        - The DMPlex object
4942 
4943   Output Parameter:
4944 . coneOrientations - The cone orientation for each point
4945 
4946   Level: developer
4947 
4948 .seealso: DMPlexGetConeSection()
4949 @*/
4950 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4951 {
4952   DM_Plex *mesh = (DM_Plex*) dm->data;
4953 
4954   PetscFunctionBegin;
4955   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4956   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4957   PetscFunctionReturn(0);
4958 }
4959 
4960 /******************************** FEM Support **********************************/
4961 
4962 /*
4963  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
4964  representing a line in the section.
4965 */
4966 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
4967 {
4968   PetscErrorCode ierr;
4969 
4970   PetscFunctionBeginHot;
4971   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
4972   if (line < 0) {
4973     *k = 0;
4974     *Nc = 0;
4975   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
4976     *k = 1;
4977   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
4978     /* An order k SEM disc has k-1 dofs on an edge */
4979     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
4980     *k = *k / *Nc + 1;
4981   }
4982   PetscFunctionReturn(0);
4983 }
4984 
4985 /*@
4986 
4987   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
4988   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
4989   section provided (or the section of the DM).
4990 
4991   Input Parameters:
4992 + dm      - The DM
4993 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
4994 - section - The PetscSection to reorder, or NULL for the default section
4995 
4996   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
4997   degree of the basis.
4998 
4999   Example:
5000   A typical interpolated single-quad mesh might order points as
5001 .vb
5002   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
5003 
5004   v4 -- e6 -- v3
5005   |           |
5006   e7    c0    e8
5007   |           |
5008   v1 -- e5 -- v2
5009 .ve
5010 
5011   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
5012   dofs in the order of points, e.g.,
5013 .vb
5014     c0 -> [0,1,2,3]
5015     v1 -> [4]
5016     ...
5017     e5 -> [8, 9]
5018 .ve
5019 
5020   which corresponds to the dofs
5021 .vb
5022     6   10  11  7
5023     13  2   3   15
5024     12  0   1   14
5025     4   8   9   5
5026 .ve
5027 
5028   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
5029 .vb
5030   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
5031 .ve
5032 
5033   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
5034 .vb
5035    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
5036 .ve
5037 
5038   Level: developer
5039 
5040 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
5041 @*/
5042 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
5043 {
5044   DMLabel        label;
5045   PetscInt       dim, depth = -1, eStart = -1, Nf;
5046   PetscBool      vertexchart;
5047   PetscErrorCode ierr;
5048 
5049   PetscFunctionBegin;
5050   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5051   if (dim < 1) PetscFunctionReturn(0);
5052   if (point < 0) {
5053     PetscInt sStart,sEnd;
5054 
5055     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
5056     point = sEnd-sStart ? sStart : point;
5057   }
5058   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5059   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
5060   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5061   if (depth == 1) {eStart = point;}
5062   else if  (depth == dim) {
5063     const PetscInt *cone;
5064 
5065     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5066     if (dim == 2) eStart = cone[0];
5067     else if (dim == 3) {
5068       const PetscInt *cone2;
5069       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
5070       eStart = cone2[0];
5071     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5072   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5073   {                             /* Determine whether the chart covers all points or just vertices. */
5074     PetscInt pStart,pEnd,cStart,cEnd;
5075     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
5076     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
5077     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
5078     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
5079   }
5080   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5081   for (PetscInt d=1; d<=dim; d++) {
5082     PetscInt k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
5083     PetscInt *perm;
5084 
5085     for (f = 0; f < Nf; ++f) {
5086       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5087       size += PetscPowInt(k+1, d)*Nc;
5088     }
5089     ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
5090     for (f = 0; f < Nf; ++f) {
5091       switch (d) {
5092       case 1:
5093         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5094         /*
5095          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
5096          We want              [ vtx0; edge of length k-1; vtx1 ]
5097          */
5098         for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
5099         for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
5100         for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
5101         foffset = offset;
5102         break;
5103       case 2:
5104         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
5105         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5106         /* The SEM order is
5107 
5108          v_lb, {e_b}, v_rb,
5109          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
5110          v_lt, reverse {e_t}, v_rt
5111          */
5112         {
5113           const PetscInt of   = 0;
5114           const PetscInt oeb  = of   + PetscSqr(k-1);
5115           const PetscInt oer  = oeb  + (k-1);
5116           const PetscInt oet  = oer  + (k-1);
5117           const PetscInt oel  = oet  + (k-1);
5118           const PetscInt ovlb = oel  + (k-1);
5119           const PetscInt ovrb = ovlb + 1;
5120           const PetscInt ovrt = ovrb + 1;
5121           const PetscInt ovlt = ovrt + 1;
5122           PetscInt       o;
5123 
5124           /* bottom */
5125           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
5126           for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5127           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
5128           /* middle */
5129           for (i = 0; i < k-1; ++i) {
5130             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
5131             for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5132             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
5133           }
5134           /* top */
5135           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
5136           for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5137           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
5138           foffset = offset;
5139         }
5140         break;
5141       case 3:
5142         /* The original hex closure is
5143 
5144          {c,
5145          f_b, f_t, f_f, f_b, f_r, f_l,
5146          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5147          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5148          */
5149         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5150         /* The SEM order is
5151          Bottom Slice
5152          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5153          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5154          v_blb, {e_bb}, v_brb,
5155 
5156          Middle Slice (j)
5157          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5158          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5159          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5160 
5161          Top Slice
5162          v_tlf, {e_tf}, v_trf,
5163          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5164          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5165          */
5166         {
5167           const PetscInt oc    = 0;
5168           const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
5169           const PetscInt oft   = ofb   + PetscSqr(k-1);
5170           const PetscInt off   = oft   + PetscSqr(k-1);
5171           const PetscInt ofk   = off   + PetscSqr(k-1);
5172           const PetscInt ofr   = ofk   + PetscSqr(k-1);
5173           const PetscInt ofl   = ofr   + PetscSqr(k-1);
5174           const PetscInt oebl  = ofl   + PetscSqr(k-1);
5175           const PetscInt oebb  = oebl  + (k-1);
5176           const PetscInt oebr  = oebb  + (k-1);
5177           const PetscInt oebf  = oebr  + (k-1);
5178           const PetscInt oetf  = oebf  + (k-1);
5179           const PetscInt oetr  = oetf  + (k-1);
5180           const PetscInt oetb  = oetr  + (k-1);
5181           const PetscInt oetl  = oetb  + (k-1);
5182           const PetscInt oerf  = oetl  + (k-1);
5183           const PetscInt oelf  = oerf  + (k-1);
5184           const PetscInt oelb  = oelf  + (k-1);
5185           const PetscInt oerb  = oelb  + (k-1);
5186           const PetscInt ovblf = oerb  + (k-1);
5187           const PetscInt ovblb = ovblf + 1;
5188           const PetscInt ovbrb = ovblb + 1;
5189           const PetscInt ovbrf = ovbrb + 1;
5190           const PetscInt ovtlf = ovbrf + 1;
5191           const PetscInt ovtrf = ovtlf + 1;
5192           const PetscInt ovtrb = ovtrf + 1;
5193           const PetscInt ovtlb = ovtrb + 1;
5194           PetscInt       o, n;
5195 
5196           /* Bottom Slice */
5197           /*   bottom */
5198           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
5199           for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5200           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
5201           /*   middle */
5202           for (i = 0; i < k-1; ++i) {
5203             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
5204             for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
5205             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
5206           }
5207           /*   top */
5208           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
5209           for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5210           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
5211 
5212           /* Middle Slice */
5213           for (j = 0; j < k-1; ++j) {
5214             /*   bottom */
5215             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
5216             for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5217             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
5218             /*   middle */
5219             for (i = 0; i < k-1; ++i) {
5220               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
5221               for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
5222               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
5223             }
5224             /*   top */
5225             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
5226             for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5227             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
5228           }
5229 
5230           /* Top Slice */
5231           /*   bottom */
5232           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
5233           for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5234           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
5235           /*   middle */
5236           for (i = 0; i < k-1; ++i) {
5237             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
5238             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
5239             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
5240           }
5241           /*   top */
5242           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
5243           for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5244           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
5245 
5246           foffset = offset;
5247         }
5248         break;
5249       default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", d);
5250       }
5251     }
5252     if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
5253     /* Check permutation */
5254     {
5255       PetscInt *check;
5256 
5257       ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
5258       for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
5259       for (i = 0; i < size; ++i) check[perm[i]] = i;
5260       for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
5261       ierr = PetscFree(check);CHKERRQ(ierr);
5262     }
5263     ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
5264   }
5265   PetscFunctionReturn(0);
5266 }
5267 
5268 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
5269 {
5270   PetscDS        prob;
5271   PetscInt       depth, Nf, h;
5272   DMLabel        label;
5273   PetscErrorCode ierr;
5274 
5275   PetscFunctionBeginHot;
5276   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
5277   Nf      = prob->Nf;
5278   label   = dm->depthLabel;
5279   *dspace = NULL;
5280   if (field < Nf) {
5281     PetscObject disc = prob->disc[field];
5282 
5283     if (disc->classid == PETSCFE_CLASSID) {
5284       PetscDualSpace dsp;
5285 
5286       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
5287       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
5288       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
5289       h    = depth - 1 - h;
5290       if (h) {
5291         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
5292       } else {
5293         *dspace = dsp;
5294       }
5295     }
5296   }
5297   PetscFunctionReturn(0);
5298 }
5299 
5300 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5301 {
5302   PetscScalar    *array, *vArray;
5303   const PetscInt *cone, *coneO;
5304   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
5305   PetscErrorCode  ierr;
5306 
5307   PetscFunctionBeginHot;
5308   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5309   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5310   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5311   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5312   if (!values || !*values) {
5313     if ((point >= pStart) && (point < pEnd)) {
5314       PetscInt dof;
5315 
5316       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5317       size += dof;
5318     }
5319     for (p = 0; p < numPoints; ++p) {
5320       const PetscInt cp = cone[p];
5321       PetscInt       dof;
5322 
5323       if ((cp < pStart) || (cp >= pEnd)) continue;
5324       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5325       size += dof;
5326     }
5327     if (!values) {
5328       if (csize) *csize = size;
5329       PetscFunctionReturn(0);
5330     }
5331     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
5332   } else {
5333     array = *values;
5334   }
5335   size = 0;
5336   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5337   if ((point >= pStart) && (point < pEnd)) {
5338     PetscInt     dof, off, d;
5339     PetscScalar *varr;
5340 
5341     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5342     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5343     varr = &vArray[off];
5344     for (d = 0; d < dof; ++d, ++offset) {
5345       array[offset] = varr[d];
5346     }
5347     size += dof;
5348   }
5349   for (p = 0; p < numPoints; ++p) {
5350     const PetscInt cp = cone[p];
5351     PetscInt       o  = coneO[p];
5352     PetscInt       dof, off, d;
5353     PetscScalar   *varr;
5354 
5355     if ((cp < pStart) || (cp >= pEnd)) continue;
5356     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5357     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
5358     varr = &vArray[off];
5359     if (o >= 0) {
5360       for (d = 0; d < dof; ++d, ++offset) {
5361         array[offset] = varr[d];
5362       }
5363     } else {
5364       for (d = dof-1; d >= 0; --d, ++offset) {
5365         array[offset] = varr[d];
5366       }
5367     }
5368     size += dof;
5369   }
5370   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5371   if (!*values) {
5372     if (csize) *csize = size;
5373     *values = array;
5374   } else {
5375     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5376     *csize = size;
5377   }
5378   PetscFunctionReturn(0);
5379 }
5380 
5381 /* Compress out points not in the section */
5382 PETSC_STATIC_INLINE PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
5383 {
5384   const PetscInt np = *numPoints;
5385   PetscInt       pStart, pEnd, p, q;
5386   PetscErrorCode ierr;
5387 
5388   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5389   for (p = 0, q = 0; p < np; ++p) {
5390     const PetscInt r = points[p*2];
5391     if ((r >= pStart) && (r < pEnd)) {
5392       points[q*2]   = r;
5393       points[q*2+1] = points[p*2+1];
5394       ++q;
5395     }
5396   }
5397   *numPoints = q;
5398   return 0;
5399 }
5400 
5401 static PetscErrorCode DMPlexTransitiveClosure_Hybrid_Internal(DM dm, PetscInt point, PetscInt np, PetscInt *numPoints, PetscInt **points)
5402 {
5403   const PetscInt *cone, *ornt;
5404   PetscInt       *pts,  *closure = NULL;
5405   PetscInt        dim, coneSize, c, d, clSize, cl;
5406   PetscErrorCode  ierr;
5407 
5408   PetscFunctionBeginHot;
5409   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5410   ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
5411   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5412   ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
5413   ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5414   ierr = DMGetWorkArray(dm, np*2, MPIU_INT, &pts);CHKERRQ(ierr);
5415   c    = 0;
5416   pts[c*2+0] = point;
5417   pts[c*2+1] = 0;
5418   ++c;
5419   for (cl = 0; cl < clSize*2; cl += 2, ++c) {pts[c*2+0] = closure[cl]; pts[c*2+1] = closure[cl+1];}
5420   ierr = DMPlexGetTransitiveClosure(dm, cone[1], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5421   for (cl = 0; cl < clSize*2; cl += 2, ++c) {pts[c*2+0] = closure[cl]; pts[c*2+1] = closure[cl+1];}
5422   ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5423   if (dim >= 2) {
5424     for (d = 2; d < coneSize; ++d, ++c) {pts[c*2+0] = cone[d]; pts[c*2+1] = ornt[d];}
5425   }
5426   if (dim >= 3) {
5427     for (d = 2; d < coneSize; ++d) {
5428       const PetscInt  fpoint = cone[d];
5429       const PetscInt *fcone;
5430       PetscInt        fconeSize, fc, i;
5431 
5432       ierr = DMPlexGetConeSize(dm, fpoint, &fconeSize);CHKERRQ(ierr);
5433       ierr = DMPlexGetCone(dm, fpoint, &fcone);CHKERRQ(ierr);
5434       for (fc = 0; fc < fconeSize; ++fc) {
5435         for (i = 0; i < c; ++i) if (pts[i*2] == fcone[fc]) break;
5436         if (i == c) {pts[c*2+0] = fcone[fc]; pts[c*2+1] = 0; ++c;}
5437       }
5438     }
5439   }
5440   if (c != np) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid closure for hybrid point %D, size %D != %D", point, c, np);
5441   *numPoints = np;
5442   *points    = pts;
5443   PetscFunctionReturn(0);
5444 }
5445 
5446 /* Compressed closure does not apply closure permutation */
5447 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5448 {
5449   const PetscInt *cla = NULL;
5450   PetscInt       np, *pts = NULL;
5451   PetscErrorCode ierr;
5452 
5453   PetscFunctionBeginHot;
5454   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
5455   if (*clPoints) {
5456     PetscInt dof, off;
5457 
5458     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
5459     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
5460     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
5461     np   = dof/2;
5462     pts  = (PetscInt *) &cla[off];
5463   } else {
5464     DMPolytopeType ct;
5465 
5466     /* Do not make the label if it does not exist */
5467     if (!dm->celltypeLabel) {ct = DM_POLYTOPE_POINT;}
5468     else                    {ierr = DMPlexGetCellType(dm, point, &ct);CHKERRQ(ierr);}
5469     switch (ct) {
5470       case DM_POLYTOPE_SEG_PRISM_TENSOR:
5471         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 9, &np, &pts);CHKERRQ(ierr);
5472         break;
5473       case DM_POLYTOPE_TRI_PRISM_TENSOR:
5474         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 21, &np, &pts);CHKERRQ(ierr);
5475         break;
5476       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
5477         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 27, &np, &pts);CHKERRQ(ierr);
5478         break;
5479       default:
5480         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
5481     }
5482     ierr = CompressPoints_Private(section, &np, pts);CHKERRQ(ierr);
5483   }
5484   *numPoints = np;
5485   *points    = pts;
5486   *clp       = cla;
5487   PetscFunctionReturn(0);
5488 }
5489 
5490 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5491 {
5492   PetscErrorCode ierr;
5493 
5494   PetscFunctionBeginHot;
5495   if (!*clPoints) {
5496     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
5497   } else {
5498     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
5499   }
5500   *numPoints = 0;
5501   *points    = NULL;
5502   *clSec     = NULL;
5503   *clPoints  = NULL;
5504   *clp       = NULL;
5505   PetscFunctionReturn(0);
5506 }
5507 
5508 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5509 {
5510   PetscInt          offset = 0, p;
5511   const PetscInt    **perms = NULL;
5512   const PetscScalar **flips = NULL;
5513   PetscErrorCode    ierr;
5514 
5515   PetscFunctionBeginHot;
5516   *size = 0;
5517   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5518   for (p = 0; p < numPoints; p++) {
5519     const PetscInt    point = points[2*p];
5520     const PetscInt    *perm = perms ? perms[p] : NULL;
5521     const PetscScalar *flip = flips ? flips[p] : NULL;
5522     PetscInt          dof, off, d;
5523     const PetscScalar *varr;
5524 
5525     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5526     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5527     varr = &vArray[off];
5528     if (clperm) {
5529       if (perm) {
5530         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
5531       } else {
5532         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
5533       }
5534       if (flip) {
5535         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
5536       }
5537     } else {
5538       if (perm) {
5539         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
5540       } else {
5541         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
5542       }
5543       if (flip) {
5544         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
5545       }
5546     }
5547     offset += dof;
5548   }
5549   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5550   *size = offset;
5551   PetscFunctionReturn(0);
5552 }
5553 
5554 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5555 {
5556   PetscInt          offset = 0, f;
5557   PetscErrorCode    ierr;
5558 
5559   PetscFunctionBeginHot;
5560   *size = 0;
5561   for (f = 0; f < numFields; ++f) {
5562     PetscInt          p;
5563     const PetscInt    **perms = NULL;
5564     const PetscScalar **flips = NULL;
5565 
5566     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5567     for (p = 0; p < numPoints; p++) {
5568       const PetscInt    point = points[2*p];
5569       PetscInt          fdof, foff, b;
5570       const PetscScalar *varr;
5571       const PetscInt    *perm = perms ? perms[p] : NULL;
5572       const PetscScalar *flip = flips ? flips[p] : NULL;
5573 
5574       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5575       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5576       varr = &vArray[foff];
5577       if (clperm) {
5578         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
5579         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
5580         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
5581       } else {
5582         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
5583         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
5584         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
5585       }
5586       offset += fdof;
5587     }
5588     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5589   }
5590   *size = offset;
5591   PetscFunctionReturn(0);
5592 }
5593 
5594 /*@C
5595   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
5596 
5597   Not collective
5598 
5599   Input Parameters:
5600 + dm - The DM
5601 . section - The section describing the layout in v, or NULL to use the default section
5602 . v - The local vector
5603 . point - The point in the DM
5604 . csize - The size of the input values array, or NULL
5605 - values - An array to use for the values, or NULL to have it allocated automatically
5606 
5607   Output Parameters:
5608 + csize - The number of values in the closure
5609 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
5610 
5611 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
5612 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
5613 $ assembly function, and a user may already have allocated storage for this operation.
5614 $
5615 $ A typical use could be
5616 $
5617 $  values = NULL;
5618 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5619 $  for (cl = 0; cl < clSize; ++cl) {
5620 $    <Compute on closure>
5621 $  }
5622 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5623 $
5624 $ or
5625 $
5626 $  PetscMalloc1(clMaxSize, &values);
5627 $  for (p = pStart; p < pEnd; ++p) {
5628 $    clSize = clMaxSize;
5629 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5630 $    for (cl = 0; cl < clSize; ++cl) {
5631 $      <Compute on closure>
5632 $    }
5633 $  }
5634 $  PetscFree(values);
5635 
5636   Fortran Notes:
5637   Since it returns an array, this routine is only available in Fortran 90, and you must
5638   include petsc.h90 in your code.
5639 
5640   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5641 
5642   Level: intermediate
5643 
5644 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5645 @*/
5646 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5647 {
5648   PetscSection       clSection;
5649   IS                 clPoints;
5650   PetscInt          *points = NULL;
5651   const PetscInt    *clp, *perm;
5652   PetscInt           depth, numFields, numPoints, asize;
5653   PetscErrorCode     ierr;
5654 
5655   PetscFunctionBeginHot;
5656   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5657   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5658   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5659   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5660   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5661   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5662   if (depth == 1 && numFields < 2) {
5663     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5664     PetscFunctionReturn(0);
5665   }
5666   /* Get points */
5667   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5668   /* Get sizes */
5669   asize = 0;
5670   for (PetscInt p = 0; p < numPoints*2; p += 2) {
5671     PetscInt dof;
5672     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5673     asize += dof;
5674   }
5675   if (values) {
5676     const PetscScalar *vArray;
5677     PetscInt          size;
5678 
5679     if (*values) {
5680       if (PetscUnlikely(*csize < asize)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %D not sufficient to hold closure size %D", *csize, asize);
5681     } else {ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, values);CHKERRQ(ierr);}
5682     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, asize, &perm);CHKERRQ(ierr);
5683     ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5684     /* Get values */
5685     if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values);CHKERRQ(ierr);}
5686     else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values);CHKERRQ(ierr);}
5687     if (PetscUnlikely(asize != size)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %D does not match Vec closure size %D", asize, size);
5688     /* Cleanup array */
5689     ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5690   }
5691   if (csize) *csize = asize;
5692   /* Cleanup points */
5693   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5694   PetscFunctionReturn(0);
5695 }
5696 
5697 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
5698 {
5699   DMLabel            depthLabel;
5700   PetscSection       clSection;
5701   IS                 clPoints;
5702   PetscScalar       *array;
5703   const PetscScalar *vArray;
5704   PetscInt          *points = NULL;
5705   const PetscInt    *clp, *perm = NULL;
5706   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
5707   PetscErrorCode     ierr;
5708 
5709   PetscFunctionBeginHot;
5710   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5711   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5712   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5713   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5714   ierr = DMPlexGetDepth(dm, &mdepth);CHKERRQ(ierr);
5715   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
5716   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5717   if (mdepth == 1 && numFields < 2) {
5718     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5719     PetscFunctionReturn(0);
5720   }
5721   /* Get points */
5722   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5723   for (clsize=0,p=0; p<Np; p++) {
5724     PetscInt dof;
5725     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
5726     clsize += dof;
5727   }
5728   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &perm);CHKERRQ(ierr);
5729   /* Filter points */
5730   for (p = 0; p < numPoints*2; p += 2) {
5731     PetscInt dep;
5732 
5733     ierr = DMLabelGetValue(depthLabel, points[p], &dep);CHKERRQ(ierr);
5734     if (dep != depth) continue;
5735     points[Np*2+0] = points[p];
5736     points[Np*2+1] = points[p+1];
5737     ++Np;
5738   }
5739   /* Get array */
5740   if (!values || !*values) {
5741     PetscInt asize = 0, dof;
5742 
5743     for (p = 0; p < Np*2; p += 2) {
5744       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5745       asize += dof;
5746     }
5747     if (!values) {
5748       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5749       if (csize) *csize = asize;
5750       PetscFunctionReturn(0);
5751     }
5752     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
5753   } else {
5754     array = *values;
5755   }
5756   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5757   /* Get values */
5758   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
5759   else               {ierr = DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array);CHKERRQ(ierr);}
5760   /* Cleanup points */
5761   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5762   /* Cleanup array */
5763   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5764   if (!*values) {
5765     if (csize) *csize = size;
5766     *values = array;
5767   } else {
5768     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5769     *csize = size;
5770   }
5771   PetscFunctionReturn(0);
5772 }
5773 
5774 /*@C
5775   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5776 
5777   Not collective
5778 
5779   Input Parameters:
5780 + dm - The DM
5781 . section - The section describing the layout in v, or NULL to use the default section
5782 . v - The local vector
5783 . point - The point in the DM
5784 . csize - The number of values in the closure, or NULL
5785 - values - The array of values, which is a borrowed array and should not be freed
5786 
5787   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
5788 
5789   Fortran Notes:
5790   Since it returns an array, this routine is only available in Fortran 90, and you must
5791   include petsc.h90 in your code.
5792 
5793   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5794 
5795   Level: intermediate
5796 
5797 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5798 @*/
5799 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5800 {
5801   PetscInt       size = 0;
5802   PetscErrorCode ierr;
5803 
5804   PetscFunctionBegin;
5805   /* Should work without recalculating size */
5806   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
5807   *values = NULL;
5808   PetscFunctionReturn(0);
5809 }
5810 
5811 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5812 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5813 
5814 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5815 {
5816   PetscInt        cdof;   /* The number of constraints on this point */
5817   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5818   PetscScalar    *a;
5819   PetscInt        off, cind = 0, k;
5820   PetscErrorCode  ierr;
5821 
5822   PetscFunctionBegin;
5823   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5824   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5825   a    = &array[off];
5826   if (!cdof || setBC) {
5827     if (clperm) {
5828       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
5829       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
5830     } else {
5831       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
5832       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
5833     }
5834   } else {
5835     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5836     if (clperm) {
5837       if (perm) {for (k = 0; k < dof; ++k) {
5838           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5839           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5840         }
5841       } else {
5842         for (k = 0; k < dof; ++k) {
5843           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5844           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5845         }
5846       }
5847     } else {
5848       if (perm) {
5849         for (k = 0; k < dof; ++k) {
5850           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5851           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5852         }
5853       } else {
5854         for (k = 0; k < dof; ++k) {
5855           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5856           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5857         }
5858       }
5859     }
5860   }
5861   PetscFunctionReturn(0);
5862 }
5863 
5864 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5865 {
5866   PetscInt        cdof;   /* The number of constraints on this point */
5867   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5868   PetscScalar    *a;
5869   PetscInt        off, cind = 0, k;
5870   PetscErrorCode  ierr;
5871 
5872   PetscFunctionBegin;
5873   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5874   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5875   a    = &array[off];
5876   if (cdof) {
5877     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5878     if (clperm) {
5879       if (perm) {
5880         for (k = 0; k < dof; ++k) {
5881           if ((cind < cdof) && (k == cdofs[cind])) {
5882             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5883             cind++;
5884           }
5885         }
5886       } else {
5887         for (k = 0; k < dof; ++k) {
5888           if ((cind < cdof) && (k == cdofs[cind])) {
5889             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5890             cind++;
5891           }
5892         }
5893       }
5894     } else {
5895       if (perm) {
5896         for (k = 0; k < dof; ++k) {
5897           if ((cind < cdof) && (k == cdofs[cind])) {
5898             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5899             cind++;
5900           }
5901         }
5902       } else {
5903         for (k = 0; k < dof; ++k) {
5904           if ((cind < cdof) && (k == cdofs[cind])) {
5905             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5906             cind++;
5907           }
5908         }
5909       }
5910     }
5911   }
5912   PetscFunctionReturn(0);
5913 }
5914 
5915 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5916 {
5917   PetscScalar    *a;
5918   PetscInt        fdof, foff, fcdof, foffset = *offset;
5919   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5920   PetscInt        cind = 0, b;
5921   PetscErrorCode  ierr;
5922 
5923   PetscFunctionBegin;
5924   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5925   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5926   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5927   a    = &array[foff];
5928   if (!fcdof || setBC) {
5929     if (clperm) {
5930       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
5931       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
5932     } else {
5933       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
5934       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
5935     }
5936   } else {
5937     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5938     if (clperm) {
5939       if (perm) {
5940         for (b = 0; b < fdof; b++) {
5941           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5942           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5943         }
5944       } else {
5945         for (b = 0; b < fdof; b++) {
5946           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5947           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
5948         }
5949       }
5950     } else {
5951       if (perm) {
5952         for (b = 0; b < fdof; b++) {
5953           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5954           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
5955         }
5956       } else {
5957         for (b = 0; b < fdof; b++) {
5958           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5959           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
5960         }
5961       }
5962     }
5963   }
5964   *offset += fdof;
5965   PetscFunctionReturn(0);
5966 }
5967 
5968 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5969 {
5970   PetscScalar    *a;
5971   PetscInt        fdof, foff, fcdof, foffset = *offset;
5972   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5973   PetscInt        Nc, cind = 0, ncind = 0, b;
5974   PetscBool       ncSet, fcSet;
5975   PetscErrorCode  ierr;
5976 
5977   PetscFunctionBegin;
5978   ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
5979   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5980   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5981   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5982   a    = &array[foff];
5983   if (fcdof) {
5984     /* We just override fcdof and fcdofs with Ncc and comps */
5985     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5986     if (clperm) {
5987       if (perm) {
5988         if (comps) {
5989           for (b = 0; b < fdof; b++) {
5990             ncSet = fcSet = PETSC_FALSE;
5991             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5992             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5993             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
5994           }
5995         } else {
5996           for (b = 0; b < fdof; b++) {
5997             if ((cind < fcdof) && (b == fcdofs[cind])) {
5998               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5999               ++cind;
6000             }
6001           }
6002         }
6003       } else {
6004         if (comps) {
6005           for (b = 0; b < fdof; b++) {
6006             ncSet = fcSet = PETSC_FALSE;
6007             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6008             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6009             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
6010           }
6011         } else {
6012           for (b = 0; b < fdof; b++) {
6013             if ((cind < fcdof) && (b == fcdofs[cind])) {
6014               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
6015               ++cind;
6016             }
6017           }
6018         }
6019       }
6020     } else {
6021       if (perm) {
6022         if (comps) {
6023           for (b = 0; b < fdof; b++) {
6024             ncSet = fcSet = PETSC_FALSE;
6025             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6026             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6027             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
6028           }
6029         } else {
6030           for (b = 0; b < fdof; b++) {
6031             if ((cind < fcdof) && (b == fcdofs[cind])) {
6032               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6033               ++cind;
6034             }
6035           }
6036         }
6037       } else {
6038         if (comps) {
6039           for (b = 0; b < fdof; b++) {
6040             ncSet = fcSet = PETSC_FALSE;
6041             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6042             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6043             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
6044           }
6045         } else {
6046           for (b = 0; b < fdof; b++) {
6047             if ((cind < fcdof) && (b == fcdofs[cind])) {
6048               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6049               ++cind;
6050             }
6051           }
6052         }
6053       }
6054     }
6055   }
6056   *offset += fdof;
6057   PetscFunctionReturn(0);
6058 }
6059 
6060 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6061 {
6062   PetscScalar    *array;
6063   const PetscInt *cone, *coneO;
6064   PetscInt        pStart, pEnd, p, numPoints, off, dof;
6065   PetscErrorCode  ierr;
6066 
6067   PetscFunctionBeginHot;
6068   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6069   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
6070   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
6071   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
6072   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6073   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
6074     const PetscInt cp = !p ? point : cone[p-1];
6075     const PetscInt o  = !p ? 0     : coneO[p-1];
6076 
6077     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
6078     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6079     /* ADD_VALUES */
6080     {
6081       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6082       PetscScalar    *a;
6083       PetscInt        cdof, coff, cind = 0, k;
6084 
6085       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
6086       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
6087       a    = &array[coff];
6088       if (!cdof) {
6089         if (o >= 0) {
6090           for (k = 0; k < dof; ++k) {
6091             a[k] += values[off+k];
6092           }
6093         } else {
6094           for (k = 0; k < dof; ++k) {
6095             a[k] += values[off+dof-k-1];
6096           }
6097         }
6098       } else {
6099         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
6100         if (o >= 0) {
6101           for (k = 0; k < dof; ++k) {
6102             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6103             a[k] += values[off+k];
6104           }
6105         } else {
6106           for (k = 0; k < dof; ++k) {
6107             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6108             a[k] += values[off+dof-k-1];
6109           }
6110         }
6111       }
6112     }
6113   }
6114   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6115   PetscFunctionReturn(0);
6116 }
6117 
6118 /*@C
6119   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6120 
6121   Not collective
6122 
6123   Input Parameters:
6124 + dm - The DM
6125 . section - The section describing the layout in v, or NULL to use the default section
6126 . v - The local vector
6127 . point - The point in the DM
6128 . values - The array of values
6129 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
6130          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
6131 
6132   Fortran Notes:
6133   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6134 
6135   Level: intermediate
6136 
6137 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6138 @*/
6139 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6140 {
6141   PetscSection    clSection;
6142   IS              clPoints;
6143   PetscScalar    *array;
6144   PetscInt       *points = NULL;
6145   const PetscInt *clp, *clperm = NULL;
6146   PetscInt        depth, numFields, numPoints, p, clsize;
6147   PetscErrorCode  ierr;
6148 
6149   PetscFunctionBeginHot;
6150   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6151   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6152   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6153   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6154   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6155   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6156   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6157     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
6158     PetscFunctionReturn(0);
6159   }
6160   /* Get points */
6161   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6162   for (clsize=0,p=0; p<numPoints; p++) {
6163     PetscInt dof;
6164     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
6165     clsize += dof;
6166   }
6167   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
6168   /* Get array */
6169   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6170   /* Get values */
6171   if (numFields > 0) {
6172     PetscInt offset = 0, f;
6173     for (f = 0; f < numFields; ++f) {
6174       const PetscInt    **perms = NULL;
6175       const PetscScalar **flips = NULL;
6176 
6177       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6178       switch (mode) {
6179       case INSERT_VALUES:
6180         for (p = 0; p < numPoints; p++) {
6181           const PetscInt    point = points[2*p];
6182           const PetscInt    *perm = perms ? perms[p] : NULL;
6183           const PetscScalar *flip = flips ? flips[p] : NULL;
6184           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
6185         } break;
6186       case INSERT_ALL_VALUES:
6187         for (p = 0; p < numPoints; p++) {
6188           const PetscInt    point = points[2*p];
6189           const PetscInt    *perm = perms ? perms[p] : NULL;
6190           const PetscScalar *flip = flips ? flips[p] : NULL;
6191           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
6192         } break;
6193       case INSERT_BC_VALUES:
6194         for (p = 0; p < numPoints; p++) {
6195           const PetscInt    point = points[2*p];
6196           const PetscInt    *perm = perms ? perms[p] : NULL;
6197           const PetscScalar *flip = flips ? flips[p] : NULL;
6198           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
6199         } break;
6200       case ADD_VALUES:
6201         for (p = 0; p < numPoints; p++) {
6202           const PetscInt    point = points[2*p];
6203           const PetscInt    *perm = perms ? perms[p] : NULL;
6204           const PetscScalar *flip = flips ? flips[p] : NULL;
6205           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
6206         } break;
6207       case ADD_ALL_VALUES:
6208         for (p = 0; p < numPoints; p++) {
6209           const PetscInt    point = points[2*p];
6210           const PetscInt    *perm = perms ? perms[p] : NULL;
6211           const PetscScalar *flip = flips ? flips[p] : NULL;
6212           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
6213         } break;
6214       case ADD_BC_VALUES:
6215         for (p = 0; p < numPoints; p++) {
6216           const PetscInt    point = points[2*p];
6217           const PetscInt    *perm = perms ? perms[p] : NULL;
6218           const PetscScalar *flip = flips ? flips[p] : NULL;
6219           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
6220         } break;
6221       default:
6222         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6223       }
6224       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6225     }
6226   } else {
6227     PetscInt dof, off;
6228     const PetscInt    **perms = NULL;
6229     const PetscScalar **flips = NULL;
6230 
6231     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6232     switch (mode) {
6233     case INSERT_VALUES:
6234       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6235         const PetscInt    point = points[2*p];
6236         const PetscInt    *perm = perms ? perms[p] : NULL;
6237         const PetscScalar *flip = flips ? flips[p] : NULL;
6238         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6239         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
6240       } break;
6241     case INSERT_ALL_VALUES:
6242       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6243         const PetscInt    point = points[2*p];
6244         const PetscInt    *perm = perms ? perms[p] : NULL;
6245         const PetscScalar *flip = flips ? flips[p] : NULL;
6246         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6247         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
6248       } break;
6249     case INSERT_BC_VALUES:
6250       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6251         const PetscInt    point = points[2*p];
6252         const PetscInt    *perm = perms ? perms[p] : NULL;
6253         const PetscScalar *flip = flips ? flips[p] : NULL;
6254         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6255         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
6256       } break;
6257     case ADD_VALUES:
6258       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6259         const PetscInt    point = points[2*p];
6260         const PetscInt    *perm = perms ? perms[p] : NULL;
6261         const PetscScalar *flip = flips ? flips[p] : NULL;
6262         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6263         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
6264       } break;
6265     case ADD_ALL_VALUES:
6266       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6267         const PetscInt    point = points[2*p];
6268         const PetscInt    *perm = perms ? perms[p] : NULL;
6269         const PetscScalar *flip = flips ? flips[p] : NULL;
6270         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6271         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
6272       } break;
6273     case ADD_BC_VALUES:
6274       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6275         const PetscInt    point = points[2*p];
6276         const PetscInt    *perm = perms ? perms[p] : NULL;
6277         const PetscScalar *flip = flips ? flips[p] : NULL;
6278         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6279         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
6280       } break;
6281     default:
6282       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6283     }
6284     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6285   }
6286   /* Cleanup points */
6287   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6288   /* Cleanup array */
6289   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6290   PetscFunctionReturn(0);
6291 }
6292 
6293 /* Check whether the given point is in the label. If not, update the offset to skip this point */
6294 PETSC_STATIC_INLINE PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset)
6295 {
6296   PetscFunctionBegin;
6297   if (label) {
6298     PetscInt       val, fdof;
6299     PetscErrorCode ierr;
6300 
6301     /* There is a problem with this:
6302          Suppose we have two label values, defining surfaces, interecting along a line in 3D. When we add cells to the label, the cells that
6303        touch both surfaces must pick a label value. Thus we miss setting values for the surface with that other value intersecting that cell.
6304        Thus I am only going to check val != -1, not val != labelId
6305     */
6306     ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
6307     if (val < 0) {
6308       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6309       *offset += fdof;
6310       PetscFunctionReturn(1);
6311     }
6312   }
6313   PetscFunctionReturn(0);
6314 }
6315 
6316 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
6317 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
6318 {
6319   PetscSection      clSection;
6320   IS                clPoints;
6321   PetscScalar       *array;
6322   PetscInt          *points = NULL;
6323   const PetscInt    *clp;
6324   PetscInt          numFields, numPoints, p;
6325   PetscInt          offset = 0, f;
6326   PetscErrorCode    ierr;
6327 
6328   PetscFunctionBeginHot;
6329   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6330   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6331   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6332   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6333   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6334   /* Get points */
6335   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6336   /* Get array */
6337   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6338   /* Get values */
6339   for (f = 0; f < numFields; ++f) {
6340     const PetscInt    **perms = NULL;
6341     const PetscScalar **flips = NULL;
6342 
6343     if (!fieldActive[f]) {
6344       for (p = 0; p < numPoints*2; p += 2) {
6345         PetscInt fdof;
6346         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6347         offset += fdof;
6348       }
6349       continue;
6350     }
6351     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6352     switch (mode) {
6353     case INSERT_VALUES:
6354       for (p = 0; p < numPoints; p++) {
6355         const PetscInt    point = points[2*p];
6356         const PetscInt    *perm = perms ? perms[p] : NULL;
6357         const PetscScalar *flip = flips ? flips[p] : NULL;
6358         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6359         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array);
6360       } break;
6361     case INSERT_ALL_VALUES:
6362       for (p = 0; p < numPoints; p++) {
6363         const PetscInt    point = points[2*p];
6364         const PetscInt    *perm = perms ? perms[p] : NULL;
6365         const PetscScalar *flip = flips ? flips[p] : NULL;
6366         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6367         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array);
6368       } break;
6369     case INSERT_BC_VALUES:
6370       for (p = 0; p < numPoints; p++) {
6371         const PetscInt    point = points[2*p];
6372         const PetscInt    *perm = perms ? perms[p] : NULL;
6373         const PetscScalar *flip = flips ? flips[p] : NULL;
6374         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6375         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array);
6376       } break;
6377     case ADD_VALUES:
6378       for (p = 0; p < numPoints; p++) {
6379         const PetscInt    point = points[2*p];
6380         const PetscInt    *perm = perms ? perms[p] : NULL;
6381         const PetscScalar *flip = flips ? flips[p] : NULL;
6382         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6383         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array);
6384       } break;
6385     case ADD_ALL_VALUES:
6386       for (p = 0; p < numPoints; p++) {
6387         const PetscInt    point = points[2*p];
6388         const PetscInt    *perm = perms ? perms[p] : NULL;
6389         const PetscScalar *flip = flips ? flips[p] : NULL;
6390         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6391         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array);
6392       } break;
6393     default:
6394       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6395     }
6396     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6397   }
6398   /* Cleanup points */
6399   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6400   /* Cleanup array */
6401   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6402   PetscFunctionReturn(0);
6403 }
6404 
6405 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
6406 {
6407   PetscMPIInt    rank;
6408   PetscInt       i, j;
6409   PetscErrorCode ierr;
6410 
6411   PetscFunctionBegin;
6412   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr);
6413   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
6414   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
6415   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
6416   numCIndices = numCIndices ? numCIndices : numRIndices;
6417   if (!values) PetscFunctionReturn(0);
6418   for (i = 0; i < numRIndices; i++) {
6419     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
6420     for (j = 0; j < numCIndices; j++) {
6421 #if defined(PETSC_USE_COMPLEX)
6422       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
6423 #else
6424       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
6425 #endif
6426     }
6427     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6428   }
6429   PetscFunctionReturn(0);
6430 }
6431 
6432 /*
6433   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
6434 
6435   Input Parameters:
6436 + section - The section for this data layout
6437 . islocal - Is the section (and thus indices being requested) local or global?
6438 . point   - The point contributing dofs with these indices
6439 . off     - The global offset of this point
6440 . loff    - The local offset of each field
6441 . setBC   - The flag determining whether to include indices of bounsary values
6442 . perm    - A permutation of the dofs on this point, or NULL
6443 - indperm - A permutation of the entire indices array, or NULL
6444 
6445   Output Parameter:
6446 . indices - Indices for dofs on this point
6447 
6448   Level: developer
6449 
6450   Note: The indices could be local or global, depending on the value of 'off'.
6451 */
6452 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
6453 {
6454   PetscInt        dof;   /* The number of unknowns on this point */
6455   PetscInt        cdof;  /* The number of constraints on this point */
6456   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6457   PetscInt        cind = 0, k;
6458   PetscErrorCode  ierr;
6459 
6460   PetscFunctionBegin;
6461   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6462   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6463   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6464   if (!cdof || setBC) {
6465     for (k = 0; k < dof; ++k) {
6466       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6467       const PetscInt ind    = indperm ? indperm[preind] : preind;
6468 
6469       indices[ind] = off + k;
6470     }
6471   } else {
6472     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6473     for (k = 0; k < dof; ++k) {
6474       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6475       const PetscInt ind    = indperm ? indperm[preind] : preind;
6476 
6477       if ((cind < cdof) && (k == cdofs[cind])) {
6478         /* Insert check for returning constrained indices */
6479         indices[ind] = -(off+k+1);
6480         ++cind;
6481       } else {
6482         indices[ind] = off + k - (islocal ? 0 : cind);
6483       }
6484     }
6485   }
6486   *loff += dof;
6487   PetscFunctionReturn(0);
6488 }
6489 
6490 /*
6491  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
6492 
6493  Input Parameters:
6494 + section - a section (global or local)
6495 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
6496 . point - point within section
6497 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
6498 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
6499 . setBC - identify constrained (boundary condition) points via involution.
6500 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
6501 . permsoff - offset
6502 - indperm - index permutation
6503 
6504  Output Parameter:
6505 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
6506 . indices - array to hold indices (as defined by section) of each dof associated with point
6507 
6508  Notes:
6509  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
6510  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
6511  in the local vector.
6512 
6513  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
6514  significant).  It is invalid to call with a global section and setBC=true.
6515 
6516  Developer Note:
6517  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
6518  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
6519  offset could be obtained from the section instead of passing it explicitly as we do now.
6520 
6521  Example:
6522  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
6523  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
6524  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
6525  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
6526 
6527  Level: developer
6528 */
6529 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6530 {
6531   PetscInt       numFields, foff, f;
6532   PetscErrorCode ierr;
6533 
6534   PetscFunctionBegin;
6535   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6536   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6537   for (f = 0, foff = 0; f < numFields; ++f) {
6538     PetscInt        fdof, cfdof;
6539     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6540     PetscInt        cind = 0, b;
6541     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6542 
6543     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6544     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6545     if (!cfdof || setBC) {
6546       for (b = 0; b < fdof; ++b) {
6547         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6548         const PetscInt ind    = indperm ? indperm[preind] : preind;
6549 
6550         indices[ind] = off+foff+b;
6551       }
6552     } else {
6553       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6554       for (b = 0; b < fdof; ++b) {
6555         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6556         const PetscInt ind    = indperm ? indperm[preind] : preind;
6557 
6558         if ((cind < cfdof) && (b == fcdofs[cind])) {
6559           indices[ind] = -(off+foff+b+1);
6560           ++cind;
6561         } else {
6562           indices[ind] = off + foff + b - (islocal ? 0 : cind);
6563         }
6564       }
6565     }
6566     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
6567     foffs[f] += fdof;
6568   }
6569   PetscFunctionReturn(0);
6570 }
6571 
6572 /*
6573   This version believes the globalSection offsets for each field, rather than just the point offset
6574 
6575  . foffs - The offset into 'indices' for each field, since it is segregated by field
6576 
6577  Notes:
6578  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
6579  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
6580 */
6581 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6582 {
6583   PetscInt       numFields, foff, f;
6584   PetscErrorCode ierr;
6585 
6586   PetscFunctionBegin;
6587   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6588   for (f = 0; f < numFields; ++f) {
6589     PetscInt        fdof, cfdof;
6590     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6591     PetscInt        cind = 0, b;
6592     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6593 
6594     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6595     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6596     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
6597     if (!cfdof) {
6598       for (b = 0; b < fdof; ++b) {
6599         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6600         const PetscInt ind    = indperm ? indperm[preind] : preind;
6601 
6602         indices[ind] = foff+b;
6603       }
6604     } else {
6605       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6606       for (b = 0; b < fdof; ++b) {
6607         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6608         const PetscInt ind    = indperm ? indperm[preind] : preind;
6609 
6610         if ((cind < cfdof) && (b == fcdofs[cind])) {
6611           indices[ind] = -(foff+b+1);
6612           ++cind;
6613         } else {
6614           indices[ind] = foff+b-cind;
6615         }
6616       }
6617     }
6618     foffs[f] += fdof;
6619   }
6620   PetscFunctionReturn(0);
6621 }
6622 
6623 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
6624 {
6625   Mat             cMat;
6626   PetscSection    aSec, cSec;
6627   IS              aIS;
6628   PetscInt        aStart = -1, aEnd = -1;
6629   const PetscInt  *anchors;
6630   PetscInt        numFields, f, p, q, newP = 0;
6631   PetscInt        newNumPoints = 0, newNumIndices = 0;
6632   PetscInt        *newPoints, *indices, *newIndices;
6633   PetscInt        maxAnchor, maxDof;
6634   PetscInt        newOffsets[32];
6635   PetscInt        *pointMatOffsets[32];
6636   PetscInt        *newPointOffsets[32];
6637   PetscScalar     *pointMat[32];
6638   PetscScalar     *newValues=NULL,*tmpValues;
6639   PetscBool       anyConstrained = PETSC_FALSE;
6640   PetscErrorCode  ierr;
6641 
6642   PetscFunctionBegin;
6643   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6644   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6645   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6646 
6647   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
6648   /* if there are point-to-point constraints */
6649   if (aSec) {
6650     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
6651     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
6652     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
6653     /* figure out how many points are going to be in the new element matrix
6654      * (we allow double counting, because it's all just going to be summed
6655      * into the global matrix anyway) */
6656     for (p = 0; p < 2*numPoints; p+=2) {
6657       PetscInt b    = points[p];
6658       PetscInt bDof = 0, bSecDof;
6659 
6660       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6661       if (!bSecDof) {
6662         continue;
6663       }
6664       if (b >= aStart && b < aEnd) {
6665         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
6666       }
6667       if (bDof) {
6668         /* this point is constrained */
6669         /* it is going to be replaced by its anchors */
6670         PetscInt bOff, q;
6671 
6672         anyConstrained = PETSC_TRUE;
6673         newNumPoints  += bDof;
6674         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
6675         for (q = 0; q < bDof; q++) {
6676           PetscInt a = anchors[bOff + q];
6677           PetscInt aDof;
6678 
6679           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
6680           newNumIndices += aDof;
6681           for (f = 0; f < numFields; ++f) {
6682             PetscInt fDof;
6683 
6684             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
6685             newOffsets[f+1] += fDof;
6686           }
6687         }
6688       }
6689       else {
6690         /* this point is not constrained */
6691         newNumPoints++;
6692         newNumIndices += bSecDof;
6693         for (f = 0; f < numFields; ++f) {
6694           PetscInt fDof;
6695 
6696           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6697           newOffsets[f+1] += fDof;
6698         }
6699       }
6700     }
6701   }
6702   if (!anyConstrained) {
6703     if (outNumPoints)  *outNumPoints  = 0;
6704     if (outNumIndices) *outNumIndices = 0;
6705     if (outPoints)     *outPoints     = NULL;
6706     if (outValues)     *outValues     = NULL;
6707     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6708     PetscFunctionReturn(0);
6709   }
6710 
6711   if (outNumPoints)  *outNumPoints  = newNumPoints;
6712   if (outNumIndices) *outNumIndices = newNumIndices;
6713 
6714   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
6715 
6716   if (!outPoints && !outValues) {
6717     if (offsets) {
6718       for (f = 0; f <= numFields; f++) {
6719         offsets[f] = newOffsets[f];
6720       }
6721     }
6722     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6723     PetscFunctionReturn(0);
6724   }
6725 
6726   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
6727 
6728   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
6729 
6730   /* workspaces */
6731   if (numFields) {
6732     for (f = 0; f < numFields; f++) {
6733       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
6734       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
6735     }
6736   }
6737   else {
6738     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
6739     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
6740   }
6741 
6742   /* get workspaces for the point-to-point matrices */
6743   if (numFields) {
6744     PetscInt totalOffset, totalMatOffset;
6745 
6746     for (p = 0; p < numPoints; p++) {
6747       PetscInt b    = points[2*p];
6748       PetscInt bDof = 0, bSecDof;
6749 
6750       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6751       if (!bSecDof) {
6752         for (f = 0; f < numFields; f++) {
6753           newPointOffsets[f][p + 1] = 0;
6754           pointMatOffsets[f][p + 1] = 0;
6755         }
6756         continue;
6757       }
6758       if (b >= aStart && b < aEnd) {
6759         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6760       }
6761       if (bDof) {
6762         for (f = 0; f < numFields; f++) {
6763           PetscInt fDof, q, bOff, allFDof = 0;
6764 
6765           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6766           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6767           for (q = 0; q < bDof; q++) {
6768             PetscInt a = anchors[bOff + q];
6769             PetscInt aFDof;
6770 
6771             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
6772             allFDof += aFDof;
6773           }
6774           newPointOffsets[f][p+1] = allFDof;
6775           pointMatOffsets[f][p+1] = fDof * allFDof;
6776         }
6777       }
6778       else {
6779         for (f = 0; f < numFields; f++) {
6780           PetscInt fDof;
6781 
6782           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6783           newPointOffsets[f][p+1] = fDof;
6784           pointMatOffsets[f][p+1] = 0;
6785         }
6786       }
6787     }
6788     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
6789       newPointOffsets[f][0] = totalOffset;
6790       pointMatOffsets[f][0] = totalMatOffset;
6791       for (p = 0; p < numPoints; p++) {
6792         newPointOffsets[f][p+1] += newPointOffsets[f][p];
6793         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
6794       }
6795       totalOffset    = newPointOffsets[f][numPoints];
6796       totalMatOffset = pointMatOffsets[f][numPoints];
6797       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
6798     }
6799   }
6800   else {
6801     for (p = 0; p < numPoints; p++) {
6802       PetscInt b    = points[2*p];
6803       PetscInt bDof = 0, bSecDof;
6804 
6805       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6806       if (!bSecDof) {
6807         newPointOffsets[0][p + 1] = 0;
6808         pointMatOffsets[0][p + 1] = 0;
6809         continue;
6810       }
6811       if (b >= aStart && b < aEnd) {
6812         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6813       }
6814       if (bDof) {
6815         PetscInt bOff, q, allDof = 0;
6816 
6817         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6818         for (q = 0; q < bDof; q++) {
6819           PetscInt a = anchors[bOff + q], aDof;
6820 
6821           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
6822           allDof += aDof;
6823         }
6824         newPointOffsets[0][p+1] = allDof;
6825         pointMatOffsets[0][p+1] = bSecDof * allDof;
6826       }
6827       else {
6828         newPointOffsets[0][p+1] = bSecDof;
6829         pointMatOffsets[0][p+1] = 0;
6830       }
6831     }
6832     newPointOffsets[0][0] = 0;
6833     pointMatOffsets[0][0] = 0;
6834     for (p = 0; p < numPoints; p++) {
6835       newPointOffsets[0][p+1] += newPointOffsets[0][p];
6836       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
6837     }
6838     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
6839   }
6840 
6841   /* output arrays */
6842   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6843 
6844   /* get the point-to-point matrices; construct newPoints */
6845   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
6846   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6847   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6848   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
6849   if (numFields) {
6850     for (p = 0, newP = 0; p < numPoints; p++) {
6851       PetscInt b    = points[2*p];
6852       PetscInt o    = points[2*p+1];
6853       PetscInt bDof = 0, bSecDof;
6854 
6855       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6856       if (!bSecDof) {
6857         continue;
6858       }
6859       if (b >= aStart && b < aEnd) {
6860         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6861       }
6862       if (bDof) {
6863         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
6864 
6865         fStart[0] = 0;
6866         fEnd[0]   = 0;
6867         for (f = 0; f < numFields; f++) {
6868           PetscInt fDof;
6869 
6870           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
6871           fStart[f+1] = fStart[f] + fDof;
6872           fEnd[f+1]   = fStart[f+1];
6873         }
6874         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6875         ierr = DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
6876 
6877         fAnchorStart[0] = 0;
6878         fAnchorEnd[0]   = 0;
6879         for (f = 0; f < numFields; f++) {
6880           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
6881 
6882           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
6883           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
6884         }
6885         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6886         for (q = 0; q < bDof; q++) {
6887           PetscInt a = anchors[bOff + q], aOff;
6888 
6889           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6890           newPoints[2*(newP + q)]     = a;
6891           newPoints[2*(newP + q) + 1] = 0;
6892           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6893           ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
6894         }
6895         newP += bDof;
6896 
6897         if (outValues) {
6898           /* get the point-to-point submatrix */
6899           for (f = 0; f < numFields; f++) {
6900             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
6901           }
6902         }
6903       }
6904       else {
6905         newPoints[2 * newP]     = b;
6906         newPoints[2 * newP + 1] = o;
6907         newP++;
6908       }
6909     }
6910   } else {
6911     for (p = 0; p < numPoints; p++) {
6912       PetscInt b    = points[2*p];
6913       PetscInt o    = points[2*p+1];
6914       PetscInt bDof = 0, bSecDof;
6915 
6916       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6917       if (!bSecDof) {
6918         continue;
6919       }
6920       if (b >= aStart && b < aEnd) {
6921         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6922       }
6923       if (bDof) {
6924         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
6925 
6926         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6927         ierr = DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
6928 
6929         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
6930         for (q = 0; q < bDof; q++) {
6931           PetscInt a = anchors[bOff + q], aOff;
6932 
6933           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6934 
6935           newPoints[2*(newP + q)]     = a;
6936           newPoints[2*(newP + q) + 1] = 0;
6937           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6938           ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
6939         }
6940         newP += bDof;
6941 
6942         /* get the point-to-point submatrix */
6943         if (outValues) {
6944           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
6945         }
6946       }
6947       else {
6948         newPoints[2 * newP]     = b;
6949         newPoints[2 * newP + 1] = o;
6950         newP++;
6951       }
6952     }
6953   }
6954 
6955   if (outValues) {
6956     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
6957     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
6958     /* multiply constraints on the right */
6959     if (numFields) {
6960       for (f = 0; f < numFields; f++) {
6961         PetscInt oldOff = offsets[f];
6962 
6963         for (p = 0; p < numPoints; p++) {
6964           PetscInt cStart = newPointOffsets[f][p];
6965           PetscInt b      = points[2 * p];
6966           PetscInt c, r, k;
6967           PetscInt dof;
6968 
6969           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
6970           if (!dof) {
6971             continue;
6972           }
6973           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
6974             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
6975             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
6976 
6977             for (r = 0; r < numIndices; r++) {
6978               for (c = 0; c < nCols; c++) {
6979                 for (k = 0; k < dof; k++) {
6980                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
6981                 }
6982               }
6983             }
6984           }
6985           else {
6986             /* copy this column as is */
6987             for (r = 0; r < numIndices; r++) {
6988               for (c = 0; c < dof; c++) {
6989                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
6990               }
6991             }
6992           }
6993           oldOff += dof;
6994         }
6995       }
6996     }
6997     else {
6998       PetscInt oldOff = 0;
6999       for (p = 0; p < numPoints; p++) {
7000         PetscInt cStart = newPointOffsets[0][p];
7001         PetscInt b      = points[2 * p];
7002         PetscInt c, r, k;
7003         PetscInt dof;
7004 
7005         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
7006         if (!dof) {
7007           continue;
7008         }
7009         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7010           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
7011           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
7012 
7013           for (r = 0; r < numIndices; r++) {
7014             for (c = 0; c < nCols; c++) {
7015               for (k = 0; k < dof; k++) {
7016                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
7017               }
7018             }
7019           }
7020         }
7021         else {
7022           /* copy this column as is */
7023           for (r = 0; r < numIndices; r++) {
7024             for (c = 0; c < dof; c++) {
7025               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7026             }
7027           }
7028         }
7029         oldOff += dof;
7030       }
7031     }
7032 
7033     if (multiplyLeft) {
7034       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
7035       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
7036       /* multiply constraints transpose on the left */
7037       if (numFields) {
7038         for (f = 0; f < numFields; f++) {
7039           PetscInt oldOff = offsets[f];
7040 
7041           for (p = 0; p < numPoints; p++) {
7042             PetscInt rStart = newPointOffsets[f][p];
7043             PetscInt b      = points[2 * p];
7044             PetscInt c, r, k;
7045             PetscInt dof;
7046 
7047             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
7048             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7049               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
7050               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
7051 
7052               for (r = 0; r < nRows; r++) {
7053                 for (c = 0; c < newNumIndices; c++) {
7054                   for (k = 0; k < dof; k++) {
7055                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7056                   }
7057                 }
7058               }
7059             }
7060             else {
7061               /* copy this row as is */
7062               for (r = 0; r < dof; r++) {
7063                 for (c = 0; c < newNumIndices; c++) {
7064                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7065                 }
7066               }
7067             }
7068             oldOff += dof;
7069           }
7070         }
7071       }
7072       else {
7073         PetscInt oldOff = 0;
7074 
7075         for (p = 0; p < numPoints; p++) {
7076           PetscInt rStart = newPointOffsets[0][p];
7077           PetscInt b      = points[2 * p];
7078           PetscInt c, r, k;
7079           PetscInt dof;
7080 
7081           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
7082           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7083             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
7084             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
7085 
7086             for (r = 0; r < nRows; r++) {
7087               for (c = 0; c < newNumIndices; c++) {
7088                 for (k = 0; k < dof; k++) {
7089                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7090                 }
7091               }
7092             }
7093           }
7094           else {
7095             /* copy this row as is */
7096             for (r = 0; r < dof; r++) {
7097               for (c = 0; c < newNumIndices; c++) {
7098                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7099               }
7100             }
7101           }
7102           oldOff += dof;
7103         }
7104       }
7105 
7106       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
7107     }
7108     else {
7109       newValues = tmpValues;
7110     }
7111   }
7112 
7113   /* clean up */
7114   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
7115   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
7116 
7117   if (numFields) {
7118     for (f = 0; f < numFields; f++) {
7119       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
7120       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
7121       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
7122     }
7123   }
7124   else {
7125     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
7126     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
7127     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
7128   }
7129   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7130 
7131   /* output */
7132   if (outPoints) {
7133     *outPoints = newPoints;
7134   }
7135   else {
7136     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
7137   }
7138   if (outValues) {
7139     *outValues = newValues;
7140   }
7141   for (f = 0; f <= numFields; f++) {
7142     offsets[f] = newOffsets[f];
7143   }
7144   PetscFunctionReturn(0);
7145 }
7146 
7147 /*@C
7148   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
7149 
7150   Not collective
7151 
7152   Input Parameters:
7153 + dm         - The DM
7154 . section    - The PetscSection describing the points (a local section)
7155 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7156 . point      - The point defining the closure
7157 - useClPerm  - Use the closure point permutation if available
7158 
7159   Output Parameters:
7160 + numIndices - The number of dof indices in the closure of point with the input sections
7161 . indices    - The dof indices
7162 . outOffsets - Array to write the field offsets into, or NULL
7163 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7164 
7165   Notes:
7166   Must call DMPlexRestoreClosureIndices() to free allocated memory
7167 
7168   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7169   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7170   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7171   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7172   indices (with the above semantics) are implied.
7173 
7174   Level: advanced
7175 
7176 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7177 @*/
7178 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7179                                        PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7180 {
7181   /* Closure ordering */
7182   PetscSection        clSection;
7183   IS                  clPoints;
7184   const PetscInt     *clp;
7185   PetscInt           *points;
7186   const PetscInt     *clperm = NULL;
7187   /* Dof permutation and sign flips */
7188   const PetscInt    **perms[32] = {NULL};
7189   const PetscScalar **flips[32] = {NULL};
7190   PetscScalar        *valCopy   = NULL;
7191   /* Hanging node constraints */
7192   PetscInt           *pointsC = NULL;
7193   PetscScalar        *valuesC = NULL;
7194   PetscInt            NclC, NiC;
7195 
7196   PetscInt           *idx;
7197   PetscInt            Nf, Ncl, Ni = 0, offsets[32], p, f;
7198   PetscBool           isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7199   PetscErrorCode      ierr;
7200 
7201   PetscFunctionBeginHot;
7202   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7203   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7204   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7205   if (numIndices) PetscValidPointer(numIndices, 6);
7206   if (indices)    PetscValidPointer(indices, 7);
7207   if (outOffsets) PetscValidPointer(outOffsets, 8);
7208   if (values)     PetscValidPointer(values, 9);
7209   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7210   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
7211   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
7212   /* 1) Get points in closure */
7213   ierr = DMPlexGetCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7214   if (useClPerm) {
7215     PetscInt depth, clsize;
7216     ierr = DMPlexGetPointDepth(dm, point, &depth);CHKERRQ(ierr);
7217     for (clsize=0,p=0; p<Ncl; p++) {
7218       PetscInt dof;
7219       ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
7220       clsize += dof;
7221     }
7222     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
7223   }
7224   /* 2) Get number of indices on these points and field offsets from section */
7225   for (p = 0; p < Ncl*2; p += 2) {
7226     PetscInt dof, fdof;
7227 
7228     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7229     for (f = 0; f < Nf; ++f) {
7230       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7231       offsets[f+1] += fdof;
7232     }
7233     Ni += dof;
7234   }
7235   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
7236   if (Nf && offsets[Nf] != Ni) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Ni);
7237   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7238   for (f = 0; f < PetscMax(1, Nf); ++f) {
7239     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7240     else    {ierr = PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7241     /* may need to apply sign changes to the element matrix */
7242     if (values && flips[f]) {
7243       PetscInt foffset = offsets[f];
7244 
7245       for (p = 0; p < Ncl; ++p) {
7246         PetscInt           pnt  = points[2*p], fdof;
7247         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7248 
7249         if (!Nf) {ierr = PetscSectionGetDof(section, pnt, &fdof);CHKERRQ(ierr);}
7250         else     {ierr = PetscSectionGetFieldDof(section, pnt, f, &fdof);CHKERRQ(ierr);}
7251         if (flip) {
7252           PetscInt i, j, k;
7253 
7254           if (!valCopy) {
7255             ierr = DMGetWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);
7256             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7257             *values = valCopy;
7258           }
7259           for (i = 0; i < fdof; ++i) {
7260             PetscScalar fval = flip[i];
7261 
7262             for (k = 0; k < Ni; ++k) {
7263               valCopy[Ni * (foffset + i) + k] *= fval;
7264               valCopy[Ni * k + (foffset + i)] *= fval;
7265             }
7266           }
7267         }
7268         foffset += fdof;
7269       }
7270     }
7271   }
7272   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7273   ierr = DMPlexAnchorsModifyMat(dm, section, Ncl, Ni, points, perms, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
7274   if (NclC) {
7275     if (valCopy) {ierr = DMRestoreWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);}
7276     for (f = 0; f < PetscMax(1, Nf); ++f) {
7277       if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7278       else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7279     }
7280     for (f = 0; f < PetscMax(1, Nf); ++f) {
7281       if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7282       else    {ierr = PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7283     }
7284     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7285     Ncl     = NclC;
7286     Ni      = NiC;
7287     points  = pointsC;
7288     if (values) *values = valuesC;
7289   }
7290   /* 5) Calculate indices */
7291   ierr = DMGetWorkArray(dm, Ni, MPIU_INT, &idx);CHKERRQ(ierr);
7292   if (Nf) {
7293     PetscInt  idxOff;
7294     PetscBool useFieldOffsets;
7295 
7296     if (outOffsets) {for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];}
7297     ierr = PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets);CHKERRQ(ierr);
7298     if (useFieldOffsets) {
7299       for (p = 0; p < Ncl; ++p) {
7300         const PetscInt pnt = points[p*2];
7301 
7302         ierr = DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx);CHKERRQ(ierr);
7303       }
7304     } else {
7305       for (p = 0; p < Ncl; ++p) {
7306         const PetscInt pnt = points[p*2];
7307 
7308         ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7309         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7310          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
7311          * global section. */
7312         ierr = DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx);CHKERRQ(ierr);
7313       }
7314     }
7315   } else {
7316     PetscInt off = 0, idxOff;
7317 
7318     for (p = 0; p < Ncl; ++p) {
7319       const PetscInt  pnt  = points[p*2];
7320       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
7321 
7322       ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7323       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7324        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
7325       ierr = DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx);CHKERRQ(ierr);
7326     }
7327   }
7328   /* 6) Cleanup */
7329   for (f = 0; f < PetscMax(1, Nf); ++f) {
7330     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7331     else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7332   }
7333   if (NclC) {
7334     ierr = DMRestoreWorkArray(dm, NclC*2, MPIU_INT, &pointsC);CHKERRQ(ierr);
7335   } else {
7336     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7337   }
7338 
7339   if (numIndices) *numIndices = Ni;
7340   if (indices)    *indices    = idx;
7341   PetscFunctionReturn(0);
7342 }
7343 
7344 /*@C
7345   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
7346 
7347   Not collective
7348 
7349   Input Parameters:
7350 + dm         - The DM
7351 . section    - The PetscSection describing the points (a local section)
7352 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7353 . point      - The point defining the closure
7354 - useClPerm  - Use the closure point permutation if available
7355 
7356   Output Parameters:
7357 + numIndices - The number of dof indices in the closure of point with the input sections
7358 . indices    - The dof indices
7359 . outOffsets - Array to write the field offsets into, or NULL
7360 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7361 
7362   Notes:
7363   If values were modified, the user is responsible for calling DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values).
7364 
7365   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7366   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7367   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7368   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7369   indices (with the above semantics) are implied.
7370 
7371   Level: advanced
7372 
7373 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7374 @*/
7375 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7376                                            PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7377 {
7378   PetscErrorCode ierr;
7379 
7380   PetscFunctionBegin;
7381   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7382   PetscValidPointer(indices, 7);
7383   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
7384   PetscFunctionReturn(0);
7385 }
7386 
7387 /*@C
7388   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7389 
7390   Not collective
7391 
7392   Input Parameters:
7393 + dm - The DM
7394 . section - The section describing the layout in v, or NULL to use the default section
7395 . globalSection - The section describing the layout in v, or NULL to use the default global section
7396 . A - The matrix
7397 . point - The point in the DM
7398 . values - The array of values
7399 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7400 
7401   Fortran Notes:
7402   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7403 
7404   Level: intermediate
7405 
7406 .seealso DMPlexMatSetClosureGeneral(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7407 @*/
7408 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7409 {
7410   DM_Plex           *mesh = (DM_Plex*) dm->data;
7411   PetscInt          *indices;
7412   PetscInt           numIndices;
7413   const PetscScalar *valuesOrig = values;
7414   PetscErrorCode     ierr;
7415 
7416   PetscFunctionBegin;
7417   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7418   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
7419   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7420   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
7421   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7422   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7423 
7424   ierr = DMPlexGetClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7425 
7426   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
7427   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7428   if (ierr) {
7429     PetscMPIInt    rank;
7430     PetscErrorCode ierr2;
7431 
7432     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7433     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7434     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
7435     ierr2 = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7436     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7437     CHKERRQ(ierr);
7438   }
7439   if (mesh->printFEM > 1) {
7440     PetscInt i;
7441     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
7442     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
7443     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7444   }
7445 
7446   ierr = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7447   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7448   PetscFunctionReturn(0);
7449 }
7450 
7451 /*@C
7452   DMPlexMatSetClosure - Set an array of the values on the closure of 'point' using a different row and column section
7453 
7454   Not collective
7455 
7456   Input Parameters:
7457 + dmRow - The DM for the row fields
7458 . sectionRow - The section describing the layout, or NULL to use the default section in dmRow
7459 . globalSectionRow - The section describing the layout, or NULL to use the default global section in dmRow
7460 . dmCol - The DM for the column fields
7461 . sectionCol - The section describing the layout, or NULL to use the default section in dmCol
7462 . globalSectionCol - The section describing the layout, or NULL to use the default global section in dmCol
7463 . A - The matrix
7464 . point - The point in the DMs
7465 . values - The array of values
7466 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7467 
7468   Level: intermediate
7469 
7470 .seealso DMPlexMatSetClosure(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7471 @*/
7472 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7473 {
7474   DM_Plex           *mesh = (DM_Plex*) dmRow->data;
7475   PetscInt          *indicesRow, *indicesCol;
7476   PetscInt           numIndicesRow, numIndicesCol;
7477   const PetscScalar *valuesOrig = values;
7478   PetscErrorCode     ierr;
7479 
7480   PetscFunctionBegin;
7481   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
7482   if (!sectionRow) {ierr = DMGetLocalSection(dmRow, &sectionRow);CHKERRQ(ierr);}
7483   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
7484   if (!globalSectionRow) {ierr = DMGetGlobalSection(dmRow, &globalSectionRow);CHKERRQ(ierr);}
7485   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
7486   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 4);
7487   if (!sectionCol) {ierr = DMGetLocalSection(dmCol, &sectionCol);CHKERRQ(ierr);}
7488   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 5);
7489   if (!globalSectionCol) {ierr = DMGetGlobalSection(dmCol, &globalSectionCol);CHKERRQ(ierr);}
7490   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 6);
7491   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7492 
7493   ierr = DMPlexGetClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7494   ierr = DMPlexGetClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7495 
7496   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr);}
7497   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values, mode);
7498   if (ierr) {
7499     PetscMPIInt    rank;
7500     PetscErrorCode ierr2;
7501 
7502     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7503     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7504     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr2);
7505     ierr2 = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7506     ierr2 = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7507     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7508     CHKERRQ(ierr);
7509   }
7510 
7511   ierr = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7512   ierr = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7513   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7514   PetscFunctionReturn(0);
7515 }
7516 
7517 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7518 {
7519   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
7520   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
7521   PetscInt       *cpoints = NULL;
7522   PetscInt       *findices, *cindices;
7523   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7524   PetscInt        foffsets[32], coffsets[32];
7525   DMPolytopeType  ct;
7526   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7527   PetscErrorCode  ierr;
7528 
7529   PetscFunctionBegin;
7530   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7531   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7532   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7533   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7534   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7535   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7536   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7537   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7538   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7539   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7540   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7541   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7542   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7543   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7544   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7545   /* Column indices */
7546   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7547   maxFPoints = numCPoints;
7548   /* Compress out points not in the section */
7549   /*   TODO: Squeeze out points with 0 dof as well */
7550   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7551   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7552     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7553       cpoints[q*2]   = cpoints[p];
7554       cpoints[q*2+1] = cpoints[p+1];
7555       ++q;
7556     }
7557   }
7558   numCPoints = q;
7559   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7560     PetscInt fdof;
7561 
7562     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7563     if (!dof) continue;
7564     for (f = 0; f < numFields; ++f) {
7565       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7566       coffsets[f+1] += fdof;
7567     }
7568     numCIndices += dof;
7569   }
7570   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7571   /* Row indices */
7572   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7573   {
7574     DMPlexCellRefiner cr;
7575     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7576     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7577     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7578   }
7579   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7580   for (r = 0, q = 0; r < numSubcells; ++r) {
7581     /* TODO Map from coarse to fine cells */
7582     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7583     /* Compress out points not in the section */
7584     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7585     for (p = 0; p < numFPoints*2; p += 2) {
7586       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7587         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7588         if (!dof) continue;
7589         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7590         if (s < q) continue;
7591         ftotpoints[q*2]   = fpoints[p];
7592         ftotpoints[q*2+1] = fpoints[p+1];
7593         ++q;
7594       }
7595     }
7596     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7597   }
7598   numFPoints = q;
7599   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7600     PetscInt fdof;
7601 
7602     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7603     if (!dof) continue;
7604     for (f = 0; f < numFields; ++f) {
7605       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7606       foffsets[f+1] += fdof;
7607     }
7608     numFIndices += dof;
7609   }
7610   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7611 
7612   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7613   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7614   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7615   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7616   if (numFields) {
7617     const PetscInt **permsF[32] = {NULL};
7618     const PetscInt **permsC[32] = {NULL};
7619 
7620     for (f = 0; f < numFields; f++) {
7621       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7622       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7623     }
7624     for (p = 0; p < numFPoints; p++) {
7625       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7626       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7627     }
7628     for (p = 0; p < numCPoints; p++) {
7629       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7630       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7631     }
7632     for (f = 0; f < numFields; f++) {
7633       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7634       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7635     }
7636   } else {
7637     const PetscInt **permsF = NULL;
7638     const PetscInt **permsC = NULL;
7639 
7640     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7641     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7642     for (p = 0, off = 0; p < numFPoints; p++) {
7643       const PetscInt *perm = permsF ? permsF[p] : NULL;
7644 
7645       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7646       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7647     }
7648     for (p = 0, off = 0; p < numCPoints; p++) {
7649       const PetscInt *perm = permsC ? permsC[p] : NULL;
7650 
7651       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7652       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7653     }
7654     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7655     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7656   }
7657   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
7658   /* TODO: flips */
7659   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
7660   if (ierr) {
7661     PetscMPIInt    rank;
7662     PetscErrorCode ierr2;
7663 
7664     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7665     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7666     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
7667     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
7668     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
7669     CHKERRQ(ierr);
7670   }
7671   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7672   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7673   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7674   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7675   PetscFunctionReturn(0);
7676 }
7677 
7678 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
7679 {
7680   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
7681   PetscInt      *cpoints = NULL;
7682   PetscInt       foffsets[32], coffsets[32];
7683   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7684   DMPolytopeType ct;
7685   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7686   PetscErrorCode ierr;
7687 
7688   PetscFunctionBegin;
7689   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7690   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7691   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7692   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7693   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7694   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7695   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7696   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7697   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7698   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7699   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7700   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7701   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7702   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7703   /* Column indices */
7704   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7705   maxFPoints = numCPoints;
7706   /* Compress out points not in the section */
7707   /*   TODO: Squeeze out points with 0 dof as well */
7708   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7709   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7710     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7711       cpoints[q*2]   = cpoints[p];
7712       cpoints[q*2+1] = cpoints[p+1];
7713       ++q;
7714     }
7715   }
7716   numCPoints = q;
7717   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7718     PetscInt fdof;
7719 
7720     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7721     if (!dof) continue;
7722     for (f = 0; f < numFields; ++f) {
7723       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7724       coffsets[f+1] += fdof;
7725     }
7726     numCIndices += dof;
7727   }
7728   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7729   /* Row indices */
7730   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7731   {
7732     DMPlexCellRefiner cr;
7733     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7734     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7735     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7736   }
7737   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7738   for (r = 0, q = 0; r < numSubcells; ++r) {
7739     /* TODO Map from coarse to fine cells */
7740     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7741     /* Compress out points not in the section */
7742     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7743     for (p = 0; p < numFPoints*2; p += 2) {
7744       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7745         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7746         if (!dof) continue;
7747         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7748         if (s < q) continue;
7749         ftotpoints[q*2]   = fpoints[p];
7750         ftotpoints[q*2+1] = fpoints[p+1];
7751         ++q;
7752       }
7753     }
7754     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7755   }
7756   numFPoints = q;
7757   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7758     PetscInt fdof;
7759 
7760     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7761     if (!dof) continue;
7762     for (f = 0; f < numFields; ++f) {
7763       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7764       foffsets[f+1] += fdof;
7765     }
7766     numFIndices += dof;
7767   }
7768   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7769 
7770   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7771   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7772   if (numFields) {
7773     const PetscInt **permsF[32] = {NULL};
7774     const PetscInt **permsC[32] = {NULL};
7775 
7776     for (f = 0; f < numFields; f++) {
7777       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7778       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7779     }
7780     for (p = 0; p < numFPoints; p++) {
7781       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7782       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7783     }
7784     for (p = 0; p < numCPoints; p++) {
7785       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7786       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7787     }
7788     for (f = 0; f < numFields; f++) {
7789       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7790       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7791     }
7792   } else {
7793     const PetscInt **permsF = NULL;
7794     const PetscInt **permsC = NULL;
7795 
7796     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7797     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7798     for (p = 0, off = 0; p < numFPoints; p++) {
7799       const PetscInt *perm = permsF ? permsF[p] : NULL;
7800 
7801       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7802       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7803     }
7804     for (p = 0, off = 0; p < numCPoints; p++) {
7805       const PetscInt *perm = permsC ? permsC[p] : NULL;
7806 
7807       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7808       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7809     }
7810     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7811     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7812   }
7813   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7814   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7815   PetscFunctionReturn(0);
7816 }
7817 
7818 /*@C
7819   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
7820 
7821   Input Parameter:
7822 . dm   - The DMPlex object
7823 
7824   Output Parameter:
7825 . cellHeight - The height of a cell
7826 
7827   Level: developer
7828 
7829 .seealso DMPlexSetVTKCellHeight()
7830 @*/
7831 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7832 {
7833   DM_Plex *mesh = (DM_Plex*) dm->data;
7834 
7835   PetscFunctionBegin;
7836   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7837   PetscValidPointer(cellHeight, 2);
7838   *cellHeight = mesh->vtkCellHeight;
7839   PetscFunctionReturn(0);
7840 }
7841 
7842 /*@C
7843   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
7844 
7845   Input Parameters:
7846 + dm   - The DMPlex object
7847 - cellHeight - The height of a cell
7848 
7849   Level: developer
7850 
7851 .seealso DMPlexGetVTKCellHeight()
7852 @*/
7853 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7854 {
7855   DM_Plex *mesh = (DM_Plex*) dm->data;
7856 
7857   PetscFunctionBegin;
7858   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7859   mesh->vtkCellHeight = cellHeight;
7860   PetscFunctionReturn(0);
7861 }
7862 
7863 /*@
7864   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
7865 
7866   Input Parameter:
7867 . dm - The DMPlex object
7868 
7869   Output Parameters:
7870 + gcStart - The first ghost cell, or NULL
7871 - gcEnd   - The upper bound on ghost cells, or NULL
7872 
7873   Level: advanced
7874 
7875 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
7876 @*/
7877 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
7878 {
7879   DMLabel        ctLabel;
7880   PetscErrorCode ierr;
7881 
7882   PetscFunctionBegin;
7883   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7884   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
7885   ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd);CHKERRQ(ierr);
7886   PetscFunctionReturn(0);
7887 }
7888 
7889 /* We can easily have a form that takes an IS instead */
7890 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
7891 {
7892   PetscSection   section, globalSection;
7893   PetscInt      *numbers, p;
7894   PetscErrorCode ierr;
7895 
7896   PetscFunctionBegin;
7897   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
7898   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
7899   for (p = pStart; p < pEnd; ++p) {
7900     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
7901   }
7902   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
7903   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
7904   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
7905   for (p = pStart; p < pEnd; ++p) {
7906     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
7907     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
7908     else                       numbers[p-pStart] += shift;
7909   }
7910   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
7911   if (globalSize) {
7912     PetscLayout layout;
7913     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
7914     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
7915     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
7916   }
7917   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
7918   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
7919   PetscFunctionReturn(0);
7920 }
7921 
7922 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
7923 {
7924   PetscInt       cellHeight, cStart, cEnd;
7925   PetscErrorCode ierr;
7926 
7927   PetscFunctionBegin;
7928   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
7929   if (includeHybrid) {ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7930   else               {ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7931   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
7932   PetscFunctionReturn(0);
7933 }
7934 
7935 /*@
7936   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
7937 
7938   Input Parameter:
7939 . dm   - The DMPlex object
7940 
7941   Output Parameter:
7942 . globalCellNumbers - Global cell numbers for all cells on this process
7943 
7944   Level: developer
7945 
7946 .seealso DMPlexGetVertexNumbering()
7947 @*/
7948 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
7949 {
7950   DM_Plex       *mesh = (DM_Plex*) dm->data;
7951   PetscErrorCode ierr;
7952 
7953   PetscFunctionBegin;
7954   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7955   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
7956   *globalCellNumbers = mesh->globalCellNumbers;
7957   PetscFunctionReturn(0);
7958 }
7959 
7960 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
7961 {
7962   PetscInt       vStart, vEnd;
7963   PetscErrorCode ierr;
7964 
7965   PetscFunctionBegin;
7966   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7967   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7968   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
7969   PetscFunctionReturn(0);
7970 }
7971 
7972 /*@
7973   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
7974 
7975   Input Parameter:
7976 . dm   - The DMPlex object
7977 
7978   Output Parameter:
7979 . globalVertexNumbers - Global vertex numbers for all vertices on this process
7980 
7981   Level: developer
7982 
7983 .seealso DMPlexGetCellNumbering()
7984 @*/
7985 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
7986 {
7987   DM_Plex       *mesh = (DM_Plex*) dm->data;
7988   PetscErrorCode ierr;
7989 
7990   PetscFunctionBegin;
7991   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7992   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
7993   *globalVertexNumbers = mesh->globalVertexNumbers;
7994   PetscFunctionReturn(0);
7995 }
7996 
7997 /*@
7998   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
7999 
8000   Input Parameter:
8001 . dm   - The DMPlex object
8002 
8003   Output Parameter:
8004 . globalPointNumbers - Global numbers for all points on this process
8005 
8006   Level: developer
8007 
8008 .seealso DMPlexGetCellNumbering()
8009 @*/
8010 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
8011 {
8012   IS             nums[4];
8013   PetscInt       depths[4], gdepths[4], starts[4];
8014   PetscInt       depth, d, shift = 0;
8015   PetscErrorCode ierr;
8016 
8017   PetscFunctionBegin;
8018   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8019   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8020   /* For unstratified meshes use dim instead of depth */
8021   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
8022   for (d = 0; d <= depth; ++d) {
8023     PetscInt end;
8024 
8025     depths[d] = depth-d;
8026     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
8027     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
8028   }
8029   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
8030   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8031   for (d = 0; d <= depth; ++d) {
8032     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
8033   }
8034   for (d = 0; d <= depth; ++d) {
8035     PetscInt pStart, pEnd, gsize;
8036 
8037     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
8038     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
8039     shift += gsize;
8040   }
8041   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
8042   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
8043   PetscFunctionReturn(0);
8044 }
8045 
8046 /*@
8047   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
8048 
8049   Input Parameter:
8050 . dm - The DMPlex object
8051 
8052   Output Parameter:
8053 . ranks - The rank field
8054 
8055   Options Database Keys:
8056 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
8057 
8058   Level: intermediate
8059 
8060 .seealso: DMView()
8061 @*/
8062 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
8063 {
8064   DM             rdm;
8065   PetscFE        fe;
8066   PetscScalar   *r;
8067   PetscMPIInt    rank;
8068   DMPolytopeType ct;
8069   PetscInt       dim, cStart, cEnd, c;
8070   PetscBool      simplex;
8071   PetscErrorCode ierr;
8072 
8073   PetscFunctionBeginUser;
8074   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8075   PetscValidPointer(ranks, 2);
8076   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
8077   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8078   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8079   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8080   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
8081   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
8082   ierr = PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
8083   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
8084   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8085   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8086   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8087   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
8088   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
8089   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
8090   for (c = cStart; c < cEnd; ++c) {
8091     PetscScalar *lr;
8092 
8093     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
8094     if (lr) *lr = rank;
8095   }
8096   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
8097   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8098   PetscFunctionReturn(0);
8099 }
8100 
8101 /*@
8102   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
8103 
8104   Input Parameters:
8105 + dm    - The DMPlex
8106 - label - The DMLabel
8107 
8108   Output Parameter:
8109 . val - The label value field
8110 
8111   Options Database Keys:
8112 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
8113 
8114   Level: intermediate
8115 
8116 .seealso: DMView()
8117 @*/
8118 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
8119 {
8120   DM             rdm;
8121   PetscFE        fe;
8122   PetscScalar   *v;
8123   PetscInt       dim, cStart, cEnd, c;
8124   PetscErrorCode ierr;
8125 
8126   PetscFunctionBeginUser;
8127   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8128   PetscValidPointer(label, 2);
8129   PetscValidPointer(val, 3);
8130   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8131   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8132   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
8133   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
8134   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8135   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8136   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8137   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8138   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
8139   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
8140   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
8141   for (c = cStart; c < cEnd; ++c) {
8142     PetscScalar *lv;
8143     PetscInt     cval;
8144 
8145     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
8146     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
8147     *lv = cval;
8148   }
8149   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
8150   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8151   PetscFunctionReturn(0);
8152 }
8153 
8154 /*@
8155   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
8156 
8157   Input Parameter:
8158 . dm - The DMPlex object
8159 
8160   Notes:
8161   This is a useful diagnostic when creating meshes programmatically.
8162 
8163   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8164 
8165   Level: developer
8166 
8167 .seealso: DMCreate(), DMSetFromOptions()
8168 @*/
8169 PetscErrorCode DMPlexCheckSymmetry(DM dm)
8170 {
8171   PetscSection    coneSection, supportSection;
8172   const PetscInt *cone, *support;
8173   PetscInt        coneSize, c, supportSize, s;
8174   PetscInt        pStart, pEnd, p, pp, csize, ssize;
8175   PetscBool       storagecheck = PETSC_TRUE;
8176   PetscErrorCode  ierr;
8177 
8178   PetscFunctionBegin;
8179   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8180   ierr = DMViewFromOptions(dm, NULL, "-sym_dm_view");CHKERRQ(ierr);
8181   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
8182   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
8183   /* Check that point p is found in the support of its cone points, and vice versa */
8184   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
8185   for (p = pStart; p < pEnd; ++p) {
8186     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
8187     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
8188     for (c = 0; c < coneSize; ++c) {
8189       PetscBool dup = PETSC_FALSE;
8190       PetscInt  d;
8191       for (d = c-1; d >= 0; --d) {
8192         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
8193       }
8194       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
8195       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
8196       for (s = 0; s < supportSize; ++s) {
8197         if (support[s] == p) break;
8198       }
8199       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
8200         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
8201         for (s = 0; s < coneSize; ++s) {
8202           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
8203         }
8204         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8205         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
8206         for (s = 0; s < supportSize; ++s) {
8207           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
8208         }
8209         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8210         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
8211         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
8212       }
8213     }
8214     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
8215     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
8216     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
8217     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
8218     for (s = 0; s < supportSize; ++s) {
8219       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
8220       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
8221       for (c = 0; c < coneSize; ++c) {
8222         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
8223         if (cone[c] != pp) { c = 0; break; }
8224         if (cone[c] == p) break;
8225       }
8226       if (c >= coneSize) {
8227         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
8228         for (c = 0; c < supportSize; ++c) {
8229           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
8230         }
8231         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8232         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
8233         for (c = 0; c < coneSize; ++c) {
8234           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
8235         }
8236         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8237         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
8238       }
8239     }
8240   }
8241   if (storagecheck) {
8242     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
8243     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
8244     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
8245   }
8246   PetscFunctionReturn(0);
8247 }
8248 
8249 /*
8250   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
8251 */
8252 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
8253 {
8254   DMPolytopeType  cct;
8255   PetscInt        ptpoints[4];
8256   const PetscInt *cone, *ccone, *ptcone;
8257   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
8258   PetscErrorCode  ierr;
8259 
8260   PetscFunctionBegin;
8261   *unsplit = 0;
8262   switch (ct) {
8263     case DM_POLYTOPE_SEG_PRISM_TENSOR:
8264       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8265       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8266       for (cp = 0; cp < coneSize; ++cp) {
8267         ierr = DMPlexGetCellType(dm, cone[cp], &cct);CHKERRQ(ierr);
8268         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
8269       }
8270       break;
8271     case DM_POLYTOPE_TRI_PRISM_TENSOR:
8272     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8273       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8274       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8275       for (cp = 0; cp < coneSize; ++cp) {
8276         ierr = DMPlexGetCone(dm, cone[cp], &ccone);CHKERRQ(ierr);
8277         ierr = DMPlexGetConeSize(dm, cone[cp], &cconeSize);CHKERRQ(ierr);
8278         for (ccp = 0; ccp < cconeSize; ++ccp) {
8279           ierr = DMPlexGetCellType(dm, ccone[ccp], &cct);CHKERRQ(ierr);
8280           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
8281             PetscInt p;
8282             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
8283             if (p == npt) ptpoints[npt++] = ccone[ccp];
8284           }
8285         }
8286       }
8287       break;
8288     default: break;
8289   }
8290   for (pt = 0; pt < npt; ++pt) {
8291     ierr = DMPlexGetCone(dm, ptpoints[pt], &ptcone);CHKERRQ(ierr);
8292     if (ptcone[0] == ptcone[1]) ++(*unsplit);
8293   }
8294   PetscFunctionReturn(0);
8295 }
8296 
8297 /*@
8298   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
8299 
8300   Input Parameters:
8301 + dm - The DMPlex object
8302 - cellHeight - Normally 0
8303 
8304   Notes:
8305   This is a useful diagnostic when creating meshes programmatically.
8306   Currently applicable only to homogeneous simplex or tensor meshes.
8307 
8308   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8309 
8310   Level: developer
8311 
8312 .seealso: DMCreate(), DMSetFromOptions()
8313 @*/
8314 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
8315 {
8316   DMPlexInterpolatedFlag interp;
8317   DMPolytopeType         ct;
8318   PetscInt               vStart, vEnd, cStart, cEnd, c;
8319   PetscErrorCode         ierr;
8320 
8321   PetscFunctionBegin;
8322   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8323   ierr = DMPlexIsInterpolated(dm, &interp);CHKERRQ(ierr);
8324   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8325   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8326   for (c = cStart; c < cEnd; ++c) {
8327     PetscInt *closure = NULL;
8328     PetscInt  coneSize, closureSize, cl, Nv = 0;
8329 
8330     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8331     if ((PetscInt) ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has no cell type", c);
8332     if (ct == DM_POLYTOPE_UNKNOWN) continue;
8333     if (interp == DMPLEX_INTERPOLATED_FULL) {
8334       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8335       if (coneSize != DMPolytopeTypeGetConeSize(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has cone size %D != %D", c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
8336     }
8337     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8338     for (cl = 0; cl < closureSize*2; cl += 2) {
8339       const PetscInt p = closure[cl];
8340       if ((p >= vStart) && (p < vEnd)) ++Nv;
8341     }
8342     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8343     /* Special Case: Tensor faces with identified vertices */
8344     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
8345       PetscInt unsplit;
8346 
8347       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8348       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
8349     }
8350     if (Nv != DMPolytopeTypeGetNumVertices(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D vertices != %D", c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
8351   }
8352   PetscFunctionReturn(0);
8353 }
8354 
8355 /*@
8356   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
8357 
8358   Not Collective
8359 
8360   Input Parameters:
8361 + dm - The DMPlex object
8362 - cellHeight - Normally 0
8363 
8364   Notes:
8365   This is a useful diagnostic when creating meshes programmatically.
8366   This routine is only relevant for meshes that are fully interpolated across all ranks.
8367   It will error out if a partially interpolated mesh is given on some rank.
8368   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
8369 
8370   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8371 
8372   Level: developer
8373 
8374 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
8375 @*/
8376 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
8377 {
8378   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
8379   PetscErrorCode ierr;
8380   DMPlexInterpolatedFlag interpEnum;
8381 
8382   PetscFunctionBegin;
8383   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8384   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
8385   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
8386   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
8387     PetscMPIInt rank;
8388     MPI_Comm    comm;
8389 
8390     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8391     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8392     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
8393   }
8394 
8395   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8396   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8397   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8398   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
8399     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
8400     for (c = cStart; c < cEnd; ++c) {
8401       const PetscInt      *cone, *ornt, *faceSizes, *faces;
8402       const DMPolytopeType *faceTypes;
8403       DMPolytopeType        ct;
8404       PetscInt              numFaces, coneSize, f;
8405       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
8406 
8407       ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8408       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8409       if (unsplit) continue;
8410       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8411       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8412       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
8413       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8414       for (cl = 0; cl < closureSize*2; cl += 2) {
8415         const PetscInt p = closure[cl];
8416         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
8417       }
8418       ierr = DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8419       if (coneSize != numFaces) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D faces but should have %D", c, DMPolytopeTypes[ct], coneSize, numFaces);
8420       for (f = 0; f < numFaces; ++f) {
8421         DMPolytopeType fct;
8422         PetscInt       *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
8423 
8424         ierr = DMPlexGetCellType(dm, cone[f], &fct);CHKERRQ(ierr);
8425         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8426         for (cl = 0; cl < fclosureSize*2; cl += 2) {
8427           const PetscInt p = fclosure[cl];
8428           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
8429         }
8430         if (fnumCorners != faceSizes[f]) SETERRQ7(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %D) of cell %D of type %s has %D vertices but should have %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
8431         for (v = 0; v < fnumCorners; ++v) {
8432           if (fclosure[v] != faces[fOff+v]) SETERRQ8(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %d) of cell %D of type %s vertex %D, %D != %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff+v]);
8433         }
8434         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8435         fOff += faceSizes[f];
8436       }
8437       ierr = DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8438       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8439     }
8440   }
8441   PetscFunctionReturn(0);
8442 }
8443 
8444 /*@
8445   DMPlexCheckGeometry - Check the geometry of mesh cells
8446 
8447   Input Parameter:
8448 . dm - The DMPlex object
8449 
8450   Notes:
8451   This is a useful diagnostic when creating meshes programmatically.
8452 
8453   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8454 
8455   Level: developer
8456 
8457 .seealso: DMCreate(), DMSetFromOptions()
8458 @*/
8459 PetscErrorCode DMPlexCheckGeometry(DM dm)
8460 {
8461   Vec            coordinates;
8462   PetscReal      detJ, J[9], refVol = 1.0;
8463   PetscReal      vol;
8464   PetscBool      periodic;
8465   PetscInt       dim, depth, dE, d, cStart, cEnd, c;
8466   PetscErrorCode ierr;
8467 
8468   PetscFunctionBegin;
8469   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8470   ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
8471   if (dim != dE) PetscFunctionReturn(0);
8472   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8473   ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
8474   for (d = 0; d < dim; ++d) refVol *= 2.0;
8475   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8476   /* Make sure local coordinates are created, because that step is collective */
8477   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
8478   for (c = cStart; c < cEnd; ++c) {
8479     DMPolytopeType ct;
8480     PetscInt       unsplit;
8481     PetscBool      ignoreZeroVol = PETSC_FALSE;
8482 
8483     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8484     switch (ct) {
8485       case DM_POLYTOPE_SEG_PRISM_TENSOR:
8486       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8487       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8488         ignoreZeroVol = PETSC_TRUE; break;
8489       default: break;
8490     }
8491     switch (ct) {
8492       case DM_POLYTOPE_TRI_PRISM:
8493       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8494       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8495       case DM_POLYTOPE_PYRAMID:
8496         continue;
8497       default: break;
8498     }
8499     ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8500     if (unsplit) continue;
8501     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
8502     if (detJ < -PETSC_SMALL || (detJ <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double) detJ);
8503     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
8504     if (depth > 1 && !periodic) {
8505       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
8506       if (vol < -PETSC_SMALL || (vol <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double) vol);
8507       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
8508     }
8509   }
8510   PetscFunctionReturn(0);
8511 }
8512 
8513 /*@
8514   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
8515 
8516   Input Parameters:
8517 . dm - The DMPlex object
8518 
8519   Notes:
8520   This is mainly intended for debugging/testing purposes.
8521   It currently checks only meshes with no partition overlapping.
8522 
8523   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8524 
8525   Level: developer
8526 
8527 .seealso: DMGetPointSF(), DMSetFromOptions()
8528 @*/
8529 PetscErrorCode DMPlexCheckPointSF(DM dm)
8530 {
8531   PetscSF         pointSF;
8532   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
8533   const PetscInt *locals, *rootdegree;
8534   PetscBool       distributed;
8535   PetscErrorCode  ierr;
8536 
8537   PetscFunctionBegin;
8538   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8539   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
8540   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
8541   if (!distributed) PetscFunctionReturn(0);
8542   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
8543   if (overlap) {
8544     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");CHKERRQ(ierr);
8545     PetscFunctionReturn(0);
8546   }
8547   if (!pointSF) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
8548   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
8549   if (nroots < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
8550   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
8551   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
8552 
8553   /* 1) check there are no faces in 2D, cells in 3D, in interface */
8554   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8555   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8556   for (l = 0; l < nleaves; ++l) {
8557     const PetscInt point = locals[l];
8558 
8559     if (point >= cStart && point < cEnd) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
8560   }
8561 
8562   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
8563   for (l = 0; l < nleaves; ++l) {
8564     const PetscInt  point = locals[l];
8565     const PetscInt *cone;
8566     PetscInt        coneSize, c, idx;
8567 
8568     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
8569     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
8570     for (c = 0; c < coneSize; ++c) {
8571       if (!rootdegree[cone[c]]) {
8572         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
8573         if (idx < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
8574       }
8575     }
8576   }
8577   PetscFunctionReturn(0);
8578 }
8579 
8580 typedef struct cell_stats
8581 {
8582   PetscReal min, max, sum, squaresum;
8583   PetscInt  count;
8584 } cell_stats_t;
8585 
8586 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
8587 {
8588   PetscInt i, N = *len;
8589 
8590   for (i = 0; i < N; i++) {
8591     cell_stats_t *A = (cell_stats_t *) a;
8592     cell_stats_t *B = (cell_stats_t *) b;
8593 
8594     B->min = PetscMin(A->min,B->min);
8595     B->max = PetscMax(A->max,B->max);
8596     B->sum += A->sum;
8597     B->squaresum += A->squaresum;
8598     B->count += A->count;
8599   }
8600 }
8601 
8602 /*@
8603   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
8604 
8605   Collective on dm
8606 
8607   Input Parameters:
8608 + dm        - The DMPlex object
8609 . output    - If true, statistics will be displayed on stdout
8610 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
8611 
8612   Notes:
8613   This is mainly intended for debugging/testing purposes.
8614 
8615   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8616 
8617   Level: developer
8618 
8619 .seealso: DMSetFromOptions(), DMPlexComputeOrthogonalQuality()
8620 @*/
8621 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
8622 {
8623   DM             dmCoarse;
8624   cell_stats_t   stats, globalStats;
8625   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
8626   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
8627   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
8628   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
8629   PetscMPIInt    rank,size;
8630   PetscErrorCode ierr;
8631 
8632   PetscFunctionBegin;
8633   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8634   stats.min   = PETSC_MAX_REAL;
8635   stats.max   = PETSC_MIN_REAL;
8636   stats.sum   = stats.squaresum = 0.;
8637   stats.count = 0;
8638 
8639   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
8640   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8641   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
8642   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
8643   ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
8644   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
8645   for (c = cStart; c < cEnd; c++) {
8646     PetscInt  i;
8647     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
8648 
8649     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
8650     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
8651     for (i = 0; i < PetscSqr(cdim); ++i) {
8652       frobJ    += J[i] * J[i];
8653       frobInvJ += invJ[i] * invJ[i];
8654     }
8655     cond2 = frobJ * frobInvJ;
8656     cond  = PetscSqrtReal(cond2);
8657 
8658     stats.min        = PetscMin(stats.min,cond);
8659     stats.max        = PetscMax(stats.max,cond);
8660     stats.sum       += cond;
8661     stats.squaresum += cond2;
8662     stats.count++;
8663     if (output && cond > limit) {
8664       PetscSection coordSection;
8665       Vec          coordsLocal;
8666       PetscScalar *coords = NULL;
8667       PetscInt     Nv, d, clSize, cl, *closure = NULL;
8668 
8669       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
8670       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
8671       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8672       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
8673       for (i = 0; i < Nv/cdim; ++i) {
8674         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
8675         for (d = 0; d < cdim; ++d) {
8676           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
8677           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
8678         }
8679         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
8680       }
8681       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8682       for (cl = 0; cl < clSize*2; cl += 2) {
8683         const PetscInt edge = closure[cl];
8684 
8685         if ((edge >= eStart) && (edge < eEnd)) {
8686           PetscReal len;
8687 
8688           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
8689           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
8690         }
8691       }
8692       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8693       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8694     }
8695   }
8696   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
8697 
8698   if (size > 1) {
8699     PetscMPIInt   blockLengths[2] = {4,1};
8700     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
8701     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
8702     MPI_Op        statReduce;
8703 
8704     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRMPI(ierr);
8705     ierr = MPI_Type_commit(&statType);CHKERRMPI(ierr);
8706     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRMPI(ierr);
8707     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRMPI(ierr);
8708     ierr = MPI_Op_free(&statReduce);CHKERRMPI(ierr);
8709     ierr = MPI_Type_free(&statType);CHKERRMPI(ierr);
8710   } else {
8711     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
8712   }
8713   if (!rank) {
8714     count = globalStats.count;
8715     min   = globalStats.min;
8716     max   = globalStats.max;
8717     mean  = globalStats.sum / globalStats.count;
8718     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
8719   }
8720 
8721   if (output) {
8722     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
8723   }
8724   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
8725 
8726   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
8727   if (dmCoarse) {
8728     PetscBool isplex;
8729 
8730     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
8731     if (isplex) {
8732       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
8733     }
8734   }
8735   PetscFunctionReturn(0);
8736 }
8737 
8738 /*@
8739   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
8740   orthogonal quality below given tolerance.
8741 
8742   Collective on dm
8743 
8744   Input Parameters:
8745 + dm   - The DMPlex object
8746 . fv   - Optional PetscFV object for pre-computed cell/face centroid information
8747 - atol - [0, 1] Absolute tolerance for tagging cells.
8748 
8749   Output Parameters:
8750 + OrthQual      - Vec containing orthogonal quality per cell
8751 - OrthQualLabel - DMLabel tagging cells below atol with DM_ADAPT_REFINE
8752 
8753   Options Database Keys:
8754 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only PETSCVIEWERASCII is
8755 supported.
8756 - -dm_plex_orthogonal_quality_vec_view - view OrthQual vector.
8757 
8758   Notes:
8759   Orthogonal quality is given by the following formula:
8760 
8761   \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]
8762 
8763   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
8764   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
8765   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
8766   calculating the cosine of the angle between these vectors.
8767 
8768   Orthogonal quality ranges from 1 (best) to 0 (worst).
8769 
8770   This routine is mainly useful for FVM, however is not restricted to only FVM. The PetscFV object is optionally used to check for
8771   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
8772 
8773   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
8774 
8775   Level: intermediate
8776 
8777 .seealso: DMPlexCheckCellShape(), DMCreateLabel()
8778 @*/
8779 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
8780 {
8781   PetscInt                nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
8782   PetscInt                *idx;
8783   PetscScalar             *oqVals;
8784   const PetscScalar       *cellGeomArr, *faceGeomArr;
8785   PetscReal               *ci, *fi, *Ai;
8786   MPI_Comm                comm;
8787   Vec                     cellgeom, facegeom;
8788   DM                      dmFace, dmCell;
8789   IS                      glob;
8790   ISLocalToGlobalMapping  ltog;
8791   PetscViewer             vwr;
8792   PetscErrorCode          ierr;
8793 
8794   PetscFunctionBegin;
8795   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8796   if (fv) {PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);}
8797   PetscValidPointer(OrthQual, 4);
8798   if (PetscUnlikelyDebug(atol < 0.0 || atol > 1.0)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Absolute tolerance %g not in [0,1]",(double)atol);
8799   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8800   ierr = DMGetDimension(dm, &nc);CHKERRQ(ierr);
8801   if (nc < 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %D)", nc);
8802   {
8803     DMPlexInterpolatedFlag interpFlag;
8804 
8805     ierr = DMPlexIsInterpolated(dm, &interpFlag);CHKERRQ(ierr);
8806     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
8807       PetscMPIInt rank;
8808 
8809       ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8810       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
8811     }
8812   }
8813   if (OrthQualLabel) {
8814     PetscValidPointer(OrthQualLabel, 5);
8815     ierr = DMCreateLabel(dm, "Orthogonal_Quality");CHKERRQ(ierr);
8816     ierr = DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel);CHKERRQ(ierr);
8817   } else {*OrthQualLabel = NULL;}
8818   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8819   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8820   ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_TRUE, &glob);CHKERRQ(ierr);
8821   ierr = ISLocalToGlobalMappingCreateIS(glob, &ltog);CHKERRQ(ierr);
8822   ierr = ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
8823   ierr = VecCreate(comm, OrthQual);CHKERRQ(ierr);
8824   ierr = VecSetType(*OrthQual, VECSTANDARD);CHKERRQ(ierr);
8825   ierr = VecSetSizes(*OrthQual, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
8826   ierr = VecSetLocalToGlobalMapping(*OrthQual, ltog);CHKERRQ(ierr);
8827   ierr = VecSetUp(*OrthQual);CHKERRQ(ierr);
8828   ierr = ISDestroy(&glob);CHKERRQ(ierr);
8829   ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
8830   ierr = DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL);CHKERRQ(ierr);
8831   ierr = VecGetArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8832   ierr = VecGetArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8833   ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
8834   ierr = VecGetDM(facegeom, &dmFace);CHKERRQ(ierr);
8835   ierr = PetscMalloc5(cEnd-cStart, &idx, cEnd-cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai);CHKERRQ(ierr);
8836   for (cell = cStart; cell < cEnd; cellIter++,cell++) {
8837     PetscInt           cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
8838     PetscInt           cellarr[2], *adj = NULL;
8839     PetscScalar        *cArr, *fArr;
8840     PetscReal          minvalc = 1.0, minvalf = 1.0;
8841     PetscFVCellGeom    *cg;
8842 
8843     idx[cellIter] = cell-cStart;
8844     cellarr[0] = cell;
8845     /* Make indexing into cellGeom easier */
8846     ierr = DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg);CHKERRQ(ierr);
8847     ierr = DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
8848     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
8849     ierr = PetscCalloc2(adjSize, &cArr, adjSize, &fArr);CHKERRQ(ierr);
8850     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++,cellneigh++) {
8851       PetscInt         i;
8852       const PetscInt   neigh = adj[cellneigh];
8853       PetscReal        normci = 0, normfi = 0, normai = 0;
8854       PetscFVCellGeom  *cgneigh;
8855       PetscFVFaceGeom  *fg;
8856 
8857       /* Don't count ourselves in the neighbor list */
8858       if (neigh == cell) continue;
8859       ierr = DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh);CHKERRQ(ierr);
8860       cellarr[1] = neigh;
8861       {
8862         PetscInt       numcovpts;
8863         const PetscInt *covpts;
8864 
8865         ierr = DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8866         ierr = DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg);CHKERRQ(ierr);
8867         ierr = DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8868       }
8869 
8870       /* Compute c_i, f_i and their norms */
8871       for (i = 0; i < nc; i++) {
8872         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
8873         fi[i] = fg->centroid[i] - cg->centroid[i];
8874         Ai[i] = fg->normal[i];
8875         normci += PetscPowReal(ci[i], 2);
8876         normfi += PetscPowReal(fi[i], 2);
8877         normai += PetscPowReal(Ai[i], 2);
8878       }
8879       normci = PetscSqrtReal(normci);
8880       normfi = PetscSqrtReal(normfi);
8881       normai = PetscSqrtReal(normai);
8882 
8883       /* Normalize and compute for each face-cell-normal pair */
8884       for (i = 0; i < nc; i++) {
8885         ci[i] = ci[i]/normci;
8886         fi[i] = fi[i]/normfi;
8887         Ai[i] = Ai[i]/normai;
8888         /* PetscAbs because I don't know if normals are guaranteed to point out */
8889         cArr[cellneighiter] += PetscAbs(Ai[i]*ci[i]);
8890         fArr[cellneighiter] += PetscAbs(Ai[i]*fi[i]);
8891       }
8892       if (PetscRealPart(cArr[cellneighiter]) < minvalc) {
8893         minvalc = PetscRealPart(cArr[cellneighiter]);
8894       }
8895       if (PetscRealPart(fArr[cellneighiter]) < minvalf) {
8896         minvalf = PetscRealPart(fArr[cellneighiter]);
8897       }
8898     }
8899     ierr = PetscFree(adj);CHKERRQ(ierr);
8900     ierr = PetscFree2(cArr, fArr);CHKERRQ(ierr);
8901     /* Defer to cell if they're equal */
8902     oqVals[cellIter] = PetscMin(minvalf, minvalc);
8903     if (OrthQualLabel) {
8904       if (PetscRealPart(oqVals[cellIter]) <= atol) {ierr = DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE);CHKERRQ(ierr);}
8905     }
8906   }
8907   ierr = VecSetValuesLocal(*OrthQual, cEnd-cStart, idx, oqVals, INSERT_VALUES);CHKERRQ(ierr);
8908   ierr = VecAssemblyBegin(*OrthQual);CHKERRQ(ierr);
8909   ierr = VecAssemblyEnd(*OrthQual);CHKERRQ(ierr);
8910   ierr = VecRestoreArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8911   ierr = VecRestoreArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8912   ierr = PetscOptionsGetViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL);CHKERRQ(ierr);
8913   if (OrthQualLabel) {
8914     if (vwr) {ierr = DMLabelView(*OrthQualLabel, vwr);CHKERRQ(ierr);}
8915   }
8916   ierr = PetscFree5(idx, oqVals, ci, fi, Ai);CHKERRQ(ierr);
8917   ierr = PetscViewerDestroy(&vwr);CHKERRQ(ierr);
8918   ierr = VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view");CHKERRQ(ierr);
8919   PetscFunctionReturn(0);
8920 }
8921 
8922 /* this is here insead of DMGetOutputDM because output DM still has constraints in the local indices that affect
8923  * interpolator construction */
8924 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
8925 {
8926   PetscSection   section, newSection, gsection;
8927   PetscSF        sf;
8928   PetscBool      hasConstraints, ghasConstraints;
8929   PetscErrorCode ierr;
8930 
8931   PetscFunctionBegin;
8932   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
8933   PetscValidPointer(odm,2);
8934   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
8935   ierr = PetscSectionHasConstraints(section, &hasConstraints);CHKERRQ(ierr);
8936   ierr = MPI_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8937   if (!ghasConstraints) {
8938     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
8939     *odm = dm;
8940     PetscFunctionReturn(0);
8941   }
8942   ierr = DMClone(dm, odm);CHKERRQ(ierr);
8943   ierr = DMCopyFields(dm, *odm);CHKERRQ(ierr);
8944   ierr = DMGetLocalSection(*odm, &newSection);CHKERRQ(ierr);
8945   ierr = DMGetPointSF(*odm, &sf);CHKERRQ(ierr);
8946   ierr = PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_FALSE, &gsection);CHKERRQ(ierr);
8947   ierr = DMSetGlobalSection(*odm, gsection);CHKERRQ(ierr);
8948   ierr = PetscSectionDestroy(&gsection);CHKERRQ(ierr);
8949   PetscFunctionReturn(0);
8950 }
8951 
8952 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
8953 {
8954   DM             dmco, dmfo;
8955   Mat            interpo;
8956   Vec            rscale;
8957   Vec            cglobalo, clocal;
8958   Vec            fglobal, fglobalo, flocal;
8959   PetscBool      regular;
8960   PetscErrorCode ierr;
8961 
8962   PetscFunctionBegin;
8963   ierr = DMGetFullDM(dmc, &dmco);CHKERRQ(ierr);
8964   ierr = DMGetFullDM(dmf, &dmfo);CHKERRQ(ierr);
8965   ierr = DMSetCoarseDM(dmfo, dmco);CHKERRQ(ierr);
8966   ierr = DMPlexGetRegularRefinement(dmf, &regular);CHKERRQ(ierr);
8967   ierr = DMPlexSetRegularRefinement(dmfo, regular);CHKERRQ(ierr);
8968   ierr = DMCreateInterpolation(dmco, dmfo, &interpo, &rscale);CHKERRQ(ierr);
8969   ierr = DMCreateGlobalVector(dmco, &cglobalo);CHKERRQ(ierr);
8970   ierr = DMCreateLocalVector(dmc, &clocal);CHKERRQ(ierr);
8971   ierr = VecSet(cglobalo, 0.);CHKERRQ(ierr);
8972   ierr = VecSet(clocal, 0.);CHKERRQ(ierr);
8973   ierr = DMCreateGlobalVector(dmf, &fglobal);CHKERRQ(ierr);
8974   ierr = DMCreateGlobalVector(dmfo, &fglobalo);CHKERRQ(ierr);
8975   ierr = DMCreateLocalVector(dmf, &flocal);CHKERRQ(ierr);
8976   ierr = VecSet(fglobal, 0.);CHKERRQ(ierr);
8977   ierr = VecSet(fglobalo, 0.);CHKERRQ(ierr);
8978   ierr = VecSet(flocal, 0.);CHKERRQ(ierr);
8979   ierr = DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL);CHKERRQ(ierr);
8980   ierr = DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8981   ierr = DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8982   ierr = MatMult(interpo, cglobalo, fglobalo);CHKERRQ(ierr);
8983   ierr = DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8984   ierr = DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8985   ierr = DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8986   ierr = DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8987   *shift = fglobal;
8988   ierr = VecDestroy(&flocal);CHKERRQ(ierr);
8989   ierr = VecDestroy(&fglobalo);CHKERRQ(ierr);
8990   ierr = VecDestroy(&clocal);CHKERRQ(ierr);
8991   ierr = VecDestroy(&cglobalo);CHKERRQ(ierr);
8992   ierr = VecDestroy(&rscale);CHKERRQ(ierr);
8993   ierr = MatDestroy(&interpo);CHKERRQ(ierr);
8994   ierr = DMDestroy(&dmfo);CHKERRQ(ierr);
8995   ierr = DMDestroy(&dmco);CHKERRQ(ierr);
8996   PetscFunctionReturn(0);
8997 }
8998 
8999 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
9000 {
9001   PetscObject    shifto;
9002   Vec            shift;
9003 
9004   PetscErrorCode ierr;
9005 
9006   PetscFunctionBegin;
9007   if (!interp) {
9008     Vec rscale;
9009 
9010     ierr = DMCreateInterpolation(coarse, fine, &interp, &rscale);CHKERRQ(ierr);
9011     ierr = VecDestroy(&rscale);CHKERRQ(ierr);
9012   } else {
9013     ierr = PetscObjectReference((PetscObject)interp);CHKERRQ(ierr);
9014   }
9015   ierr = PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto);CHKERRQ(ierr);
9016   if (!shifto) {
9017     ierr = DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift);CHKERRQ(ierr);
9018     ierr = PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject) shift);CHKERRQ(ierr);
9019     shifto = (PetscObject) shift;
9020     ierr = VecDestroy(&shift);CHKERRQ(ierr);
9021   }
9022   shift = (Vec) shifto;
9023   ierr = MatInterpolate(interp, coarseSol, fineSol);CHKERRQ(ierr);
9024   ierr = VecAXPY(fineSol, 1.0, shift);CHKERRQ(ierr);
9025   ierr = MatDestroy(&interp);CHKERRQ(ierr);
9026   PetscFunctionReturn(0);
9027 }
9028 
9029 /* Pointwise interpolation
9030      Just code FEM for now
9031      u^f = I u^c
9032      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
9033      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
9034      I_{ij} = psi^f_i phi^c_j
9035 */
9036 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
9037 {
9038   PetscSection   gsc, gsf;
9039   PetscInt       m, n;
9040   void          *ctx;
9041   DM             cdm;
9042   PetscBool      regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
9043   PetscErrorCode ierr;
9044 
9045   PetscFunctionBegin;
9046   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9047   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9048   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9049   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9050 
9051   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
9052   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
9053   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9054   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
9055   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9056 
9057   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9058   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9059   if (!isRefined || (regular && cdm == dmCoarse)) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx);CHKERRQ(ierr);}
9060   else                                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
9061   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
9062   if (scaling) {
9063     /* Use naive scaling */
9064     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
9065   }
9066   PetscFunctionReturn(0);
9067 }
9068 
9069 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
9070 {
9071   PetscErrorCode ierr;
9072   VecScatter     ctx;
9073 
9074   PetscFunctionBegin;
9075   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
9076   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
9077   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
9078   PetscFunctionReturn(0);
9079 }
9080 
9081 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux,
9082                                 const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
9083                                 const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
9084                                 PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
9085 {
9086   g0[0] = 1.0;
9087 }
9088 
9089 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
9090 {
9091   PetscSection   gsc, gsf;
9092   PetscInt       m, n;
9093   void          *ctx;
9094   DM             cdm;
9095   PetscBool      regular;
9096   PetscErrorCode ierr;
9097 
9098   PetscFunctionBegin;
9099   if (dmFine == dmCoarse) {
9100     DM       dmc;
9101     PetscDS  ds;
9102     Vec      u;
9103     IS       cellIS;
9104     PetscFormKey key;
9105     PetscInt depth;
9106 
9107     ierr = DMClone(dmFine, &dmc);CHKERRQ(ierr);
9108     ierr = DMCopyDisc(dmFine, dmc);CHKERRQ(ierr);
9109     ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
9110     ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
9111     ierr = DMCreateMatrix(dmc, mass);CHKERRQ(ierr);
9112     ierr = DMGetGlobalVector(dmc, &u);CHKERRQ(ierr);
9113     ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
9114     ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
9115     ierr = MatZeroEntries(*mass);CHKERRQ(ierr);
9116     key.label = NULL;
9117     key.value = 0;
9118     key.field = 0;
9119     key.part  = 0;
9120     ierr = DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL);CHKERRQ(ierr);
9121     ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9122     ierr = DMRestoreGlobalVector(dmc, &u);CHKERRQ(ierr);
9123     ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9124   } else {
9125     ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9126     ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9127     ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9128     ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9129 
9130     ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
9131     ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9132     ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
9133     ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9134 
9135     ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9136     ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9137     if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9138     else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9139   }
9140   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
9141   PetscFunctionReturn(0);
9142 }
9143 
9144 /*@
9145   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9146 
9147   Input Parameter:
9148 . dm - The DMPlex object
9149 
9150   Output Parameter:
9151 . regular - The flag
9152 
9153   Level: intermediate
9154 
9155 .seealso: DMPlexSetRegularRefinement()
9156 @*/
9157 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
9158 {
9159   PetscFunctionBegin;
9160   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9161   PetscValidPointer(regular, 2);
9162   *regular = ((DM_Plex *) dm->data)->regularRefinement;
9163   PetscFunctionReturn(0);
9164 }
9165 
9166 /*@
9167   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9168 
9169   Input Parameters:
9170 + dm - The DMPlex object
9171 - regular - The flag
9172 
9173   Level: intermediate
9174 
9175 .seealso: DMPlexGetRegularRefinement()
9176 @*/
9177 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
9178 {
9179   PetscFunctionBegin;
9180   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9181   ((DM_Plex *) dm->data)->regularRefinement = regular;
9182   PetscFunctionReturn(0);
9183 }
9184 
9185 /*@
9186   DMPlexGetCellRefinerType - Get the strategy for refining a cell
9187 
9188   Input Parameter:
9189 . dm - The DMPlex object
9190 
9191   Output Parameter:
9192 . cr - The strategy number
9193 
9194   Level: intermediate
9195 
9196 .seealso: DMPlexSetCellRefinerType(), DMPlexSetRegularRefinement()
9197 @*/
9198 PetscErrorCode DMPlexGetCellRefinerType(DM dm, DMPlexCellRefinerType *cr)
9199 {
9200   PetscFunctionBegin;
9201   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9202   PetscValidPointer(cr, 2);
9203   *cr = ((DM_Plex *) dm->data)->cellRefiner;
9204   PetscFunctionReturn(0);
9205 }
9206 
9207 /*@
9208   DMPlexSetCellRefinerType - Set the strategy for refining a cell
9209 
9210   Input Parameters:
9211 + dm - The DMPlex object
9212 - cr - The strategy number
9213 
9214   Level: intermediate
9215 
9216 .seealso: DMPlexGetCellRefinerType(), DMPlexGetRegularRefinement()
9217 @*/
9218 PetscErrorCode DMPlexSetCellRefinerType(DM dm, DMPlexCellRefinerType cr)
9219 {
9220   PetscFunctionBegin;
9221   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9222   ((DM_Plex *) dm->data)->cellRefiner = cr;
9223   PetscFunctionReturn(0);
9224 }
9225 
9226 /* anchors */
9227 /*@
9228   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
9229   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
9230 
9231   not collective
9232 
9233   Input Parameters:
9234 . dm - The DMPlex object
9235 
9236   Output Parameters:
9237 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
9238 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
9239 
9240   Level: intermediate
9241 
9242 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
9243 @*/
9244 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
9245 {
9246   DM_Plex *plex = (DM_Plex *)dm->data;
9247   PetscErrorCode ierr;
9248 
9249   PetscFunctionBegin;
9250   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9251   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
9252   if (anchorSection) *anchorSection = plex->anchorSection;
9253   if (anchorIS) *anchorIS = plex->anchorIS;
9254   PetscFunctionReturn(0);
9255 }
9256 
9257 /*@
9258   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
9259   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
9260   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
9261 
9262   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
9263   DMGetConstraints() and filling in the entries in the constraint matrix.
9264 
9265   collective on dm
9266 
9267   Input Parameters:
9268 + dm - The DMPlex object
9269 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9270 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9271 
9272   The reference counts of anchorSection and anchorIS are incremented.
9273 
9274   Level: intermediate
9275 
9276 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
9277 @*/
9278 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
9279 {
9280   DM_Plex        *plex = (DM_Plex *)dm->data;
9281   PetscMPIInt    result;
9282   PetscErrorCode ierr;
9283 
9284   PetscFunctionBegin;
9285   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9286   if (anchorSection) {
9287     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
9288     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRMPI(ierr);
9289     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
9290   }
9291   if (anchorIS) {
9292     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
9293     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRMPI(ierr);
9294     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
9295   }
9296 
9297   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
9298   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
9299   plex->anchorSection = anchorSection;
9300 
9301   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
9302   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
9303   plex->anchorIS = anchorIS;
9304 
9305   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
9306     PetscInt size, a, pStart, pEnd;
9307     const PetscInt *anchors;
9308 
9309     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9310     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
9311     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
9312     for (a = 0; a < size; a++) {
9313       PetscInt p;
9314 
9315       p = anchors[a];
9316       if (p >= pStart && p < pEnd) {
9317         PetscInt dof;
9318 
9319         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9320         if (dof) {
9321           PetscErrorCode ierr2;
9322 
9323           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
9324           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
9325         }
9326       }
9327     }
9328     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
9329   }
9330   /* reset the generic constraints */
9331   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
9332   PetscFunctionReturn(0);
9333 }
9334 
9335 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
9336 {
9337   PetscSection anchorSection;
9338   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
9339   PetscErrorCode ierr;
9340 
9341   PetscFunctionBegin;
9342   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9343   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9344   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
9345   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9346   if (numFields) {
9347     PetscInt f;
9348     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
9349 
9350     for (f = 0; f < numFields; f++) {
9351       PetscInt numComp;
9352 
9353       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
9354       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
9355     }
9356   }
9357   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9358   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9359   pStart = PetscMax(pStart,sStart);
9360   pEnd   = PetscMin(pEnd,sEnd);
9361   pEnd   = PetscMax(pStart,pEnd);
9362   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
9363   for (p = pStart; p < pEnd; p++) {
9364     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9365     if (dof) {
9366       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
9367       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
9368       for (f = 0; f < numFields; f++) {
9369         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
9370         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
9371       }
9372     }
9373   }
9374   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
9375   ierr = PetscObjectSetName((PetscObject) *cSec, "Constraint Section");CHKERRQ(ierr);
9376   PetscFunctionReturn(0);
9377 }
9378 
9379 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
9380 {
9381   PetscSection   aSec;
9382   PetscInt       pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
9383   const PetscInt *anchors;
9384   PetscInt       numFields, f;
9385   IS             aIS;
9386   PetscErrorCode ierr;
9387   MatType        mtype;
9388   PetscBool      iscuda,iskokkos;
9389 
9390   PetscFunctionBegin;
9391   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9392   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
9393   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
9394   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
9395   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
9396   ierr = PetscStrcmp(dm->mattype,MATSEQAIJCUSPARSE,&iscuda);CHKERRQ(ierr);
9397   if (!iscuda) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJCUSPARSE,&iscuda);CHKERRQ(ierr); }
9398   ierr = PetscStrcmp(dm->mattype,MATSEQAIJKOKKOS,&iskokkos);CHKERRQ(ierr);
9399   if (!iskokkos) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJKOKKOS,&iskokkos);CHKERRQ(ierr); }
9400   if (iscuda) mtype = MATSEQAIJCUSPARSE;
9401   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
9402   else mtype = MATSEQAIJ;
9403   ierr = MatSetType(*cMat,mtype);CHKERRQ(ierr);
9404   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
9405   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
9406   /* cSec will be a subset of aSec and section */
9407   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
9408   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9409   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
9410   i[0] = 0;
9411   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9412   for (p = pStart; p < pEnd; p++) {
9413     PetscInt rDof, rOff, r;
9414 
9415     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9416     if (!rDof) continue;
9417     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9418     if (numFields) {
9419       for (f = 0; f < numFields; f++) {
9420         annz = 0;
9421         for (r = 0; r < rDof; r++) {
9422           a = anchors[rOff + r];
9423           if (a < sStart || a >= sEnd) continue;
9424           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9425           annz += aDof;
9426         }
9427         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9428         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
9429         for (q = 0; q < dof; q++) {
9430           i[off + q + 1] = i[off + q] + annz;
9431         }
9432       }
9433     }
9434     else {
9435       annz = 0;
9436       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9437       for (q = 0; q < dof; q++) {
9438         a = anchors[rOff + q];
9439         if (a < sStart || a >= sEnd) continue;
9440         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9441         annz += aDof;
9442       }
9443       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9444       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
9445       for (q = 0; q < dof; q++) {
9446         i[off + q + 1] = i[off + q] + annz;
9447       }
9448     }
9449   }
9450   nnz = i[m];
9451   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
9452   offset = 0;
9453   for (p = pStart; p < pEnd; p++) {
9454     if (numFields) {
9455       for (f = 0; f < numFields; f++) {
9456         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9457         for (q = 0; q < dof; q++) {
9458           PetscInt rDof, rOff, r;
9459           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9460           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9461           for (r = 0; r < rDof; r++) {
9462             PetscInt s;
9463 
9464             a = anchors[rOff + r];
9465             if (a < sStart || a >= sEnd) continue;
9466             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9467             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
9468             for (s = 0; s < aDof; s++) {
9469               j[offset++] = aOff + s;
9470             }
9471           }
9472         }
9473       }
9474     }
9475     else {
9476       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9477       for (q = 0; q < dof; q++) {
9478         PetscInt rDof, rOff, r;
9479         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9480         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9481         for (r = 0; r < rDof; r++) {
9482           PetscInt s;
9483 
9484           a = anchors[rOff + r];
9485           if (a < sStart || a >= sEnd) continue;
9486           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9487           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
9488           for (s = 0; s < aDof; s++) {
9489             j[offset++] = aOff + s;
9490           }
9491         }
9492       }
9493     }
9494   }
9495   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
9496   ierr = PetscFree(i);CHKERRQ(ierr);
9497   ierr = PetscFree(j);CHKERRQ(ierr);
9498   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
9499   PetscFunctionReturn(0);
9500 }
9501 
9502 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
9503 {
9504   DM_Plex        *plex = (DM_Plex *)dm->data;
9505   PetscSection   anchorSection, section, cSec;
9506   Mat            cMat;
9507   PetscErrorCode ierr;
9508 
9509   PetscFunctionBegin;
9510   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9511   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9512   if (anchorSection) {
9513     PetscInt Nf;
9514 
9515     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
9516     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
9517     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
9518     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
9519     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
9520     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
9521     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
9522     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
9523   }
9524   PetscFunctionReturn(0);
9525 }
9526 
9527 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
9528 {
9529   IS             subis;
9530   PetscSection   section, subsection;
9531   PetscErrorCode ierr;
9532 
9533   PetscFunctionBegin;
9534   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9535   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
9536   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
9537   /* Create subdomain */
9538   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
9539   /* Create submodel */
9540   ierr = DMPlexGetSubpointIS(*subdm, &subis);CHKERRQ(ierr);
9541   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
9542   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
9543   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
9544   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
9545   /* Create map from submodel to global model */
9546   if (is) {
9547     PetscSection    sectionGlobal, subsectionGlobal;
9548     IS              spIS;
9549     const PetscInt *spmap;
9550     PetscInt       *subIndices;
9551     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
9552     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
9553 
9554     ierr = DMPlexGetSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
9555     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
9556     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
9557     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
9558     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
9559     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
9560     for (p = pStart; p < pEnd; ++p) {
9561       PetscInt gdof, pSubSize  = 0;
9562 
9563       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
9564       if (gdof > 0) {
9565         for (f = 0; f < Nf; ++f) {
9566           PetscInt fdof, fcdof;
9567 
9568           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
9569           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
9570           pSubSize += fdof-fcdof;
9571         }
9572         subSize += pSubSize;
9573         if (pSubSize) {
9574           if (bs < 0) {
9575             bs = pSubSize;
9576           } else if (bs != pSubSize) {
9577             /* Layout does not admit a pointwise block size */
9578             bs = 1;
9579           }
9580         }
9581       }
9582     }
9583     /* Must have same blocksize on all procs (some might have no points) */
9584     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
9585     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
9586     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
9587     else                            {bs = bsMinMax[0];}
9588     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
9589     for (p = pStart; p < pEnd; ++p) {
9590       PetscInt gdof, goff;
9591 
9592       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
9593       if (gdof > 0) {
9594         const PetscInt point = spmap[p];
9595 
9596         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
9597         for (f = 0; f < Nf; ++f) {
9598           PetscInt fdof, fcdof, fc, f2, poff = 0;
9599 
9600           /* Can get rid of this loop by storing field information in the global section */
9601           for (f2 = 0; f2 < f; ++f2) {
9602             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
9603             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
9604             poff += fdof-fcdof;
9605           }
9606           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
9607           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
9608           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
9609             subIndices[subOff] = goff+poff+fc;
9610           }
9611         }
9612       }
9613     }
9614     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
9615     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
9616     if (bs > 1) {
9617       /* We need to check that the block size does not come from non-contiguous fields */
9618       PetscInt i, j, set = 1;
9619       for (i = 0; i < subSize; i += bs) {
9620         for (j = 0; j < bs; ++j) {
9621           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
9622         }
9623       }
9624       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
9625     }
9626     /* Attach nullspace */
9627     for (f = 0; f < Nf; ++f) {
9628       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
9629       if ((*subdm)->nullspaceConstructors[f]) break;
9630     }
9631     if (f < Nf) {
9632       MatNullSpace nullSpace;
9633       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace);CHKERRQ(ierr);
9634 
9635       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
9636       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
9637     }
9638   }
9639   PetscFunctionReturn(0);
9640 }
9641 
9642 /*@
9643   DMPlexMonitorThroughput - Report the cell throughput of FE integration
9644 
9645   Input Parameter:
9646 - dm - The DM
9647 
9648   Level: developer
9649 
9650   Options Database Keys:
9651 . -dm_plex_monitor_throughput - Activate the monitor
9652 
9653 .seealso: DMSetFromOptions(), DMPlexCreate()
9654 @*/
9655 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
9656 {
9657 #if defined(PETSC_USE_LOG)
9658   PetscStageLog      stageLog;
9659   PetscLogEvent      event;
9660   PetscLogStage      stage;
9661   PetscEventPerfInfo eventInfo;
9662   PetscReal          cellRate, flopRate;
9663   PetscInt           cStart, cEnd, Nf, N;
9664   const char        *name;
9665   PetscErrorCode     ierr;
9666 #endif
9667 
9668   PetscFunctionBegin;
9669   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9670 #if defined(PETSC_USE_LOG)
9671   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
9672   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
9673   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
9674   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
9675   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
9676   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
9677   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
9678   N        = (cEnd - cStart)*Nf*eventInfo.count;
9679   flopRate = eventInfo.flops/eventInfo.time;
9680   cellRate = N/eventInfo.time;
9681   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6));CHKERRQ(ierr);
9682 #else
9683   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
9684 #endif
9685   PetscFunctionReturn(0);
9686 }
9687