xref: /petsc/src/dm/impls/plex/plex.c (revision 9fca9976580a661c418fd9281b99affbad84f0a0)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 #include <petscdmplextransform.h>
10 
11 /* Logging support */
12 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF,DMPLEX_LocatePoints,DMPLEX_TopologyView,DMPLEX_LabelsView,DMPLEX_CoordinatesView,DMPLEX_SectionView,DMPLEX_GlobalVectorView,DMPLEX_LocalVectorView,DMPLEX_TopologyLoad,DMPLEX_LabelsLoad,DMPLEX_CoordinatesLoad,DMPLEX_SectionLoad,DMPLEX_GlobalVectorLoad,DMPLEX_LocalVectorLoad;
13 
14 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
15 
16 /*@
17   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
18 
19   Input Parameter:
20 . dm      - The DMPlex object
21 
22   Output Parameter:
23 . simplex - Flag checking for a simplex
24 
25   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
26   If the mesh has no cells, this returns PETSC_FALSE.
27 
28   Level: intermediate
29 
30 .seealso `DMPlexGetSimplexOrBoxCells()`, `DMPlexGetCellType()`, `DMPlexGetHeightStratum()`, `DMPolytopeTypeGetNumVertices()`
31 @*/
32 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
33 {
34   DMPolytopeType ct;
35   PetscInt       cStart, cEnd;
36 
37   PetscFunctionBegin;
38   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
39   if (cEnd <= cStart) {*simplex = PETSC_FALSE; PetscFunctionReturn(0);}
40   PetscCall(DMPlexGetCellType(dm, cStart, &ct));
41   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
42   PetscFunctionReturn(0);
43 }
44 
45 /*@
46   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
47 
48   Input Parameters:
49 + dm     - The DMPlex object
50 - height - The cell height in the Plex, 0 is the default
51 
52   Output Parameters:
53 + cStart - The first "normal" cell
54 - cEnd   - The upper bound on "normal"" cells
55 
56   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
57 
58   Level: developer
59 
60 .seealso `DMPlexConstructGhostCells()`, `DMPlexGetGhostCellStratum()`
61 @*/
62 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
63 {
64   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
65   PetscInt       cS, cE, c;
66 
67   PetscFunctionBegin;
68   PetscCall(DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE));
69   for (c = cS; c < cE; ++c) {
70     DMPolytopeType cct;
71 
72     PetscCall(DMPlexGetCellType(dm, c, &cct));
73     if ((PetscInt) cct < 0) break;
74     switch (cct) {
75       case DM_POLYTOPE_POINT:
76       case DM_POLYTOPE_SEGMENT:
77       case DM_POLYTOPE_TRIANGLE:
78       case DM_POLYTOPE_QUADRILATERAL:
79       case DM_POLYTOPE_TETRAHEDRON:
80       case DM_POLYTOPE_HEXAHEDRON:
81         ct = cct;
82         break;
83       default: break;
84     }
85     if (ct != DM_POLYTOPE_UNKNOWN) break;
86   }
87   if (ct != DM_POLYTOPE_UNKNOWN) {
88     DMLabel ctLabel;
89 
90     PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
91     PetscCall(DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE));
92   }
93   if (cStart) *cStart = cS;
94   if (cEnd)   *cEnd   = cE;
95   PetscFunctionReturn(0);
96 }
97 
98 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
99 {
100   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
101   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
102 
103   PetscFunctionBegin;
104   *ft  = PETSC_VTK_INVALID;
105   PetscCall(DMGetCoordinateDim(dm, &cdim));
106   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
107   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
108   PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
109   if (field >= 0) {
110     if ((vStart >= pStart) && (vStart < pEnd)) PetscCall(PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]));
111     if ((cStart >= pStart) && (cStart < pEnd)) PetscCall(PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]));
112   } else {
113     if ((vStart >= pStart) && (vStart < pEnd)) PetscCall(PetscSectionGetDof(section, vStart, &vcdof[0]));
114     if ((cStart >= pStart) && (cStart < pEnd)) PetscCall(PetscSectionGetDof(section, cStart, &vcdof[1]));
115   }
116   PetscCallMPI(MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
117   if (globalvcdof[0]) {
118     *sStart = vStart;
119     *sEnd   = vEnd;
120     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
121     else                        *ft = PETSC_VTK_POINT_FIELD;
122   } else if (globalvcdof[1]) {
123     *sStart = cStart;
124     *sEnd   = cEnd;
125     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
126     else                        *ft = PETSC_VTK_CELL_FIELD;
127   } else {
128     if (field >= 0) {
129       const char *fieldname;
130 
131       PetscCall(PetscSectionGetFieldName(section, field, &fieldname));
132       PetscCall(PetscInfo((PetscObject) dm, "Could not classify VTK output type of section field %" PetscInt_FMT " \"%s\"\n", field, fieldname));
133     } else {
134       PetscCall(PetscInfo((PetscObject) dm, "Could not classify VTK output type of section\n"));
135     }
136   }
137   PetscFunctionReturn(0);
138 }
139 
140 /*@
141   DMPlexVecView1D - Plot many 1D solutions on the same line graph
142 
143   Collective on dm
144 
145   Input Parameters:
146 + dm - The DMPlex
147 . n  - The number of vectors
148 . u  - The array of local vectors
149 - viewer - The Draw viewer
150 
151   Level: advanced
152 
153 .seealso: `VecViewFromOptions()`, `VecView()`
154 @*/
155 PetscErrorCode DMPlexVecView1D(DM dm, PetscInt n, Vec u[], PetscViewer viewer)
156 {
157   PetscDS            ds;
158   PetscDraw          draw = NULL;
159   PetscDrawLG        lg;
160   Vec                coordinates;
161   const PetscScalar *coords, **sol;
162   PetscReal         *vals;
163   PetscInt          *Nc;
164   PetscInt           Nf, f, c, Nl, l, i, vStart, vEnd, v;
165   char             **names;
166 
167   PetscFunctionBegin;
168   PetscCall(DMGetDS(dm, &ds));
169   PetscCall(PetscDSGetNumFields(ds, &Nf));
170   PetscCall(PetscDSGetTotalComponents(ds, &Nl));
171   PetscCall(PetscDSGetComponents(ds, &Nc));
172 
173   PetscCall(PetscViewerDrawGetDraw(viewer, 0, &draw));
174   if (!draw) PetscFunctionReturn(0);
175   PetscCall(PetscDrawLGCreate(draw, n*Nl, &lg));
176 
177   PetscCall(PetscMalloc3(n, &sol, n*Nl, &names, n*Nl, &vals));
178   for (i = 0, l = 0; i < n; ++i) {
179     const char *vname;
180 
181     PetscCall(PetscObjectGetName((PetscObject) u[i], &vname));
182     for (f = 0; f < Nf; ++f) {
183       PetscObject disc;
184       const char *fname;
185       char        tmpname[PETSC_MAX_PATH_LEN];
186 
187       PetscCall(PetscDSGetDiscretization(ds, f, &disc));
188       /* TODO Create names for components */
189       for (c = 0; c < Nc[f]; ++c, ++l) {
190         PetscCall(PetscObjectGetName(disc, &fname));
191         PetscCall(PetscStrcpy(tmpname, vname));
192         PetscCall(PetscStrlcat(tmpname, ":", PETSC_MAX_PATH_LEN));
193         PetscCall(PetscStrlcat(tmpname, fname, PETSC_MAX_PATH_LEN));
194         PetscCall(PetscStrallocpy(tmpname, &names[l]));
195       }
196     }
197   }
198   PetscCall(PetscDrawLGSetLegend(lg, (const char *const *) names));
199   /* Just add P_1 support for now */
200   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
201   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
202   PetscCall(VecGetArrayRead(coordinates, &coords));
203   for (i = 0; i < n; ++i) PetscCall(VecGetArrayRead(u[i], &sol[i]));
204   for (v = vStart; v < vEnd; ++v) {
205     PetscScalar *x, *svals;
206 
207     PetscCall(DMPlexPointLocalRead(dm, v, coords, &x));
208     for (i = 0; i < n; ++i) {
209       PetscCall(DMPlexPointLocalRead(dm, v, sol[i], &svals));
210       for (l = 0; l < Nl; ++l) vals[i*Nl + l] = PetscRealPart(svals[l]);
211     }
212     PetscCall(PetscDrawLGAddCommonPoint(lg, PetscRealPart(x[0]), vals));
213   }
214   PetscCall(VecRestoreArrayRead(coordinates, &coords));
215   for (i = 0; i < n; ++i) PetscCall(VecRestoreArrayRead(u[i], &sol[i]));
216   for (l = 0; l < n*Nl; ++l) PetscCall(PetscFree(names[l]));
217   PetscCall(PetscFree3(sol, names, vals));
218 
219   PetscCall(PetscDrawLGDraw(lg));
220   PetscCall(PetscDrawLGDestroy(&lg));
221   PetscFunctionReturn(0);
222 }
223 
224 static PetscErrorCode VecView_Plex_Local_Draw_1D(Vec u, PetscViewer viewer)
225 {
226   DM             dm;
227 
228   PetscFunctionBegin;
229   PetscCall(VecGetDM(u, &dm));
230   PetscCall(DMPlexVecView1D(dm, 1, &u, viewer));
231   PetscFunctionReturn(0);
232 }
233 
234 static PetscErrorCode VecView_Plex_Local_Draw_2D(Vec v, PetscViewer viewer)
235 {
236   DM                 dm;
237   PetscSection       s;
238   PetscDraw          draw, popup;
239   DM                 cdm;
240   PetscSection       coordSection;
241   Vec                coordinates;
242   const PetscScalar *coords, *array;
243   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
244   PetscReal          vbound[2], time;
245   PetscBool          flg;
246   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
247   const char        *name;
248   char               title[PETSC_MAX_PATH_LEN];
249 
250   PetscFunctionBegin;
251   PetscCall(PetscViewerDrawGetDraw(viewer, 0, &draw));
252   PetscCall(VecGetDM(v, &dm));
253   PetscCall(DMGetCoordinateDim(dm, &dim));
254   PetscCall(DMGetLocalSection(dm, &s));
255   PetscCall(PetscSectionGetNumFields(s, &Nf));
256   PetscCall(DMGetCoarsenLevel(dm, &level));
257   PetscCall(DMGetCoordinateDM(dm, &cdm));
258   PetscCall(DMGetLocalSection(cdm, &coordSection));
259   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
260   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
261   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
262 
263   PetscCall(PetscObjectGetName((PetscObject) v, &name));
264   PetscCall(DMGetOutputSequenceNumber(dm, &step, &time));
265 
266   PetscCall(VecGetLocalSize(coordinates, &N));
267   PetscCall(VecGetArrayRead(coordinates, &coords));
268   for (c = 0; c < N; c += dim) {
269     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
270     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
271   }
272   PetscCall(VecRestoreArrayRead(coordinates, &coords));
273   PetscCall(PetscDrawClear(draw));
274 
275   /* Could implement something like DMDASelectFields() */
276   for (f = 0; f < Nf; ++f) {
277     DM   fdm = dm;
278     Vec  fv  = v;
279     IS   fis;
280     char prefix[PETSC_MAX_PATH_LEN];
281     const char *fname;
282 
283     PetscCall(PetscSectionGetFieldComponents(s, f, &Nc));
284     PetscCall(PetscSectionGetFieldName(s, f, &fname));
285 
286     if (v->hdr.prefix) PetscCall(PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix)));
287     else               {prefix[0] = '\0';}
288     if (Nf > 1) {
289       PetscCall(DMCreateSubDM(dm, 1, &f, &fis, &fdm));
290       PetscCall(VecGetSubVector(v, fis, &fv));
291       PetscCall(PetscStrlcat(prefix, fname,sizeof(prefix)));
292       PetscCall(PetscStrlcat(prefix, "_",sizeof(prefix)));
293     }
294     for (comp = 0; comp < Nc; ++comp, ++w) {
295       PetscInt nmax = 2;
296 
297       PetscCall(PetscViewerDrawGetDraw(viewer, w, &draw));
298       if (Nc > 1) PetscCall(PetscSNPrintf(title, sizeof(title), "%s:%s_%" PetscInt_FMT " Step: %" PetscInt_FMT " Time: %.4g", name, fname, comp, step, (double)time));
299       else        PetscCall(PetscSNPrintf(title, sizeof(title), "%s:%s Step: %" PetscInt_FMT " Time: %.4g", name, fname, step, (double)time));
300       PetscCall(PetscDrawSetTitle(draw, title));
301 
302       /* TODO Get max and min only for this component */
303       PetscCall(PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg));
304       if (!flg) {
305         PetscCall(VecMin(fv, NULL, &vbound[0]));
306         PetscCall(VecMax(fv, NULL, &vbound[1]));
307         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
308       }
309       PetscCall(PetscDrawGetPopup(draw, &popup));
310       PetscCall(PetscDrawScalePopup(popup, vbound[0], vbound[1]));
311       PetscCall(PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]));
312 
313       PetscCall(VecGetArrayRead(fv, &array));
314       for (c = cStart; c < cEnd; ++c) {
315         PetscScalar *coords = NULL, *a = NULL;
316         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
317 
318         PetscCall(DMPlexPointLocalRead(fdm, c, array, &a));
319         if (a) {
320           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
321           color[1] = color[2] = color[3] = color[0];
322         } else {
323           PetscScalar *vals = NULL;
324           PetscInt     numVals, va;
325 
326           PetscCall(DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals));
327           PetscCheck(numVals % Nc == 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %" PetscInt_FMT " does not divide the number of values in the closure %" PetscInt_FMT, Nc, numVals);
328           switch (numVals/Nc) {
329           case 3: /* P1 Triangle */
330           case 4: /* P1 Quadrangle */
331             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
332             break;
333           case 6: /* P2 Triangle */
334           case 8: /* P2 Quadrangle */
335             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
336             break;
337           default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %" PetscInt_FMT " cannot be handled", numVals/Nc);
338           }
339           PetscCall(DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals));
340         }
341         PetscCall(DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords));
342         switch (numCoords) {
343         case 6:
344         case 12: /* Localized triangle */
345           PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]));
346           break;
347         case 8:
348         case 16: /* Localized quadrilateral */
349           PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]));
350           PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]));
351           break;
352         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %" PetscInt_FMT " coordinates", numCoords);
353         }
354         PetscCall(DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords));
355       }
356       PetscCall(VecRestoreArrayRead(fv, &array));
357       PetscCall(PetscDrawFlush(draw));
358       PetscCall(PetscDrawPause(draw));
359       PetscCall(PetscDrawSave(draw));
360     }
361     if (Nf > 1) {
362       PetscCall(VecRestoreSubVector(v, fis, &fv));
363       PetscCall(ISDestroy(&fis));
364       PetscCall(DMDestroy(&fdm));
365     }
366   }
367   PetscFunctionReturn(0);
368 }
369 
370 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
371 {
372   DM        dm;
373   PetscDraw draw;
374   PetscInt  dim;
375   PetscBool isnull;
376 
377   PetscFunctionBegin;
378   PetscCall(PetscViewerDrawGetDraw(viewer, 0, &draw));
379   PetscCall(PetscDrawIsNull(draw, &isnull));
380   if (isnull) PetscFunctionReturn(0);
381 
382   PetscCall(VecGetDM(v, &dm));
383   PetscCall(DMGetCoordinateDim(dm, &dim));
384   switch (dim) {
385   case 1: PetscCall(VecView_Plex_Local_Draw_1D(v, viewer));break;
386   case 2: PetscCall(VecView_Plex_Local_Draw_2D(v, viewer));break;
387   default: SETERRQ(PetscObjectComm((PetscObject) v), PETSC_ERR_SUP, "Cannot draw meshes of dimension %" PetscInt_FMT ". Try PETSCVIEWERGLVIS", dim);
388   }
389   PetscFunctionReturn(0);
390 }
391 
392 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
393 {
394   DM                      dm;
395   Vec                     locv;
396   const char              *name;
397   PetscSection            section;
398   PetscInt                pStart, pEnd;
399   PetscInt                numFields;
400   PetscViewerVTKFieldType ft;
401 
402   PetscFunctionBegin;
403   PetscCall(VecGetDM(v, &dm));
404   PetscCall(DMCreateLocalVector(dm, &locv)); /* VTK viewer requires exclusive ownership of the vector */
405   PetscCall(PetscObjectGetName((PetscObject) v, &name));
406   PetscCall(PetscObjectSetName((PetscObject) locv, name));
407   PetscCall(VecCopy(v, locv));
408   PetscCall(DMGetLocalSection(dm, &section));
409   PetscCall(PetscSectionGetNumFields(section, &numFields));
410   if (!numFields) {
411     PetscCall(DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft));
412     PetscCall(PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv));
413   } else {
414     PetscInt f;
415 
416     for (f = 0; f < numFields; f++) {
417       PetscCall(DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft));
418       if (ft == PETSC_VTK_INVALID) continue;
419       PetscCall(PetscObjectReference((PetscObject)locv));
420       PetscCall(PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv));
421     }
422     PetscCall(VecDestroy(&locv));
423   }
424   PetscFunctionReturn(0);
425 }
426 
427 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
428 {
429   DM             dm;
430   PetscBool      isvtk, ishdf5, isdraw, isglvis;
431 
432   PetscFunctionBegin;
433   PetscCall(VecGetDM(v, &dm));
434   PetscCheck(dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
435   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk));
436   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5));
437   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw));
438   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis));
439   if (isvtk || ishdf5 || isdraw || isglvis) {
440     PetscInt    i,numFields;
441     PetscObject fe;
442     PetscBool   fem = PETSC_FALSE;
443     Vec         locv = v;
444     const char  *name;
445     PetscInt    step;
446     PetscReal   time;
447 
448     PetscCall(DMGetNumFields(dm, &numFields));
449     for (i=0; i<numFields; i++) {
450       PetscCall(DMGetField(dm, i, NULL, &fe));
451       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
452     }
453     if (fem) {
454       PetscObject isZero;
455 
456       PetscCall(DMGetLocalVector(dm, &locv));
457       PetscCall(PetscObjectGetName((PetscObject) v, &name));
458       PetscCall(PetscObjectSetName((PetscObject) locv, name));
459       PetscCall(PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero));
460       PetscCall(PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero));
461       PetscCall(VecCopy(v, locv));
462       PetscCall(DMGetOutputSequenceNumber(dm, NULL, &time));
463       PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL));
464     }
465     if (isvtk) {
466       PetscCall(VecView_Plex_Local_VTK(locv, viewer));
467     } else if (ishdf5) {
468 #if defined(PETSC_HAVE_HDF5)
469       PetscCall(VecView_Plex_Local_HDF5_Internal(locv, viewer));
470 #else
471       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
472 #endif
473     } else if (isdraw) {
474       PetscCall(VecView_Plex_Local_Draw(locv, viewer));
475     } else if (isglvis) {
476       PetscCall(DMGetOutputSequenceNumber(dm, &step, NULL));
477       PetscCall(PetscViewerGLVisSetSnapId(viewer, step));
478       PetscCall(VecView_GLVis(locv, viewer));
479     }
480     if (fem) {
481       PetscCall(PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL));
482       PetscCall(DMRestoreLocalVector(dm, &locv));
483     }
484   } else {
485     PetscBool isseq;
486 
487     PetscCall(PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq));
488     if (isseq) PetscCall(VecView_Seq(v, viewer));
489     else       PetscCall(VecView_MPI(v, viewer));
490   }
491   PetscFunctionReturn(0);
492 }
493 
494 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
495 {
496   DM        dm;
497   PetscBool isvtk, ishdf5, isdraw, isglvis, isexodusii;
498 
499   PetscFunctionBegin;
500   PetscCall(VecGetDM(v, &dm));
501   PetscCheck(dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
502   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk));
503   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5));
504   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw));
505   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis));
506   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii));
507   if (isvtk || isdraw || isglvis) {
508     Vec         locv;
509     PetscObject isZero;
510     const char *name;
511 
512     PetscCall(DMGetLocalVector(dm, &locv));
513     PetscCall(PetscObjectGetName((PetscObject) v, &name));
514     PetscCall(PetscObjectSetName((PetscObject) locv, name));
515     PetscCall(DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv));
516     PetscCall(DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv));
517     PetscCall(PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero));
518     PetscCall(PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero));
519     PetscCall(VecView_Plex_Local(locv, viewer));
520     PetscCall(PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL));
521     PetscCall(DMRestoreLocalVector(dm, &locv));
522   } else if (ishdf5) {
523 #if defined(PETSC_HAVE_HDF5)
524     PetscCall(VecView_Plex_HDF5_Internal(v, viewer));
525 #else
526     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
527 #endif
528   } else if (isexodusii) {
529 #if defined(PETSC_HAVE_EXODUSII)
530     PetscCall(VecView_PlexExodusII_Internal(v, viewer));
531 #else
532     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
533 #endif
534   } else {
535     PetscBool isseq;
536 
537     PetscCall(PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq));
538     if (isseq) PetscCall(VecView_Seq(v, viewer));
539     else       PetscCall(VecView_MPI(v, viewer));
540   }
541   PetscFunctionReturn(0);
542 }
543 
544 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
545 {
546   DM                dm;
547   MPI_Comm          comm;
548   PetscViewerFormat format;
549   Vec               v;
550   PetscBool         isvtk, ishdf5;
551 
552   PetscFunctionBegin;
553   PetscCall(VecGetDM(originalv, &dm));
554   PetscCall(PetscObjectGetComm((PetscObject) originalv, &comm));
555   PetscCheck(dm,comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
556   PetscCall(PetscViewerGetFormat(viewer, &format));
557   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5));
558   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk));
559   if (format == PETSC_VIEWER_NATIVE) {
560     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
561     /* this need a better fix */
562     if (dm->useNatural) {
563       if (dm->sfNatural) {
564         const char *vecname;
565         PetscInt    n, nroots;
566 
567         PetscCall(VecGetLocalSize(originalv, &n));
568         PetscCall(PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL));
569         if (n == nroots) {
570           PetscCall(DMGetGlobalVector(dm, &v));
571           PetscCall(DMPlexGlobalToNaturalBegin(dm, originalv, v));
572           PetscCall(DMPlexGlobalToNaturalEnd(dm, originalv, v));
573           PetscCall(PetscObjectGetName((PetscObject) originalv, &vecname));
574           PetscCall(PetscObjectSetName((PetscObject) v, vecname));
575         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
576       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
577     } else v = originalv;
578   } else v = originalv;
579 
580   if (ishdf5) {
581 #if defined(PETSC_HAVE_HDF5)
582     PetscCall(VecView_Plex_HDF5_Native_Internal(v, viewer));
583 #else
584     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
585 #endif
586   } else if (isvtk) {
587     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
588   } else {
589     PetscBool isseq;
590 
591     PetscCall(PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq));
592     if (isseq) PetscCall(VecView_Seq(v, viewer));
593     else       PetscCall(VecView_MPI(v, viewer));
594   }
595   if (v != originalv) PetscCall(DMRestoreGlobalVector(dm, &v));
596   PetscFunctionReturn(0);
597 }
598 
599 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
600 {
601   DM             dm;
602   PetscBool      ishdf5;
603 
604   PetscFunctionBegin;
605   PetscCall(VecGetDM(v, &dm));
606   PetscCheck(dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
607   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5));
608   if (ishdf5) {
609     DM          dmBC;
610     Vec         gv;
611     const char *name;
612 
613     PetscCall(DMGetOutputDM(dm, &dmBC));
614     PetscCall(DMGetGlobalVector(dmBC, &gv));
615     PetscCall(PetscObjectGetName((PetscObject) v, &name));
616     PetscCall(PetscObjectSetName((PetscObject) gv, name));
617     PetscCall(VecLoad_Default(gv, viewer));
618     PetscCall(DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v));
619     PetscCall(DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v));
620     PetscCall(DMRestoreGlobalVector(dmBC, &gv));
621   } else {
622     PetscCall(VecLoad_Default(v, viewer));
623   }
624   PetscFunctionReturn(0);
625 }
626 
627 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
628 {
629   DM             dm;
630   PetscBool      ishdf5,isexodusii;
631 
632   PetscFunctionBegin;
633   PetscCall(VecGetDM(v, &dm));
634   PetscCheck(dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
635   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5));
636   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii));
637   if (ishdf5) {
638 #if defined(PETSC_HAVE_HDF5)
639     PetscCall(VecLoad_Plex_HDF5_Internal(v, viewer));
640 #else
641     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
642 #endif
643   } else if (isexodusii) {
644 #if defined(PETSC_HAVE_EXODUSII)
645     PetscCall(VecLoad_PlexExodusII_Internal(v, viewer));
646 #else
647     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
648 #endif
649   } else {
650     PetscCall(VecLoad_Default(v, viewer));
651   }
652   PetscFunctionReturn(0);
653 }
654 
655 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
656 {
657   DM                dm;
658   PetscViewerFormat format;
659   PetscBool         ishdf5;
660 
661   PetscFunctionBegin;
662   PetscCall(VecGetDM(originalv, &dm));
663   PetscCheck(dm,PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
664   PetscCall(PetscViewerGetFormat(viewer, &format));
665   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5));
666   if (format == PETSC_VIEWER_NATIVE) {
667     if (dm->useNatural) {
668       if (dm->sfNatural) {
669         if (ishdf5) {
670 #if defined(PETSC_HAVE_HDF5)
671           Vec         v;
672           const char *vecname;
673 
674           PetscCall(DMGetGlobalVector(dm, &v));
675           PetscCall(PetscObjectGetName((PetscObject) originalv, &vecname));
676           PetscCall(PetscObjectSetName((PetscObject) v, vecname));
677           PetscCall(VecLoad_Plex_HDF5_Native_Internal(v, viewer));
678           PetscCall(DMPlexNaturalToGlobalBegin(dm, v, originalv));
679           PetscCall(DMPlexNaturalToGlobalEnd(dm, v, originalv));
680           PetscCall(DMRestoreGlobalVector(dm, &v));
681 #else
682           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
683 #endif
684         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
685       }
686     } else {
687       PetscCall(VecLoad_Default(originalv, viewer));
688     }
689   }
690   PetscFunctionReturn(0);
691 }
692 
693 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
694 {
695   PetscSection       coordSection;
696   Vec                coordinates;
697   DMLabel            depthLabel, celltypeLabel;
698   const char        *name[4];
699   const PetscScalar *a;
700   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
701 
702   PetscFunctionBegin;
703   PetscCall(DMGetDimension(dm, &dim));
704   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
705   PetscCall(DMGetCoordinateSection(dm, &coordSection));
706   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
707   PetscCall(DMPlexGetCellTypeLabel(dm, &celltypeLabel));
708   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
709   PetscCall(PetscSectionGetChart(coordSection, &pStart, &pEnd));
710   PetscCall(VecGetArrayRead(coordinates, &a));
711   name[0]     = "vertex";
712   name[1]     = "edge";
713   name[dim-1] = "face";
714   name[dim]   = "cell";
715   for (c = cStart; c < cEnd; ++c) {
716     PetscInt *closure = NULL;
717     PetscInt  closureSize, cl, ct;
718 
719     PetscCall(DMLabelGetValue(celltypeLabel, c, &ct));
720     PetscCall(PetscViewerASCIIPrintf(viewer, "Geometry for cell %" PetscInt_FMT " polytope type %s:\n", c, DMPolytopeTypes[ct]));
721     PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
722     PetscCall(PetscViewerASCIIPushTab(viewer));
723     for (cl = 0; cl < closureSize*2; cl += 2) {
724       PetscInt point = closure[cl], depth, dof, off, d, p;
725 
726       if ((point < pStart) || (point >= pEnd)) continue;
727       PetscCall(PetscSectionGetDof(coordSection, point, &dof));
728       if (!dof) continue;
729       PetscCall(DMLabelGetValue(depthLabel, point, &depth));
730       PetscCall(PetscSectionGetOffset(coordSection, point, &off));
731       PetscCall(PetscViewerASCIIPrintf(viewer, "%s %" PetscInt_FMT " coords:", name[depth], point));
732       for (p = 0; p < dof/dim; ++p) {
733         PetscCall(PetscViewerASCIIPrintf(viewer, " ("));
734         for (d = 0; d < dim; ++d) {
735           if (d > 0) PetscCall(PetscViewerASCIIPrintf(viewer, ", "));
736           PetscCall(PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d])));
737         }
738         PetscCall(PetscViewerASCIIPrintf(viewer, ")"));
739       }
740       PetscCall(PetscViewerASCIIPrintf(viewer, "\n"));
741     }
742     PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
743     PetscCall(PetscViewerASCIIPopTab(viewer));
744   }
745   PetscCall(VecRestoreArrayRead(coordinates, &a));
746   PetscFunctionReturn(0);
747 }
748 
749 typedef enum {CS_CARTESIAN, CS_POLAR, CS_CYLINDRICAL, CS_SPHERICAL} CoordSystem;
750 const char *CoordSystems[] = {"cartesian", "polar", "cylindrical", "spherical", "CoordSystem", "CS_", NULL};
751 
752 static PetscErrorCode DMPlexView_Ascii_Coordinates(PetscViewer viewer, CoordSystem cs, PetscInt dim, const PetscScalar x[])
753 {
754   PetscInt       i;
755 
756   PetscFunctionBegin;
757   if (dim > 3) {
758     for (i = 0; i < dim; ++i) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double) PetscRealPart(x[i])));
759   } else {
760     PetscReal coords[3], trcoords[3] = {0., 0., 0.};
761 
762     for (i = 0; i < dim; ++i) coords[i] = PetscRealPart(x[i]);
763     switch (cs) {
764       case CS_CARTESIAN: for (i = 0; i < dim; ++i) trcoords[i] = coords[i];break;
765       case CS_POLAR:
766         PetscCheck(dim == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Polar coordinates are for 2 dimension, not %" PetscInt_FMT, dim);
767         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
768         trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
769         break;
770       case CS_CYLINDRICAL:
771         PetscCheck(dim == 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cylindrical coordinates are for 3 dimension, not %" PetscInt_FMT, dim);
772         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
773         trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
774         trcoords[2] = coords[2];
775         break;
776       case CS_SPHERICAL:
777         PetscCheck(dim == 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Spherical coordinates are for 3 dimension, not %" PetscInt_FMT, dim);
778         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]) + PetscSqr(coords[2]));
779         trcoords[1] = PetscAtan2Real(PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1])), coords[2]);
780         trcoords[2] = PetscAtan2Real(coords[1], coords[0]);
781         break;
782     }
783     for (i = 0; i < dim; ++i) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double) trcoords[i]));
784   }
785   PetscFunctionReturn(0);
786 }
787 
788 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
789 {
790   DM_Plex          *mesh = (DM_Plex*) dm->data;
791   DM                cdm;
792   PetscSection      coordSection;
793   Vec               coordinates;
794   PetscViewerFormat format;
795 
796   PetscFunctionBegin;
797   PetscCall(DMGetCoordinateDM(dm, &cdm));
798   PetscCall(DMGetLocalSection(cdm, &coordSection));
799   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
800   PetscCall(PetscViewerGetFormat(viewer, &format));
801   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
802     const char *name;
803     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
804     PetscInt    pStart, pEnd, p, numLabels, l;
805     PetscMPIInt rank, size;
806 
807     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
808     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
809     PetscCall(PetscObjectGetName((PetscObject) dm, &name));
810     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
811     PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
812     PetscCall(DMGetDimension(dm, &dim));
813     PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
814     if (name) PetscCall(PetscViewerASCIIPrintf(viewer, "%s in %" PetscInt_FMT " dimension%s:\n", name, dim, dim == 1 ? "" : "s"));
815     else      PetscCall(PetscViewerASCIIPrintf(viewer, "Mesh in %" PetscInt_FMT " dimension%s:\n", dim, dim == 1 ? "" : "s"));
816     if (cellHeight) PetscCall(PetscViewerASCIIPrintf(viewer, "  Cells are at height %" PetscInt_FMT "\n", cellHeight));
817     PetscCall(PetscViewerASCIIPrintf(viewer, "Supports:\n"));
818     PetscCall(PetscViewerASCIIPushSynchronized(viewer));
819     PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %" PetscInt_FMT "\n", rank, maxSupportSize));
820     for (p = pStart; p < pEnd; ++p) {
821       PetscInt dof, off, s;
822 
823       PetscCall(PetscSectionGetDof(mesh->supportSection, p, &dof));
824       PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off));
825       for (s = off; s < off+dof; ++s) {
826         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %" PetscInt_FMT " ----> %" PetscInt_FMT "\n", rank, p, mesh->supports[s]));
827       }
828     }
829     PetscCall(PetscViewerFlush(viewer));
830     PetscCall(PetscViewerASCIIPrintf(viewer, "Cones:\n"));
831     PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %" PetscInt_FMT "\n", rank, maxConeSize));
832     for (p = pStart; p < pEnd; ++p) {
833       PetscInt dof, off, c;
834 
835       PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
836       PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
837       for (c = off; c < off+dof; ++c) {
838         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %" PetscInt_FMT " <---- %" PetscInt_FMT " (%" PetscInt_FMT ")\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]));
839       }
840     }
841     PetscCall(PetscViewerFlush(viewer));
842     PetscCall(PetscViewerASCIIPopSynchronized(viewer));
843     if (coordSection && coordinates) {
844       CoordSystem        cs = CS_CARTESIAN;
845       const PetscScalar *array;
846       PetscInt           Nf, Nc, pStart, pEnd, p;
847       PetscMPIInt        rank;
848       const char        *name;
849 
850       PetscCall(PetscOptionsGetEnum(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_coord_system", CoordSystems, (PetscEnum *) &cs, NULL));
851       PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)viewer), &rank));
852       PetscCall(PetscSectionGetNumFields(coordSection, &Nf));
853       PetscCheck(Nf == 1,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Coordinate section should have 1 field, not %" PetscInt_FMT, Nf);
854       PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &Nc));
855       PetscCall(PetscSectionGetChart(coordSection, &pStart, &pEnd));
856       PetscCall(PetscObjectGetName((PetscObject) coordinates, &name));
857       PetscCall(PetscViewerASCIIPrintf(viewer, "%s with %" PetscInt_FMT " fields\n", name, Nf));
858       PetscCall(PetscViewerASCIIPrintf(viewer, "  field 0 with %" PetscInt_FMT " components\n", Nc));
859       if (cs != CS_CARTESIAN) PetscCall(PetscViewerASCIIPrintf(viewer, "  output coordinate system: %s\n", CoordSystems[cs]));
860 
861       PetscCall(VecGetArrayRead(coordinates, &array));
862       PetscCall(PetscViewerASCIIPushSynchronized(viewer));
863       PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "Process %d:\n", rank));
864       for (p = pStart; p < pEnd; ++p) {
865         PetscInt dof, off;
866 
867         PetscCall(PetscSectionGetDof(coordSection, p, &dof));
868         PetscCall(PetscSectionGetOffset(coordSection, p, &off));
869         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "  (%4" PetscInt_FMT ") dim %2" PetscInt_FMT " offset %3" PetscInt_FMT, p, dof, off));
870         PetscCall(DMPlexView_Ascii_Coordinates(viewer, cs, dof, &array[off]));
871         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\n"));
872       }
873       PetscCall(PetscViewerFlush(viewer));
874       PetscCall(PetscViewerASCIIPopSynchronized(viewer));
875       PetscCall(VecRestoreArrayRead(coordinates, &array));
876     }
877     PetscCall(DMGetNumLabels(dm, &numLabels));
878     if (numLabels) PetscCall(PetscViewerASCIIPrintf(viewer, "Labels:\n"));
879     for (l = 0; l < numLabels; ++l) {
880       DMLabel     label;
881       PetscBool   isdepth;
882       const char *name;
883 
884       PetscCall(DMGetLabelName(dm, l, &name));
885       PetscCall(PetscStrcmp(name, "depth", &isdepth));
886       if (isdepth) continue;
887       PetscCall(DMGetLabel(dm, name, &label));
888       PetscCall(DMLabelView(label, viewer));
889     }
890     if (size > 1) {
891       PetscSF sf;
892 
893       PetscCall(DMGetPointSF(dm, &sf));
894       PetscCall(PetscSFView(sf, viewer));
895     }
896     PetscCall(PetscViewerFlush(viewer));
897   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
898     const char  *name, *color;
899     const char  *defcolors[3]  = {"gray", "orange", "green"};
900     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
901     char         lname[PETSC_MAX_PATH_LEN];
902     PetscReal    scale         = 2.0;
903     PetscReal    tikzscale     = 1.0;
904     PetscBool    useNumbers    = PETSC_TRUE, drawNumbers[4], drawColors[4], useLabels, useColors, plotEdges, drawHasse = PETSC_FALSE;
905     double       tcoords[3];
906     PetscScalar *coords;
907     PetscInt     numLabels, l, numColors, numLColors, dim, d, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p, n;
908     PetscMPIInt  rank, size;
909     char         **names, **colors, **lcolors;
910     PetscBool    flg, lflg;
911     PetscBT      wp = NULL;
912     PetscInt     pEnd, pStart;
913 
914     PetscCall(DMGetDimension(dm, &dim));
915     PetscCall(DMPlexGetDepth(dm, &depth));
916     PetscCall(DMGetNumLabels(dm, &numLabels));
917     numLabels  = PetscMax(numLabels, 10);
918     numColors  = 10;
919     numLColors = 10;
920     PetscCall(PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors));
921     PetscCall(PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL));
922     PetscCall(PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL));
923     PetscCall(PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL));
924     for (d = 0; d < 4; ++d) drawNumbers[d] = useNumbers;
925     for (d = 0; d < 4; ++d) drawColors[d]  = PETSC_TRUE;
926     n = 4;
927     PetscCall(PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers_depth", drawNumbers, &n, &flg));
928     PetscCheck(!flg || n == dim+1,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %" PetscInt_FMT " != %" PetscInt_FMT " dim+1", n, dim+1);
929     PetscCall(PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors_depth", drawColors, &n, &flg));
930     PetscCheck(!flg || n == dim+1,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %" PetscInt_FMT " != %" PetscInt_FMT " dim+1", n, dim+1);
931     PetscCall(PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels));
932     if (!useLabels) numLabels = 0;
933     PetscCall(PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors));
934     if (!useColors) {
935       numColors = 3;
936       for (c = 0; c < numColors; ++c) PetscCall(PetscStrallocpy(defcolors[c], &colors[c]));
937     }
938     PetscCall(PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors));
939     if (!useColors) {
940       numLColors = 4;
941       for (c = 0; c < numLColors; ++c) PetscCall(PetscStrallocpy(deflcolors[c], &lcolors[c]));
942     }
943     PetscCall(PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg));
944     plotEdges = (PetscBool)(depth > 1 && drawNumbers[1] && dim < 3);
945     PetscCall(PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg));
946     PetscCheck(!flg || !plotEdges || depth >= dim,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
947     if (depth < dim) plotEdges = PETSC_FALSE;
948     PetscCall(PetscOptionsGetBool(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_hasse", &drawHasse, NULL));
949 
950     /* filter points with labelvalue != labeldefaultvalue */
951     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
952     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
953     PetscCall(DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd));
954     PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
955     if (lflg) {
956       DMLabel lbl;
957 
958       PetscCall(DMGetLabel(dm, lname, &lbl));
959       if (lbl) {
960         PetscInt val, defval;
961 
962         PetscCall(DMLabelGetDefaultValue(lbl, &defval));
963         PetscCall(PetscBTCreate(pEnd-pStart, &wp));
964         for (c = pStart;  c < pEnd; c++) {
965           PetscInt *closure = NULL;
966           PetscInt  closureSize;
967 
968           PetscCall(DMLabelGetValue(lbl, c, &val));
969           if (val == defval) continue;
970 
971           PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
972           for (p = 0; p < closureSize*2; p += 2) {
973             PetscCall(PetscBTSet(wp, closure[p] - pStart));
974           }
975           PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
976         }
977       }
978     }
979 
980     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
981     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
982     PetscCall(PetscObjectGetName((PetscObject) dm, &name));
983     PetscCall(PetscViewerASCIIPrintf(viewer, "\
984 \\documentclass[tikz]{standalone}\n\n\
985 \\usepackage{pgflibraryshapes}\n\
986 \\usetikzlibrary{backgrounds}\n\
987 \\usetikzlibrary{arrows}\n\
988 \\begin{document}\n"));
989     if (size > 1) {
990       PetscCall(PetscViewerASCIIPrintf(viewer, "%s for process ", name));
991       for (p = 0; p < size; ++p) {
992         if (p) PetscCall(PetscViewerASCIIPrintf(viewer, (p == size-1) ? ", and " :  ", "));
993         PetscCall(PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%" PetscInt_FMT "}", colors[p%numColors], p));
994       }
995       PetscCall(PetscViewerASCIIPrintf(viewer, ".\n\n\n"));
996     }
997     if (drawHasse) {
998       PetscInt maxStratum = PetscMax(vEnd-vStart, PetscMax(eEnd-eStart, cEnd-cStart));
999 
1000       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vStart}{%" PetscInt_FMT "}\n", vStart));
1001       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vEnd}{%" PetscInt_FMT "}\n", vEnd-1));
1002       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numVertices}{%" PetscInt_FMT "}\n", vEnd-vStart));
1003       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vShift}{%.2f}\n", 3 + (maxStratum-(vEnd-vStart))/2.));
1004       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eStart}{%" PetscInt_FMT "}\n", eStart));
1005       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eEnd}{%" PetscInt_FMT "}\n", eEnd-1));
1006       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eShift}{%.2f}\n", 3 + (maxStratum-(eEnd-eStart))/2.));
1007       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numEdges}{%" PetscInt_FMT "}\n", eEnd-eStart));
1008       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cStart}{%" PetscInt_FMT "}\n", cStart));
1009       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cEnd}{%" PetscInt_FMT "}\n", cEnd-1));
1010       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numCells}{%" PetscInt_FMT "}\n", cEnd-cStart));
1011       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cShift}{%.2f}\n", 3 + (maxStratum-(cEnd-cStart))/2.));
1012     }
1013     PetscCall(PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale));
1014 
1015     /* Plot vertices */
1016     PetscCall(VecGetArray(coordinates, &coords));
1017     PetscCall(PetscViewerASCIIPushSynchronized(viewer));
1018     for (v = vStart; v < vEnd; ++v) {
1019       PetscInt  off, dof, d;
1020       PetscBool isLabeled = PETSC_FALSE;
1021 
1022       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
1023       PetscCall(PetscSectionGetDof(coordSection, v, &dof));
1024       PetscCall(PetscSectionGetOffset(coordSection, v, &off));
1025       PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\path ("));
1026       PetscCheck(dof <= 3,PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %" PetscInt_FMT " has dof %" PetscInt_FMT " > 3",v,dof);
1027       for (d = 0; d < dof; ++d) {
1028         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
1029         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1030       }
1031       /* Rotate coordinates since PGF makes z point out of the page instead of up */
1032       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1033       for (d = 0; d < dof; ++d) {
1034         if (d > 0) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ","));
1035         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]));
1036       }
1037       if (drawHasse) color = colors[0%numColors];
1038       else           color = colors[rank%numColors];
1039       for (l = 0; l < numLabels; ++l) {
1040         PetscInt val;
1041         PetscCall(DMGetLabelValue(dm, names[l], v, &val));
1042         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
1043       }
1044       if (drawNumbers[0]) {
1045         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [draw,shape=circle,color=%s] {%" PetscInt_FMT "};\n", v, rank, color, v));
1046       } else if (drawColors[0]) {
1047         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color));
1048       } else {
1049         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [] {};\n", v, rank));
1050       }
1051     }
1052     PetscCall(VecRestoreArray(coordinates, &coords));
1053     PetscCall(PetscViewerFlush(viewer));
1054     /* Plot edges */
1055     if (plotEdges) {
1056       PetscCall(VecGetArray(coordinates, &coords));
1057       PetscCall(PetscViewerASCIIPrintf(viewer, "\\path\n"));
1058       for (e = eStart; e < eEnd; ++e) {
1059         const PetscInt *cone;
1060         PetscInt        coneSize, offA, offB, dof, d;
1061 
1062         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
1063         PetscCall(DMPlexGetConeSize(dm, e, &coneSize));
1064         PetscCheck(coneSize == 2,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " cone should have two vertices, not %" PetscInt_FMT, e, coneSize);
1065         PetscCall(DMPlexGetCone(dm, e, &cone));
1066         PetscCall(PetscSectionGetDof(coordSection, cone[0], &dof));
1067         PetscCall(PetscSectionGetOffset(coordSection, cone[0], &offA));
1068         PetscCall(PetscSectionGetOffset(coordSection, cone[1], &offB));
1069         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "("));
1070         for (d = 0; d < dof; ++d) {
1071           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
1072           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1073         }
1074         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1075         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1076         for (d = 0; d < dof; ++d) {
1077           if (d > 0) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ","));
1078           PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]));
1079         }
1080         if (drawHasse) color = colors[1%numColors];
1081         else           color = colors[rank%numColors];
1082         for (l = 0; l < numLabels; ++l) {
1083           PetscInt val;
1084           PetscCall(DMGetLabelValue(dm, names[l], v, &val));
1085           if (val >= 0) {color = lcolors[l%numLColors]; break;}
1086         }
1087         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [draw,shape=circle,color=%s] {%" PetscInt_FMT "} --\n", e, rank, color, e));
1088       }
1089       PetscCall(VecRestoreArray(coordinates, &coords));
1090       PetscCall(PetscViewerFlush(viewer));
1091       PetscCall(PetscViewerASCIIPrintf(viewer, "(0,0);\n"));
1092     }
1093     /* Plot cells */
1094     if (dim == 3 || !drawNumbers[1]) {
1095       for (e = eStart; e < eEnd; ++e) {
1096         const PetscInt *cone;
1097 
1098         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
1099         color = colors[rank%numColors];
1100         for (l = 0; l < numLabels; ++l) {
1101           PetscInt val;
1102           PetscCall(DMGetLabelValue(dm, names[l], e, &val));
1103           if (val >= 0) {color = lcolors[l%numLColors]; break;}
1104         }
1105         PetscCall(DMPlexGetCone(dm, e, &cone));
1106         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%" PetscInt_FMT "_%d) -- (%" PetscInt_FMT "_%d);\n", color, cone[0], rank, cone[1], rank));
1107       }
1108     } else {
1109        DMPolytopeType ct;
1110 
1111       /* Drawing a 2D polygon */
1112       for (c = cStart; c < cEnd; ++c) {
1113         if (wp && !PetscBTLookup(wp, c - pStart)) continue;
1114         PetscCall(DMPlexGetCellType(dm, c, &ct));
1115         if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR ||
1116             ct == DM_POLYTOPE_TRI_PRISM_TENSOR ||
1117             ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
1118           const PetscInt *cone;
1119           PetscInt        coneSize, e;
1120 
1121           PetscCall(DMPlexGetCone(dm, c, &cone));
1122           PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
1123           for (e = 0; e < coneSize; ++e) {
1124             const PetscInt *econe;
1125 
1126             PetscCall(DMPlexGetCone(dm, cone[e], &econe));
1127             PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%" PetscInt_FMT "_%d) -- (%" PetscInt_FMT "_%d) -- (%" PetscInt_FMT "_%d);\n", colors[rank%numColors], econe[0], rank, cone[e], rank, econe[1], rank));
1128           }
1129         } else {
1130           PetscInt *closure = NULL;
1131           PetscInt  closureSize, Nv = 0, v;
1132 
1133           PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
1134           for (p = 0; p < closureSize*2; p += 2) {
1135             const PetscInt point = closure[p];
1136 
1137             if ((point >= vStart) && (point < vEnd)) closure[Nv++] = point;
1138           }
1139           PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]));
1140           for (v = 0; v <= Nv; ++v) {
1141             const PetscInt vertex = closure[v%Nv];
1142 
1143             if (v > 0) {
1144               if (plotEdges) {
1145                 const PetscInt *edge;
1146                 PetscInt        endpoints[2], ne;
1147 
1148                 endpoints[0] = closure[v-1]; endpoints[1] = vertex;
1149                 PetscCall(DMPlexGetJoin(dm, 2, endpoints, &ne, &edge));
1150                 PetscCheck(ne == 1,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find edge for vertices %" PetscInt_FMT ", %" PetscInt_FMT, endpoints[0], endpoints[1]);
1151                 PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, " -- (%" PetscInt_FMT "_%d) -- ", edge[0], rank));
1152                 PetscCall(DMPlexRestoreJoin(dm, 2, endpoints, &ne, &edge));
1153               } else {
1154                 PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, " -- "));
1155               }
1156             }
1157             PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "(%" PetscInt_FMT "_%d)", vertex, rank));
1158           }
1159           PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ";\n"));
1160           PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
1161         }
1162       }
1163     }
1164     PetscCall(VecGetArray(coordinates, &coords));
1165     for (c = cStart; c < cEnd; ++c) {
1166       double    ccoords[3] = {0.0, 0.0, 0.0};
1167       PetscBool isLabeled  = PETSC_FALSE;
1168       PetscInt *closure    = NULL;
1169       PetscInt  closureSize, dof, d, n = 0;
1170 
1171       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
1172       PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
1173       PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\path ("));
1174       for (p = 0; p < closureSize*2; p += 2) {
1175         const PetscInt point = closure[p];
1176         PetscInt       off;
1177 
1178         if ((point < vStart) || (point >= vEnd)) continue;
1179         PetscCall(PetscSectionGetDof(coordSection, point, &dof));
1180         PetscCall(PetscSectionGetOffset(coordSection, point, &off));
1181         for (d = 0; d < dof; ++d) {
1182           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
1183           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1184         }
1185         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1186         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1187         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
1188         ++n;
1189       }
1190       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
1191       PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
1192       for (d = 0; d < dof; ++d) {
1193         if (d > 0) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ","));
1194         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]));
1195       }
1196       if (drawHasse) color = colors[depth%numColors];
1197       else           color = colors[rank%numColors];
1198       for (l = 0; l < numLabels; ++l) {
1199         PetscInt val;
1200         PetscCall(DMGetLabelValue(dm, names[l], c, &val));
1201         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
1202       }
1203       if (drawNumbers[dim]) {
1204         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [draw,shape=circle,color=%s] {%" PetscInt_FMT "};\n", c, rank, color, c));
1205       } else if (drawColors[dim]) {
1206         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color));
1207       } else {
1208         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [] {};\n", c, rank));
1209       }
1210     }
1211     PetscCall(VecRestoreArray(coordinates, &coords));
1212     if (drawHasse) {
1213       color = colors[depth%numColors];
1214       PetscCall(PetscViewerASCIIPrintf(viewer, "%% Cells\n"));
1215       PetscCall(PetscViewerASCIIPrintf(viewer, "\\foreach \\c in {\\cStart,...,\\cEnd}\n"));
1216       PetscCall(PetscViewerASCIIPrintf(viewer, "{\n"));
1217       PetscCall(PetscViewerASCIIPrintf(viewer, "  \\node(\\c_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\cShift+\\c-\\cStart,0) {\\c};\n", rank, color));
1218       PetscCall(PetscViewerASCIIPrintf(viewer, "}\n"));
1219 
1220       color = colors[1%numColors];
1221       PetscCall(PetscViewerASCIIPrintf(viewer, "%% Edges\n"));
1222       PetscCall(PetscViewerASCIIPrintf(viewer, "\\foreach \\e in {\\eStart,...,\\eEnd}\n"));
1223       PetscCall(PetscViewerASCIIPrintf(viewer, "{\n"));
1224       PetscCall(PetscViewerASCIIPrintf(viewer, "  \\node(\\e_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\eShift+\\e-\\eStart,1) {\\e};\n", rank, color));
1225       PetscCall(PetscViewerASCIIPrintf(viewer, "}\n"));
1226 
1227       color = colors[0%numColors];
1228       PetscCall(PetscViewerASCIIPrintf(viewer, "%% Vertices\n"));
1229       PetscCall(PetscViewerASCIIPrintf(viewer, "\\foreach \\v in {\\vStart,...,\\vEnd}\n"));
1230       PetscCall(PetscViewerASCIIPrintf(viewer, "{\n"));
1231       PetscCall(PetscViewerASCIIPrintf(viewer, "  \\node(\\v_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\vShift+\\v-\\vStart,2) {\\v};\n", rank, color));
1232       PetscCall(PetscViewerASCIIPrintf(viewer, "}\n"));
1233 
1234       for (p = pStart; p < pEnd; ++p) {
1235         const PetscInt *cone;
1236         PetscInt        coneSize, cp;
1237 
1238         PetscCall(DMPlexGetCone(dm, p, &cone));
1239         PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
1240         for (cp = 0; cp < coneSize; ++cp) {
1241           PetscCall(PetscViewerASCIIPrintf(viewer, "\\draw[->, shorten >=1pt] (%" PetscInt_FMT "_%d) -- (%" PetscInt_FMT "_%d);\n", cone[cp], rank, p, rank));
1242         }
1243       }
1244     }
1245     PetscCall(PetscViewerFlush(viewer));
1246     PetscCall(PetscViewerASCIIPopSynchronized(viewer));
1247     PetscCall(PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n"));
1248     PetscCall(PetscViewerASCIIPrintf(viewer, "\\end{document}\n"));
1249     for (l = 0; l < numLabels;  ++l) PetscCall(PetscFree(names[l]));
1250     for (c = 0; c < numColors;  ++c) PetscCall(PetscFree(colors[c]));
1251     for (c = 0; c < numLColors; ++c) PetscCall(PetscFree(lcolors[c]));
1252     PetscCall(PetscFree3(names, colors, lcolors));
1253     PetscCall(PetscBTDestroy(&wp));
1254   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
1255     Vec                    cown,acown;
1256     VecScatter             sct;
1257     ISLocalToGlobalMapping g2l;
1258     IS                     gid,acis;
1259     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
1260     MPI_Group              ggroup,ngroup;
1261     PetscScalar            *array,nid;
1262     const PetscInt         *idxs;
1263     PetscInt               *idxs2,*start,*adjacency,*work;
1264     PetscInt64             lm[3],gm[3];
1265     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
1266     PetscMPIInt            d1,d2,rank;
1267 
1268     PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
1269     PetscCallMPI(MPI_Comm_rank(comm,&rank));
1270 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1271     PetscCallMPI(MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm));
1272 #endif
1273     if (ncomm != MPI_COMM_NULL) {
1274       PetscCallMPI(MPI_Comm_group(comm,&ggroup));
1275       PetscCallMPI(MPI_Comm_group(ncomm,&ngroup));
1276       d1   = 0;
1277       PetscCallMPI(MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2));
1278       nid  = d2;
1279       PetscCallMPI(MPI_Group_free(&ggroup));
1280       PetscCallMPI(MPI_Group_free(&ngroup));
1281       PetscCallMPI(MPI_Comm_free(&ncomm));
1282     } else nid = 0.0;
1283 
1284     /* Get connectivity */
1285     PetscCall(DMPlexGetVTKCellHeight(dm,&cellHeight));
1286     PetscCall(DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid));
1287 
1288     /* filter overlapped local cells */
1289     PetscCall(DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd));
1290     PetscCall(ISGetIndices(gid,&idxs));
1291     PetscCall(ISGetLocalSize(gid,&cum));
1292     PetscCall(PetscMalloc1(cum,&idxs2));
1293     for (c = cStart, cum = 0; c < cEnd; c++) {
1294       if (idxs[c-cStart] < 0) continue;
1295       idxs2[cum++] = idxs[c-cStart];
1296     }
1297     PetscCall(ISRestoreIndices(gid,&idxs));
1298     PetscCheck(numVertices == cum,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %" PetscInt_FMT " != %" PetscInt_FMT,numVertices,cum);
1299     PetscCall(ISDestroy(&gid));
1300     PetscCall(ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid));
1301 
1302     /* support for node-aware cell locality */
1303     PetscCall(ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis));
1304     PetscCall(VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown));
1305     PetscCall(VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown));
1306     PetscCall(VecGetArray(cown,&array));
1307     for (c = 0; c < numVertices; c++) array[c] = nid;
1308     PetscCall(VecRestoreArray(cown,&array));
1309     PetscCall(VecScatterCreate(cown,acis,acown,NULL,&sct));
1310     PetscCall(VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD));
1311     PetscCall(VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD));
1312     PetscCall(ISDestroy(&acis));
1313     PetscCall(VecScatterDestroy(&sct));
1314     PetscCall(VecDestroy(&cown));
1315 
1316     /* compute edgeCut */
1317     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1318     PetscCall(PetscMalloc1(cum,&work));
1319     PetscCall(ISLocalToGlobalMappingCreateIS(gid,&g2l));
1320     PetscCall(ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH));
1321     PetscCall(ISDestroy(&gid));
1322     PetscCall(VecGetArray(acown,&array));
1323     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1324       PetscInt totl;
1325 
1326       totl = start[c+1]-start[c];
1327       PetscCall(ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work));
1328       for (i = 0; i < totl; i++) {
1329         if (work[i] < 0) {
1330           ect  += 1;
1331           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1332         }
1333       }
1334     }
1335     PetscCall(PetscFree(work));
1336     PetscCall(VecRestoreArray(acown,&array));
1337     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1338     lm[1] = -numVertices;
1339     PetscCall(MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm));
1340     PetscCall(PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %" PetscInt_FMT ", min %" PetscInt_FMT,-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]));
1341     lm[0] = ect; /* edgeCut */
1342     lm[1] = ectn; /* node-aware edgeCut */
1343     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1344     PetscCall(MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm));
1345     PetscCall(PetscViewerASCIIPrintf(viewer,", empty %" PetscInt_FMT ")\n",(PetscInt)gm[2]));
1346 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1347     PetscCall(PetscViewerASCIIPrintf(viewer,"  Edge Cut: %" PetscInt_FMT " (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.));
1348 #else
1349     PetscCall(PetscViewerASCIIPrintf(viewer,"  Edge Cut: %" PetscInt_FMT " (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0));
1350 #endif
1351     PetscCall(ISLocalToGlobalMappingDestroy(&g2l));
1352     PetscCall(PetscFree(start));
1353     PetscCall(PetscFree(adjacency));
1354     PetscCall(VecDestroy(&acown));
1355   } else {
1356     const char    *name;
1357     PetscInt      *sizes, *hybsizes, *ghostsizes;
1358     PetscInt       locDepth, depth, cellHeight, dim, d;
1359     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1360     PetscInt       numLabels, l, maxSize = 17;
1361     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1362     MPI_Comm       comm;
1363     PetscMPIInt    size, rank;
1364 
1365     PetscCall(PetscObjectGetComm((PetscObject) dm, &comm));
1366     PetscCallMPI(MPI_Comm_size(comm, &size));
1367     PetscCallMPI(MPI_Comm_rank(comm, &rank));
1368     PetscCall(DMGetDimension(dm, &dim));
1369     PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
1370     PetscCall(PetscObjectGetName((PetscObject) dm, &name));
1371     if (name) PetscCall(PetscViewerASCIIPrintf(viewer, "%s in %" PetscInt_FMT " dimension%s:\n", name, dim, dim == 1 ? "" : "s"));
1372     else      PetscCall(PetscViewerASCIIPrintf(viewer, "Mesh in %" PetscInt_FMT " dimension%s:\n", dim, dim == 1 ? "" : "s"));
1373     if (cellHeight) PetscCall(PetscViewerASCIIPrintf(viewer, "  Cells are at height %" PetscInt_FMT "\n", cellHeight));
1374     PetscCall(DMPlexGetDepth(dm, &locDepth));
1375     PetscCall(MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm));
1376     PetscCall(DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd));
1377     gcNum = gcEnd - gcStart;
1378     if (size < maxSize) PetscCall(PetscCalloc3(size, &sizes, size, &hybsizes, size, &ghostsizes));
1379     else                PetscCall(PetscCalloc3(3,    &sizes, 3,    &hybsizes, 3,    &ghostsizes));
1380     for (d = 0; d <= depth; d++) {
1381       PetscInt Nc[2] = {0, 0}, ict;
1382 
1383       PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
1384       if (pStart < pEnd) PetscCall(DMPlexGetCellType(dm, pStart, &ct0));
1385       ict  = ct0;
1386       PetscCallMPI(MPI_Bcast(&ict, 1, MPIU_INT, 0, comm));
1387       ct0  = (DMPolytopeType) ict;
1388       for (p = pStart; p < pEnd; ++p) {
1389         DMPolytopeType ct;
1390 
1391         PetscCall(DMPlexGetCellType(dm, p, &ct));
1392         if (ct == ct0) ++Nc[0];
1393         else           ++Nc[1];
1394       }
1395       if (size < maxSize) {
1396         PetscCallMPI(MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm));
1397         PetscCallMPI(MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm));
1398         if (d == depth) PetscCallMPI(MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm));
1399         PetscCall(PetscViewerASCIIPrintf(viewer, "  Number of %" PetscInt_FMT "-cells per rank:", (depth == 1) && d ? dim : d));
1400         for (p = 0; p < size; ++p) {
1401           if (rank == 0) {
1402             PetscCall(PetscViewerASCIIPrintf(viewer, " %" PetscInt_FMT, sizes[p]+hybsizes[p]));
1403             if (hybsizes[p]   > 0) PetscCall(PetscViewerASCIIPrintf(viewer, " (%" PetscInt_FMT ")", hybsizes[p]));
1404             if (ghostsizes[p] > 0) PetscCall(PetscViewerASCIIPrintf(viewer, " [%" PetscInt_FMT "]", ghostsizes[p]));
1405           }
1406         }
1407       } else {
1408         PetscInt locMinMax[2];
1409 
1410         locMinMax[0] = Nc[0]+Nc[1]; locMinMax[1] = Nc[0]+Nc[1];
1411         PetscCall(PetscGlobalMinMaxInt(comm, locMinMax, sizes));
1412         locMinMax[0] = Nc[1]; locMinMax[1] = Nc[1];
1413         PetscCall(PetscGlobalMinMaxInt(comm, locMinMax, hybsizes));
1414         if (d == depth) {
1415           locMinMax[0] = gcNum; locMinMax[1] = gcNum;
1416           PetscCall(PetscGlobalMinMaxInt(comm, locMinMax, ghostsizes));
1417         }
1418         PetscCall(PetscViewerASCIIPrintf(viewer, "  Min/Max of %" PetscInt_FMT "-cells per rank:", (depth == 1) && d ? dim : d));
1419         PetscCall(PetscViewerASCIIPrintf(viewer, " %" PetscInt_FMT "/%" PetscInt_FMT, sizes[0], sizes[1]));
1420         if (hybsizes[0]   > 0) PetscCall(PetscViewerASCIIPrintf(viewer, " (%" PetscInt_FMT "/%" PetscInt_FMT ")", hybsizes[0], hybsizes[1]));
1421         if (ghostsizes[0] > 0) PetscCall(PetscViewerASCIIPrintf(viewer, " [%" PetscInt_FMT "/%" PetscInt_FMT "]", ghostsizes[0], ghostsizes[1]));
1422       }
1423       PetscCall(PetscViewerASCIIPrintf(viewer, "\n"));
1424     }
1425     PetscCall(PetscFree3(sizes, hybsizes, ghostsizes));
1426     {
1427       const PetscReal      *maxCell;
1428       const PetscReal      *L;
1429       const DMBoundaryType *bd;
1430       PetscBool             per, localized;
1431 
1432       PetscCall(DMGetPeriodicity(dm, &per, &maxCell, &L, &bd));
1433       PetscCall(DMGetCoordinatesLocalized(dm, &localized));
1434       if (per) {
1435         PetscCall(PetscViewerASCIIPrintf(viewer, "Periodic mesh ("));
1436         PetscCall(PetscViewerASCIIUseTabs(viewer, PETSC_FALSE));
1437         for (d = 0; d < dim; ++d) {
1438           if (bd && d > 0) PetscCall(PetscViewerASCIIPrintf(viewer, ", "));
1439           if (bd)    PetscCall(PetscViewerASCIIPrintf(viewer, "%s", DMBoundaryTypes[bd[d]]));
1440         }
1441         PetscCall(PetscViewerASCIIPrintf(viewer, ") coordinates %s\n", localized ? "localized" : "not localized"));
1442         PetscCall(PetscViewerASCIIUseTabs(viewer, PETSC_TRUE));
1443       }
1444     }
1445     PetscCall(DMGetNumLabels(dm, &numLabels));
1446     if (numLabels) PetscCall(PetscViewerASCIIPrintf(viewer, "Labels:\n"));
1447     for (l = 0; l < numLabels; ++l) {
1448       DMLabel         label;
1449       const char     *name;
1450       IS              valueIS;
1451       const PetscInt *values;
1452       PetscInt        numValues, v;
1453 
1454       PetscCall(DMGetLabelName(dm, l, &name));
1455       PetscCall(DMGetLabel(dm, name, &label));
1456       PetscCall(DMLabelGetNumValues(label, &numValues));
1457       PetscCall(PetscViewerASCIIPrintf(viewer, "  %s: %" PetscInt_FMT " strata with value/size (", name, numValues));
1458       PetscCall(DMLabelGetValueIS(label, &valueIS));
1459       PetscCall(ISGetIndices(valueIS, &values));
1460       PetscCall(PetscViewerASCIIUseTabs(viewer, PETSC_FALSE));
1461       for (v = 0; v < numValues; ++v) {
1462         PetscInt size;
1463 
1464         PetscCall(DMLabelGetStratumSize(label, values[v], &size));
1465         if (v > 0) PetscCall(PetscViewerASCIIPrintf(viewer, ", "));
1466         PetscCall(PetscViewerASCIIPrintf(viewer, "%" PetscInt_FMT " (%" PetscInt_FMT ")", values[v], size));
1467       }
1468       PetscCall(PetscViewerASCIIPrintf(viewer, ")\n"));
1469       PetscCall(PetscViewerASCIIUseTabs(viewer, PETSC_TRUE));
1470       PetscCall(ISRestoreIndices(valueIS, &values));
1471       PetscCall(ISDestroy(&valueIS));
1472     }
1473     {
1474       char    **labelNames;
1475       PetscInt  Nl = numLabels;
1476       PetscBool flg;
1477 
1478       PetscCall(PetscMalloc1(Nl, &labelNames));
1479       PetscCall(PetscOptionsGetStringArray(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_plex_view_labels", labelNames, &Nl, &flg));
1480       for (l = 0; l < Nl; ++l) {
1481         DMLabel label;
1482 
1483         PetscCall(DMHasLabel(dm, labelNames[l], &flg));
1484         if (flg) {
1485           PetscCall(DMGetLabel(dm, labelNames[l], &label));
1486           PetscCall(DMLabelView(label, viewer));
1487         }
1488         PetscCall(PetscFree(labelNames[l]));
1489       }
1490       PetscCall(PetscFree(labelNames));
1491     }
1492     /* If no fields are specified, people do not want to see adjacency */
1493     if (dm->Nf) {
1494       PetscInt f;
1495 
1496       for (f = 0; f < dm->Nf; ++f) {
1497         const char *name;
1498 
1499         PetscCall(PetscObjectGetName(dm->fields[f].disc, &name));
1500         if (numLabels) PetscCall(PetscViewerASCIIPrintf(viewer, "Field %s:\n", name));
1501         PetscCall(PetscViewerASCIIPushTab(viewer));
1502         if (dm->fields[f].label) PetscCall(DMLabelView(dm->fields[f].label, viewer));
1503         if (dm->fields[f].adjacency[0]) {
1504           if (dm->fields[f].adjacency[1]) PetscCall(PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n"));
1505           else                            PetscCall(PetscViewerASCIIPrintf(viewer, "adjacency FVM\n"));
1506         } else {
1507           if (dm->fields[f].adjacency[1]) PetscCall(PetscViewerASCIIPrintf(viewer, "adjacency FEM\n"));
1508           else                            PetscCall(PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n"));
1509         }
1510         PetscCall(PetscViewerASCIIPopTab(viewer));
1511       }
1512     }
1513     PetscCall(DMGetCoarseDM(dm, &cdm));
1514     if (cdm) {
1515       PetscCall(PetscViewerASCIIPushTab(viewer));
1516       PetscCall(DMPlexView_Ascii(cdm, viewer));
1517       PetscCall(PetscViewerASCIIPopTab(viewer));
1518     }
1519   }
1520   PetscFunctionReturn(0);
1521 }
1522 
1523 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1524 {
1525   DMPolytopeType ct;
1526   PetscMPIInt    rank;
1527   PetscInt       cdim;
1528 
1529   PetscFunctionBegin;
1530   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
1531   PetscCall(DMPlexGetCellType(dm, cell, &ct));
1532   PetscCall(DMGetCoordinateDim(dm, &cdim));
1533   switch (ct) {
1534   case DM_POLYTOPE_SEGMENT:
1535   case DM_POLYTOPE_POINT_PRISM_TENSOR:
1536     switch (cdim) {
1537     case 1:
1538     {
1539       const PetscReal y  = 0.5;  /* TODO Put it in the middle of the viewport */
1540       const PetscReal dy = 0.05; /* TODO Make it a fraction of the total length */
1541 
1542       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), y,    PetscRealPart(coords[1]), y,    PETSC_DRAW_BLACK));
1543       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), y+dy, PetscRealPart(coords[0]), y-dy, PETSC_DRAW_BLACK));
1544       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[1]), y+dy, PetscRealPart(coords[1]), y-dy, PETSC_DRAW_BLACK));
1545     }
1546     break;
1547     case 2:
1548     {
1549       const PetscReal dx = (PetscRealPart(coords[3]) - PetscRealPart(coords[1]));
1550       const PetscReal dy = (PetscRealPart(coords[2]) - PetscRealPart(coords[0]));
1551       const PetscReal l  = 0.1/PetscSqrtReal(dx*dx + dy*dy);
1552 
1553       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK));
1554       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0])+l*dx, PetscRealPart(coords[1])+l*dy, PetscRealPart(coords[0])-l*dx, PetscRealPart(coords[1])-l*dy, PETSC_DRAW_BLACK));
1555       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[2])+l*dx, PetscRealPart(coords[3])+l*dy, PetscRealPart(coords[2])-l*dx, PetscRealPart(coords[3])-l*dy, PETSC_DRAW_BLACK));
1556     }
1557     break;
1558     default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of dimension %" PetscInt_FMT, cdim);
1559     }
1560     break;
1561   case DM_POLYTOPE_TRIANGLE:
1562     PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1563                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1564                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1565                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2));
1566     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK));
1567     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK));
1568     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK));
1569     break;
1570   case DM_POLYTOPE_QUADRILATERAL:
1571     PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1572                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1573                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1574                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2));
1575     PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1576                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1577                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1578                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2));
1579     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK));
1580     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK));
1581     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK));
1582     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK));
1583     break;
1584   default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1585   }
1586   PetscFunctionReturn(0);
1587 }
1588 
1589 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1590 {
1591   DMPolytopeType ct;
1592   PetscReal      centroid[2] = {0., 0.};
1593   PetscMPIInt    rank;
1594   PetscInt       fillColor, v, e, d;
1595 
1596   PetscFunctionBegin;
1597   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
1598   PetscCall(DMPlexGetCellType(dm, cell, &ct));
1599   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2;
1600   switch (ct) {
1601   case DM_POLYTOPE_TRIANGLE:
1602     {
1603       PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1604 
1605       for (v = 0; v < 3; ++v) {centroid[0] += PetscRealPart(coords[v*2+0])/3.;centroid[1] += PetscRealPart(coords[v*2+1])/3.;}
1606       for (e = 0; e < 3; ++e) {
1607         refCoords[0] = refVertices[e*2+0];
1608         refCoords[1] = refVertices[e*2+1];
1609         for (d = 1; d <= edgeDiv; ++d) {
1610           refCoords[d*2+0] = refCoords[0] + (refVertices[(e+1)%3 * 2 + 0] - refCoords[0])*d/edgeDiv;
1611           refCoords[d*2+1] = refCoords[1] + (refVertices[(e+1)%3 * 2 + 1] - refCoords[1])*d/edgeDiv;
1612         }
1613         PetscCall(DMPlexReferenceToCoordinates(dm, cell, edgeDiv+1, refCoords, edgeCoords));
1614         for (d = 0; d < edgeDiv; ++d) {
1615           PetscCall(PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], fillColor, fillColor, fillColor));
1616           PetscCall(PetscDrawLine(draw, edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], PETSC_DRAW_BLACK));
1617         }
1618       }
1619     }
1620     break;
1621   default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1622   }
1623   PetscFunctionReturn(0);
1624 }
1625 
1626 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1627 {
1628   PetscDraw          draw;
1629   DM                 cdm;
1630   PetscSection       coordSection;
1631   Vec                coordinates;
1632   const PetscScalar *coords;
1633   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1634   PetscReal         *refCoords, *edgeCoords;
1635   PetscBool          isnull, drawAffine = PETSC_TRUE;
1636   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N, edgeDiv = 4;
1637 
1638   PetscFunctionBegin;
1639   PetscCall(DMGetCoordinateDim(dm, &dim));
1640   PetscCheck(dim <= 2,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %" PetscInt_FMT, dim);
1641   PetscCall(PetscOptionsGetBool(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL));
1642   if (!drawAffine) PetscCall(PetscMalloc2((edgeDiv+1)*dim, &refCoords, (edgeDiv+1)*dim, &edgeCoords));
1643   PetscCall(DMGetCoordinateDM(dm, &cdm));
1644   PetscCall(DMGetLocalSection(cdm, &coordSection));
1645   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
1646   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1647   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
1648 
1649   PetscCall(PetscViewerDrawGetDraw(viewer, 0, &draw));
1650   PetscCall(PetscDrawIsNull(draw, &isnull));
1651   if (isnull) PetscFunctionReturn(0);
1652   PetscCall(PetscDrawSetTitle(draw, "Mesh"));
1653 
1654   PetscCall(VecGetLocalSize(coordinates, &N));
1655   PetscCall(VecGetArrayRead(coordinates, &coords));
1656   for (c = 0; c < N; c += dim) {
1657     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1658     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1659   }
1660   PetscCall(VecRestoreArrayRead(coordinates, &coords));
1661   PetscCall(MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm)));
1662   PetscCall(MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm)));
1663   PetscCall(PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]));
1664   PetscCall(PetscDrawClear(draw));
1665 
1666   for (c = cStart; c < cEnd; ++c) {
1667     PetscScalar *coords = NULL;
1668     PetscInt     numCoords;
1669 
1670     PetscCall(DMPlexVecGetClosureAtDepth_Internal(dm, coordSection, coordinates, c, 0, &numCoords, &coords));
1671     if (drawAffine) {
1672       PetscCall(DMPlexDrawCell(dm, draw, c, coords));
1673     } else {
1674       PetscCall(DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords));
1675     }
1676     PetscCall(DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords));
1677   }
1678   if (!drawAffine) PetscCall(PetscFree2(refCoords, edgeCoords));
1679   PetscCall(PetscDrawFlush(draw));
1680   PetscCall(PetscDrawPause(draw));
1681   PetscCall(PetscDrawSave(draw));
1682   PetscFunctionReturn(0);
1683 }
1684 
1685 #if defined(PETSC_HAVE_EXODUSII)
1686 #include <exodusII.h>
1687 #include <petscviewerexodusii.h>
1688 #endif
1689 
1690 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1691 {
1692   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1693   char           name[PETSC_MAX_PATH_LEN];
1694 
1695   PetscFunctionBegin;
1696   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1697   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1698   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii));
1699   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk));
1700   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5));
1701   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw));
1702   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis));
1703   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus));
1704   if (iascii) {
1705     PetscViewerFormat format;
1706     PetscCall(PetscViewerGetFormat(viewer, &format));
1707     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1708       PetscCall(DMPlexView_GLVis(dm, viewer));
1709     } else {
1710       PetscCall(DMPlexView_Ascii(dm, viewer));
1711     }
1712   } else if (ishdf5) {
1713 #if defined(PETSC_HAVE_HDF5)
1714     PetscCall(DMPlexView_HDF5_Internal(dm, viewer));
1715 #else
1716     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1717 #endif
1718   } else if (isvtk) {
1719     PetscCall(DMPlexVTKWriteAll((PetscObject) dm,viewer));
1720   } else if (isdraw) {
1721     PetscCall(DMPlexView_Draw(dm, viewer));
1722   } else if (isglvis) {
1723     PetscCall(DMPlexView_GLVis(dm, viewer));
1724 #if defined(PETSC_HAVE_EXODUSII)
1725   } else if (isexodus) {
1726 /*
1727       exodusII requires that all sets be part of exactly one cell set.
1728       If the dm does not have a "Cell Sets" label defined, we create one
1729       with ID 1, containig all cells.
1730       Note that if the Cell Sets label is defined but does not cover all cells,
1731       we may still have a problem. This should probably be checked here or in the viewer;
1732     */
1733     PetscInt numCS;
1734     PetscCall(DMGetLabelSize(dm,"Cell Sets",&numCS));
1735     if (!numCS) {
1736       PetscInt cStart, cEnd, c;
1737       PetscCall(DMCreateLabel(dm, "Cell Sets"));
1738       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
1739       for (c = cStart; c < cEnd; ++c) PetscCall(DMSetLabelValue(dm, "Cell Sets", c, 1));
1740     }
1741     PetscCall(DMView_PlexExodusII(dm, viewer));
1742 #endif
1743   } else {
1744     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1745   }
1746   /* Optionally view the partition */
1747   PetscCall(PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg));
1748   if (flg) {
1749     Vec ranks;
1750     PetscCall(DMPlexCreateRankField(dm, &ranks));
1751     PetscCall(VecView(ranks, viewer));
1752     PetscCall(VecDestroy(&ranks));
1753   }
1754   /* Optionally view a label */
1755   PetscCall(PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, sizeof(name), &flg));
1756   if (flg) {
1757     DMLabel label;
1758     Vec     val;
1759 
1760     PetscCall(DMGetLabel(dm, name, &label));
1761     PetscCheck(label,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1762     PetscCall(DMPlexCreateLabelField(dm, label, &val));
1763     PetscCall(VecView(val, viewer));
1764     PetscCall(VecDestroy(&val));
1765   }
1766   PetscFunctionReturn(0);
1767 }
1768 
1769 /*@
1770   DMPlexTopologyView - Saves a DMPlex topology into a file
1771 
1772   Collective on DM
1773 
1774   Input Parameters:
1775 + dm     - The DM whose topology is to be saved
1776 - viewer - The PetscViewer for saving
1777 
1778   Level: advanced
1779 
1780 .seealso: `DMView()`, `DMPlexCoordinatesView()`, `DMPlexLabelsView()`, `DMPlexTopologyLoad()`
1781 @*/
1782 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1783 {
1784   PetscBool      ishdf5;
1785 
1786   PetscFunctionBegin;
1787   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1788   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1789   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5));
1790   PetscCall(PetscLogEventBegin(DMPLEX_TopologyView,viewer,0,0,0));
1791   if (ishdf5) {
1792 #if defined(PETSC_HAVE_HDF5)
1793     PetscViewerFormat format;
1794     PetscCall(PetscViewerGetFormat(viewer, &format));
1795     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1796       IS globalPointNumbering;
1797 
1798       PetscCall(DMPlexCreatePointNumbering(dm, &globalPointNumbering));
1799       PetscCall(DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer));
1800       PetscCall(ISDestroy(&globalPointNumbering));
1801     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1802 #else
1803     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1804 #endif
1805   }
1806   PetscCall(PetscLogEventEnd(DMPLEX_TopologyView,viewer,0,0,0));
1807   PetscFunctionReturn(0);
1808 }
1809 
1810 /*@
1811   DMPlexCoordinatesView - Saves DMPlex coordinates into a file
1812 
1813   Collective on DM
1814 
1815   Input Parameters:
1816 + dm     - The DM whose coordinates are to be saved
1817 - viewer - The PetscViewer for saving
1818 
1819   Level: advanced
1820 
1821 .seealso: `DMView()`, `DMPlexTopologyView()`, `DMPlexLabelsView()`, `DMPlexCoordinatesLoad()`
1822 @*/
1823 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
1824 {
1825   PetscBool      ishdf5;
1826 
1827   PetscFunctionBegin;
1828   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1829   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1830   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5));
1831   PetscCall(PetscLogEventBegin(DMPLEX_CoordinatesView,viewer,0,0,0));
1832   if (ishdf5) {
1833 #if defined(PETSC_HAVE_HDF5)
1834     PetscViewerFormat format;
1835     PetscCall(PetscViewerGetFormat(viewer, &format));
1836     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1837       PetscCall(DMPlexCoordinatesView_HDF5_Internal(dm, viewer));
1838     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1839 #else
1840     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1841 #endif
1842   }
1843   PetscCall(PetscLogEventEnd(DMPLEX_CoordinatesView,viewer,0,0,0));
1844   PetscFunctionReturn(0);
1845 }
1846 
1847 /*@
1848   DMPlexLabelsView - Saves DMPlex labels into a file
1849 
1850   Collective on DM
1851 
1852   Input Parameters:
1853 + dm     - The DM whose labels are to be saved
1854 - viewer - The PetscViewer for saving
1855 
1856   Level: advanced
1857 
1858 .seealso: `DMView()`, `DMPlexTopologyView()`, `DMPlexCoordinatesView()`, `DMPlexLabelsLoad()`
1859 @*/
1860 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
1861 {
1862   PetscBool      ishdf5;
1863 
1864   PetscFunctionBegin;
1865   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1866   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1867   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5));
1868   PetscCall(PetscLogEventBegin(DMPLEX_LabelsView,viewer,0,0,0));
1869   if (ishdf5) {
1870 #if defined(PETSC_HAVE_HDF5)
1871     IS                globalPointNumbering;
1872     PetscViewerFormat format;
1873 
1874     PetscCall(PetscViewerGetFormat(viewer, &format));
1875     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1876       PetscCall(DMPlexCreatePointNumbering(dm, &globalPointNumbering));
1877       PetscCall(DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer));
1878       PetscCall(ISDestroy(&globalPointNumbering));
1879     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1880 #else
1881     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1882 #endif
1883   }
1884   PetscCall(PetscLogEventEnd(DMPLEX_LabelsView,viewer,0,0,0));
1885   PetscFunctionReturn(0);
1886 }
1887 
1888 /*@
1889   DMPlexSectionView - Saves a section associated with a DMPlex
1890 
1891   Collective on DM
1892 
1893   Input Parameters:
1894 + dm         - The DM that contains the topology on which the section to be saved is defined
1895 . viewer     - The PetscViewer for saving
1896 - sectiondm  - The DM that contains the section to be saved
1897 
1898   Level: advanced
1899 
1900   Notes:
1901   This function is a wrapper around PetscSectionView(); in addition to the raw section, it saves information that associates the section points to the topology (dm) points. When the topology (dm) and the section are later loaded with DMPlexTopologyLoad() and DMPlexSectionLoad(), respectively, this information is used to match section points with topology points.
1902 
1903   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1904 
1905 .seealso: `DMView()`, `DMPlexTopologyView()`, `DMPlexCoordinatesView()`, `DMPlexLabelsView()`, `DMPlexGlobalVectorView()`, `DMPlexLocalVectorView()`, `PetscSectionView()`, `DMPlexSectionLoad()`
1906 @*/
1907 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
1908 {
1909   PetscBool      ishdf5;
1910 
1911   PetscFunctionBegin;
1912   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1913   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1914   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1915   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5));
1916   PetscCall(PetscLogEventBegin(DMPLEX_SectionView,viewer,0,0,0));
1917   if (ishdf5) {
1918 #if defined(PETSC_HAVE_HDF5)
1919     PetscCall(DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm));
1920 #else
1921     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1922 #endif
1923   }
1924   PetscCall(PetscLogEventEnd(DMPLEX_SectionView,viewer,0,0,0));
1925   PetscFunctionReturn(0);
1926 }
1927 
1928 /*@
1929   DMPlexGlobalVectorView - Saves a global vector
1930 
1931   Collective on DM
1932 
1933   Input Parameters:
1934 + dm        - The DM that represents the topology
1935 . viewer    - The PetscViewer to save data with
1936 . sectiondm - The DM that contains the global section on which vec is defined
1937 - vec       - The global vector to be saved
1938 
1939   Level: advanced
1940 
1941   Notes:
1942   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1943 
1944   Typical calling sequence
1945 $       DMCreate(PETSC_COMM_WORLD, &dm);
1946 $       DMSetType(dm, DMPLEX);
1947 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1948 $       DMClone(dm, &sectiondm);
1949 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1950 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1951 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1952 $       PetscSectionSetChart(section, pStart, pEnd);
1953 $       PetscSectionSetUp(section);
1954 $       DMSetLocalSection(sectiondm, section);
1955 $       PetscSectionDestroy(&section);
1956 $       DMGetGlobalVector(sectiondm, &vec);
1957 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1958 $       DMPlexTopologyView(dm, viewer);
1959 $       DMPlexSectionView(dm, viewer, sectiondm);
1960 $       DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
1961 $       DMRestoreGlobalVector(sectiondm, &vec);
1962 $       DMDestroy(&sectiondm);
1963 $       DMDestroy(&dm);
1964 
1965 .seealso: `DMPlexTopologyView()`, `DMPlexSectionView()`, `DMPlexLocalVectorView()`, `DMPlexGlobalVectorLoad()`, `DMPlexLocalVectorLoad()`
1966 @*/
1967 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1968 {
1969   PetscBool       ishdf5;
1970 
1971   PetscFunctionBegin;
1972   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1973   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1974   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1975   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1976   /* Check consistency */
1977   {
1978     PetscSection  section;
1979     PetscBool     includesConstraints;
1980     PetscInt      m, m1;
1981 
1982     PetscCall(VecGetLocalSize(vec, &m1));
1983     PetscCall(DMGetGlobalSection(sectiondm, &section));
1984     PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
1985     if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
1986     else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
1987     PetscCheck(m1 == m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%" PetscInt_FMT ") != global section storage size (%" PetscInt_FMT ")", m1, m);
1988   }
1989   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
1990   PetscCall(PetscLogEventBegin(DMPLEX_GlobalVectorView,viewer,0,0,0));
1991   if (ishdf5) {
1992 #if defined(PETSC_HAVE_HDF5)
1993     PetscCall(DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec));
1994 #else
1995     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1996 #endif
1997   }
1998   PetscCall(PetscLogEventEnd(DMPLEX_GlobalVectorView,viewer,0,0,0));
1999   PetscFunctionReturn(0);
2000 }
2001 
2002 /*@
2003   DMPlexLocalVectorView - Saves a local vector
2004 
2005   Collective on DM
2006 
2007   Input Parameters:
2008 + dm        - The DM that represents the topology
2009 . viewer    - The PetscViewer to save data with
2010 . sectiondm - The DM that contains the local section on which vec is defined; may be the same as dm
2011 - vec       - The local vector to be saved
2012 
2013   Level: advanced
2014 
2015   Notes:
2016   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2017 
2018   Typical calling sequence
2019 $       DMCreate(PETSC_COMM_WORLD, &dm);
2020 $       DMSetType(dm, DMPLEX);
2021 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2022 $       DMClone(dm, &sectiondm);
2023 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2024 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
2025 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
2026 $       PetscSectionSetChart(section, pStart, pEnd);
2027 $       PetscSectionSetUp(section);
2028 $       DMSetLocalSection(sectiondm, section);
2029 $       DMGetLocalVector(sectiondm, &vec);
2030 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2031 $       DMPlexTopologyView(dm, viewer);
2032 $       DMPlexSectionView(dm, viewer, sectiondm);
2033 $       DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
2034 $       DMRestoreLocalVector(sectiondm, &vec);
2035 $       DMDestroy(&sectiondm);
2036 $       DMDestroy(&dm);
2037 
2038 .seealso: `DMPlexTopologyView()`, `DMPlexSectionView()`, `DMPlexGlobalVectorView()`, `DMPlexGlobalVectorLoad()`, `DMPlexLocalVectorLoad()`
2039 @*/
2040 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
2041 {
2042   PetscBool       ishdf5;
2043 
2044   PetscFunctionBegin;
2045   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2046   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2047   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2048   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
2049   /* Check consistency */
2050   {
2051     PetscSection  section;
2052     PetscBool     includesConstraints;
2053     PetscInt      m, m1;
2054 
2055     PetscCall(VecGetLocalSize(vec, &m1));
2056     PetscCall(DMGetLocalSection(sectiondm, &section));
2057     PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
2058     if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
2059     else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
2060     PetscCheck(m1 == m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%" PetscInt_FMT ") != local section storage size (%" PetscInt_FMT ")", m1, m);
2061   }
2062   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2063   PetscCall(PetscLogEventBegin(DMPLEX_LocalVectorView,viewer,0,0,0));
2064   if (ishdf5) {
2065 #if defined(PETSC_HAVE_HDF5)
2066     PetscCall(DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec));
2067 #else
2068     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2069 #endif
2070   }
2071   PetscCall(PetscLogEventEnd(DMPLEX_LocalVectorView,viewer,0,0,0));
2072   PetscFunctionReturn(0);
2073 }
2074 
2075 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
2076 {
2077   PetscBool      ishdf5;
2078 
2079   PetscFunctionBegin;
2080   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2081   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2082   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5));
2083   if (ishdf5) {
2084 #if defined(PETSC_HAVE_HDF5)
2085     PetscViewerFormat format;
2086     PetscCall(PetscViewerGetFormat(viewer, &format));
2087     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
2088       PetscCall(DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer));
2089     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2090       PetscCall(DMPlexLoad_HDF5_Internal(dm, viewer));
2091     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2092     PetscFunctionReturn(0);
2093 #else
2094     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2095 #endif
2096   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
2097 }
2098 
2099 /*@
2100   DMPlexTopologyLoad - Loads a topology into a DMPlex
2101 
2102   Collective on DM
2103 
2104   Input Parameters:
2105 + dm     - The DM into which the topology is loaded
2106 - viewer - The PetscViewer for the saved topology
2107 
2108   Output Parameters:
2109 . globalToLocalPointSF - The PetscSF that pushes points in [0, N) to the associated points in the loaded plex, where N is the global number of points; NULL if unneeded
2110 
2111   Level: advanced
2112 
2113 .seealso: `DMLoad()`, `DMPlexCoordinatesLoad()`, `DMPlexLabelsLoad()`, `DMView()`, `PetscViewerHDF5Open()`, `PetscViewerPushFormat()`
2114 @*/
2115 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
2116 {
2117   PetscBool      ishdf5;
2118 
2119   PetscFunctionBegin;
2120   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2121   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2122   if (globalToLocalPointSF) PetscValidPointer(globalToLocalPointSF, 3);
2123   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5));
2124   PetscCall(PetscLogEventBegin(DMPLEX_TopologyLoad,viewer,0,0,0));
2125   if (ishdf5) {
2126 #if defined(PETSC_HAVE_HDF5)
2127     PetscViewerFormat format;
2128     PetscCall(PetscViewerGetFormat(viewer, &format));
2129     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2130       PetscCall(DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF));
2131     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2132 #else
2133     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2134 #endif
2135   }
2136   PetscCall(PetscLogEventEnd(DMPLEX_TopologyLoad,viewer,0,0,0));
2137   PetscFunctionReturn(0);
2138 }
2139 
2140 /*@
2141   DMPlexCoordinatesLoad - Loads coordinates into a DMPlex
2142 
2143   Collective on DM
2144 
2145   Input Parameters:
2146 + dm     - The DM into which the coordinates are loaded
2147 . viewer - The PetscViewer for the saved coordinates
2148 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2149 
2150   Level: advanced
2151 
2152 .seealso: `DMLoad()`, `DMPlexTopologyLoad()`, `DMPlexLabelsLoad()`, `DMView()`, `PetscViewerHDF5Open()`, `PetscViewerPushFormat()`
2153 @*/
2154 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer, PetscSF globalToLocalPointSF)
2155 {
2156   PetscBool      ishdf5;
2157 
2158   PetscFunctionBegin;
2159   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2160   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2161   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 3);
2162   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5));
2163   PetscCall(PetscLogEventBegin(DMPLEX_CoordinatesLoad,viewer,0,0,0));
2164   if (ishdf5) {
2165 #if defined(PETSC_HAVE_HDF5)
2166     PetscViewerFormat format;
2167     PetscCall(PetscViewerGetFormat(viewer, &format));
2168     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2169       PetscCall(DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF));
2170     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2171 #else
2172     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2173 #endif
2174   }
2175   PetscCall(PetscLogEventEnd(DMPLEX_CoordinatesLoad,viewer,0,0,0));
2176   PetscFunctionReturn(0);
2177 }
2178 
2179 /*@
2180   DMPlexLabelsLoad - Loads labels into a DMPlex
2181 
2182   Collective on DM
2183 
2184   Input Parameters:
2185 + dm     - The DM into which the labels are loaded
2186 . viewer - The PetscViewer for the saved labels
2187 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2188 
2189   Level: advanced
2190 
2191   Notes:
2192   The PetscSF argument must not be NULL if the DM is distributed, otherwise an error occurs.
2193 
2194 .seealso: `DMLoad()`, `DMPlexTopologyLoad()`, `DMPlexCoordinatesLoad()`, `DMView()`, `PetscViewerHDF5Open()`, `PetscViewerPushFormat()`
2195 @*/
2196 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer, PetscSF globalToLocalPointSF)
2197 {
2198   PetscBool      ishdf5;
2199 
2200   PetscFunctionBegin;
2201   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2202   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2203   if (globalToLocalPointSF) PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 3);
2204   PetscCall(PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5));
2205   PetscCall(PetscLogEventBegin(DMPLEX_LabelsLoad,viewer,0,0,0));
2206   if (ishdf5) {
2207 #if defined(PETSC_HAVE_HDF5)
2208     PetscViewerFormat format;
2209 
2210     PetscCall(PetscViewerGetFormat(viewer, &format));
2211     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2212       PetscCall(DMPlexLabelsLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF));
2213     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2214 #else
2215     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2216 #endif
2217   }
2218   PetscCall(PetscLogEventEnd(DMPLEX_LabelsLoad,viewer,0,0,0));
2219   PetscFunctionReturn(0);
2220 }
2221 
2222 /*@
2223   DMPlexSectionLoad - Loads section into a DMPlex
2224 
2225   Collective on DM
2226 
2227   Input Parameters:
2228 + dm          - The DM that represents the topology
2229 . viewer      - The PetscViewer that represents the on-disk section (sectionA)
2230 . sectiondm   - The DM into which the on-disk section (sectionA) is migrated
2231 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2232 
2233   Output Parameters
2234 + globalDofSF - The SF that migrates any on-disk Vec data associated with sectionA into a global Vec associated with the sectiondm's global section (NULL if not needed)
2235 - localDofSF  - The SF that migrates any on-disk Vec data associated with sectionA into a local Vec associated with the sectiondm's local section (NULL if not needed)
2236 
2237   Level: advanced
2238 
2239   Notes:
2240   This function is a wrapper around PetscSectionLoad(); it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in dm. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
2241 
2242   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2243 
2244   The output parameter, globalDofSF (localDofSF), can later be used with DMPlexGlobalVectorLoad() (DMPlexLocalVectorLoad()) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
2245 
2246   Example using 2 processes:
2247 $  NX (number of points on dm): 4
2248 $  sectionA                   : the on-disk section
2249 $  vecA                       : a vector associated with sectionA
2250 $  sectionB                   : sectiondm's local section constructed in this function
2251 $  vecB (local)               : a vector associated with sectiondm's local section
2252 $  vecB (global)              : a vector associated with sectiondm's global section
2253 $
2254 $                                     rank 0    rank 1
2255 $  vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
2256 $  sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
2257 $  sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
2258 $  sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
2259 $  [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
2260 $  sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
2261 $  sectionB->atlasDof             :     1 0 1 | 1 3
2262 $  sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
2263 $  vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
2264 $  vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
2265 $
2266 $  where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
2267 
2268 .seealso: `DMLoad()`, `DMPlexTopologyLoad()`, `DMPlexCoordinatesLoad()`, `DMPlexLabelsLoad()`, `DMPlexGlobalVectorLoad()`, `DMPlexLocalVectorLoad()`, `PetscSectionLoad()`, `DMPlexSectionView()`
2269 @*/
2270 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
2271 {
2272   PetscBool      ishdf5;
2273 
2274   PetscFunctionBegin;
2275   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2276   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2277   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2278   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
2279   if (globalDofSF) PetscValidPointer(globalDofSF, 5);
2280   if (localDofSF) PetscValidPointer(localDofSF, 6);
2281   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5));
2282   PetscCall(PetscLogEventBegin(DMPLEX_SectionLoad,viewer,0,0,0));
2283   if (ishdf5) {
2284 #if defined(PETSC_HAVE_HDF5)
2285     PetscCall(DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF));
2286 #else
2287     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2288 #endif
2289   }
2290   PetscCall(PetscLogEventEnd(DMPLEX_SectionLoad,viewer,0,0,0));
2291   PetscFunctionReturn(0);
2292 }
2293 
2294 /*@
2295   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
2296 
2297   Collective on DM
2298 
2299   Input Parameters:
2300 + dm        - The DM that represents the topology
2301 . viewer    - The PetscViewer that represents the on-disk vector data
2302 . sectiondm - The DM that contains the global section on which vec is defined
2303 . sf        - The SF that migrates the on-disk vector data into vec
2304 - vec       - The global vector to set values of
2305 
2306   Level: advanced
2307 
2308   Notes:
2309   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2310 
2311   Typical calling sequence
2312 $       DMCreate(PETSC_COMM_WORLD, &dm);
2313 $       DMSetType(dm, DMPLEX);
2314 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2315 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2316 $       DMClone(dm, &sectiondm);
2317 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2318 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
2319 $       DMGetGlobalVector(sectiondm, &vec);
2320 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2321 $       DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
2322 $       DMRestoreGlobalVector(sectiondm, &vec);
2323 $       PetscSFDestroy(&gsf);
2324 $       PetscSFDestroy(&sfX);
2325 $       DMDestroy(&sectiondm);
2326 $       DMDestroy(&dm);
2327 
2328 .seealso: `DMPlexTopologyLoad()`, `DMPlexSectionLoad()`, `DMPlexLocalVectorLoad()`, `DMPlexGlobalVectorView()`, `DMPlexLocalVectorView()`
2329 @*/
2330 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2331 {
2332   PetscBool       ishdf5;
2333 
2334   PetscFunctionBegin;
2335   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2336   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2337   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2338   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2339   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2340   /* Check consistency */
2341   {
2342     PetscSection  section;
2343     PetscBool     includesConstraints;
2344     PetscInt      m, m1;
2345 
2346     PetscCall(VecGetLocalSize(vec, &m1));
2347     PetscCall(DMGetGlobalSection(sectiondm, &section));
2348     PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
2349     if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
2350     else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
2351     PetscCheck(m1 == m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%" PetscInt_FMT ") != global section storage size (%" PetscInt_FMT ")", m1, m);
2352   }
2353   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5));
2354   PetscCall(PetscLogEventBegin(DMPLEX_GlobalVectorLoad,viewer,0,0,0));
2355   if (ishdf5) {
2356 #if defined(PETSC_HAVE_HDF5)
2357     PetscCall(DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec));
2358 #else
2359     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2360 #endif
2361   }
2362   PetscCall(PetscLogEventEnd(DMPLEX_GlobalVectorLoad,viewer,0,0,0));
2363   PetscFunctionReturn(0);
2364 }
2365 
2366 /*@
2367   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2368 
2369   Collective on DM
2370 
2371   Input Parameters:
2372 + dm        - The DM that represents the topology
2373 . viewer    - The PetscViewer that represents the on-disk vector data
2374 . sectiondm - The DM that contains the local section on which vec is defined
2375 . sf        - The SF that migrates the on-disk vector data into vec
2376 - vec       - The local vector to set values of
2377 
2378   Level: advanced
2379 
2380   Notes:
2381   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2382 
2383   Typical calling sequence
2384 $       DMCreate(PETSC_COMM_WORLD, &dm);
2385 $       DMSetType(dm, DMPLEX);
2386 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2387 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2388 $       DMClone(dm, &sectiondm);
2389 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2390 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2391 $       DMGetLocalVector(sectiondm, &vec);
2392 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2393 $       DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2394 $       DMRestoreLocalVector(sectiondm, &vec);
2395 $       PetscSFDestroy(&lsf);
2396 $       PetscSFDestroy(&sfX);
2397 $       DMDestroy(&sectiondm);
2398 $       DMDestroy(&dm);
2399 
2400 .seealso: `DMPlexTopologyLoad()`, `DMPlexSectionLoad()`, `DMPlexGlobalVectorLoad()`, `DMPlexGlobalVectorView()`, `DMPlexLocalVectorView()`
2401 @*/
2402 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2403 {
2404   PetscBool       ishdf5;
2405 
2406   PetscFunctionBegin;
2407   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2408   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2409   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2410   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2411   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2412   /* Check consistency */
2413   {
2414     PetscSection  section;
2415     PetscBool     includesConstraints;
2416     PetscInt      m, m1;
2417 
2418     PetscCall(VecGetLocalSize(vec, &m1));
2419     PetscCall(DMGetLocalSection(sectiondm, &section));
2420     PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
2421     if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
2422     else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
2423     PetscCheck(m1 == m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%" PetscInt_FMT ") != local section storage size (%" PetscInt_FMT ")", m1, m);
2424   }
2425   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5));
2426   PetscCall(PetscLogEventBegin(DMPLEX_LocalVectorLoad,viewer,0,0,0));
2427   if (ishdf5) {
2428 #if defined(PETSC_HAVE_HDF5)
2429     PetscCall(DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec));
2430 #else
2431     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2432 #endif
2433   }
2434   PetscCall(PetscLogEventEnd(DMPLEX_LocalVectorLoad,viewer,0,0,0));
2435   PetscFunctionReturn(0);
2436 }
2437 
2438 PetscErrorCode DMDestroy_Plex(DM dm)
2439 {
2440   DM_Plex       *mesh = (DM_Plex*) dm->data;
2441 
2442   PetscFunctionBegin;
2443   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL));
2444   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL));
2445   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL));
2446   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C", NULL));
2447   if (--mesh->refct > 0) PetscFunctionReturn(0);
2448   PetscCall(PetscSectionDestroy(&mesh->coneSection));
2449   PetscCall(PetscFree(mesh->cones));
2450   PetscCall(PetscFree(mesh->coneOrientations));
2451   PetscCall(PetscSectionDestroy(&mesh->supportSection));
2452   PetscCall(PetscSectionDestroy(&mesh->subdomainSection));
2453   PetscCall(PetscFree(mesh->supports));
2454   PetscCall(PetscFree(mesh->facesTmp));
2455   PetscCall(PetscFree(mesh->tetgenOpts));
2456   PetscCall(PetscFree(mesh->triangleOpts));
2457   PetscCall(PetscFree(mesh->transformType));
2458   PetscCall(PetscPartitionerDestroy(&mesh->partitioner));
2459   PetscCall(DMLabelDestroy(&mesh->subpointMap));
2460   PetscCall(ISDestroy(&mesh->subpointIS));
2461   PetscCall(ISDestroy(&mesh->globalVertexNumbers));
2462   PetscCall(ISDestroy(&mesh->globalCellNumbers));
2463   PetscCall(PetscSectionDestroy(&mesh->anchorSection));
2464   PetscCall(ISDestroy(&mesh->anchorIS));
2465   PetscCall(PetscSectionDestroy(&mesh->parentSection));
2466   PetscCall(PetscFree(mesh->parents));
2467   PetscCall(PetscFree(mesh->childIDs));
2468   PetscCall(PetscSectionDestroy(&mesh->childSection));
2469   PetscCall(PetscFree(mesh->children));
2470   PetscCall(DMDestroy(&mesh->referenceTree));
2471   PetscCall(PetscGridHashDestroy(&mesh->lbox));
2472   PetscCall(PetscFree(mesh->neighbors));
2473   if (mesh->metricCtx) PetscCall(PetscFree(mesh->metricCtx));
2474   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2475   PetscCall(PetscFree(mesh));
2476   PetscFunctionReturn(0);
2477 }
2478 
2479 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2480 {
2481   PetscSection           sectionGlobal;
2482   PetscInt               bs = -1, mbs;
2483   PetscInt               localSize, localStart = 0;
2484   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2485   MatType                mtype;
2486   ISLocalToGlobalMapping ltog;
2487 
2488   PetscFunctionBegin;
2489   PetscCall(MatInitializePackage());
2490   mtype = dm->mattype;
2491   PetscCall(DMGetGlobalSection(dm, &sectionGlobal));
2492   /* PetscCall(PetscSectionGetStorageSize(sectionGlobal, &localSize)); */
2493   PetscCall(PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize));
2494   PetscCallMPI(MPI_Exscan(&localSize, &localStart, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) dm)));
2495   PetscCall(MatCreate(PetscObjectComm((PetscObject)dm), J));
2496   PetscCall(MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE));
2497   PetscCall(MatSetType(*J, mtype));
2498   PetscCall(MatSetFromOptions(*J));
2499   PetscCall(MatGetBlockSize(*J, &mbs));
2500   if (mbs > 1) bs = mbs;
2501   PetscCall(PetscStrcmp(mtype, MATSHELL, &isShell));
2502   PetscCall(PetscStrcmp(mtype, MATBAIJ, &isBlock));
2503   PetscCall(PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock));
2504   PetscCall(PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock));
2505   PetscCall(PetscStrcmp(mtype, MATSBAIJ, &isSymBlock));
2506   PetscCall(PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock));
2507   PetscCall(PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock));
2508   PetscCall(PetscStrcmp(mtype, MATIS, &isMatIS));
2509   if (!isShell) {
2510     PetscBool fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2511     PetscInt  *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *pblocks;
2512     PetscInt  pStart, pEnd, p, dof, cdof;
2513 
2514     PetscCall(DMGetLocalToGlobalMapping(dm,&ltog));
2515 
2516     PetscCall(PetscCalloc1(localSize, &pblocks));
2517     PetscCall(PetscSectionGetChart(sectionGlobal, &pStart, &pEnd));
2518     for (p = pStart; p < pEnd; ++p) {
2519       PetscInt bdof, offset;
2520 
2521       PetscCall(PetscSectionGetDof(sectionGlobal, p, &dof));
2522       PetscCall(PetscSectionGetOffset(sectionGlobal, p, &offset));
2523       PetscCall(PetscSectionGetConstraintDof(sectionGlobal, p, &cdof));
2524       for (PetscInt i=0; i < dof - cdof; i++)
2525         pblocks[offset - localStart + i] = dof - cdof;
2526       dof  = dof < 0 ? -(dof+1) : dof;
2527       bdof = cdof && (dof-cdof) ? 1 : dof;
2528       if (dof) {
2529         if (bs < 0)          {bs = bdof;}
2530         else if (bs != bdof) {bs = 1;}
2531       }
2532     }
2533     /* Must have same blocksize on all procs (some might have no points) */
2534     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs;
2535     bsLocal[1] = bs;
2536     PetscCall(PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax));
2537     if (bsMinMax[0] != bsMinMax[1]) bs = 1;
2538     else bs = bsMinMax[0];
2539     bs = PetscMax(1,bs);
2540     PetscCall(MatSetLocalToGlobalMapping(*J,ltog,ltog));
2541     if (dm->prealloc_skip) { // User will likely use MatSetPreallocationCOO(), but still set structural parameters
2542       PetscCall(MatSetBlockSize(*J, bs));
2543       PetscCall(MatSetUp(*J));
2544     } else {
2545       PetscCall(PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu));
2546       PetscCall(DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix));
2547       PetscCall(PetscFree4(dnz, onz, dnzu, onzu));
2548     }
2549     { // Consolidate blocks
2550       PetscInt nblocks = 0;
2551       for (PetscInt i=0; i<localSize; i += PetscMax(1, pblocks[i])) {
2552         if (pblocks[i] == 0) continue;
2553         pblocks[nblocks++] = pblocks[i]; // nblocks always <= i
2554         for (PetscInt j=1; j<pblocks[i]; j++) {
2555            PetscCheck(pblocks[i+j] == pblocks[i], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Block of size %" PetscInt_FMT " mismatches entry %" PetscInt_FMT, pblocks[i], pblocks[i+j]);
2556         }
2557       }
2558       PetscCall(MatSetVariableBlockSizes(*J, nblocks, pblocks));
2559     }
2560     PetscCall(PetscFree(pblocks));
2561   }
2562   PetscCall(MatSetDM(*J, dm));
2563   PetscFunctionReturn(0);
2564 }
2565 
2566 /*@
2567   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2568 
2569   Not collective
2570 
2571   Input Parameter:
2572 . mesh - The DMPlex
2573 
2574   Output Parameters:
2575 . subsection - The subdomain section
2576 
2577   Level: developer
2578 
2579 .seealso:
2580 @*/
2581 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2582 {
2583   DM_Plex       *mesh = (DM_Plex*) dm->data;
2584 
2585   PetscFunctionBegin;
2586   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2587   if (!mesh->subdomainSection) {
2588     PetscSection section;
2589     PetscSF      sf;
2590 
2591     PetscCall(PetscSFCreate(PETSC_COMM_SELF,&sf));
2592     PetscCall(DMGetLocalSection(dm,&section));
2593     PetscCall(PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection));
2594     PetscCall(PetscSFDestroy(&sf));
2595   }
2596   *subsection = mesh->subdomainSection;
2597   PetscFunctionReturn(0);
2598 }
2599 
2600 /*@
2601   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
2602 
2603   Not collective
2604 
2605   Input Parameter:
2606 . mesh - The DMPlex
2607 
2608   Output Parameters:
2609 + pStart - The first mesh point
2610 - pEnd   - The upper bound for mesh points
2611 
2612   Level: beginner
2613 
2614 .seealso: `DMPlexCreate()`, `DMPlexSetChart()`
2615 @*/
2616 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2617 {
2618   DM_Plex       *mesh = (DM_Plex*) dm->data;
2619 
2620   PetscFunctionBegin;
2621   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2622   PetscCall(PetscSectionGetChart(mesh->coneSection, pStart, pEnd));
2623   PetscFunctionReturn(0);
2624 }
2625 
2626 /*@
2627   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
2628 
2629   Not collective
2630 
2631   Input Parameters:
2632 + mesh - The DMPlex
2633 . pStart - The first mesh point
2634 - pEnd   - The upper bound for mesh points
2635 
2636   Output Parameters:
2637 
2638   Level: beginner
2639 
2640 .seealso: `DMPlexCreate()`, `DMPlexGetChart()`
2641 @*/
2642 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2643 {
2644   DM_Plex       *mesh = (DM_Plex*) dm->data;
2645 
2646   PetscFunctionBegin;
2647   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2648   PetscCall(PetscSectionSetChart(mesh->coneSection, pStart, pEnd));
2649   PetscCall(PetscSectionSetChart(mesh->supportSection, pStart, pEnd));
2650   PetscFunctionReturn(0);
2651 }
2652 
2653 /*@
2654   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2655 
2656   Not collective
2657 
2658   Input Parameters:
2659 + mesh - The DMPlex
2660 - p - The point, which must lie in the chart set with DMPlexSetChart()
2661 
2662   Output Parameter:
2663 . size - The cone size for point p
2664 
2665   Level: beginner
2666 
2667 .seealso: `DMPlexCreate()`, `DMPlexSetConeSize()`, `DMPlexSetChart()`
2668 @*/
2669 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2670 {
2671   DM_Plex       *mesh = (DM_Plex*) dm->data;
2672 
2673   PetscFunctionBegin;
2674   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2675   PetscValidIntPointer(size, 3);
2676   PetscCall(PetscSectionGetDof(mesh->coneSection, p, size));
2677   PetscFunctionReturn(0);
2678 }
2679 
2680 /*@
2681   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2682 
2683   Not collective
2684 
2685   Input Parameters:
2686 + mesh - The DMPlex
2687 . p - The point, which must lie in the chart set with DMPlexSetChart()
2688 - size - The cone size for point p
2689 
2690   Output Parameter:
2691 
2692   Note:
2693   This should be called after DMPlexSetChart().
2694 
2695   Level: beginner
2696 
2697 .seealso: `DMPlexCreate()`, `DMPlexGetConeSize()`, `DMPlexSetChart()`
2698 @*/
2699 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
2700 {
2701   DM_Plex       *mesh = (DM_Plex*) dm->data;
2702 
2703   PetscFunctionBegin;
2704   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2705   PetscCall(PetscSectionSetDof(mesh->coneSection, p, size));
2706   PetscFunctionReturn(0);
2707 }
2708 
2709 /*@
2710   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
2711 
2712   Not collective
2713 
2714   Input Parameters:
2715 + mesh - The DMPlex
2716 . p - The point, which must lie in the chart set with DMPlexSetChart()
2717 - size - The additional cone size for point p
2718 
2719   Output Parameter:
2720 
2721   Note:
2722   This should be called after DMPlexSetChart().
2723 
2724   Level: beginner
2725 
2726 .seealso: `DMPlexCreate()`, `DMPlexSetConeSize()`, `DMPlexGetConeSize()`, `DMPlexSetChart()`
2727 @*/
2728 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
2729 {
2730   DM_Plex       *mesh = (DM_Plex*) dm->data;
2731   PetscFunctionBegin;
2732   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2733   PetscCall(PetscSectionAddDof(mesh->coneSection, p, size));
2734   PetscFunctionReturn(0);
2735 }
2736 
2737 /*@C
2738   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
2739 
2740   Not collective
2741 
2742   Input Parameters:
2743 + dm - The DMPlex
2744 - p - The point, which must lie in the chart set with DMPlexSetChart()
2745 
2746   Output Parameter:
2747 . cone - An array of points which are on the in-edges for point p
2748 
2749   Level: beginner
2750 
2751   Fortran Notes:
2752   Since it returns an array, this routine is only available in Fortran 90, and you must
2753   include petsc.h90 in your code.
2754   You must also call DMPlexRestoreCone() after you finish using the returned array.
2755   DMPlexRestoreCone() is not needed/available in C.
2756 
2757 .seealso: `DMPlexGetConeSize()`, `DMPlexSetCone()`, `DMPlexGetConeTuple()`, `DMPlexSetChart()`
2758 @*/
2759 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
2760 {
2761   DM_Plex       *mesh = (DM_Plex*) dm->data;
2762   PetscInt       off;
2763 
2764   PetscFunctionBegin;
2765   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2766   PetscValidPointer(cone, 3);
2767   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
2768   *cone = &mesh->cones[off];
2769   PetscFunctionReturn(0);
2770 }
2771 
2772 /*@C
2773   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
2774 
2775   Not collective
2776 
2777   Input Parameters:
2778 + dm - The DMPlex
2779 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
2780 
2781   Output Parameters:
2782 + pConesSection - PetscSection describing the layout of pCones
2783 - pCones - An array of points which are on the in-edges for the point set p
2784 
2785   Level: intermediate
2786 
2787 .seealso: `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexGetConeRecursive()`, `DMPlexSetChart()`
2788 @*/
2789 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
2790 {
2791   PetscSection        cs, newcs;
2792   PetscInt            *cones;
2793   PetscInt            *newarr=NULL;
2794   PetscInt            n;
2795 
2796   PetscFunctionBegin;
2797   PetscCall(DMPlexGetCones(dm, &cones));
2798   PetscCall(DMPlexGetConeSection(dm, &cs));
2799   PetscCall(PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL));
2800   if (pConesSection) *pConesSection = newcs;
2801   if (pCones) {
2802     PetscCall(PetscSectionGetStorageSize(newcs, &n));
2803     PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones));
2804   }
2805   PetscFunctionReturn(0);
2806 }
2807 
2808 /*@
2809   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
2810 
2811   Not collective
2812 
2813   Input Parameters:
2814 + dm - The DMPlex
2815 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2816 
2817   Output Parameter:
2818 . expandedPoints - An array of vertices recursively expanded from input points
2819 
2820   Level: advanced
2821 
2822   Notes:
2823   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
2824   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
2825 
2826 .seealso: `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexGetConeTuple()`, `DMPlexGetConeRecursive()`, `DMPlexRestoreConeRecursive()`, `DMPlexGetDepth()`
2827 @*/
2828 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
2829 {
2830   IS                  *expandedPointsAll;
2831   PetscInt            depth;
2832 
2833   PetscFunctionBegin;
2834   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2835   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2836   PetscValidPointer(expandedPoints, 3);
2837   PetscCall(DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL));
2838   *expandedPoints = expandedPointsAll[0];
2839   PetscCall(PetscObjectReference((PetscObject)expandedPointsAll[0]));
2840   PetscCall(DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL));
2841   PetscFunctionReturn(0);
2842 }
2843 
2844 /*@
2845   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
2846 
2847   Not collective
2848 
2849   Input Parameters:
2850 + dm - The DMPlex
2851 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2852 
2853   Output Parameters:
2854 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2855 . expandedPoints - (optional) An array of index sets with recursively expanded cones
2856 - sections - (optional) An array of sections which describe mappings from points to their cone points
2857 
2858   Level: advanced
2859 
2860   Notes:
2861   Like DMPlexGetConeTuple() but recursive.
2862 
2863   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
2864   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
2865 
2866   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
2867   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
2868   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
2869 
2870 .seealso: `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexGetConeTuple()`, `DMPlexRestoreConeRecursive()`, `DMPlexGetConeRecursiveVertices()`, `DMPlexGetDepth()`
2871 @*/
2872 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2873 {
2874   const PetscInt      *arr0=NULL, *cone=NULL;
2875   PetscInt            *arr=NULL, *newarr=NULL;
2876   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
2877   IS                  *expandedPoints_;
2878   PetscSection        *sections_;
2879 
2880   PetscFunctionBegin;
2881   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2882   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2883   if (depth) PetscValidIntPointer(depth, 3);
2884   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
2885   if (sections) PetscValidPointer(sections, 5);
2886   PetscCall(ISGetLocalSize(points, &n));
2887   PetscCall(ISGetIndices(points, &arr0));
2888   PetscCall(DMPlexGetDepth(dm, &depth_));
2889   PetscCall(PetscCalloc1(depth_, &expandedPoints_));
2890   PetscCall(PetscCalloc1(depth_, &sections_));
2891   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
2892   for (d=depth_-1; d>=0; d--) {
2893     PetscCall(PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]));
2894     PetscCall(PetscSectionSetChart(sections_[d], 0, n));
2895     for (i=0; i<n; i++) {
2896       PetscCall(DMPlexGetDepthStratum(dm, d+1, &start, &end));
2897       if (arr[i] >= start && arr[i] < end) {
2898         PetscCall(DMPlexGetConeSize(dm, arr[i], &cn));
2899         PetscCall(PetscSectionSetDof(sections_[d], i, cn));
2900       } else {
2901         PetscCall(PetscSectionSetDof(sections_[d], i, 1));
2902       }
2903     }
2904     PetscCall(PetscSectionSetUp(sections_[d]));
2905     PetscCall(PetscSectionGetStorageSize(sections_[d], &newn));
2906     PetscCall(PetscMalloc1(newn, &newarr));
2907     for (i=0; i<n; i++) {
2908       PetscCall(PetscSectionGetDof(sections_[d], i, &cn));
2909       PetscCall(PetscSectionGetOffset(sections_[d], i, &co));
2910       if (cn > 1) {
2911         PetscCall(DMPlexGetCone(dm, arr[i], &cone));
2912         PetscCall(PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt)));
2913       } else {
2914         newarr[co] = arr[i];
2915       }
2916     }
2917     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]));
2918     arr = newarr;
2919     n = newn;
2920   }
2921   PetscCall(ISRestoreIndices(points, &arr0));
2922   *depth = depth_;
2923   if (expandedPoints) *expandedPoints = expandedPoints_;
2924   else {
2925     for (d=0; d<depth_; d++) PetscCall(ISDestroy(&expandedPoints_[d]));
2926     PetscCall(PetscFree(expandedPoints_));
2927   }
2928   if (sections) *sections = sections_;
2929   else {
2930     for (d=0; d<depth_; d++) PetscCall(PetscSectionDestroy(&sections_[d]));
2931     PetscCall(PetscFree(sections_));
2932   }
2933   PetscFunctionReturn(0);
2934 }
2935 
2936 /*@
2937   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
2938 
2939   Not collective
2940 
2941   Input Parameters:
2942 + dm - The DMPlex
2943 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2944 
2945   Output Parameters:
2946 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2947 . expandedPoints - (optional) An array of recursively expanded cones
2948 - sections - (optional) An array of sections which describe mappings from points to their cone points
2949 
2950   Level: advanced
2951 
2952   Notes:
2953   See DMPlexGetConeRecursive() for details.
2954 
2955 .seealso: `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexGetConeTuple()`, `DMPlexGetConeRecursive()`, `DMPlexGetConeRecursiveVertices()`, `DMPlexGetDepth()`
2956 @*/
2957 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2958 {
2959   PetscInt            d, depth_;
2960 
2961   PetscFunctionBegin;
2962   PetscCall(DMPlexGetDepth(dm, &depth_));
2963   PetscCheck(!depth || *depth == depth_,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
2964   if (depth) *depth = 0;
2965   if (expandedPoints) {
2966     for (d=0; d<depth_; d++) PetscCall(ISDestroy(&((*expandedPoints)[d])));
2967     PetscCall(PetscFree(*expandedPoints));
2968   }
2969   if (sections)  {
2970     for (d=0; d<depth_; d++) PetscCall(PetscSectionDestroy(&((*sections)[d])));
2971     PetscCall(PetscFree(*sections));
2972   }
2973   PetscFunctionReturn(0);
2974 }
2975 
2976 /*@
2977   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
2978 
2979   Not collective
2980 
2981   Input Parameters:
2982 + mesh - The DMPlex
2983 . p - The point, which must lie in the chart set with DMPlexSetChart()
2984 - cone - An array of points which are on the in-edges for point p
2985 
2986   Output Parameter:
2987 
2988   Note:
2989   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2990 
2991   Level: beginner
2992 
2993 .seealso: `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`, `DMPlexSetSupport()`, `DMPlexSetSupportSize()`
2994 @*/
2995 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
2996 {
2997   DM_Plex       *mesh = (DM_Plex*) dm->data;
2998   PetscInt       pStart, pEnd;
2999   PetscInt       dof, off, c;
3000 
3001   PetscFunctionBegin;
3002   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3003   PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd));
3004   PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3005   if (dof) PetscValidIntPointer(cone, 3);
3006   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3007   PetscCheck(!(p < pStart) && !(p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3008   for (c = 0; c < dof; ++c) {
3009     PetscCheck(!(cone[c] < pStart) && !(cone[c] >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", cone[c], pStart, pEnd);
3010     mesh->cones[off+c] = cone[c];
3011   }
3012   PetscFunctionReturn(0);
3013 }
3014 
3015 /*@C
3016   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
3017 
3018   Not collective
3019 
3020   Input Parameters:
3021 + mesh - The DMPlex
3022 - p - The point, which must lie in the chart set with DMPlexSetChart()
3023 
3024   Output Parameter:
3025 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
3026                     integer giving the prescription for cone traversal.
3027 
3028   Level: beginner
3029 
3030   Notes:
3031   The number indexes the symmetry transformations for the cell type (see manual). Orientation 0 is always
3032   the identity transformation. Negative orientation indicates reflection so that -(o+1) is the reflection
3033   of o, however it is not necessarily the inverse. To get the inverse, use DMPolytopeTypeComposeOrientationInv()
3034   with the identity.
3035 
3036   Fortran Notes:
3037   Since it returns an array, this routine is only available in Fortran 90, and you must
3038   include petsc.h90 in your code.
3039   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
3040   DMPlexRestoreConeOrientation() is not needed/available in C.
3041 
3042 .seealso: `DMPolytopeTypeComposeOrientation()`, `DMPolytopeTypeComposeOrientationInv()`, `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetCone()`, `DMPlexSetChart()`
3043 @*/
3044 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
3045 {
3046   DM_Plex       *mesh = (DM_Plex*) dm->data;
3047   PetscInt       off;
3048 
3049   PetscFunctionBegin;
3050   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3051   if (PetscDefined(USE_DEBUG)) {
3052     PetscInt dof;
3053     PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3054     if (dof) PetscValidPointer(coneOrientation, 3);
3055   }
3056   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3057 
3058   *coneOrientation = &mesh->coneOrientations[off];
3059   PetscFunctionReturn(0);
3060 }
3061 
3062 /*@
3063   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
3064 
3065   Not collective
3066 
3067   Input Parameters:
3068 + mesh - The DMPlex
3069 . p - The point, which must lie in the chart set with DMPlexSetChart()
3070 - coneOrientation - An array of orientations
3071   Output Parameter:
3072 
3073   Notes:
3074   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
3075 
3076   The meaning of coneOrientation is detailed in DMPlexGetConeOrientation().
3077 
3078   Level: beginner
3079 
3080 .seealso: `DMPlexCreate()`, `DMPlexGetConeOrientation()`, `DMPlexSetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`
3081 @*/
3082 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
3083 {
3084   DM_Plex       *mesh = (DM_Plex*) dm->data;
3085   PetscInt       pStart, pEnd;
3086   PetscInt       dof, off, c;
3087 
3088   PetscFunctionBegin;
3089   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3090   PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd));
3091   PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3092   if (dof) PetscValidIntPointer(coneOrientation, 3);
3093   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3094   PetscCheck(!(p < pStart) && !(p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3095   for (c = 0; c < dof; ++c) {
3096     PetscInt cdof, o = coneOrientation[c];
3097 
3098     PetscCall(PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof));
3099     PetscCheck(!o || (o >= -(cdof+1) && o < cdof),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ". %" PetscInt_FMT ")", o, -(cdof+1), cdof);
3100     mesh->coneOrientations[off+c] = o;
3101   }
3102   PetscFunctionReturn(0);
3103 }
3104 
3105 /*@
3106   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
3107 
3108   Not collective
3109 
3110   Input Parameters:
3111 + mesh - The DMPlex
3112 . p - The point, which must lie in the chart set with DMPlexSetChart()
3113 . conePos - The local index in the cone where the point should be put
3114 - conePoint - The mesh point to insert
3115 
3116   Level: beginner
3117 
3118 .seealso: `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`
3119 @*/
3120 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
3121 {
3122   DM_Plex       *mesh = (DM_Plex*) dm->data;
3123   PetscInt       pStart, pEnd;
3124   PetscInt       dof, off;
3125 
3126   PetscFunctionBegin;
3127   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3128   PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd));
3129   PetscCheck(!(p < pStart) && !(p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3130   PetscCheck(!(conePoint < pStart) && !(conePoint >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", conePoint, pStart, pEnd);
3131   PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3132   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3133   PetscCheck(!(conePos < 0) && !(conePos >= dof),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %" PetscInt_FMT " of point %" PetscInt_FMT " is not in the valid range [0, %" PetscInt_FMT ")", conePos, p, dof);
3134   mesh->cones[off+conePos] = conePoint;
3135   PetscFunctionReturn(0);
3136 }
3137 
3138 /*@
3139   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
3140 
3141   Not collective
3142 
3143   Input Parameters:
3144 + mesh - The DMPlex
3145 . p - The point, which must lie in the chart set with DMPlexSetChart()
3146 . conePos - The local index in the cone where the point should be put
3147 - coneOrientation - The point orientation to insert
3148 
3149   Level: beginner
3150 
3151   Notes:
3152   The meaning of coneOrientation values is detailed in DMPlexGetConeOrientation().
3153 
3154 .seealso: `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`
3155 @*/
3156 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
3157 {
3158   DM_Plex       *mesh = (DM_Plex*) dm->data;
3159   PetscInt       pStart, pEnd;
3160   PetscInt       dof, off;
3161 
3162   PetscFunctionBegin;
3163   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3164   PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd));
3165   PetscCheck(!(p < pStart) && !(p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3166   PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3167   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3168   PetscCheck(!(conePos < 0) && !(conePos >= dof),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %" PetscInt_FMT " of point %" PetscInt_FMT " is not in the valid range [0, %" PetscInt_FMT ")", conePos, p, dof);
3169   mesh->coneOrientations[off+conePos] = coneOrientation;
3170   PetscFunctionReturn(0);
3171 }
3172 
3173 /*@
3174   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
3175 
3176   Not collective
3177 
3178   Input Parameters:
3179 + mesh - The DMPlex
3180 - p - The point, which must lie in the chart set with DMPlexSetChart()
3181 
3182   Output Parameter:
3183 . size - The support size for point p
3184 
3185   Level: beginner
3186 
3187 .seealso: `DMPlexCreate()`, `DMPlexSetConeSize()`, `DMPlexSetChart()`, `DMPlexGetConeSize()`
3188 @*/
3189 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
3190 {
3191   DM_Plex       *mesh = (DM_Plex*) dm->data;
3192 
3193   PetscFunctionBegin;
3194   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3195   PetscValidIntPointer(size, 3);
3196   PetscCall(PetscSectionGetDof(mesh->supportSection, p, size));
3197   PetscFunctionReturn(0);
3198 }
3199 
3200 /*@
3201   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
3202 
3203   Not collective
3204 
3205   Input Parameters:
3206 + mesh - The DMPlex
3207 . p - The point, which must lie in the chart set with DMPlexSetChart()
3208 - size - The support size for point p
3209 
3210   Output Parameter:
3211 
3212   Note:
3213   This should be called after DMPlexSetChart().
3214 
3215   Level: beginner
3216 
3217 .seealso: `DMPlexCreate()`, `DMPlexGetSupportSize()`, `DMPlexSetChart()`
3218 @*/
3219 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
3220 {
3221   DM_Plex       *mesh = (DM_Plex*) dm->data;
3222 
3223   PetscFunctionBegin;
3224   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3225   PetscCall(PetscSectionSetDof(mesh->supportSection, p, size));
3226   PetscFunctionReturn(0);
3227 }
3228 
3229 /*@C
3230   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
3231 
3232   Not collective
3233 
3234   Input Parameters:
3235 + mesh - The DMPlex
3236 - p - The point, which must lie in the chart set with DMPlexSetChart()
3237 
3238   Output Parameter:
3239 . support - An array of points which are on the out-edges for point p
3240 
3241   Level: beginner
3242 
3243   Fortran Notes:
3244   Since it returns an array, this routine is only available in Fortran 90, and you must
3245   include petsc.h90 in your code.
3246   You must also call DMPlexRestoreSupport() after you finish using the returned array.
3247   DMPlexRestoreSupport() is not needed/available in C.
3248 
3249 .seealso: `DMPlexGetSupportSize()`, `DMPlexSetSupport()`, `DMPlexGetCone()`, `DMPlexSetChart()`
3250 @*/
3251 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
3252 {
3253   DM_Plex       *mesh = (DM_Plex*) dm->data;
3254   PetscInt       off;
3255 
3256   PetscFunctionBegin;
3257   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3258   PetscValidPointer(support, 3);
3259   PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off));
3260   *support = &mesh->supports[off];
3261   PetscFunctionReturn(0);
3262 }
3263 
3264 /*@
3265   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
3266 
3267   Not collective
3268 
3269   Input Parameters:
3270 + mesh - The DMPlex
3271 . p - The point, which must lie in the chart set with DMPlexSetChart()
3272 - support - An array of points which are on the out-edges for point p
3273 
3274   Output Parameter:
3275 
3276   Note:
3277   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
3278 
3279   Level: beginner
3280 
3281 .seealso: `DMPlexSetCone()`, `DMPlexSetConeSize()`, `DMPlexCreate()`, `DMPlexGetSupport()`, `DMPlexSetChart()`, `DMPlexSetSupportSize()`, `DMSetUp()`
3282 @*/
3283 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
3284 {
3285   DM_Plex       *mesh = (DM_Plex*) dm->data;
3286   PetscInt       pStart, pEnd;
3287   PetscInt       dof, off, c;
3288 
3289   PetscFunctionBegin;
3290   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3291   PetscCall(PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd));
3292   PetscCall(PetscSectionGetDof(mesh->supportSection, p, &dof));
3293   if (dof) PetscValidIntPointer(support, 3);
3294   PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off));
3295   PetscCheck(!(p < pStart) && !(p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3296   for (c = 0; c < dof; ++c) {
3297     PetscCheck(!(support[c] < pStart) && !(support[c] >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", support[c], pStart, pEnd);
3298     mesh->supports[off+c] = support[c];
3299   }
3300   PetscFunctionReturn(0);
3301 }
3302 
3303 /*@
3304   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
3305 
3306   Not collective
3307 
3308   Input Parameters:
3309 + mesh - The DMPlex
3310 . p - The point, which must lie in the chart set with DMPlexSetChart()
3311 . supportPos - The local index in the cone where the point should be put
3312 - supportPoint - The mesh point to insert
3313 
3314   Level: beginner
3315 
3316 .seealso: `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`
3317 @*/
3318 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3319 {
3320   DM_Plex       *mesh = (DM_Plex*) dm->data;
3321   PetscInt       pStart, pEnd;
3322   PetscInt       dof, off;
3323 
3324   PetscFunctionBegin;
3325   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3326   PetscCall(PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd));
3327   PetscCall(PetscSectionGetDof(mesh->supportSection, p, &dof));
3328   PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off));
3329   PetscCheck(!(p < pStart) && !(p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3330   PetscCheck(!(supportPoint < pStart) && !(supportPoint >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", supportPoint, pStart, pEnd);
3331   PetscCheck(supportPos < dof,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %" PetscInt_FMT " of point %" PetscInt_FMT " is not in the valid range [0, %" PetscInt_FMT ")", supportPos, p, dof);
3332   mesh->supports[off+supportPos] = supportPoint;
3333   PetscFunctionReturn(0);
3334 }
3335 
3336 /* Converts an orientation o in the current numbering to the previous scheme used in Plex */
3337 PetscInt DMPolytopeConvertNewOrientation_Internal(DMPolytopeType ct, PetscInt o)
3338 {
3339   switch (ct) {
3340     case DM_POLYTOPE_SEGMENT:
3341       if (o == -1) return -2;
3342       break;
3343     case DM_POLYTOPE_TRIANGLE:
3344       if (o == -3) return -1;
3345       if (o == -2) return -3;
3346       if (o == -1) return -2;
3347       break;
3348     case DM_POLYTOPE_QUADRILATERAL:
3349       if (o == -4) return -2;
3350       if (o == -3) return -1;
3351       if (o == -2) return -4;
3352       if (o == -1) return -3;
3353       break;
3354     default: return o;
3355   }
3356   return o;
3357 }
3358 
3359 /* Converts an orientation o in the previous scheme used in Plex to the current numbering */
3360 PetscInt DMPolytopeConvertOldOrientation_Internal(DMPolytopeType ct, PetscInt o)
3361 {
3362   switch (ct) {
3363     case DM_POLYTOPE_SEGMENT:
3364       if ((o == -2) || (o == 1)) return -1;
3365       if (o == -1) return 0;
3366       break;
3367     case DM_POLYTOPE_TRIANGLE:
3368       if (o == -3) return -2;
3369       if (o == -2) return -1;
3370       if (o == -1) return -3;
3371       break;
3372     case DM_POLYTOPE_QUADRILATERAL:
3373       if (o == -4) return -2;
3374       if (o == -3) return -1;
3375       if (o == -2) return -4;
3376       if (o == -1) return -3;
3377       break;
3378     default: return o;
3379   }
3380   return o;
3381 }
3382 
3383 /* Takes in a mesh whose orientations are in the previous scheme and converts them all to the current numbering */
3384 PetscErrorCode DMPlexConvertOldOrientations_Internal(DM dm)
3385 {
3386   PetscInt       pStart, pEnd, p;
3387 
3388   PetscFunctionBegin;
3389   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3390   for (p = pStart; p < pEnd; ++p) {
3391     const PetscInt *cone, *ornt;
3392     PetscInt        coneSize, c;
3393 
3394     PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
3395     PetscCall(DMPlexGetCone(dm, p, &cone));
3396     PetscCall(DMPlexGetConeOrientation(dm, p, &ornt));
3397     for (c = 0; c < coneSize; ++c) {
3398       DMPolytopeType ct;
3399       const PetscInt o = ornt[c];
3400 
3401       PetscCall(DMPlexGetCellType(dm, cone[c], &ct));
3402       switch (ct) {
3403         case DM_POLYTOPE_SEGMENT:
3404           if ((o == -2) || (o == 1)) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -1));
3405           if (o == -1) PetscCall(DMPlexInsertConeOrientation(dm, p, c, 0));
3406           break;
3407         case DM_POLYTOPE_TRIANGLE:
3408           if (o == -3) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -2));
3409           if (o == -2) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -1));
3410           if (o == -1) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -3));
3411           break;
3412         case DM_POLYTOPE_QUADRILATERAL:
3413           if (o == -4) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -2));
3414           if (o == -3) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -1));
3415           if (o == -2) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -4));
3416           if (o == -1) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -3));
3417           break;
3418         default: break;
3419       }
3420     }
3421   }
3422   PetscFunctionReturn(0);
3423 }
3424 
3425 static PetscErrorCode DMPlexGetTransitiveClosure_Depth1_Private(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3426 {
3427   DMPolytopeType  ct = DM_POLYTOPE_UNKNOWN;
3428   PetscInt       *closure;
3429   const PetscInt *tmp = NULL, *tmpO = NULL;
3430   PetscInt        off = 0, tmpSize, t;
3431 
3432   PetscFunctionBeginHot;
3433   if (ornt) {
3434     PetscCall(DMPlexGetCellType(dm, p, &ct));
3435     if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3436   }
3437   if (*points) {
3438     closure = *points;
3439   } else {
3440     PetscInt maxConeSize, maxSupportSize;
3441     PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
3442     PetscCall(DMGetWorkArray(dm, 2*(PetscMax(maxConeSize, maxSupportSize)+1), MPIU_INT, &closure));
3443   }
3444   if (useCone) {
3445     PetscCall(DMPlexGetConeSize(dm, p, &tmpSize));
3446     PetscCall(DMPlexGetCone(dm, p, &tmp));
3447     PetscCall(DMPlexGetConeOrientation(dm, p, &tmpO));
3448   } else {
3449     PetscCall(DMPlexGetSupportSize(dm, p, &tmpSize));
3450     PetscCall(DMPlexGetSupport(dm, p, &tmp));
3451   }
3452   if (ct == DM_POLYTOPE_UNKNOWN) {
3453     closure[off++] = p;
3454     closure[off++] = 0;
3455     for (t = 0; t < tmpSize; ++t) {
3456       closure[off++] = tmp[t];
3457       closure[off++] = tmpO ? tmpO[t] : 0;
3458     }
3459   } else {
3460     const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, ornt);
3461 
3462     /* We assume that cells with a valid type have faces with a valid type */
3463     closure[off++] = p;
3464     closure[off++] = ornt;
3465     for (t = 0; t < tmpSize; ++t) {
3466       DMPolytopeType ft;
3467 
3468       PetscCall(DMPlexGetCellType(dm, tmp[t], &ft));
3469       closure[off++] = tmp[arr[t]];
3470       closure[off++] = tmpO ? DMPolytopeTypeComposeOrientation(ft, ornt, tmpO[t]) : 0;
3471     }
3472   }
3473   if (numPoints) *numPoints = tmpSize+1;
3474   if (points)    *points    = closure;
3475   PetscFunctionReturn(0);
3476 }
3477 
3478 /* We need a special tensor verison becasue we want to allow duplicate points in the endcaps for hybrid cells */
3479 static PetscErrorCode DMPlexTransitiveClosure_Tensor_Internal(DM dm, PetscInt point, DMPolytopeType ct, PetscInt o, PetscBool useCone, PetscInt *numPoints, PetscInt **points)
3480 {
3481   const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, o);
3482   const PetscInt *cone, *ornt;
3483   PetscInt       *pts,  *closure = NULL;
3484   DMPolytopeType  ft;
3485   PetscInt        maxConeSize, maxSupportSize, coneSeries, supportSeries, maxSize;
3486   PetscInt        dim, coneSize, c, d, clSize, cl;
3487 
3488   PetscFunctionBeginHot;
3489   PetscCall(DMGetDimension(dm, &dim));
3490   PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3491   PetscCall(DMPlexGetCone(dm, point, &cone));
3492   PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3493   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
3494   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    dim+1)-1)/(maxConeSize-1))    : dim+1;
3495   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, dim+1)-1)/(maxSupportSize-1)) : dim+1;
3496   maxSize       = PetscMax(coneSeries, supportSeries);
3497   if (*points) {pts  = *points;}
3498   else         PetscCall(DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &pts));
3499   c    = 0;
3500   pts[c++] = point;
3501   pts[c++] = o;
3502   PetscCall(DMPlexGetCellType(dm, cone[arr[0*2+0]], &ft));
3503   PetscCall(DMPlexGetTransitiveClosure_Internal(dm, cone[arr[0*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[0*2+1], ornt[0]), useCone, &clSize, &closure));
3504   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3505   PetscCall(DMPlexGetTransitiveClosure_Internal(dm, cone[arr[1*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[1*2+1], ornt[1]), useCone, &clSize, &closure));
3506   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3507   PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], useCone, &clSize, &closure));
3508   for (d = 2; d < coneSize; ++d) {
3509     PetscCall(DMPlexGetCellType(dm, cone[arr[d*2+0]], &ft));
3510     pts[c++] = cone[arr[d*2+0]];
3511     pts[c++] = DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]);
3512   }
3513   if (dim >= 3) {
3514     for (d = 2; d < coneSize; ++d) {
3515       const PetscInt  fpoint = cone[arr[d*2+0]];
3516       const PetscInt *fcone, *fornt;
3517       PetscInt        fconeSize, fc, i;
3518 
3519       PetscCall(DMPlexGetCellType(dm, fpoint, &ft));
3520       const PetscInt *farr = DMPolytopeTypeGetArrangment(ft, DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]));
3521       PetscCall(DMPlexGetConeSize(dm, fpoint, &fconeSize));
3522       PetscCall(DMPlexGetCone(dm, fpoint, &fcone));
3523       PetscCall(DMPlexGetConeOrientation(dm, fpoint, &fornt));
3524       for (fc = 0; fc < fconeSize; ++fc) {
3525         const PetscInt cp = fcone[farr[fc*2+0]];
3526         const PetscInt co = farr[fc*2+1];
3527 
3528         for (i = 0; i < c; i += 2) if (pts[i] == cp) break;
3529         if (i == c) {
3530           PetscCall(DMPlexGetCellType(dm, cp, &ft));
3531           pts[c++] = cp;
3532           pts[c++] = DMPolytopeTypeComposeOrientation(ft, co, fornt[farr[fc*2+0]]);
3533         }
3534       }
3535     }
3536   }
3537   *numPoints = c/2;
3538   *points    = pts;
3539   PetscFunctionReturn(0);
3540 }
3541 
3542 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3543 {
3544   DMPolytopeType ct;
3545   PetscInt      *closure, *fifo;
3546   PetscInt       closureSize = 0, fifoStart = 0, fifoSize = 0;
3547   PetscInt       maxConeSize, maxSupportSize, coneSeries, supportSeries;
3548   PetscInt       depth, maxSize;
3549 
3550   PetscFunctionBeginHot;
3551   PetscCall(DMPlexGetDepth(dm, &depth));
3552   if (depth == 1) {
3553     PetscCall(DMPlexGetTransitiveClosure_Depth1_Private(dm, p, ornt, useCone, numPoints, points));
3554     PetscFunctionReturn(0);
3555   }
3556   PetscCall(DMPlexGetCellType(dm, p, &ct));
3557   if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3558   if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
3559     PetscCall(DMPlexTransitiveClosure_Tensor_Internal(dm, p, ct, ornt, useCone, numPoints, points));
3560     PetscFunctionReturn(0);
3561   }
3562   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
3563   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    depth+1)-1)/(maxConeSize-1))    : depth+1;
3564   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, depth+1)-1)/(maxSupportSize-1)) : depth+1;
3565   maxSize       = PetscMax(coneSeries, supportSeries);
3566   PetscCall(DMGetWorkArray(dm, 3*maxSize, MPIU_INT, &fifo));
3567   if (*points) {closure = *points;}
3568   else         PetscCall(DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &closure));
3569   closure[closureSize++] = p;
3570   closure[closureSize++] = ornt;
3571   fifo[fifoSize++]       = p;
3572   fifo[fifoSize++]       = ornt;
3573   fifo[fifoSize++]       = ct;
3574   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3575   while (fifoSize - fifoStart) {
3576     const PetscInt       q    = fifo[fifoStart++];
3577     const PetscInt       o    = fifo[fifoStart++];
3578     const DMPolytopeType qt   = (DMPolytopeType) fifo[fifoStart++];
3579     const PetscInt      *qarr = DMPolytopeTypeGetArrangment(qt, o);
3580     const PetscInt      *tmp, *tmpO;
3581     PetscInt             tmpSize, t;
3582 
3583     if (PetscDefined(USE_DEBUG)) {
3584       PetscInt nO = DMPolytopeTypeGetNumArrangments(qt)/2;
3585       PetscCheck(!o || !(o >= nO || o < -nO),PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid orientation %" PetscInt_FMT " not in [%" PetscInt_FMT ",%" PetscInt_FMT ") for %s %" PetscInt_FMT, o, -nO, nO, DMPolytopeTypes[qt], q);
3586     }
3587     if (useCone) {
3588       PetscCall(DMPlexGetConeSize(dm, q, &tmpSize));
3589       PetscCall(DMPlexGetCone(dm, q, &tmp));
3590       PetscCall(DMPlexGetConeOrientation(dm, q, &tmpO));
3591     } else {
3592       PetscCall(DMPlexGetSupportSize(dm, q, &tmpSize));
3593       PetscCall(DMPlexGetSupport(dm, q, &tmp));
3594       tmpO = NULL;
3595     }
3596     for (t = 0; t < tmpSize; ++t) {
3597       const PetscInt ip = useCone && qarr ? qarr[t*2]   : t;
3598       const PetscInt io = useCone && qarr ? qarr[t*2+1] : 0;
3599       const PetscInt cp = tmp[ip];
3600       PetscCall(DMPlexGetCellType(dm, cp, &ct));
3601       const PetscInt co = tmpO ? DMPolytopeTypeComposeOrientation(ct, io, tmpO[ip]) : 0;
3602       PetscInt       c;
3603 
3604       /* Check for duplicate */
3605       for (c = 0; c < closureSize; c += 2) {
3606         if (closure[c] == cp) break;
3607       }
3608       if (c == closureSize) {
3609         closure[closureSize++] = cp;
3610         closure[closureSize++] = co;
3611         fifo[fifoSize++]       = cp;
3612         fifo[fifoSize++]       = co;
3613         fifo[fifoSize++]       = ct;
3614       }
3615     }
3616   }
3617   PetscCall(DMRestoreWorkArray(dm, 3*maxSize, MPIU_INT, &fifo));
3618   if (numPoints) *numPoints = closureSize/2;
3619   if (points)    *points    = closure;
3620   PetscFunctionReturn(0);
3621 }
3622 
3623 /*@C
3624   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
3625 
3626   Not collective
3627 
3628   Input Parameters:
3629 + dm      - The DMPlex
3630 . p       - The mesh point
3631 - useCone - PETSC_TRUE for the closure, otherwise return the star
3632 
3633   Input/Output Parameter:
3634 . points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...];
3635            if NULL on input, internal storage will be returned, otherwise the provided array is used
3636 
3637   Output Parameter:
3638 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3639 
3640   Note:
3641   If using internal storage (points is NULL on input), each call overwrites the last output.
3642 
3643   Fortran Notes:
3644   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3645 
3646   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3647 
3648   Level: beginner
3649 
3650 .seealso: `DMPlexRestoreTransitiveClosure()`, `DMPlexCreate()`, `DMPlexSetCone()`, `DMPlexSetChart()`, `DMPlexGetCone()`
3651 @*/
3652 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3653 {
3654   PetscFunctionBeginHot;
3655   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3656   if (numPoints) PetscValidIntPointer(numPoints, 4);
3657   if (points)    PetscValidPointer(points, 5);
3658   PetscCall(DMPlexGetTransitiveClosure_Internal(dm, p, 0, useCone, numPoints, points));
3659   PetscFunctionReturn(0);
3660 }
3661 
3662 /*@C
3663   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
3664 
3665   Not collective
3666 
3667   Input Parameters:
3668 + dm        - The DMPlex
3669 . p         - The mesh point
3670 . useCone   - PETSC_TRUE for the closure, otherwise return the star
3671 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3672 - points    - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3673 
3674   Note:
3675   If not using internal storage (points is not NULL on input), this call is unnecessary
3676 
3677   Fortran Notes:
3678   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3679 
3680   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3681 
3682   Level: beginner
3683 
3684 .seealso: `DMPlexGetTransitiveClosure()`, `DMPlexCreate()`, `DMPlexSetCone()`, `DMPlexSetChart()`, `DMPlexGetCone()`
3685 @*/
3686 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3687 {
3688   PetscFunctionBeginHot;
3689   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3690   if (numPoints) *numPoints = 0;
3691   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_INT, points));
3692   PetscFunctionReturn(0);
3693 }
3694 
3695 /*@
3696   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
3697 
3698   Not collective
3699 
3700   Input Parameter:
3701 . mesh - The DMPlex
3702 
3703   Output Parameters:
3704 + maxConeSize - The maximum number of in-edges
3705 - maxSupportSize - The maximum number of out-edges
3706 
3707   Level: beginner
3708 
3709 .seealso: `DMPlexCreate()`, `DMPlexSetConeSize()`, `DMPlexSetChart()`
3710 @*/
3711 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
3712 {
3713   DM_Plex *mesh = (DM_Plex*) dm->data;
3714 
3715   PetscFunctionBegin;
3716   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3717   if (maxConeSize) {
3718     PetscCall(PetscSectionGetMaxDof(mesh->coneSection, maxConeSize));
3719   }
3720   if (maxSupportSize) {
3721     PetscCall(PetscSectionGetMaxDof(mesh->supportSection, maxSupportSize));
3722   }
3723   PetscFunctionReturn(0);
3724 }
3725 
3726 PetscErrorCode DMSetUp_Plex(DM dm)
3727 {
3728   DM_Plex       *mesh = (DM_Plex*) dm->data;
3729   PetscInt       size, maxSupportSize;
3730 
3731   PetscFunctionBegin;
3732   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3733   PetscCall(PetscSectionSetUp(mesh->coneSection));
3734   PetscCall(PetscSectionGetStorageSize(mesh->coneSection, &size));
3735   PetscCall(PetscMalloc1(size, &mesh->cones));
3736   PetscCall(PetscCalloc1(size, &mesh->coneOrientations));
3737   PetscCall(PetscLogObjectMemory((PetscObject) dm, size*2*sizeof(PetscInt)));
3738   PetscCall(PetscSectionGetMaxDof(mesh->supportSection, &maxSupportSize));
3739   if (maxSupportSize) {
3740     PetscCall(PetscSectionSetUp(mesh->supportSection));
3741     PetscCall(PetscSectionGetStorageSize(mesh->supportSection, &size));
3742     PetscCall(PetscMalloc1(size, &mesh->supports));
3743     PetscCall(PetscLogObjectMemory((PetscObject) dm, size*sizeof(PetscInt)));
3744   }
3745   PetscFunctionReturn(0);
3746 }
3747 
3748 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
3749 {
3750   PetscFunctionBegin;
3751   if (subdm) PetscCall(DMClone(dm, subdm));
3752   PetscCall(DMCreateSectionSubDM(dm, numFields, fields, is, subdm));
3753   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
3754   if (dm->useNatural && dm->sfMigration) {
3755     PetscSF        sfMigrationInv,sfNatural;
3756     PetscSection   section, sectionSeq;
3757 
3758     (*subdm)->sfMigration = dm->sfMigration;
3759     PetscCall(PetscObjectReference((PetscObject) dm->sfMigration));
3760     PetscCall(DMGetLocalSection((*subdm), &section));
3761     PetscCall(PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv));
3762     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq));
3763     PetscCall(PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq));
3764 
3765     PetscCall(DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural));
3766     (*subdm)->sfNatural = sfNatural;
3767     PetscCall(PetscSectionDestroy(&sectionSeq));
3768     PetscCall(PetscSFDestroy(&sfMigrationInv));
3769   }
3770   PetscFunctionReturn(0);
3771 }
3772 
3773 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
3774 {
3775   PetscInt       i = 0;
3776 
3777   PetscFunctionBegin;
3778   PetscCall(DMClone(dms[0], superdm));
3779   PetscCall(DMCreateSectionSuperDM(dms, len, is, superdm));
3780   (*superdm)->useNatural = PETSC_FALSE;
3781   for (i = 0; i < len; i++) {
3782     if (dms[i]->useNatural && dms[i]->sfMigration) {
3783       PetscSF        sfMigrationInv,sfNatural;
3784       PetscSection   section, sectionSeq;
3785 
3786       (*superdm)->sfMigration = dms[i]->sfMigration;
3787       PetscCall(PetscObjectReference((PetscObject) dms[i]->sfMigration));
3788       (*superdm)->useNatural = PETSC_TRUE;
3789       PetscCall(DMGetLocalSection((*superdm), &section));
3790       PetscCall(PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv));
3791       PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq));
3792       PetscCall(PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq));
3793 
3794       PetscCall(DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural));
3795       (*superdm)->sfNatural = sfNatural;
3796       PetscCall(PetscSectionDestroy(&sectionSeq));
3797       PetscCall(PetscSFDestroy(&sfMigrationInv));
3798       break;
3799     }
3800   }
3801   PetscFunctionReturn(0);
3802 }
3803 
3804 /*@
3805   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
3806 
3807   Not collective
3808 
3809   Input Parameter:
3810 . mesh - The DMPlex
3811 
3812   Output Parameter:
3813 
3814   Note:
3815   This should be called after all calls to DMPlexSetCone()
3816 
3817   Level: beginner
3818 
3819 .seealso: `DMPlexCreate()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMPlexSetCone()`
3820 @*/
3821 PetscErrorCode DMPlexSymmetrize(DM dm)
3822 {
3823   DM_Plex       *mesh = (DM_Plex*) dm->data;
3824   PetscInt      *offsets;
3825   PetscInt       supportSize;
3826   PetscInt       pStart, pEnd, p;
3827 
3828   PetscFunctionBegin;
3829   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3830   PetscCheck(!mesh->supports,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
3831   PetscCall(PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0));
3832   /* Calculate support sizes */
3833   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3834   for (p = pStart; p < pEnd; ++p) {
3835     PetscInt dof, off, c;
3836 
3837     PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3838     PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3839     for (c = off; c < off+dof; ++c) {
3840       PetscCall(PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1));
3841     }
3842   }
3843   PetscCall(PetscSectionSetUp(mesh->supportSection));
3844   /* Calculate supports */
3845   PetscCall(PetscSectionGetStorageSize(mesh->supportSection, &supportSize));
3846   PetscCall(PetscMalloc1(supportSize, &mesh->supports));
3847   PetscCall(PetscCalloc1(pEnd - pStart, &offsets));
3848   for (p = pStart; p < pEnd; ++p) {
3849     PetscInt dof, off, c;
3850 
3851     PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3852     PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3853     for (c = off; c < off+dof; ++c) {
3854       const PetscInt q = mesh->cones[c];
3855       PetscInt       offS;
3856 
3857       PetscCall(PetscSectionGetOffset(mesh->supportSection, q, &offS));
3858 
3859       mesh->supports[offS+offsets[q]] = p;
3860       ++offsets[q];
3861     }
3862   }
3863   PetscCall(PetscFree(offsets));
3864   PetscCall(PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0));
3865   PetscFunctionReturn(0);
3866 }
3867 
3868 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
3869 {
3870   IS             stratumIS;
3871 
3872   PetscFunctionBegin;
3873   if (pStart >= pEnd) PetscFunctionReturn(0);
3874   if (PetscDefined(USE_DEBUG)) {
3875     PetscInt  qStart, qEnd, numLevels, level;
3876     PetscBool overlap = PETSC_FALSE;
3877     PetscCall(DMLabelGetNumValues(label, &numLevels));
3878     for (level = 0; level < numLevels; level++) {
3879       PetscCall(DMLabelGetStratumBounds(label, level, &qStart, &qEnd));
3880       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
3881     }
3882     PetscCheck(!overlap,PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %" PetscInt_FMT " range [%" PetscInt_FMT ",%" PetscInt_FMT ") overlaps with depth %" PetscInt_FMT " range [%" PetscInt_FMT ",%" PetscInt_FMT ")", depth, pStart, pEnd, level, qStart, qEnd);
3883   }
3884   PetscCall(ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS));
3885   PetscCall(DMLabelSetStratumIS(label, depth, stratumIS));
3886   PetscCall(ISDestroy(&stratumIS));
3887   PetscFunctionReturn(0);
3888 }
3889 
3890 /*@
3891   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
3892   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
3893   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
3894   the DAG.
3895 
3896   Collective on dm
3897 
3898   Input Parameter:
3899 . mesh - The DMPlex
3900 
3901   Output Parameter:
3902 
3903   Notes:
3904   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
3905   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
3906   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
3907   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
3908   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
3909 
3910   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
3911   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
3912   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
3913   to interpolate only that one (e0), so that
3914 $  cone(c0) = {e0, v2}
3915 $  cone(e0) = {v0, v1}
3916   If DMPlexStratify() is run on this mesh, it will give depths
3917 $  depth 0 = {v0, v1, v2}
3918 $  depth 1 = {e0, c0}
3919   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
3920 
3921   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
3922 
3923   Level: beginner
3924 
3925 .seealso: `DMPlexCreate()`, `DMPlexSymmetrize()`, `DMPlexComputeCellTypes()`
3926 @*/
3927 PetscErrorCode DMPlexStratify(DM dm)
3928 {
3929   DM_Plex       *mesh = (DM_Plex*) dm->data;
3930   DMLabel        label;
3931   PetscInt       pStart, pEnd, p;
3932   PetscInt       numRoots = 0, numLeaves = 0;
3933 
3934   PetscFunctionBegin;
3935   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3936   PetscCall(PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0));
3937 
3938   /* Create depth label */
3939   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3940   PetscCall(DMCreateLabel(dm, "depth"));
3941   PetscCall(DMPlexGetDepthLabel(dm, &label));
3942 
3943   {
3944     /* Initialize roots and count leaves */
3945     PetscInt sMin = PETSC_MAX_INT;
3946     PetscInt sMax = PETSC_MIN_INT;
3947     PetscInt coneSize, supportSize;
3948 
3949     for (p = pStart; p < pEnd; ++p) {
3950       PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
3951       PetscCall(DMPlexGetSupportSize(dm, p, &supportSize));
3952       if (!coneSize && supportSize) {
3953         sMin = PetscMin(p, sMin);
3954         sMax = PetscMax(p, sMax);
3955         ++numRoots;
3956       } else if (!supportSize && coneSize) {
3957         ++numLeaves;
3958       } else if (!supportSize && !coneSize) {
3959         /* Isolated points */
3960         sMin = PetscMin(p, sMin);
3961         sMax = PetscMax(p, sMax);
3962       }
3963     }
3964     PetscCall(DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1));
3965   }
3966 
3967   if (numRoots + numLeaves == (pEnd - pStart)) {
3968     PetscInt sMin = PETSC_MAX_INT;
3969     PetscInt sMax = PETSC_MIN_INT;
3970     PetscInt coneSize, supportSize;
3971 
3972     for (p = pStart; p < pEnd; ++p) {
3973       PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
3974       PetscCall(DMPlexGetSupportSize(dm, p, &supportSize));
3975       if (!supportSize && coneSize) {
3976         sMin = PetscMin(p, sMin);
3977         sMax = PetscMax(p, sMax);
3978       }
3979     }
3980     PetscCall(DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1));
3981   } else {
3982     PetscInt level = 0;
3983     PetscInt qStart, qEnd, q;
3984 
3985     PetscCall(DMLabelGetStratumBounds(label, level, &qStart, &qEnd));
3986     while (qEnd > qStart) {
3987       PetscInt sMin = PETSC_MAX_INT;
3988       PetscInt sMax = PETSC_MIN_INT;
3989 
3990       for (q = qStart; q < qEnd; ++q) {
3991         const PetscInt *support;
3992         PetscInt        supportSize, s;
3993 
3994         PetscCall(DMPlexGetSupportSize(dm, q, &supportSize));
3995         PetscCall(DMPlexGetSupport(dm, q, &support));
3996         for (s = 0; s < supportSize; ++s) {
3997           sMin = PetscMin(support[s], sMin);
3998           sMax = PetscMax(support[s], sMax);
3999         }
4000       }
4001       PetscCall(DMLabelGetNumValues(label, &level));
4002       PetscCall(DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1));
4003       PetscCall(DMLabelGetStratumBounds(label, level, &qStart, &qEnd));
4004     }
4005   }
4006   { /* just in case there is an empty process */
4007     PetscInt numValues, maxValues = 0, v;
4008 
4009     PetscCall(DMLabelGetNumValues(label, &numValues));
4010     PetscCallMPI(MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm)));
4011     for (v = numValues; v < maxValues; v++) {
4012       PetscCall(DMLabelAddStratum(label, v));
4013     }
4014   }
4015   PetscCall(PetscObjectStateGet((PetscObject) label, &mesh->depthState));
4016   PetscCall(PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0));
4017   PetscFunctionReturn(0);
4018 }
4019 
4020 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
4021 {
4022   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
4023   PetscInt       dim, depth, pheight, coneSize;
4024 
4025   PetscFunctionBeginHot;
4026   PetscCall(DMGetDimension(dm, &dim));
4027   PetscCall(DMPlexGetDepth(dm, &depth));
4028   PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
4029   pheight = depth - pdepth;
4030   if (depth <= 1) {
4031     switch (pdepth) {
4032       case 0: ct = DM_POLYTOPE_POINT;break;
4033       case 1:
4034         switch (coneSize) {
4035           case 2: ct = DM_POLYTOPE_SEGMENT;break;
4036           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4037           case 4:
4038           switch (dim) {
4039             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
4040             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
4041             default: break;
4042           }
4043           break;
4044         case 5: ct = DM_POLYTOPE_PYRAMID;break;
4045         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
4046         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
4047         default: break;
4048       }
4049     }
4050   } else {
4051     if (pdepth == 0) {
4052       ct = DM_POLYTOPE_POINT;
4053     } else if (pheight == 0) {
4054       switch (dim) {
4055         case 1:
4056           switch (coneSize) {
4057             case 2: ct = DM_POLYTOPE_SEGMENT;break;
4058             default: break;
4059           }
4060           break;
4061         case 2:
4062           switch (coneSize) {
4063             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4064             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
4065             default: break;
4066           }
4067           break;
4068         case 3:
4069           switch (coneSize) {
4070             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
4071             case 5:
4072             {
4073               const PetscInt *cone;
4074               PetscInt        faceConeSize;
4075 
4076               PetscCall(DMPlexGetCone(dm, p, &cone));
4077               PetscCall(DMPlexGetConeSize(dm, cone[0], &faceConeSize));
4078               switch (faceConeSize) {
4079                 case 3: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
4080                 case 4: ct = DM_POLYTOPE_PYRAMID;break;
4081               }
4082             }
4083             break;
4084             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
4085             default: break;
4086           }
4087           break;
4088         default: break;
4089       }
4090     } else if (pheight > 0) {
4091       switch (coneSize) {
4092         case 2: ct = DM_POLYTOPE_SEGMENT;break;
4093         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4094         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
4095         default: break;
4096       }
4097     }
4098   }
4099   *pt = ct;
4100   PetscFunctionReturn(0);
4101 }
4102 
4103 /*@
4104   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
4105 
4106   Collective on dm
4107 
4108   Input Parameter:
4109 . mesh - The DMPlex
4110 
4111   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
4112 
4113   Level: developer
4114 
4115   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
4116   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
4117   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
4118 
4119 .seealso: `DMPlexCreate()`, `DMPlexSymmetrize()`, `DMPlexStratify()`, `DMGetLabel()`, `DMCreateLabel()`
4120 @*/
4121 PetscErrorCode DMPlexComputeCellTypes(DM dm)
4122 {
4123   DM_Plex       *mesh;
4124   DMLabel        ctLabel;
4125   PetscInt       pStart, pEnd, p;
4126 
4127   PetscFunctionBegin;
4128   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4129   mesh = (DM_Plex *) dm->data;
4130   PetscCall(DMCreateLabel(dm, "celltype"));
4131   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
4132   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4133   for (p = pStart; p < pEnd; ++p) {
4134     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
4135     PetscInt       pdepth;
4136 
4137     PetscCall(DMPlexGetPointDepth(dm, p, &pdepth));
4138     PetscCall(DMPlexComputeCellType_Internal(dm, p, pdepth, &ct));
4139     PetscCheck(ct != DM_POLYTOPE_UNKNOWN,PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %" PetscInt_FMT " is screwed up", p);
4140     PetscCall(DMLabelSetValue(ctLabel, p, ct));
4141   }
4142   PetscCall(PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState));
4143   PetscCall(PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view"));
4144   PetscFunctionReturn(0);
4145 }
4146 
4147 /*@C
4148   DMPlexGetJoin - Get an array for the join of the set of points
4149 
4150   Not Collective
4151 
4152   Input Parameters:
4153 + dm - The DMPlex object
4154 . numPoints - The number of input points for the join
4155 - points - The input points
4156 
4157   Output Parameters:
4158 + numCoveredPoints - The number of points in the join
4159 - coveredPoints - The points in the join
4160 
4161   Level: intermediate
4162 
4163   Note: Currently, this is restricted to a single level join
4164 
4165   Fortran Notes:
4166   Since it returns an array, this routine is only available in Fortran 90, and you must
4167   include petsc.h90 in your code.
4168 
4169   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4170 
4171 .seealso: `DMPlexRestoreJoin()`, `DMPlexGetMeet()`
4172 @*/
4173 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4174 {
4175   DM_Plex       *mesh = (DM_Plex*) dm->data;
4176   PetscInt      *join[2];
4177   PetscInt       joinSize, i = 0;
4178   PetscInt       dof, off, p, c, m;
4179   PetscInt       maxSupportSize;
4180 
4181   PetscFunctionBegin;
4182   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4183   PetscValidIntPointer(points, 3);
4184   PetscValidIntPointer(numCoveredPoints, 4);
4185   PetscValidPointer(coveredPoints, 5);
4186   PetscCall(PetscSectionGetMaxDof(mesh->supportSection, &maxSupportSize));
4187   PetscCall(DMGetWorkArray(dm, maxSupportSize, MPIU_INT, &join[0]));
4188   PetscCall(DMGetWorkArray(dm, maxSupportSize, MPIU_INT, &join[1]));
4189   /* Copy in support of first point */
4190   PetscCall(PetscSectionGetDof(mesh->supportSection, points[0], &dof));
4191   PetscCall(PetscSectionGetOffset(mesh->supportSection, points[0], &off));
4192   for (joinSize = 0; joinSize < dof; ++joinSize) {
4193     join[i][joinSize] = mesh->supports[off+joinSize];
4194   }
4195   /* Check each successive support */
4196   for (p = 1; p < numPoints; ++p) {
4197     PetscInt newJoinSize = 0;
4198 
4199     PetscCall(PetscSectionGetDof(mesh->supportSection, points[p], &dof));
4200     PetscCall(PetscSectionGetOffset(mesh->supportSection, points[p], &off));
4201     for (c = 0; c < dof; ++c) {
4202       const PetscInt point = mesh->supports[off+c];
4203 
4204       for (m = 0; m < joinSize; ++m) {
4205         if (point == join[i][m]) {
4206           join[1-i][newJoinSize++] = point;
4207           break;
4208         }
4209       }
4210     }
4211     joinSize = newJoinSize;
4212     i        = 1-i;
4213   }
4214   *numCoveredPoints = joinSize;
4215   *coveredPoints    = join[i];
4216   PetscCall(DMRestoreWorkArray(dm, maxSupportSize, MPIU_INT, &join[1-i]));
4217   PetscFunctionReturn(0);
4218 }
4219 
4220 /*@C
4221   DMPlexRestoreJoin - Restore an array for the join of the set of points
4222 
4223   Not Collective
4224 
4225   Input Parameters:
4226 + dm - The DMPlex object
4227 . numPoints - The number of input points for the join
4228 - points - The input points
4229 
4230   Output Parameters:
4231 + numCoveredPoints - The number of points in the join
4232 - coveredPoints - The points in the join
4233 
4234   Fortran Notes:
4235   Since it returns an array, this routine is only available in Fortran 90, and you must
4236   include petsc.h90 in your code.
4237 
4238   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4239 
4240   Level: intermediate
4241 
4242 .seealso: `DMPlexGetJoin()`, `DMPlexGetFullJoin()`, `DMPlexGetMeet()`
4243 @*/
4244 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4245 {
4246   PetscFunctionBegin;
4247   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4248   if (points) PetscValidIntPointer(points,3);
4249   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4250   PetscValidPointer(coveredPoints, 5);
4251   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints));
4252   if (numCoveredPoints) *numCoveredPoints = 0;
4253   PetscFunctionReturn(0);
4254 }
4255 
4256 /*@C
4257   DMPlexGetFullJoin - Get an array for the join of the set of points
4258 
4259   Not Collective
4260 
4261   Input Parameters:
4262 + dm - The DMPlex object
4263 . numPoints - The number of input points for the join
4264 - points - The input points
4265 
4266   Output Parameters:
4267 + numCoveredPoints - The number of points in the join
4268 - coveredPoints - The points in the join
4269 
4270   Fortran Notes:
4271   Since it returns an array, this routine is only available in Fortran 90, and you must
4272   include petsc.h90 in your code.
4273 
4274   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4275 
4276   Level: intermediate
4277 
4278 .seealso: `DMPlexGetJoin()`, `DMPlexRestoreJoin()`, `DMPlexGetMeet()`
4279 @*/
4280 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4281 {
4282   PetscInt      *offsets, **closures;
4283   PetscInt      *join[2];
4284   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
4285   PetscInt       p, d, c, m, ms;
4286 
4287   PetscFunctionBegin;
4288   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4289   PetscValidIntPointer(points, 3);
4290   PetscValidIntPointer(numCoveredPoints, 4);
4291   PetscValidPointer(coveredPoints, 5);
4292 
4293   PetscCall(DMPlexGetDepth(dm, &depth));
4294   PetscCall(PetscCalloc1(numPoints, &closures));
4295   PetscCall(DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets));
4296   PetscCall(DMPlexGetMaxSizes(dm, NULL, &ms));
4297   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
4298   PetscCall(DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]));
4299   PetscCall(DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]));
4300 
4301   for (p = 0; p < numPoints; ++p) {
4302     PetscInt closureSize;
4303 
4304     PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]));
4305 
4306     offsets[p*(depth+2)+0] = 0;
4307     for (d = 0; d < depth+1; ++d) {
4308       PetscInt pStart, pEnd, i;
4309 
4310       PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
4311       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
4312         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4313           offsets[p*(depth+2)+d+1] = i;
4314           break;
4315         }
4316       }
4317       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
4318     }
4319     PetscCheck(offsets[p*(depth+2)+depth+1] == closureSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %" PetscInt_FMT " should be %" PetscInt_FMT, offsets[p*(depth+2)+depth+1], closureSize);
4320   }
4321   for (d = 0; d < depth+1; ++d) {
4322     PetscInt dof;
4323 
4324     /* Copy in support of first point */
4325     dof = offsets[d+1] - offsets[d];
4326     for (joinSize = 0; joinSize < dof; ++joinSize) {
4327       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
4328     }
4329     /* Check each successive cone */
4330     for (p = 1; p < numPoints && joinSize; ++p) {
4331       PetscInt newJoinSize = 0;
4332 
4333       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
4334       for (c = 0; c < dof; ++c) {
4335         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
4336 
4337         for (m = 0; m < joinSize; ++m) {
4338           if (point == join[i][m]) {
4339             join[1-i][newJoinSize++] = point;
4340             break;
4341           }
4342         }
4343       }
4344       joinSize = newJoinSize;
4345       i        = 1-i;
4346     }
4347     if (joinSize) break;
4348   }
4349   *numCoveredPoints = joinSize;
4350   *coveredPoints    = join[i];
4351   for (p = 0; p < numPoints; ++p) {
4352     PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]));
4353   }
4354   PetscCall(PetscFree(closures));
4355   PetscCall(DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets));
4356   PetscCall(DMRestoreWorkArray(dm, ms, MPIU_INT, &join[1-i]));
4357   PetscFunctionReturn(0);
4358 }
4359 
4360 /*@C
4361   DMPlexGetMeet - Get an array for the meet of the set of points
4362 
4363   Not Collective
4364 
4365   Input Parameters:
4366 + dm - The DMPlex object
4367 . numPoints - The number of input points for the meet
4368 - points - The input points
4369 
4370   Output Parameters:
4371 + numCoveredPoints - The number of points in the meet
4372 - coveredPoints - The points in the meet
4373 
4374   Level: intermediate
4375 
4376   Note: Currently, this is restricted to a single level meet
4377 
4378   Fortran Notes:
4379   Since it returns an array, this routine is only available in Fortran 90, and you must
4380   include petsc.h90 in your code.
4381 
4382   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4383 
4384 .seealso: `DMPlexRestoreMeet()`, `DMPlexGetJoin()`
4385 @*/
4386 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
4387 {
4388   DM_Plex       *mesh = (DM_Plex*) dm->data;
4389   PetscInt      *meet[2];
4390   PetscInt       meetSize, i = 0;
4391   PetscInt       dof, off, p, c, m;
4392   PetscInt       maxConeSize;
4393 
4394   PetscFunctionBegin;
4395   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4396   PetscValidIntPointer(points, 3);
4397   PetscValidIntPointer(numCoveringPoints, 4);
4398   PetscValidPointer(coveringPoints, 5);
4399   PetscCall(PetscSectionGetMaxDof(mesh->coneSection, &maxConeSize));
4400   PetscCall(DMGetWorkArray(dm, maxConeSize, MPIU_INT, &meet[0]));
4401   PetscCall(DMGetWorkArray(dm, maxConeSize, MPIU_INT, &meet[1]));
4402   /* Copy in cone of first point */
4403   PetscCall(PetscSectionGetDof(mesh->coneSection, points[0], &dof));
4404   PetscCall(PetscSectionGetOffset(mesh->coneSection, points[0], &off));
4405   for (meetSize = 0; meetSize < dof; ++meetSize) {
4406     meet[i][meetSize] = mesh->cones[off+meetSize];
4407   }
4408   /* Check each successive cone */
4409   for (p = 1; p < numPoints; ++p) {
4410     PetscInt newMeetSize = 0;
4411 
4412     PetscCall(PetscSectionGetDof(mesh->coneSection, points[p], &dof));
4413     PetscCall(PetscSectionGetOffset(mesh->coneSection, points[p], &off));
4414     for (c = 0; c < dof; ++c) {
4415       const PetscInt point = mesh->cones[off+c];
4416 
4417       for (m = 0; m < meetSize; ++m) {
4418         if (point == meet[i][m]) {
4419           meet[1-i][newMeetSize++] = point;
4420           break;
4421         }
4422       }
4423     }
4424     meetSize = newMeetSize;
4425     i        = 1-i;
4426   }
4427   *numCoveringPoints = meetSize;
4428   *coveringPoints    = meet[i];
4429   PetscCall(DMRestoreWorkArray(dm, maxConeSize, MPIU_INT, &meet[1-i]));
4430   PetscFunctionReturn(0);
4431 }
4432 
4433 /*@C
4434   DMPlexRestoreMeet - Restore an array for the meet of the set of points
4435 
4436   Not Collective
4437 
4438   Input Parameters:
4439 + dm - The DMPlex object
4440 . numPoints - The number of input points for the meet
4441 - points - The input points
4442 
4443   Output Parameters:
4444 + numCoveredPoints - The number of points in the meet
4445 - coveredPoints - The points in the meet
4446 
4447   Level: intermediate
4448 
4449   Fortran Notes:
4450   Since it returns an array, this routine is only available in Fortran 90, and you must
4451   include petsc.h90 in your code.
4452 
4453   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4454 
4455 .seealso: `DMPlexGetMeet()`, `DMPlexGetFullMeet()`, `DMPlexGetJoin()`
4456 @*/
4457 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4458 {
4459   PetscFunctionBegin;
4460   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4461   if (points) PetscValidIntPointer(points,3);
4462   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4463   PetscValidPointer(coveredPoints,5);
4464   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints));
4465   if (numCoveredPoints) *numCoveredPoints = 0;
4466   PetscFunctionReturn(0);
4467 }
4468 
4469 /*@C
4470   DMPlexGetFullMeet - Get an array for the meet of the set of points
4471 
4472   Not Collective
4473 
4474   Input Parameters:
4475 + dm - The DMPlex object
4476 . numPoints - The number of input points for the meet
4477 - points - The input points
4478 
4479   Output Parameters:
4480 + numCoveredPoints - The number of points in the meet
4481 - coveredPoints - The points in the meet
4482 
4483   Level: intermediate
4484 
4485   Fortran Notes:
4486   Since it returns an array, this routine is only available in Fortran 90, and you must
4487   include petsc.h90 in your code.
4488 
4489   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4490 
4491 .seealso: `DMPlexGetMeet()`, `DMPlexRestoreMeet()`, `DMPlexGetJoin()`
4492 @*/
4493 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4494 {
4495   PetscInt      *offsets, **closures;
4496   PetscInt      *meet[2];
4497   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
4498   PetscInt       p, h, c, m, mc;
4499 
4500   PetscFunctionBegin;
4501   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4502   PetscValidIntPointer(points, 3);
4503   PetscValidIntPointer(numCoveredPoints, 4);
4504   PetscValidPointer(coveredPoints, 5);
4505 
4506   PetscCall(DMPlexGetDepth(dm, &height));
4507   PetscCall(PetscMalloc1(numPoints, &closures));
4508   PetscCall(DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets));
4509   PetscCall(DMPlexGetMaxSizes(dm, &mc, NULL));
4510   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
4511   PetscCall(DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]));
4512   PetscCall(DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]));
4513 
4514   for (p = 0; p < numPoints; ++p) {
4515     PetscInt closureSize;
4516 
4517     PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]));
4518 
4519     offsets[p*(height+2)+0] = 0;
4520     for (h = 0; h < height+1; ++h) {
4521       PetscInt pStart, pEnd, i;
4522 
4523       PetscCall(DMPlexGetHeightStratum(dm, h, &pStart, &pEnd));
4524       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
4525         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4526           offsets[p*(height+2)+h+1] = i;
4527           break;
4528         }
4529       }
4530       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
4531     }
4532     PetscCheck(offsets[p*(height+2)+height+1] == closureSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %" PetscInt_FMT " should be %" PetscInt_FMT, offsets[p*(height+2)+height+1], closureSize);
4533   }
4534   for (h = 0; h < height+1; ++h) {
4535     PetscInt dof;
4536 
4537     /* Copy in cone of first point */
4538     dof = offsets[h+1] - offsets[h];
4539     for (meetSize = 0; meetSize < dof; ++meetSize) {
4540       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
4541     }
4542     /* Check each successive cone */
4543     for (p = 1; p < numPoints && meetSize; ++p) {
4544       PetscInt newMeetSize = 0;
4545 
4546       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
4547       for (c = 0; c < dof; ++c) {
4548         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
4549 
4550         for (m = 0; m < meetSize; ++m) {
4551           if (point == meet[i][m]) {
4552             meet[1-i][newMeetSize++] = point;
4553             break;
4554           }
4555         }
4556       }
4557       meetSize = newMeetSize;
4558       i        = 1-i;
4559     }
4560     if (meetSize) break;
4561   }
4562   *numCoveredPoints = meetSize;
4563   *coveredPoints    = meet[i];
4564   for (p = 0; p < numPoints; ++p) {
4565     PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]));
4566   }
4567   PetscCall(PetscFree(closures));
4568   PetscCall(DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets));
4569   PetscCall(DMRestoreWorkArray(dm, mc, MPIU_INT, &meet[1-i]));
4570   PetscFunctionReturn(0);
4571 }
4572 
4573 /*@C
4574   DMPlexEqual - Determine if two DMs have the same topology
4575 
4576   Not Collective
4577 
4578   Input Parameters:
4579 + dmA - A DMPlex object
4580 - dmB - A DMPlex object
4581 
4582   Output Parameters:
4583 . equal - PETSC_TRUE if the topologies are identical
4584 
4585   Level: intermediate
4586 
4587   Notes:
4588   We are not solving graph isomorphism, so we do not permutation.
4589 
4590 .seealso: `DMPlexGetCone()`
4591 @*/
4592 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
4593 {
4594   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
4595 
4596   PetscFunctionBegin;
4597   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4598   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4599   PetscValidBoolPointer(equal, 3);
4600 
4601   *equal = PETSC_FALSE;
4602   PetscCall(DMPlexGetDepth(dmA, &depth));
4603   PetscCall(DMPlexGetDepth(dmB, &depthB));
4604   if (depth != depthB) PetscFunctionReturn(0);
4605   PetscCall(DMPlexGetChart(dmA, &pStart,  &pEnd));
4606   PetscCall(DMPlexGetChart(dmB, &pStartB, &pEndB));
4607   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
4608   for (p = pStart; p < pEnd; ++p) {
4609     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
4610     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
4611 
4612     PetscCall(DMPlexGetConeSize(dmA, p, &coneSize));
4613     PetscCall(DMPlexGetCone(dmA, p, &cone));
4614     PetscCall(DMPlexGetConeOrientation(dmA, p, &ornt));
4615     PetscCall(DMPlexGetConeSize(dmB, p, &coneSizeB));
4616     PetscCall(DMPlexGetCone(dmB, p, &coneB));
4617     PetscCall(DMPlexGetConeOrientation(dmB, p, &orntB));
4618     if (coneSize != coneSizeB) PetscFunctionReturn(0);
4619     for (c = 0; c < coneSize; ++c) {
4620       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
4621       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
4622     }
4623     PetscCall(DMPlexGetSupportSize(dmA, p, &supportSize));
4624     PetscCall(DMPlexGetSupport(dmA, p, &support));
4625     PetscCall(DMPlexGetSupportSize(dmB, p, &supportSizeB));
4626     PetscCall(DMPlexGetSupport(dmB, p, &supportB));
4627     if (supportSize != supportSizeB) PetscFunctionReturn(0);
4628     for (s = 0; s < supportSize; ++s) {
4629       if (support[s] != supportB[s]) PetscFunctionReturn(0);
4630     }
4631   }
4632   *equal = PETSC_TRUE;
4633   PetscFunctionReturn(0);
4634 }
4635 
4636 /*@C
4637   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
4638 
4639   Not Collective
4640 
4641   Input Parameters:
4642 + dm         - The DMPlex
4643 . cellDim    - The cell dimension
4644 - numCorners - The number of vertices on a cell
4645 
4646   Output Parameters:
4647 . numFaceVertices - The number of vertices on a face
4648 
4649   Level: developer
4650 
4651   Notes:
4652   Of course this can only work for a restricted set of symmetric shapes
4653 
4654 .seealso: `DMPlexGetCone()`
4655 @*/
4656 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
4657 {
4658   MPI_Comm       comm;
4659 
4660   PetscFunctionBegin;
4661   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
4662   PetscValidIntPointer(numFaceVertices,4);
4663   switch (cellDim) {
4664   case 0:
4665     *numFaceVertices = 0;
4666     break;
4667   case 1:
4668     *numFaceVertices = 1;
4669     break;
4670   case 2:
4671     switch (numCorners) {
4672     case 3: /* triangle */
4673       *numFaceVertices = 2; /* Edge has 2 vertices */
4674       break;
4675     case 4: /* quadrilateral */
4676       *numFaceVertices = 2; /* Edge has 2 vertices */
4677       break;
4678     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
4679       *numFaceVertices = 3; /* Edge has 3 vertices */
4680       break;
4681     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
4682       *numFaceVertices = 3; /* Edge has 3 vertices */
4683       break;
4684     default:
4685       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %" PetscInt_FMT " for dimension %" PetscInt_FMT, numCorners, cellDim);
4686     }
4687     break;
4688   case 3:
4689     switch (numCorners) {
4690     case 4: /* tetradehdron */
4691       *numFaceVertices = 3; /* Face has 3 vertices */
4692       break;
4693     case 6: /* tet cohesive cells */
4694       *numFaceVertices = 4; /* Face has 4 vertices */
4695       break;
4696     case 8: /* hexahedron */
4697       *numFaceVertices = 4; /* Face has 4 vertices */
4698       break;
4699     case 9: /* tet cohesive Lagrange cells */
4700       *numFaceVertices = 6; /* Face has 6 vertices */
4701       break;
4702     case 10: /* quadratic tetrahedron */
4703       *numFaceVertices = 6; /* Face has 6 vertices */
4704       break;
4705     case 12: /* hex cohesive Lagrange cells */
4706       *numFaceVertices = 6; /* Face has 6 vertices */
4707       break;
4708     case 18: /* quadratic tet cohesive Lagrange cells */
4709       *numFaceVertices = 6; /* Face has 6 vertices */
4710       break;
4711     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
4712       *numFaceVertices = 9; /* Face has 9 vertices */
4713       break;
4714     default:
4715       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %" PetscInt_FMT " for dimension %" PetscInt_FMT, numCorners, cellDim);
4716     }
4717     break;
4718   default:
4719     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %" PetscInt_FMT, cellDim);
4720   }
4721   PetscFunctionReturn(0);
4722 }
4723 
4724 /*@
4725   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4726 
4727   Not Collective
4728 
4729   Input Parameter:
4730 . dm    - The DMPlex object
4731 
4732   Output Parameter:
4733 . depthLabel - The DMLabel recording point depth
4734 
4735   Level: developer
4736 
4737 .seealso: `DMPlexGetDepth()`, `DMPlexGetHeightStratum()`, `DMPlexGetDepthStratum()`, `DMPlexGetPointDepth()`,
4738 @*/
4739 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4740 {
4741   PetscFunctionBegin;
4742   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4743   PetscValidPointer(depthLabel, 2);
4744   *depthLabel = dm->depthLabel;
4745   PetscFunctionReturn(0);
4746 }
4747 
4748 /*@
4749   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4750 
4751   Not Collective
4752 
4753   Input Parameter:
4754 . dm    - The DMPlex object
4755 
4756   Output Parameter:
4757 . depth - The number of strata (breadth first levels) in the DAG
4758 
4759   Level: developer
4760 
4761   Notes:
4762   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
4763   The point depth is described more in detail in DMPlexGetDepthStratum().
4764   An empty mesh gives -1.
4765 
4766 .seealso: `DMPlexGetDepthLabel()`, `DMPlexGetDepthStratum()`, `DMPlexGetPointDepth()`, `DMPlexSymmetrize()`
4767 @*/
4768 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4769 {
4770   DMLabel        label;
4771   PetscInt       d = 0;
4772 
4773   PetscFunctionBegin;
4774   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4775   PetscValidIntPointer(depth, 2);
4776   PetscCall(DMPlexGetDepthLabel(dm, &label));
4777   if (label) PetscCall(DMLabelGetNumValues(label, &d));
4778   *depth = d-1;
4779   PetscFunctionReturn(0);
4780 }
4781 
4782 /*@
4783   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4784 
4785   Not Collective
4786 
4787   Input Parameters:
4788 + dm    - The DMPlex object
4789 - depth - The requested depth
4790 
4791   Output Parameters:
4792 + start - The first point at this depth
4793 - end   - One beyond the last point at this depth
4794 
4795   Notes:
4796   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
4797   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
4798   higher dimension, e.g., "edges".
4799 
4800   Level: developer
4801 
4802 .seealso: `DMPlexGetHeightStratum()`, `DMPlexGetDepth()`, `DMPlexGetDepthLabel()`, `DMPlexGetPointDepth()`, `DMPlexSymmetrize()`, `DMPlexInterpolate()`
4803 @*/
4804 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt depth, PetscInt *start, PetscInt *end)
4805 {
4806   DMLabel        label;
4807   PetscInt       pStart, pEnd;
4808 
4809   PetscFunctionBegin;
4810   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4811   if (start) {PetscValidIntPointer(start, 3); *start = 0;}
4812   if (end)   {PetscValidIntPointer(end,   4); *end   = 0;}
4813   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4814   if (pStart == pEnd) PetscFunctionReturn(0);
4815   if (depth < 0) {
4816     if (start) *start = pStart;
4817     if (end)   *end   = pEnd;
4818     PetscFunctionReturn(0);
4819   }
4820   PetscCall(DMPlexGetDepthLabel(dm, &label));
4821   PetscCheck(label,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4822   PetscCall(DMLabelGetStratumBounds(label, depth, start, end));
4823   PetscFunctionReturn(0);
4824 }
4825 
4826 /*@
4827   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4828 
4829   Not Collective
4830 
4831   Input Parameters:
4832 + dm     - The DMPlex object
4833 - height - The requested height
4834 
4835   Output Parameters:
4836 + start - The first point at this height
4837 - end   - One beyond the last point at this height
4838 
4839   Notes:
4840   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
4841   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
4842   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
4843 
4844   Level: developer
4845 
4846 .seealso: `DMPlexGetDepthStratum()`, `DMPlexGetDepth()`, `DMPlexGetPointHeight()`
4847 @*/
4848 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt height, PetscInt *start, PetscInt *end)
4849 {
4850   DMLabel        label;
4851   PetscInt       depth, pStart, pEnd;
4852 
4853   PetscFunctionBegin;
4854   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4855   if (start) {PetscValidIntPointer(start, 3); *start = 0;}
4856   if (end)   {PetscValidIntPointer(end,   4); *end   = 0;}
4857   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4858   if (pStart == pEnd) PetscFunctionReturn(0);
4859   if (height < 0) {
4860     if (start) *start = pStart;
4861     if (end)   *end   = pEnd;
4862     PetscFunctionReturn(0);
4863   }
4864   PetscCall(DMPlexGetDepthLabel(dm, &label));
4865   PetscCheck(label,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4866   PetscCall(DMLabelGetNumValues(label, &depth));
4867   PetscCall(DMLabelGetStratumBounds(label, depth-1-height, start, end));
4868   PetscFunctionReturn(0);
4869 }
4870 
4871 /*@
4872   DMPlexGetPointDepth - Get the depth of a given point
4873 
4874   Not Collective
4875 
4876   Input Parameters:
4877 + dm    - The DMPlex object
4878 - point - The point
4879 
4880   Output Parameter:
4881 . depth - The depth of the point
4882 
4883   Level: intermediate
4884 
4885 .seealso: `DMPlexGetCellType()`, `DMPlexGetDepthLabel()`, `DMPlexGetDepth()`, `DMPlexGetPointHeight()`
4886 @*/
4887 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
4888 {
4889   PetscFunctionBegin;
4890   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4891   PetscValidIntPointer(depth, 3);
4892   PetscCall(DMLabelGetValue(dm->depthLabel, point, depth));
4893   PetscFunctionReturn(0);
4894 }
4895 
4896 /*@
4897   DMPlexGetPointHeight - Get the height of a given point
4898 
4899   Not Collective
4900 
4901   Input Parameters:
4902 + dm    - The DMPlex object
4903 - point - The point
4904 
4905   Output Parameter:
4906 . height - The height of the point
4907 
4908   Level: intermediate
4909 
4910 .seealso: `DMPlexGetCellType()`, `DMPlexGetDepthLabel()`, `DMPlexGetDepth()`, `DMPlexGetPointDepth()`
4911 @*/
4912 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
4913 {
4914   PetscInt       n, pDepth;
4915 
4916   PetscFunctionBegin;
4917   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4918   PetscValidIntPointer(height, 3);
4919   PetscCall(DMLabelGetNumValues(dm->depthLabel, &n));
4920   PetscCall(DMLabelGetValue(dm->depthLabel, point, &pDepth));
4921   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
4922   PetscFunctionReturn(0);
4923 }
4924 
4925 /*@
4926   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
4927 
4928   Not Collective
4929 
4930   Input Parameter:
4931 . dm - The DMPlex object
4932 
4933   Output Parameter:
4934 . celltypeLabel - The DMLabel recording cell polytope type
4935 
4936   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
4937   DMCreateLabel(dm, "celltype") beforehand.
4938 
4939   Level: developer
4940 
4941 .seealso: `DMPlexGetCellType()`, `DMPlexGetDepthLabel()`, `DMCreateLabel()`
4942 @*/
4943 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
4944 {
4945   PetscFunctionBegin;
4946   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4947   PetscValidPointer(celltypeLabel, 2);
4948   if (!dm->celltypeLabel) PetscCall(DMPlexComputeCellTypes(dm));
4949   *celltypeLabel = dm->celltypeLabel;
4950   PetscFunctionReturn(0);
4951 }
4952 
4953 /*@
4954   DMPlexGetCellType - Get the polytope type of a given cell
4955 
4956   Not Collective
4957 
4958   Input Parameters:
4959 + dm   - The DMPlex object
4960 - cell - The cell
4961 
4962   Output Parameter:
4963 . celltype - The polytope type of the cell
4964 
4965   Level: intermediate
4966 
4967 .seealso: `DMPlexGetCellTypeLabel()`, `DMPlexGetDepthLabel()`, `DMPlexGetDepth()`
4968 @*/
4969 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
4970 {
4971   DMLabel        label;
4972   PetscInt       ct;
4973 
4974   PetscFunctionBegin;
4975   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4976   PetscValidPointer(celltype, 3);
4977   PetscCall(DMPlexGetCellTypeLabel(dm, &label));
4978   PetscCall(DMLabelGetValue(label, cell, &ct));
4979   PetscCheck(ct >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %" PetscInt_FMT " has not been assigned a cell type", cell);
4980   *celltype = (DMPolytopeType) ct;
4981   PetscFunctionReturn(0);
4982 }
4983 
4984 /*@
4985   DMPlexSetCellType - Set the polytope type of a given cell
4986 
4987   Not Collective
4988 
4989   Input Parameters:
4990 + dm   - The DMPlex object
4991 . cell - The cell
4992 - celltype - The polytope type of the cell
4993 
4994   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
4995   is executed. This function will override the computed type. However, if automatic classification will not succeed
4996   and a user wants to manually specify all types, the classification must be disabled by calling
4997   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
4998 
4999   Level: advanced
5000 
5001 .seealso: `DMPlexGetCellTypeLabel()`, `DMPlexGetDepthLabel()`, `DMPlexGetDepth()`, `DMPlexComputeCellTypes()`, `DMCreateLabel()`
5002 @*/
5003 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
5004 {
5005   DMLabel        label;
5006 
5007   PetscFunctionBegin;
5008   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5009   PetscCall(DMPlexGetCellTypeLabel(dm, &label));
5010   PetscCall(DMLabelSetValue(label, cell, celltype));
5011   PetscFunctionReturn(0);
5012 }
5013 
5014 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
5015 {
5016   PetscSection   section, s;
5017   Mat            m;
5018   PetscInt       maxHeight;
5019 
5020   PetscFunctionBegin;
5021   PetscCall(DMClone(dm, cdm));
5022   PetscCall(DMPlexGetMaxProjectionHeight(dm, &maxHeight));
5023   PetscCall(DMPlexSetMaxProjectionHeight(*cdm, maxHeight));
5024   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section));
5025   PetscCall(DMSetLocalSection(*cdm, section));
5026   PetscCall(PetscSectionDestroy(&section));
5027   PetscCall(PetscSectionCreate(PETSC_COMM_SELF, &s));
5028   PetscCall(MatCreate(PETSC_COMM_SELF, &m));
5029   PetscCall(DMSetDefaultConstraints(*cdm, s, m, NULL));
5030   PetscCall(PetscSectionDestroy(&s));
5031   PetscCall(MatDestroy(&m));
5032 
5033   PetscCall(DMSetNumFields(*cdm, 1));
5034   PetscCall(DMCreateDS(*cdm));
5035   PetscFunctionReturn(0);
5036 }
5037 
5038 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
5039 {
5040   Vec            coordsLocal;
5041   DM             coordsDM;
5042 
5043   PetscFunctionBegin;
5044   *field = NULL;
5045   PetscCall(DMGetCoordinatesLocal(dm,&coordsLocal));
5046   PetscCall(DMGetCoordinateDM(dm,&coordsDM));
5047   if (coordsLocal && coordsDM) {
5048     PetscCall(DMFieldCreateDS(coordsDM, 0, coordsLocal, field));
5049   }
5050   PetscFunctionReturn(0);
5051 }
5052 
5053 /*@C
5054   DMPlexGetConeSection - Return a section which describes the layout of cone data
5055 
5056   Not Collective
5057 
5058   Input Parameters:
5059 . dm        - The DMPlex object
5060 
5061   Output Parameter:
5062 . section - The PetscSection object
5063 
5064   Level: developer
5065 
5066 .seealso: `DMPlexGetSupportSection()`, `DMPlexGetCones()`, `DMPlexGetConeOrientations()`
5067 @*/
5068 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
5069 {
5070   DM_Plex *mesh = (DM_Plex*) dm->data;
5071 
5072   PetscFunctionBegin;
5073   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5074   if (section) *section = mesh->coneSection;
5075   PetscFunctionReturn(0);
5076 }
5077 
5078 /*@C
5079   DMPlexGetSupportSection - Return a section which describes the layout of support data
5080 
5081   Not Collective
5082 
5083   Input Parameters:
5084 . dm        - The DMPlex object
5085 
5086   Output Parameter:
5087 . section - The PetscSection object
5088 
5089   Level: developer
5090 
5091 .seealso: `DMPlexGetConeSection()`
5092 @*/
5093 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
5094 {
5095   DM_Plex *mesh = (DM_Plex*) dm->data;
5096 
5097   PetscFunctionBegin;
5098   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5099   if (section) *section = mesh->supportSection;
5100   PetscFunctionReturn(0);
5101 }
5102 
5103 /*@C
5104   DMPlexGetCones - Return cone data
5105 
5106   Not Collective
5107 
5108   Input Parameters:
5109 . dm        - The DMPlex object
5110 
5111   Output Parameter:
5112 . cones - The cone for each point
5113 
5114   Level: developer
5115 
5116 .seealso: `DMPlexGetConeSection()`
5117 @*/
5118 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
5119 {
5120   DM_Plex *mesh = (DM_Plex*) dm->data;
5121 
5122   PetscFunctionBegin;
5123   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5124   if (cones) *cones = mesh->cones;
5125   PetscFunctionReturn(0);
5126 }
5127 
5128 /*@C
5129   DMPlexGetConeOrientations - Return cone orientation data
5130 
5131   Not Collective
5132 
5133   Input Parameters:
5134 . dm        - The DMPlex object
5135 
5136   Output Parameter:
5137 . coneOrientations - The array of cone orientations for all points
5138 
5139   Level: developer
5140 
5141   Notes:
5142   The PetscSection returned by DMPlexGetConeSection() partitions coneOrientations into cone orientations of particular points as returned by DMPlexGetConeOrientation().
5143 
5144   The meaning of coneOrientations values is detailed in DMPlexGetConeOrientation().
5145 
5146 .seealso: `DMPlexGetConeSection()`, `DMPlexGetConeOrientation()`
5147 @*/
5148 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
5149 {
5150   DM_Plex *mesh = (DM_Plex*) dm->data;
5151 
5152   PetscFunctionBegin;
5153   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5154   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
5155   PetscFunctionReturn(0);
5156 }
5157 
5158 /******************************** FEM Support **********************************/
5159 
5160 /*
5161  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
5162  representing a line in the section.
5163 */
5164 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
5165 {
5166   PetscFunctionBeginHot;
5167   PetscCall(PetscSectionGetFieldComponents(section, field, Nc));
5168   if (line < 0) {
5169     *k = 0;
5170     *Nc = 0;
5171   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
5172     *k = 1;
5173   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
5174     /* An order k SEM disc has k-1 dofs on an edge */
5175     PetscCall(PetscSectionGetFieldDof(section, line, field, k));
5176     *k = *k / *Nc + 1;
5177   }
5178   PetscFunctionReturn(0);
5179 }
5180 
5181 /*@
5182 
5183   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
5184   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
5185   section provided (or the section of the DM).
5186 
5187   Input Parameters:
5188 + dm      - The DM
5189 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
5190 - section - The PetscSection to reorder, or NULL for the default section
5191 
5192   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
5193   degree of the basis.
5194 
5195   Example:
5196   A typical interpolated single-quad mesh might order points as
5197 .vb
5198   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
5199 
5200   v4 -- e6 -- v3
5201   |           |
5202   e7    c0    e8
5203   |           |
5204   v1 -- e5 -- v2
5205 .ve
5206 
5207   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
5208   dofs in the order of points, e.g.,
5209 .vb
5210     c0 -> [0,1,2,3]
5211     v1 -> [4]
5212     ...
5213     e5 -> [8, 9]
5214 .ve
5215 
5216   which corresponds to the dofs
5217 .vb
5218     6   10  11  7
5219     13  2   3   15
5220     12  0   1   14
5221     4   8   9   5
5222 .ve
5223 
5224   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
5225 .vb
5226   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
5227 .ve
5228 
5229   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
5230 .vb
5231    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
5232 .ve
5233 
5234   Level: developer
5235 
5236 .seealso: `DMGetLocalSection()`, `PetscSectionSetClosurePermutation()`, `DMSetGlobalSection()`
5237 @*/
5238 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
5239 {
5240   DMLabel        label;
5241   PetscInt       dim, depth = -1, eStart = -1, Nf;
5242   PetscBool      vertexchart;
5243 
5244   PetscFunctionBegin;
5245   PetscCall(DMGetDimension(dm, &dim));
5246   if (dim < 1) PetscFunctionReturn(0);
5247   if (point < 0) {
5248     PetscInt sStart,sEnd;
5249 
5250     PetscCall(DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd));
5251     point = sEnd-sStart ? sStart : point;
5252   }
5253   PetscCall(DMPlexGetDepthLabel(dm, &label));
5254   if (point >= 0) PetscCall(DMLabelGetValue(label, point, &depth));
5255   if (!section) PetscCall(DMGetLocalSection(dm, &section));
5256   if (depth == 1) {eStart = point;}
5257   else if  (depth == dim) {
5258     const PetscInt *cone;
5259 
5260     PetscCall(DMPlexGetCone(dm, point, &cone));
5261     if (dim == 2) eStart = cone[0];
5262     else if (dim == 3) {
5263       const PetscInt *cone2;
5264       PetscCall(DMPlexGetCone(dm, cone[0], &cone2));
5265       eStart = cone2[0];
5266     } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " of depth %" PetscInt_FMT " cannot be used to bootstrap spectral ordering for dim %" PetscInt_FMT, point, depth, dim);
5267   } else PetscCheck(depth < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " of depth %" PetscInt_FMT " cannot be used to bootstrap spectral ordering for dim %" PetscInt_FMT, point, depth, dim);
5268   {                             /* Determine whether the chart covers all points or just vertices. */
5269     PetscInt pStart,pEnd,cStart,cEnd;
5270     PetscCall(DMPlexGetDepthStratum(dm,0,&pStart,&pEnd));
5271     PetscCall(PetscSectionGetChart(section,&cStart,&cEnd));
5272     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE;      /* Only vertices are in the chart */
5273     else if (cStart <= point && point < cEnd) vertexchart = PETSC_FALSE; /* Some interpolated points exist in the chart */
5274     else vertexchart = PETSC_TRUE;                                       /* Some interpolated points are not in chart; assume dofs only at cells and vertices */
5275   }
5276   PetscCall(PetscSectionGetNumFields(section, &Nf));
5277   for (PetscInt d=1; d<=dim; d++) {
5278     PetscInt k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
5279     PetscInt *perm;
5280 
5281     for (f = 0; f < Nf; ++f) {
5282       PetscCall(PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k));
5283       size += PetscPowInt(k+1, d)*Nc;
5284     }
5285     PetscCall(PetscMalloc1(size, &perm));
5286     for (f = 0; f < Nf; ++f) {
5287       switch (d) {
5288       case 1:
5289         PetscCall(PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k));
5290         /*
5291          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
5292          We want              [ vtx0; edge of length k-1; vtx1 ]
5293          */
5294         for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
5295         for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
5296         for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
5297         foffset = offset;
5298         break;
5299       case 2:
5300         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
5301         PetscCall(PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k));
5302         /* The SEM order is
5303 
5304          v_lb, {e_b}, v_rb,
5305          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
5306          v_lt, reverse {e_t}, v_rt
5307          */
5308         {
5309           const PetscInt of   = 0;
5310           const PetscInt oeb  = of   + PetscSqr(k-1);
5311           const PetscInt oer  = oeb  + (k-1);
5312           const PetscInt oet  = oer  + (k-1);
5313           const PetscInt oel  = oet  + (k-1);
5314           const PetscInt ovlb = oel  + (k-1);
5315           const PetscInt ovrb = ovlb + 1;
5316           const PetscInt ovrt = ovrb + 1;
5317           const PetscInt ovlt = ovrt + 1;
5318           PetscInt       o;
5319 
5320           /* bottom */
5321           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
5322           for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5323           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
5324           /* middle */
5325           for (i = 0; i < k-1; ++i) {
5326             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
5327             for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5328             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
5329           }
5330           /* top */
5331           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
5332           for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5333           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
5334           foffset = offset;
5335         }
5336         break;
5337       case 3:
5338         /* The original hex closure is
5339 
5340          {c,
5341          f_b, f_t, f_f, f_b, f_r, f_l,
5342          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5343          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5344          */
5345         PetscCall(PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k));
5346         /* The SEM order is
5347          Bottom Slice
5348          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5349          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5350          v_blb, {e_bb}, v_brb,
5351 
5352          Middle Slice (j)
5353          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5354          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5355          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5356 
5357          Top Slice
5358          v_tlf, {e_tf}, v_trf,
5359          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5360          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5361          */
5362         {
5363           const PetscInt oc    = 0;
5364           const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
5365           const PetscInt oft   = ofb   + PetscSqr(k-1);
5366           const PetscInt off   = oft   + PetscSqr(k-1);
5367           const PetscInt ofk   = off   + PetscSqr(k-1);
5368           const PetscInt ofr   = ofk   + PetscSqr(k-1);
5369           const PetscInt ofl   = ofr   + PetscSqr(k-1);
5370           const PetscInt oebl  = ofl   + PetscSqr(k-1);
5371           const PetscInt oebb  = oebl  + (k-1);
5372           const PetscInt oebr  = oebb  + (k-1);
5373           const PetscInt oebf  = oebr  + (k-1);
5374           const PetscInt oetf  = oebf  + (k-1);
5375           const PetscInt oetr  = oetf  + (k-1);
5376           const PetscInt oetb  = oetr  + (k-1);
5377           const PetscInt oetl  = oetb  + (k-1);
5378           const PetscInt oerf  = oetl  + (k-1);
5379           const PetscInt oelf  = oerf  + (k-1);
5380           const PetscInt oelb  = oelf  + (k-1);
5381           const PetscInt oerb  = oelb  + (k-1);
5382           const PetscInt ovblf = oerb  + (k-1);
5383           const PetscInt ovblb = ovblf + 1;
5384           const PetscInt ovbrb = ovblb + 1;
5385           const PetscInt ovbrf = ovbrb + 1;
5386           const PetscInt ovtlf = ovbrf + 1;
5387           const PetscInt ovtrf = ovtlf + 1;
5388           const PetscInt ovtrb = ovtrf + 1;
5389           const PetscInt ovtlb = ovtrb + 1;
5390           PetscInt       o, n;
5391 
5392           /* Bottom Slice */
5393           /*   bottom */
5394           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
5395           for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5396           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
5397           /*   middle */
5398           for (i = 0; i < k-1; ++i) {
5399             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
5400             for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
5401             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
5402           }
5403           /*   top */
5404           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
5405           for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5406           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
5407 
5408           /* Middle Slice */
5409           for (j = 0; j < k-1; ++j) {
5410             /*   bottom */
5411             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
5412             for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5413             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
5414             /*   middle */
5415             for (i = 0; i < k-1; ++i) {
5416               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
5417               for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
5418               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
5419             }
5420             /*   top */
5421             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
5422             for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5423             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
5424           }
5425 
5426           /* Top Slice */
5427           /*   bottom */
5428           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
5429           for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5430           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
5431           /*   middle */
5432           for (i = 0; i < k-1; ++i) {
5433             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
5434             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
5435             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
5436           }
5437           /*   top */
5438           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
5439           for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5440           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
5441 
5442           foffset = offset;
5443         }
5444         break;
5445       default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %" PetscInt_FMT, d);
5446       }
5447     }
5448     PetscCheck(offset == size,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %" PetscInt_FMT " != %" PetscInt_FMT, offset, size);
5449     /* Check permutation */
5450     {
5451       PetscInt *check;
5452 
5453       PetscCall(PetscMalloc1(size, &check));
5454       for (i = 0; i < size; ++i) {
5455         check[i] = -1;
5456         PetscCheck(perm[i] >= 0 && perm[i] < size,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%" PetscInt_FMT "] = %" PetscInt_FMT, i, perm[i]);
5457       }
5458       for (i = 0; i < size; ++i) check[perm[i]] = i;
5459       for (i = 0; i < size; ++i) PetscCheck(check[i] >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %" PetscInt_FMT, i);
5460       PetscCall(PetscFree(check));
5461     }
5462     PetscCall(PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size, PETSC_OWN_POINTER, perm));
5463     if (d == dim) { // Add permutation for localized (in case this is a coordinate DM)
5464       PetscInt *loc_perm;
5465       PetscCall(PetscMalloc1(size*2, &loc_perm));
5466       for (PetscInt i=0; i<size; i++) {
5467         loc_perm[i] = perm[i];
5468         loc_perm[size+i] = size + perm[i];
5469       }
5470       PetscCall(PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size*2, PETSC_OWN_POINTER, loc_perm));
5471     }
5472   }
5473   PetscFunctionReturn(0);
5474 }
5475 
5476 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
5477 {
5478   PetscDS        prob;
5479   PetscInt       depth, Nf, h;
5480   DMLabel        label;
5481 
5482   PetscFunctionBeginHot;
5483   PetscCall(DMGetDS(dm, &prob));
5484   Nf      = prob->Nf;
5485   label   = dm->depthLabel;
5486   *dspace = NULL;
5487   if (field < Nf) {
5488     PetscObject disc = prob->disc[field];
5489 
5490     if (disc->classid == PETSCFE_CLASSID) {
5491       PetscDualSpace dsp;
5492 
5493       PetscCall(PetscFEGetDualSpace((PetscFE)disc,&dsp));
5494       PetscCall(DMLabelGetNumValues(label,&depth));
5495       PetscCall(DMLabelGetValue(label,point,&h));
5496       h    = depth - 1 - h;
5497       if (h) {
5498         PetscCall(PetscDualSpaceGetHeightSubspace(dsp,h,dspace));
5499       } else {
5500         *dspace = dsp;
5501       }
5502     }
5503   }
5504   PetscFunctionReturn(0);
5505 }
5506 
5507 static inline PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5508 {
5509   PetscScalar    *array, *vArray;
5510   const PetscInt *cone, *coneO;
5511   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
5512 
5513   PetscFunctionBeginHot;
5514   PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
5515   PetscCall(DMPlexGetConeSize(dm, point, &numPoints));
5516   PetscCall(DMPlexGetCone(dm, point, &cone));
5517   PetscCall(DMPlexGetConeOrientation(dm, point, &coneO));
5518   if (!values || !*values) {
5519     if ((point >= pStart) && (point < pEnd)) {
5520       PetscInt dof;
5521 
5522       PetscCall(PetscSectionGetDof(section, point, &dof));
5523       size += dof;
5524     }
5525     for (p = 0; p < numPoints; ++p) {
5526       const PetscInt cp = cone[p];
5527       PetscInt       dof;
5528 
5529       if ((cp < pStart) || (cp >= pEnd)) continue;
5530       PetscCall(PetscSectionGetDof(section, cp, &dof));
5531       size += dof;
5532     }
5533     if (!values) {
5534       if (csize) *csize = size;
5535       PetscFunctionReturn(0);
5536     }
5537     PetscCall(DMGetWorkArray(dm, size, MPIU_SCALAR, &array));
5538   } else {
5539     array = *values;
5540   }
5541   size = 0;
5542   PetscCall(VecGetArray(v, &vArray));
5543   if ((point >= pStart) && (point < pEnd)) {
5544     PetscInt     dof, off, d;
5545     PetscScalar *varr;
5546 
5547     PetscCall(PetscSectionGetDof(section, point, &dof));
5548     PetscCall(PetscSectionGetOffset(section, point, &off));
5549     varr = &vArray[off];
5550     for (d = 0; d < dof; ++d, ++offset) {
5551       array[offset] = varr[d];
5552     }
5553     size += dof;
5554   }
5555   for (p = 0; p < numPoints; ++p) {
5556     const PetscInt cp = cone[p];
5557     PetscInt       o  = coneO[p];
5558     PetscInt       dof, off, d;
5559     PetscScalar   *varr;
5560 
5561     if ((cp < pStart) || (cp >= pEnd)) continue;
5562     PetscCall(PetscSectionGetDof(section, cp, &dof));
5563     PetscCall(PetscSectionGetOffset(section, cp, &off));
5564     varr = &vArray[off];
5565     if (o >= 0) {
5566       for (d = 0; d < dof; ++d, ++offset) {
5567         array[offset] = varr[d];
5568       }
5569     } else {
5570       for (d = dof-1; d >= 0; --d, ++offset) {
5571         array[offset] = varr[d];
5572       }
5573     }
5574     size += dof;
5575   }
5576   PetscCall(VecRestoreArray(v, &vArray));
5577   if (!*values) {
5578     if (csize) *csize = size;
5579     *values = array;
5580   } else {
5581     PetscCheck(size <= *csize,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %" PetscInt_FMT " < actual size %" PetscInt_FMT, *csize, size);
5582     *csize = size;
5583   }
5584   PetscFunctionReturn(0);
5585 }
5586 
5587 /* Compress out points not in the section */
5588 static inline PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
5589 {
5590   const PetscInt np = *numPoints;
5591   PetscInt       pStart, pEnd, p, q;
5592 
5593   PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
5594   for (p = 0, q = 0; p < np; ++p) {
5595     const PetscInt r = points[p*2];
5596     if ((r >= pStart) && (r < pEnd)) {
5597       points[q*2]   = r;
5598       points[q*2+1] = points[p*2+1];
5599       ++q;
5600     }
5601   }
5602   *numPoints = q;
5603   return 0;
5604 }
5605 
5606 /* Compressed closure does not apply closure permutation */
5607 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5608 {
5609   const PetscInt *cla = NULL;
5610   PetscInt       np, *pts = NULL;
5611 
5612   PetscFunctionBeginHot;
5613   PetscCall(PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints));
5614   if (*clPoints) {
5615     PetscInt dof, off;
5616 
5617     PetscCall(PetscSectionGetDof(*clSec, point, &dof));
5618     PetscCall(PetscSectionGetOffset(*clSec, point, &off));
5619     PetscCall(ISGetIndices(*clPoints, &cla));
5620     np   = dof/2;
5621     pts  = (PetscInt *) &cla[off];
5622   } else {
5623     PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts));
5624     PetscCall(CompressPoints_Private(section, &np, pts));
5625   }
5626   *numPoints = np;
5627   *points    = pts;
5628   *clp       = cla;
5629   PetscFunctionReturn(0);
5630 }
5631 
5632 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5633 {
5634   PetscFunctionBeginHot;
5635   if (!*clPoints) {
5636     PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points));
5637   } else {
5638     PetscCall(ISRestoreIndices(*clPoints, clp));
5639   }
5640   *numPoints = 0;
5641   *points    = NULL;
5642   *clSec     = NULL;
5643   *clPoints  = NULL;
5644   *clp       = NULL;
5645   PetscFunctionReturn(0);
5646 }
5647 
5648 static inline PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5649 {
5650   PetscInt          offset = 0, p;
5651   const PetscInt    **perms = NULL;
5652   const PetscScalar **flips = NULL;
5653 
5654   PetscFunctionBeginHot;
5655   *size = 0;
5656   PetscCall(PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips));
5657   for (p = 0; p < numPoints; p++) {
5658     const PetscInt    point = points[2*p];
5659     const PetscInt    *perm = perms ? perms[p] : NULL;
5660     const PetscScalar *flip = flips ? flips[p] : NULL;
5661     PetscInt          dof, off, d;
5662     const PetscScalar *varr;
5663 
5664     PetscCall(PetscSectionGetDof(section, point, &dof));
5665     PetscCall(PetscSectionGetOffset(section, point, &off));
5666     varr = &vArray[off];
5667     if (clperm) {
5668       if (perm) {
5669         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
5670       } else {
5671         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
5672       }
5673       if (flip) {
5674         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
5675       }
5676     } else {
5677       if (perm) {
5678         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
5679       } else {
5680         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
5681       }
5682       if (flip) {
5683         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
5684       }
5685     }
5686     offset += dof;
5687   }
5688   PetscCall(PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips));
5689   *size = offset;
5690   PetscFunctionReturn(0);
5691 }
5692 
5693 static inline PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5694 {
5695   PetscInt          offset = 0, f;
5696 
5697   PetscFunctionBeginHot;
5698   *size = 0;
5699   for (f = 0; f < numFields; ++f) {
5700     PetscInt          p;
5701     const PetscInt    **perms = NULL;
5702     const PetscScalar **flips = NULL;
5703 
5704     PetscCall(PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips));
5705     for (p = 0; p < numPoints; p++) {
5706       const PetscInt    point = points[2*p];
5707       PetscInt          fdof, foff, b;
5708       const PetscScalar *varr;
5709       const PetscInt    *perm = perms ? perms[p] : NULL;
5710       const PetscScalar *flip = flips ? flips[p] : NULL;
5711 
5712       PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
5713       PetscCall(PetscSectionGetFieldOffset(section, point, f, &foff));
5714       varr = &vArray[foff];
5715       if (clperm) {
5716         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
5717         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
5718         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
5719       } else {
5720         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
5721         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
5722         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
5723       }
5724       offset += fdof;
5725     }
5726     PetscCall(PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips));
5727   }
5728   *size = offset;
5729   PetscFunctionReturn(0);
5730 }
5731 
5732 /*@C
5733   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
5734 
5735   Not collective
5736 
5737   Input Parameters:
5738 + dm - The DM
5739 . section - The section describing the layout in v, or NULL to use the default section
5740 . v - The local vector
5741 - point - The point in the DM
5742 
5743   Input/Output Parameters:
5744 + csize  - The size of the input values array, or NULL; on output the number of values in the closure
5745 - values - An array to use for the values, or NULL to have it allocated automatically;
5746            if the user provided NULL, it is a borrowed array and should not be freed
5747 
5748 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
5749 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
5750 $ assembly function, and a user may already have allocated storage for this operation.
5751 $
5752 $ A typical use could be
5753 $
5754 $  values = NULL;
5755 $  PetscCall(DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values));
5756 $  for (cl = 0; cl < clSize; ++cl) {
5757 $    <Compute on closure>
5758 $  }
5759 $  PetscCall(DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values));
5760 $
5761 $ or
5762 $
5763 $  PetscMalloc1(clMaxSize, &values);
5764 $  for (p = pStart; p < pEnd; ++p) {
5765 $    clSize = clMaxSize;
5766 $    PetscCall(DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values));
5767 $    for (cl = 0; cl < clSize; ++cl) {
5768 $      <Compute on closure>
5769 $    }
5770 $  }
5771 $  PetscFree(values);
5772 
5773   Fortran Notes:
5774   Since it returns an array, this routine is only available in Fortran 90, and you must
5775   include petsc.h90 in your code.
5776 
5777   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5778 
5779   Level: intermediate
5780 
5781 .seealso `DMPlexVecRestoreClosure()`, `DMPlexVecSetClosure()`, `DMPlexMatSetClosure()`
5782 @*/
5783 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5784 {
5785   PetscSection       clSection;
5786   IS                 clPoints;
5787   PetscInt          *points = NULL;
5788   const PetscInt    *clp, *perm;
5789   PetscInt           depth, numFields, numPoints, asize;
5790 
5791   PetscFunctionBeginHot;
5792   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5793   if (!section) PetscCall(DMGetLocalSection(dm, &section));
5794   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5795   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5796   PetscCall(DMPlexGetDepth(dm, &depth));
5797   PetscCall(PetscSectionGetNumFields(section, &numFields));
5798   if (depth == 1 && numFields < 2) {
5799     PetscCall(DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values));
5800     PetscFunctionReturn(0);
5801   }
5802   /* Get points */
5803   PetscCall(DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp));
5804   /* Get sizes */
5805   asize = 0;
5806   for (PetscInt p = 0; p < numPoints*2; p += 2) {
5807     PetscInt dof;
5808     PetscCall(PetscSectionGetDof(section, points[p], &dof));
5809     asize += dof;
5810   }
5811   if (values) {
5812     const PetscScalar *vArray;
5813     PetscInt          size;
5814 
5815     if (*values) {
5816       PetscCheck(*csize >= asize,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %" PetscInt_FMT " not sufficient to hold closure size %" PetscInt_FMT, *csize, asize);
5817     } else PetscCall(DMGetWorkArray(dm, asize, MPIU_SCALAR, values));
5818     PetscCall(PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, asize, &perm));
5819     PetscCall(VecGetArrayRead(v, &vArray));
5820     /* Get values */
5821     if (numFields > 0) PetscCall(DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values));
5822     else               PetscCall(DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values));
5823     PetscCheck(asize == size,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %" PetscInt_FMT " does not match Vec closure size %" PetscInt_FMT, asize, size);
5824     /* Cleanup array */
5825     PetscCall(VecRestoreArrayRead(v, &vArray));
5826   }
5827   if (csize) *csize = asize;
5828   /* Cleanup points */
5829   PetscCall(DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp));
5830   PetscFunctionReturn(0);
5831 }
5832 
5833 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
5834 {
5835   DMLabel            depthLabel;
5836   PetscSection       clSection;
5837   IS                 clPoints;
5838   PetscScalar       *array;
5839   const PetscScalar *vArray;
5840   PetscInt          *points = NULL;
5841   const PetscInt    *clp, *perm = NULL;
5842   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
5843 
5844   PetscFunctionBeginHot;
5845   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5846   if (!section) PetscCall(DMGetLocalSection(dm, &section));
5847   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5848   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5849   PetscCall(DMPlexGetDepth(dm, &mdepth));
5850   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
5851   PetscCall(PetscSectionGetNumFields(section, &numFields));
5852   if (mdepth == 1 && numFields < 2) {
5853     PetscCall(DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values));
5854     PetscFunctionReturn(0);
5855   }
5856   /* Get points */
5857   PetscCall(DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp));
5858   for (clsize=0,p=0; p<Np; p++) {
5859     PetscInt dof;
5860     PetscCall(PetscSectionGetDof(section, points[2*p], &dof));
5861     clsize += dof;
5862   }
5863   PetscCall(PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &perm));
5864   /* Filter points */
5865   for (p = 0; p < numPoints*2; p += 2) {
5866     PetscInt dep;
5867 
5868     PetscCall(DMLabelGetValue(depthLabel, points[p], &dep));
5869     if (dep != depth) continue;
5870     points[Np*2+0] = points[p];
5871     points[Np*2+1] = points[p+1];
5872     ++Np;
5873   }
5874   /* Get array */
5875   if (!values || !*values) {
5876     PetscInt asize = 0, dof;
5877 
5878     for (p = 0; p < Np*2; p += 2) {
5879       PetscCall(PetscSectionGetDof(section, points[p], &dof));
5880       asize += dof;
5881     }
5882     if (!values) {
5883       PetscCall(DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp));
5884       if (csize) *csize = asize;
5885       PetscFunctionReturn(0);
5886     }
5887     PetscCall(DMGetWorkArray(dm, asize, MPIU_SCALAR, &array));
5888   } else {
5889     array = *values;
5890   }
5891   PetscCall(VecGetArrayRead(v, &vArray));
5892   /* Get values */
5893   if (numFields > 0) PetscCall(DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array));
5894   else               PetscCall(DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array));
5895   /* Cleanup points */
5896   PetscCall(DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp));
5897   /* Cleanup array */
5898   PetscCall(VecRestoreArrayRead(v, &vArray));
5899   if (!*values) {
5900     if (csize) *csize = size;
5901     *values = array;
5902   } else {
5903     PetscCheck(size <= *csize,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %" PetscInt_FMT " < actual size %" PetscInt_FMT, *csize, size);
5904     *csize = size;
5905   }
5906   PetscFunctionReturn(0);
5907 }
5908 
5909 /*@C
5910   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5911 
5912   Not collective
5913 
5914   Input Parameters:
5915 + dm - The DM
5916 . section - The section describing the layout in v, or NULL to use the default section
5917 . v - The local vector
5918 . point - The point in the DM
5919 . csize - The number of values in the closure, or NULL
5920 - values - The array of values, which is a borrowed array and should not be freed
5921 
5922   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
5923 
5924   Fortran Notes:
5925   Since it returns an array, this routine is only available in Fortran 90, and you must
5926   include petsc.h90 in your code.
5927 
5928   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5929 
5930   Level: intermediate
5931 
5932 .seealso `DMPlexVecGetClosure()`, `DMPlexVecSetClosure()`, `DMPlexMatSetClosure()`
5933 @*/
5934 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5935 {
5936   PetscInt       size = 0;
5937 
5938   PetscFunctionBegin;
5939   /* Should work without recalculating size */
5940   PetscCall(DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values));
5941   *values = NULL;
5942   PetscFunctionReturn(0);
5943 }
5944 
5945 static inline void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5946 static inline void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5947 
5948 static inline PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5949 {
5950   PetscInt        cdof;   /* The number of constraints on this point */
5951   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5952   PetscScalar    *a;
5953   PetscInt        off, cind = 0, k;
5954 
5955   PetscFunctionBegin;
5956   PetscCall(PetscSectionGetConstraintDof(section, point, &cdof));
5957   PetscCall(PetscSectionGetOffset(section, point, &off));
5958   a    = &array[off];
5959   if (!cdof || setBC) {
5960     if (clperm) {
5961       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
5962       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
5963     } else {
5964       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
5965       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
5966     }
5967   } else {
5968     PetscCall(PetscSectionGetConstraintIndices(section, point, &cdofs));
5969     if (clperm) {
5970       if (perm) {for (k = 0; k < dof; ++k) {
5971           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5972           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5973         }
5974       } else {
5975         for (k = 0; k < dof; ++k) {
5976           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5977           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5978         }
5979       }
5980     } else {
5981       if (perm) {
5982         for (k = 0; k < dof; ++k) {
5983           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5984           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5985         }
5986       } else {
5987         for (k = 0; k < dof; ++k) {
5988           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5989           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5990         }
5991       }
5992     }
5993   }
5994   PetscFunctionReturn(0);
5995 }
5996 
5997 static inline PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5998 {
5999   PetscInt        cdof;   /* The number of constraints on this point */
6000   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6001   PetscScalar    *a;
6002   PetscInt        off, cind = 0, k;
6003 
6004   PetscFunctionBegin;
6005   PetscCall(PetscSectionGetConstraintDof(section, point, &cdof));
6006   PetscCall(PetscSectionGetOffset(section, point, &off));
6007   a    = &array[off];
6008   if (cdof) {
6009     PetscCall(PetscSectionGetConstraintIndices(section, point, &cdofs));
6010     if (clperm) {
6011       if (perm) {
6012         for (k = 0; k < dof; ++k) {
6013           if ((cind < cdof) && (k == cdofs[cind])) {
6014             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
6015             cind++;
6016           }
6017         }
6018       } else {
6019         for (k = 0; k < dof; ++k) {
6020           if ((cind < cdof) && (k == cdofs[cind])) {
6021             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
6022             cind++;
6023           }
6024         }
6025       }
6026     } else {
6027       if (perm) {
6028         for (k = 0; k < dof; ++k) {
6029           if ((cind < cdof) && (k == cdofs[cind])) {
6030             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
6031             cind++;
6032           }
6033         }
6034       } else {
6035         for (k = 0; k < dof; ++k) {
6036           if ((cind < cdof) && (k == cdofs[cind])) {
6037             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
6038             cind++;
6039           }
6040         }
6041       }
6042     }
6043   }
6044   PetscFunctionReturn(0);
6045 }
6046 
6047 static inline PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6048 {
6049   PetscScalar    *a;
6050   PetscInt        fdof, foff, fcdof, foffset = *offset;
6051   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6052   PetscInt        cind = 0, b;
6053 
6054   PetscFunctionBegin;
6055   PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
6056   PetscCall(PetscSectionGetFieldConstraintDof(section, point, f, &fcdof));
6057   PetscCall(PetscSectionGetFieldOffset(section, point, f, &foff));
6058   a    = &array[foff];
6059   if (!fcdof || setBC) {
6060     if (clperm) {
6061       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
6062       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
6063     } else {
6064       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
6065       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
6066     }
6067   } else {
6068     PetscCall(PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs));
6069     if (clperm) {
6070       if (perm) {
6071         for (b = 0; b < fdof; b++) {
6072           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6073           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
6074         }
6075       } else {
6076         for (b = 0; b < fdof; b++) {
6077           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6078           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
6079         }
6080       }
6081     } else {
6082       if (perm) {
6083         for (b = 0; b < fdof; b++) {
6084           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6085           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6086         }
6087       } else {
6088         for (b = 0; b < fdof; b++) {
6089           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6090           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6091         }
6092       }
6093     }
6094   }
6095   *offset += fdof;
6096   PetscFunctionReturn(0);
6097 }
6098 
6099 static inline PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6100 {
6101   PetscScalar    *a;
6102   PetscInt        fdof, foff, fcdof, foffset = *offset;
6103   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6104   PetscInt        Nc, cind = 0, ncind = 0, b;
6105   PetscBool       ncSet, fcSet;
6106 
6107   PetscFunctionBegin;
6108   PetscCall(PetscSectionGetFieldComponents(section, f, &Nc));
6109   PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
6110   PetscCall(PetscSectionGetFieldConstraintDof(section, point, f, &fcdof));
6111   PetscCall(PetscSectionGetFieldOffset(section, point, f, &foff));
6112   a    = &array[foff];
6113   if (fcdof) {
6114     /* We just override fcdof and fcdofs with Ncc and comps */
6115     PetscCall(PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs));
6116     if (clperm) {
6117       if (perm) {
6118         if (comps) {
6119           for (b = 0; b < fdof; b++) {
6120             ncSet = fcSet = PETSC_FALSE;
6121             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6122             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6123             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
6124           }
6125         } else {
6126           for (b = 0; b < fdof; b++) {
6127             if ((cind < fcdof) && (b == fcdofs[cind])) {
6128               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
6129               ++cind;
6130             }
6131           }
6132         }
6133       } else {
6134         if (comps) {
6135           for (b = 0; b < fdof; b++) {
6136             ncSet = fcSet = PETSC_FALSE;
6137             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6138             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6139             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
6140           }
6141         } else {
6142           for (b = 0; b < fdof; b++) {
6143             if ((cind < fcdof) && (b == fcdofs[cind])) {
6144               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
6145               ++cind;
6146             }
6147           }
6148         }
6149       }
6150     } else {
6151       if (perm) {
6152         if (comps) {
6153           for (b = 0; b < fdof; b++) {
6154             ncSet = fcSet = PETSC_FALSE;
6155             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6156             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6157             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
6158           }
6159         } else {
6160           for (b = 0; b < fdof; b++) {
6161             if ((cind < fcdof) && (b == fcdofs[cind])) {
6162               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6163               ++cind;
6164             }
6165           }
6166         }
6167       } else {
6168         if (comps) {
6169           for (b = 0; b < fdof; b++) {
6170             ncSet = fcSet = PETSC_FALSE;
6171             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6172             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6173             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
6174           }
6175         } else {
6176           for (b = 0; b < fdof; b++) {
6177             if ((cind < fcdof) && (b == fcdofs[cind])) {
6178               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6179               ++cind;
6180             }
6181           }
6182         }
6183       }
6184     }
6185   }
6186   *offset += fdof;
6187   PetscFunctionReturn(0);
6188 }
6189 
6190 static inline PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6191 {
6192   PetscScalar    *array;
6193   const PetscInt *cone, *coneO;
6194   PetscInt        pStart, pEnd, p, numPoints, off, dof;
6195 
6196   PetscFunctionBeginHot;
6197   PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
6198   PetscCall(DMPlexGetConeSize(dm, point, &numPoints));
6199   PetscCall(DMPlexGetCone(dm, point, &cone));
6200   PetscCall(DMPlexGetConeOrientation(dm, point, &coneO));
6201   PetscCall(VecGetArray(v, &array));
6202   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
6203     const PetscInt cp = !p ? point : cone[p-1];
6204     const PetscInt o  = !p ? 0     : coneO[p-1];
6205 
6206     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
6207     PetscCall(PetscSectionGetDof(section, cp, &dof));
6208     /* ADD_VALUES */
6209     {
6210       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6211       PetscScalar    *a;
6212       PetscInt        cdof, coff, cind = 0, k;
6213 
6214       PetscCall(PetscSectionGetConstraintDof(section, cp, &cdof));
6215       PetscCall(PetscSectionGetOffset(section, cp, &coff));
6216       a    = &array[coff];
6217       if (!cdof) {
6218         if (o >= 0) {
6219           for (k = 0; k < dof; ++k) {
6220             a[k] += values[off+k];
6221           }
6222         } else {
6223           for (k = 0; k < dof; ++k) {
6224             a[k] += values[off+dof-k-1];
6225           }
6226         }
6227       } else {
6228         PetscCall(PetscSectionGetConstraintIndices(section, cp, &cdofs));
6229         if (o >= 0) {
6230           for (k = 0; k < dof; ++k) {
6231             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6232             a[k] += values[off+k];
6233           }
6234         } else {
6235           for (k = 0; k < dof; ++k) {
6236             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6237             a[k] += values[off+dof-k-1];
6238           }
6239         }
6240       }
6241     }
6242   }
6243   PetscCall(VecRestoreArray(v, &array));
6244   PetscFunctionReturn(0);
6245 }
6246 
6247 /*@C
6248   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6249 
6250   Not collective
6251 
6252   Input Parameters:
6253 + dm - The DM
6254 . section - The section describing the layout in v, or NULL to use the default section
6255 . v - The local vector
6256 . point - The point in the DM
6257 . values - The array of values
6258 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
6259          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
6260 
6261   Fortran Notes:
6262   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6263 
6264   Level: intermediate
6265 
6266 .seealso `DMPlexVecGetClosure()`, `DMPlexMatSetClosure()`
6267 @*/
6268 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6269 {
6270   PetscSection    clSection;
6271   IS              clPoints;
6272   PetscScalar    *array;
6273   PetscInt       *points = NULL;
6274   const PetscInt *clp, *clperm = NULL;
6275   PetscInt        depth, numFields, numPoints, p, clsize;
6276 
6277   PetscFunctionBeginHot;
6278   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6279   if (!section) PetscCall(DMGetLocalSection(dm, &section));
6280   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6281   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6282   PetscCall(DMPlexGetDepth(dm, &depth));
6283   PetscCall(PetscSectionGetNumFields(section, &numFields));
6284   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6285     PetscCall(DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode));
6286     PetscFunctionReturn(0);
6287   }
6288   /* Get points */
6289   PetscCall(DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp));
6290   for (clsize=0,p=0; p<numPoints; p++) {
6291     PetscInt dof;
6292     PetscCall(PetscSectionGetDof(section, points[2*p], &dof));
6293     clsize += dof;
6294   }
6295   PetscCall(PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm));
6296   /* Get array */
6297   PetscCall(VecGetArray(v, &array));
6298   /* Get values */
6299   if (numFields > 0) {
6300     PetscInt offset = 0, f;
6301     for (f = 0; f < numFields; ++f) {
6302       const PetscInt    **perms = NULL;
6303       const PetscScalar **flips = NULL;
6304 
6305       PetscCall(PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips));
6306       switch (mode) {
6307       case INSERT_VALUES:
6308         for (p = 0; p < numPoints; p++) {
6309           const PetscInt    point = points[2*p];
6310           const PetscInt    *perm = perms ? perms[p] : NULL;
6311           const PetscScalar *flip = flips ? flips[p] : NULL;
6312           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
6313         } break;
6314       case INSERT_ALL_VALUES:
6315         for (p = 0; p < numPoints; p++) {
6316           const PetscInt    point = points[2*p];
6317           const PetscInt    *perm = perms ? perms[p] : NULL;
6318           const PetscScalar *flip = flips ? flips[p] : NULL;
6319           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
6320         } break;
6321       case INSERT_BC_VALUES:
6322         for (p = 0; p < numPoints; p++) {
6323           const PetscInt    point = points[2*p];
6324           const PetscInt    *perm = perms ? perms[p] : NULL;
6325           const PetscScalar *flip = flips ? flips[p] : NULL;
6326           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
6327         } break;
6328       case ADD_VALUES:
6329         for (p = 0; p < numPoints; p++) {
6330           const PetscInt    point = points[2*p];
6331           const PetscInt    *perm = perms ? perms[p] : NULL;
6332           const PetscScalar *flip = flips ? flips[p] : NULL;
6333           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
6334         } break;
6335       case ADD_ALL_VALUES:
6336         for (p = 0; p < numPoints; p++) {
6337           const PetscInt    point = points[2*p];
6338           const PetscInt    *perm = perms ? perms[p] : NULL;
6339           const PetscScalar *flip = flips ? flips[p] : NULL;
6340           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
6341         } break;
6342       case ADD_BC_VALUES:
6343         for (p = 0; p < numPoints; p++) {
6344           const PetscInt    point = points[2*p];
6345           const PetscInt    *perm = perms ? perms[p] : NULL;
6346           const PetscScalar *flip = flips ? flips[p] : NULL;
6347           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
6348         } break;
6349       default:
6350         SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6351       }
6352       PetscCall(PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips));
6353     }
6354   } else {
6355     PetscInt dof, off;
6356     const PetscInt    **perms = NULL;
6357     const PetscScalar **flips = NULL;
6358 
6359     PetscCall(PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips));
6360     switch (mode) {
6361     case INSERT_VALUES:
6362       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6363         const PetscInt    point = points[2*p];
6364         const PetscInt    *perm = perms ? perms[p] : NULL;
6365         const PetscScalar *flip = flips ? flips[p] : NULL;
6366         PetscCall(PetscSectionGetDof(section, point, &dof));
6367         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
6368       } break;
6369     case INSERT_ALL_VALUES:
6370       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6371         const PetscInt    point = points[2*p];
6372         const PetscInt    *perm = perms ? perms[p] : NULL;
6373         const PetscScalar *flip = flips ? flips[p] : NULL;
6374         PetscCall(PetscSectionGetDof(section, point, &dof));
6375         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
6376       } break;
6377     case INSERT_BC_VALUES:
6378       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6379         const PetscInt    point = points[2*p];
6380         const PetscInt    *perm = perms ? perms[p] : NULL;
6381         const PetscScalar *flip = flips ? flips[p] : NULL;
6382         PetscCall(PetscSectionGetDof(section, point, &dof));
6383         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
6384       } break;
6385     case ADD_VALUES:
6386       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6387         const PetscInt    point = points[2*p];
6388         const PetscInt    *perm = perms ? perms[p] : NULL;
6389         const PetscScalar *flip = flips ? flips[p] : NULL;
6390         PetscCall(PetscSectionGetDof(section, point, &dof));
6391         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
6392       } break;
6393     case ADD_ALL_VALUES:
6394       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6395         const PetscInt    point = points[2*p];
6396         const PetscInt    *perm = perms ? perms[p] : NULL;
6397         const PetscScalar *flip = flips ? flips[p] : NULL;
6398         PetscCall(PetscSectionGetDof(section, point, &dof));
6399         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
6400       } break;
6401     case ADD_BC_VALUES:
6402       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6403         const PetscInt    point = points[2*p];
6404         const PetscInt    *perm = perms ? perms[p] : NULL;
6405         const PetscScalar *flip = flips ? flips[p] : NULL;
6406         PetscCall(PetscSectionGetDof(section, point, &dof));
6407         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
6408       } break;
6409     default:
6410       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6411     }
6412     PetscCall(PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips));
6413   }
6414   /* Cleanup points */
6415   PetscCall(DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp));
6416   /* Cleanup array */
6417   PetscCall(VecRestoreArray(v, &array));
6418   PetscFunctionReturn(0);
6419 }
6420 
6421 /* Check whether the given point is in the label. If not, update the offset to skip this point */
6422 static inline PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset)
6423 {
6424   PetscFunctionBegin;
6425   if (label) {
6426     PetscInt       val, fdof;
6427 
6428     /* There is a problem with this:
6429          Suppose we have two label values, defining surfaces, interecting along a line in 3D. When we add cells to the label, the cells that
6430        touch both surfaces must pick a label value. Thus we miss setting values for the surface with that other value intersecting that cell.
6431        Thus I am only going to check val != -1, not val != labelId
6432     */
6433     PetscCall(DMLabelGetValue(label, point, &val));
6434     if (val < 0) {
6435       PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
6436       *offset += fdof;
6437       PetscFunctionReturn(1);
6438     }
6439   }
6440   PetscFunctionReturn(0);
6441 }
6442 
6443 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
6444 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
6445 {
6446   PetscSection    clSection;
6447   IS              clPoints;
6448   PetscScalar    *array;
6449   PetscInt       *points = NULL;
6450   const PetscInt *clp;
6451   PetscInt        numFields, numPoints, p;
6452   PetscInt        offset = 0, f;
6453 
6454   PetscFunctionBeginHot;
6455   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6456   if (!section) PetscCall(DMGetLocalSection(dm, &section));
6457   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6458   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6459   PetscCall(PetscSectionGetNumFields(section, &numFields));
6460   /* Get points */
6461   PetscCall(DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp));
6462   /* Get array */
6463   PetscCall(VecGetArray(v, &array));
6464   /* Get values */
6465   for (f = 0; f < numFields; ++f) {
6466     const PetscInt    **perms = NULL;
6467     const PetscScalar **flips = NULL;
6468 
6469     if (!fieldActive[f]) {
6470       for (p = 0; p < numPoints*2; p += 2) {
6471         PetscInt fdof;
6472         PetscCall(PetscSectionGetFieldDof(section, points[p], f, &fdof));
6473         offset += fdof;
6474       }
6475       continue;
6476     }
6477     PetscCall(PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips));
6478     switch (mode) {
6479     case INSERT_VALUES:
6480       for (p = 0; p < numPoints; p++) {
6481         const PetscInt    point = points[2*p];
6482         const PetscInt    *perm = perms ? perms[p] : NULL;
6483         const PetscScalar *flip = flips ? flips[p] : NULL;
6484         if (CheckPoint_Private(label, labelId, section, point, f, &offset)) continue;
6485         PetscCall(updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array));
6486       } break;
6487     case INSERT_ALL_VALUES:
6488       for (p = 0; p < numPoints; p++) {
6489         const PetscInt    point = points[2*p];
6490         const PetscInt    *perm = perms ? perms[p] : NULL;
6491         const PetscScalar *flip = flips ? flips[p] : NULL;
6492         if (CheckPoint_Private(label, labelId, section, point, f, &offset)) continue;
6493         PetscCall(updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array));
6494       } break;
6495     case INSERT_BC_VALUES:
6496       for (p = 0; p < numPoints; p++) {
6497         const PetscInt    point = points[2*p];
6498         const PetscInt    *perm = perms ? perms[p] : NULL;
6499         const PetscScalar *flip = flips ? flips[p] : NULL;
6500         if (CheckPoint_Private(label, labelId, section, point, f, &offset)) continue;
6501         PetscCall(updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array));
6502       } break;
6503     case ADD_VALUES:
6504       for (p = 0; p < numPoints; p++) {
6505         const PetscInt    point = points[2*p];
6506         const PetscInt    *perm = perms ? perms[p] : NULL;
6507         const PetscScalar *flip = flips ? flips[p] : NULL;
6508         if (CheckPoint_Private(label, labelId, section, point, f, &offset)) continue;
6509         PetscCall(updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array));
6510       } break;
6511     case ADD_ALL_VALUES:
6512       for (p = 0; p < numPoints; p++) {
6513         const PetscInt    point = points[2*p];
6514         const PetscInt    *perm = perms ? perms[p] : NULL;
6515         const PetscScalar *flip = flips ? flips[p] : NULL;
6516         if (CheckPoint_Private(label, labelId, section, point, f, &offset)) continue;
6517         PetscCall(updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array));
6518       } break;
6519     default:
6520       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6521     }
6522     PetscCall(PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips));
6523   }
6524   /* Cleanup points */
6525   PetscCall(DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp));
6526   /* Cleanup array */
6527   PetscCall(VecRestoreArray(v, &array));
6528   PetscFunctionReturn(0);
6529 }
6530 
6531 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
6532 {
6533   PetscMPIInt    rank;
6534   PetscInt       i, j;
6535 
6536   PetscFunctionBegin;
6537   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank));
6538   PetscCall(PetscViewerASCIIPrintf(viewer, "[%d]mat for point %" PetscInt_FMT "\n", rank, point));
6539   for (i = 0; i < numRIndices; i++) PetscCall(PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%" PetscInt_FMT "] = %" PetscInt_FMT "\n", rank, i, rindices[i]));
6540   for (i = 0; i < numCIndices; i++) PetscCall(PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%" PetscInt_FMT "] = %" PetscInt_FMT "\n", rank, i, cindices[i]));
6541   numCIndices = numCIndices ? numCIndices : numRIndices;
6542   if (!values) PetscFunctionReturn(0);
6543   for (i = 0; i < numRIndices; i++) {
6544     PetscCall(PetscViewerASCIIPrintf(viewer, "[%d]", rank));
6545     for (j = 0; j < numCIndices; j++) {
6546 #if defined(PETSC_USE_COMPLEX)
6547       PetscCall(PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j])));
6548 #else
6549       PetscCall(PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]));
6550 #endif
6551     }
6552     PetscCall(PetscViewerASCIIPrintf(viewer, "\n"));
6553   }
6554   PetscFunctionReturn(0);
6555 }
6556 
6557 /*
6558   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
6559 
6560   Input Parameters:
6561 + section - The section for this data layout
6562 . islocal - Is the section (and thus indices being requested) local or global?
6563 . point   - The point contributing dofs with these indices
6564 . off     - The global offset of this point
6565 . loff    - The local offset of each field
6566 . setBC   - The flag determining whether to include indices of boundary values
6567 . perm    - A permutation of the dofs on this point, or NULL
6568 - indperm - A permutation of the entire indices array, or NULL
6569 
6570   Output Parameter:
6571 . indices - Indices for dofs on this point
6572 
6573   Level: developer
6574 
6575   Note: The indices could be local or global, depending on the value of 'off'.
6576 */
6577 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
6578 {
6579   PetscInt        dof;   /* The number of unknowns on this point */
6580   PetscInt        cdof;  /* The number of constraints on this point */
6581   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6582   PetscInt        cind = 0, k;
6583 
6584   PetscFunctionBegin;
6585   PetscCheck(islocal || !setBC,PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6586   PetscCall(PetscSectionGetDof(section, point, &dof));
6587   PetscCall(PetscSectionGetConstraintDof(section, point, &cdof));
6588   if (!cdof || setBC) {
6589     for (k = 0; k < dof; ++k) {
6590       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6591       const PetscInt ind    = indperm ? indperm[preind] : preind;
6592 
6593       indices[ind] = off + k;
6594     }
6595   } else {
6596     PetscCall(PetscSectionGetConstraintIndices(section, point, &cdofs));
6597     for (k = 0; k < dof; ++k) {
6598       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6599       const PetscInt ind    = indperm ? indperm[preind] : preind;
6600 
6601       if ((cind < cdof) && (k == cdofs[cind])) {
6602         /* Insert check for returning constrained indices */
6603         indices[ind] = -(off+k+1);
6604         ++cind;
6605       } else {
6606         indices[ind] = off + k - (islocal ? 0 : cind);
6607       }
6608     }
6609   }
6610   *loff += dof;
6611   PetscFunctionReturn(0);
6612 }
6613 
6614 /*
6615  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
6616 
6617  Input Parameters:
6618 + section - a section (global or local)
6619 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
6620 . point - point within section
6621 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
6622 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
6623 . setBC - identify constrained (boundary condition) points via involution.
6624 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
6625 . permsoff - offset
6626 - indperm - index permutation
6627 
6628  Output Parameter:
6629 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
6630 . indices - array to hold indices (as defined by section) of each dof associated with point
6631 
6632  Notes:
6633  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
6634  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
6635  in the local vector.
6636 
6637  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
6638  significant).  It is invalid to call with a global section and setBC=true.
6639 
6640  Developer Note:
6641  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
6642  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
6643  offset could be obtained from the section instead of passing it explicitly as we do now.
6644 
6645  Example:
6646  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
6647  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
6648  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
6649  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
6650 
6651  Level: developer
6652 */
6653 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6654 {
6655   PetscInt       numFields, foff, f;
6656 
6657   PetscFunctionBegin;
6658   PetscCheck(islocal || !setBC,PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6659   PetscCall(PetscSectionGetNumFields(section, &numFields));
6660   for (f = 0, foff = 0; f < numFields; ++f) {
6661     PetscInt        fdof, cfdof;
6662     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6663     PetscInt        cind = 0, b;
6664     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6665 
6666     PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
6667     PetscCall(PetscSectionGetFieldConstraintDof(section, point, f, &cfdof));
6668     if (!cfdof || setBC) {
6669       for (b = 0; b < fdof; ++b) {
6670         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6671         const PetscInt ind    = indperm ? indperm[preind] : preind;
6672 
6673         indices[ind] = off+foff+b;
6674       }
6675     } else {
6676       PetscCall(PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs));
6677       for (b = 0; b < fdof; ++b) {
6678         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6679         const PetscInt ind    = indperm ? indperm[preind] : preind;
6680 
6681         if ((cind < cfdof) && (b == fcdofs[cind])) {
6682           indices[ind] = -(off+foff+b+1);
6683           ++cind;
6684         } else {
6685           indices[ind] = off + foff + b - (islocal ? 0 : cind);
6686         }
6687       }
6688     }
6689     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
6690     foffs[f] += fdof;
6691   }
6692   PetscFunctionReturn(0);
6693 }
6694 
6695 /*
6696   This version believes the globalSection offsets for each field, rather than just the point offset
6697 
6698  . foffs - The offset into 'indices' for each field, since it is segregated by field
6699 
6700  Notes:
6701  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
6702  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
6703 */
6704 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6705 {
6706   PetscInt       numFields, foff, f;
6707 
6708   PetscFunctionBegin;
6709   PetscCall(PetscSectionGetNumFields(section, &numFields));
6710   for (f = 0; f < numFields; ++f) {
6711     PetscInt        fdof, cfdof;
6712     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6713     PetscInt        cind = 0, b;
6714     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6715 
6716     PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
6717     PetscCall(PetscSectionGetFieldConstraintDof(section, point, f, &cfdof));
6718     PetscCall(PetscSectionGetFieldOffset(globalSection, point, f, &foff));
6719     if (!cfdof) {
6720       for (b = 0; b < fdof; ++b) {
6721         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6722         const PetscInt ind    = indperm ? indperm[preind] : preind;
6723 
6724         indices[ind] = foff+b;
6725       }
6726     } else {
6727       PetscCall(PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs));
6728       for (b = 0; b < fdof; ++b) {
6729         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6730         const PetscInt ind    = indperm ? indperm[preind] : preind;
6731 
6732         if ((cind < cfdof) && (b == fcdofs[cind])) {
6733           indices[ind] = -(foff+b+1);
6734           ++cind;
6735         } else {
6736           indices[ind] = foff+b-cind;
6737         }
6738       }
6739     }
6740     foffs[f] += fdof;
6741   }
6742   PetscFunctionReturn(0);
6743 }
6744 
6745 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
6746 {
6747   Mat             cMat;
6748   PetscSection    aSec, cSec;
6749   IS              aIS;
6750   PetscInt        aStart = -1, aEnd = -1;
6751   const PetscInt  *anchors;
6752   PetscInt        numFields, f, p, q, newP = 0;
6753   PetscInt        newNumPoints = 0, newNumIndices = 0;
6754   PetscInt        *newPoints, *indices, *newIndices;
6755   PetscInt        maxAnchor, maxDof;
6756   PetscInt        newOffsets[32];
6757   PetscInt        *pointMatOffsets[32];
6758   PetscInt        *newPointOffsets[32];
6759   PetscScalar     *pointMat[32];
6760   PetscScalar     *newValues=NULL,*tmpValues;
6761   PetscBool       anyConstrained = PETSC_FALSE;
6762 
6763   PetscFunctionBegin;
6764   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6765   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6766   PetscCall(PetscSectionGetNumFields(section, &numFields));
6767 
6768   PetscCall(DMPlexGetAnchors(dm,&aSec,&aIS));
6769   /* if there are point-to-point constraints */
6770   if (aSec) {
6771     PetscCall(PetscArrayzero(newOffsets, 32));
6772     PetscCall(ISGetIndices(aIS,&anchors));
6773     PetscCall(PetscSectionGetChart(aSec,&aStart,&aEnd));
6774     /* figure out how many points are going to be in the new element matrix
6775      * (we allow double counting, because it's all just going to be summed
6776      * into the global matrix anyway) */
6777     for (p = 0; p < 2*numPoints; p+=2) {
6778       PetscInt b    = points[p];
6779       PetscInt bDof = 0, bSecDof;
6780 
6781       PetscCall(PetscSectionGetDof(section,b,&bSecDof));
6782       if (!bSecDof) {
6783         continue;
6784       }
6785       if (b >= aStart && b < aEnd) {
6786         PetscCall(PetscSectionGetDof(aSec,b,&bDof));
6787       }
6788       if (bDof) {
6789         /* this point is constrained */
6790         /* it is going to be replaced by its anchors */
6791         PetscInt bOff, q;
6792 
6793         anyConstrained = PETSC_TRUE;
6794         newNumPoints  += bDof;
6795         PetscCall(PetscSectionGetOffset(aSec,b,&bOff));
6796         for (q = 0; q < bDof; q++) {
6797           PetscInt a = anchors[bOff + q];
6798           PetscInt aDof;
6799 
6800           PetscCall(PetscSectionGetDof(section,a,&aDof));
6801           newNumIndices += aDof;
6802           for (f = 0; f < numFields; ++f) {
6803             PetscInt fDof;
6804 
6805             PetscCall(PetscSectionGetFieldDof(section, a, f, &fDof));
6806             newOffsets[f+1] += fDof;
6807           }
6808         }
6809       }
6810       else {
6811         /* this point is not constrained */
6812         newNumPoints++;
6813         newNumIndices += bSecDof;
6814         for (f = 0; f < numFields; ++f) {
6815           PetscInt fDof;
6816 
6817           PetscCall(PetscSectionGetFieldDof(section, b, f, &fDof));
6818           newOffsets[f+1] += fDof;
6819         }
6820       }
6821     }
6822   }
6823   if (!anyConstrained) {
6824     if (outNumPoints)  *outNumPoints  = 0;
6825     if (outNumIndices) *outNumIndices = 0;
6826     if (outPoints)     *outPoints     = NULL;
6827     if (outValues)     *outValues     = NULL;
6828     if (aSec) PetscCall(ISRestoreIndices(aIS,&anchors));
6829     PetscFunctionReturn(0);
6830   }
6831 
6832   if (outNumPoints)  *outNumPoints  = newNumPoints;
6833   if (outNumIndices) *outNumIndices = newNumIndices;
6834 
6835   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
6836 
6837   if (!outPoints && !outValues) {
6838     if (offsets) {
6839       for (f = 0; f <= numFields; f++) {
6840         offsets[f] = newOffsets[f];
6841       }
6842     }
6843     if (aSec) PetscCall(ISRestoreIndices(aIS,&anchors));
6844     PetscFunctionReturn(0);
6845   }
6846 
6847   PetscCheck(!numFields || newOffsets[numFields] == newNumIndices,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, newOffsets[numFields], newNumIndices);
6848 
6849   PetscCall(DMGetDefaultConstraints(dm, &cSec, &cMat, NULL));
6850 
6851   /* workspaces */
6852   if (numFields) {
6853     for (f = 0; f < numFields; f++) {
6854       PetscCall(DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]));
6855       PetscCall(DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]));
6856     }
6857   }
6858   else {
6859     PetscCall(DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]));
6860     PetscCall(DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]));
6861   }
6862 
6863   /* get workspaces for the point-to-point matrices */
6864   if (numFields) {
6865     PetscInt totalOffset, totalMatOffset;
6866 
6867     for (p = 0; p < numPoints; p++) {
6868       PetscInt b    = points[2*p];
6869       PetscInt bDof = 0, bSecDof;
6870 
6871       PetscCall(PetscSectionGetDof(section,b,&bSecDof));
6872       if (!bSecDof) {
6873         for (f = 0; f < numFields; f++) {
6874           newPointOffsets[f][p + 1] = 0;
6875           pointMatOffsets[f][p + 1] = 0;
6876         }
6877         continue;
6878       }
6879       if (b >= aStart && b < aEnd) {
6880         PetscCall(PetscSectionGetDof(aSec, b, &bDof));
6881       }
6882       if (bDof) {
6883         for (f = 0; f < numFields; f++) {
6884           PetscInt fDof, q, bOff, allFDof = 0;
6885 
6886           PetscCall(PetscSectionGetFieldDof(section, b, f, &fDof));
6887           PetscCall(PetscSectionGetOffset(aSec, b, &bOff));
6888           for (q = 0; q < bDof; q++) {
6889             PetscInt a = anchors[bOff + q];
6890             PetscInt aFDof;
6891 
6892             PetscCall(PetscSectionGetFieldDof(section, a, f, &aFDof));
6893             allFDof += aFDof;
6894           }
6895           newPointOffsets[f][p+1] = allFDof;
6896           pointMatOffsets[f][p+1] = fDof * allFDof;
6897         }
6898       }
6899       else {
6900         for (f = 0; f < numFields; f++) {
6901           PetscInt fDof;
6902 
6903           PetscCall(PetscSectionGetFieldDof(section, b, f, &fDof));
6904           newPointOffsets[f][p+1] = fDof;
6905           pointMatOffsets[f][p+1] = 0;
6906         }
6907       }
6908     }
6909     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
6910       newPointOffsets[f][0] = totalOffset;
6911       pointMatOffsets[f][0] = totalMatOffset;
6912       for (p = 0; p < numPoints; p++) {
6913         newPointOffsets[f][p+1] += newPointOffsets[f][p];
6914         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
6915       }
6916       totalOffset    = newPointOffsets[f][numPoints];
6917       totalMatOffset = pointMatOffsets[f][numPoints];
6918       PetscCall(DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]));
6919     }
6920   }
6921   else {
6922     for (p = 0; p < numPoints; p++) {
6923       PetscInt b    = points[2*p];
6924       PetscInt bDof = 0, bSecDof;
6925 
6926       PetscCall(PetscSectionGetDof(section,b,&bSecDof));
6927       if (!bSecDof) {
6928         newPointOffsets[0][p + 1] = 0;
6929         pointMatOffsets[0][p + 1] = 0;
6930         continue;
6931       }
6932       if (b >= aStart && b < aEnd) {
6933         PetscCall(PetscSectionGetDof(aSec, b, &bDof));
6934       }
6935       if (bDof) {
6936         PetscInt bOff, q, allDof = 0;
6937 
6938         PetscCall(PetscSectionGetOffset(aSec, b, &bOff));
6939         for (q = 0; q < bDof; q++) {
6940           PetscInt a = anchors[bOff + q], aDof;
6941 
6942           PetscCall(PetscSectionGetDof(section, a, &aDof));
6943           allDof += aDof;
6944         }
6945         newPointOffsets[0][p+1] = allDof;
6946         pointMatOffsets[0][p+1] = bSecDof * allDof;
6947       }
6948       else {
6949         newPointOffsets[0][p+1] = bSecDof;
6950         pointMatOffsets[0][p+1] = 0;
6951       }
6952     }
6953     newPointOffsets[0][0] = 0;
6954     pointMatOffsets[0][0] = 0;
6955     for (p = 0; p < numPoints; p++) {
6956       newPointOffsets[0][p+1] += newPointOffsets[0][p];
6957       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
6958     }
6959     PetscCall(DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]));
6960   }
6961 
6962   /* output arrays */
6963   PetscCall(DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints));
6964 
6965   /* get the point-to-point matrices; construct newPoints */
6966   PetscCall(PetscSectionGetMaxDof(aSec, &maxAnchor));
6967   PetscCall(PetscSectionGetMaxDof(section, &maxDof));
6968   PetscCall(DMGetWorkArray(dm,maxDof,MPIU_INT,&indices));
6969   PetscCall(DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices));
6970   if (numFields) {
6971     for (p = 0, newP = 0; p < numPoints; p++) {
6972       PetscInt b    = points[2*p];
6973       PetscInt o    = points[2*p+1];
6974       PetscInt bDof = 0, bSecDof;
6975 
6976       PetscCall(PetscSectionGetDof(section, b, &bSecDof));
6977       if (!bSecDof) {
6978         continue;
6979       }
6980       if (b >= aStart && b < aEnd) {
6981         PetscCall(PetscSectionGetDof(aSec, b, &bDof));
6982       }
6983       if (bDof) {
6984         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
6985 
6986         fStart[0] = 0;
6987         fEnd[0]   = 0;
6988         for (f = 0; f < numFields; f++) {
6989           PetscInt fDof;
6990 
6991           PetscCall(PetscSectionGetFieldDof(cSec, b, f, &fDof));
6992           fStart[f+1] = fStart[f] + fDof;
6993           fEnd[f+1]   = fStart[f+1];
6994         }
6995         PetscCall(PetscSectionGetOffset(cSec, b, &bOff));
6996         PetscCall(DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices));
6997 
6998         fAnchorStart[0] = 0;
6999         fAnchorEnd[0]   = 0;
7000         for (f = 0; f < numFields; f++) {
7001           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
7002 
7003           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
7004           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
7005         }
7006         PetscCall(PetscSectionGetOffset(aSec, b, &bOff));
7007         for (q = 0; q < bDof; q++) {
7008           PetscInt a = anchors[bOff + q], aOff;
7009 
7010           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
7011           newPoints[2*(newP + q)]     = a;
7012           newPoints[2*(newP + q) + 1] = 0;
7013           PetscCall(PetscSectionGetOffset(section, a, &aOff));
7014           PetscCall(DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices));
7015         }
7016         newP += bDof;
7017 
7018         if (outValues) {
7019           /* get the point-to-point submatrix */
7020           for (f = 0; f < numFields; f++) {
7021             PetscCall(MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]));
7022           }
7023         }
7024       }
7025       else {
7026         newPoints[2 * newP]     = b;
7027         newPoints[2 * newP + 1] = o;
7028         newP++;
7029       }
7030     }
7031   } else {
7032     for (p = 0; p < numPoints; p++) {
7033       PetscInt b    = points[2*p];
7034       PetscInt o    = points[2*p+1];
7035       PetscInt bDof = 0, bSecDof;
7036 
7037       PetscCall(PetscSectionGetDof(section, b, &bSecDof));
7038       if (!bSecDof) {
7039         continue;
7040       }
7041       if (b >= aStart && b < aEnd) {
7042         PetscCall(PetscSectionGetDof(aSec, b, &bDof));
7043       }
7044       if (bDof) {
7045         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
7046 
7047         PetscCall(PetscSectionGetOffset(cSec, b, &bOff));
7048         PetscCall(DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices));
7049 
7050         PetscCall(PetscSectionGetOffset (aSec, b, &bOff));
7051         for (q = 0; q < bDof; q++) {
7052           PetscInt a = anchors[bOff + q], aOff;
7053 
7054           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
7055 
7056           newPoints[2*(newP + q)]     = a;
7057           newPoints[2*(newP + q) + 1] = 0;
7058           PetscCall(PetscSectionGetOffset(section, a, &aOff));
7059           PetscCall(DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices));
7060         }
7061         newP += bDof;
7062 
7063         /* get the point-to-point submatrix */
7064         if (outValues) {
7065           PetscCall(MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]));
7066         }
7067       }
7068       else {
7069         newPoints[2 * newP]     = b;
7070         newPoints[2 * newP + 1] = o;
7071         newP++;
7072       }
7073     }
7074   }
7075 
7076   if (outValues) {
7077     PetscCall(DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues));
7078     PetscCall(PetscArrayzero(tmpValues,newNumIndices*numIndices));
7079     /* multiply constraints on the right */
7080     if (numFields) {
7081       for (f = 0; f < numFields; f++) {
7082         PetscInt oldOff = offsets[f];
7083 
7084         for (p = 0; p < numPoints; p++) {
7085           PetscInt cStart = newPointOffsets[f][p];
7086           PetscInt b      = points[2 * p];
7087           PetscInt c, r, k;
7088           PetscInt dof;
7089 
7090           PetscCall(PetscSectionGetFieldDof(section,b,f,&dof));
7091           if (!dof) {
7092             continue;
7093           }
7094           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7095             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
7096             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
7097 
7098             for (r = 0; r < numIndices; r++) {
7099               for (c = 0; c < nCols; c++) {
7100                 for (k = 0; k < dof; k++) {
7101                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
7102                 }
7103               }
7104             }
7105           }
7106           else {
7107             /* copy this column as is */
7108             for (r = 0; r < numIndices; r++) {
7109               for (c = 0; c < dof; c++) {
7110                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7111               }
7112             }
7113           }
7114           oldOff += dof;
7115         }
7116       }
7117     }
7118     else {
7119       PetscInt oldOff = 0;
7120       for (p = 0; p < numPoints; p++) {
7121         PetscInt cStart = newPointOffsets[0][p];
7122         PetscInt b      = points[2 * p];
7123         PetscInt c, r, k;
7124         PetscInt dof;
7125 
7126         PetscCall(PetscSectionGetDof(section,b,&dof));
7127         if (!dof) {
7128           continue;
7129         }
7130         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7131           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
7132           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
7133 
7134           for (r = 0; r < numIndices; r++) {
7135             for (c = 0; c < nCols; c++) {
7136               for (k = 0; k < dof; k++) {
7137                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
7138               }
7139             }
7140           }
7141         }
7142         else {
7143           /* copy this column as is */
7144           for (r = 0; r < numIndices; r++) {
7145             for (c = 0; c < dof; c++) {
7146               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7147             }
7148           }
7149         }
7150         oldOff += dof;
7151       }
7152     }
7153 
7154     if (multiplyLeft) {
7155       PetscCall(DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues));
7156       PetscCall(PetscArrayzero(newValues,newNumIndices*newNumIndices));
7157       /* multiply constraints transpose on the left */
7158       if (numFields) {
7159         for (f = 0; f < numFields; f++) {
7160           PetscInt oldOff = offsets[f];
7161 
7162           for (p = 0; p < numPoints; p++) {
7163             PetscInt rStart = newPointOffsets[f][p];
7164             PetscInt b      = points[2 * p];
7165             PetscInt c, r, k;
7166             PetscInt dof;
7167 
7168             PetscCall(PetscSectionGetFieldDof(section,b,f,&dof));
7169             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7170               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
7171               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
7172 
7173               for (r = 0; r < nRows; r++) {
7174                 for (c = 0; c < newNumIndices; c++) {
7175                   for (k = 0; k < dof; k++) {
7176                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7177                   }
7178                 }
7179               }
7180             }
7181             else {
7182               /* copy this row as is */
7183               for (r = 0; r < dof; r++) {
7184                 for (c = 0; c < newNumIndices; c++) {
7185                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7186                 }
7187               }
7188             }
7189             oldOff += dof;
7190           }
7191         }
7192       }
7193       else {
7194         PetscInt oldOff = 0;
7195 
7196         for (p = 0; p < numPoints; p++) {
7197           PetscInt rStart = newPointOffsets[0][p];
7198           PetscInt b      = points[2 * p];
7199           PetscInt c, r, k;
7200           PetscInt dof;
7201 
7202           PetscCall(PetscSectionGetDof(section,b,&dof));
7203           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7204             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
7205             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
7206 
7207             for (r = 0; r < nRows; r++) {
7208               for (c = 0; c < newNumIndices; c++) {
7209                 for (k = 0; k < dof; k++) {
7210                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7211                 }
7212               }
7213             }
7214           }
7215           else {
7216             /* copy this row as is */
7217             for (r = 0; r < dof; r++) {
7218               for (c = 0; c < newNumIndices; c++) {
7219                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7220               }
7221             }
7222           }
7223           oldOff += dof;
7224         }
7225       }
7226 
7227       PetscCall(DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues));
7228     }
7229     else {
7230       newValues = tmpValues;
7231     }
7232   }
7233 
7234   /* clean up */
7235   PetscCall(DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices));
7236   PetscCall(DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices));
7237 
7238   if (numFields) {
7239     for (f = 0; f < numFields; f++) {
7240       PetscCall(DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]));
7241       PetscCall(DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]));
7242       PetscCall(DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]));
7243     }
7244   }
7245   else {
7246     PetscCall(DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]));
7247     PetscCall(DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]));
7248     PetscCall(DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]));
7249   }
7250   PetscCall(ISRestoreIndices(aIS,&anchors));
7251 
7252   /* output */
7253   if (outPoints) {
7254     *outPoints = newPoints;
7255   }
7256   else {
7257     PetscCall(DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints));
7258   }
7259   if (outValues) {
7260     *outValues = newValues;
7261   }
7262   for (f = 0; f <= numFields; f++) {
7263     offsets[f] = newOffsets[f];
7264   }
7265   PetscFunctionReturn(0);
7266 }
7267 
7268 /*@C
7269   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
7270 
7271   Not collective
7272 
7273   Input Parameters:
7274 + dm         - The DM
7275 . section    - The PetscSection describing the points (a local section)
7276 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7277 . point      - The point defining the closure
7278 - useClPerm  - Use the closure point permutation if available
7279 
7280   Output Parameters:
7281 + numIndices - The number of dof indices in the closure of point with the input sections
7282 . indices    - The dof indices
7283 . outOffsets - Array to write the field offsets into, or NULL
7284 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7285 
7286   Notes:
7287   Must call DMPlexRestoreClosureIndices() to free allocated memory
7288 
7289   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7290   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7291   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7292   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7293   indices (with the above semantics) are implied.
7294 
7295   Level: advanced
7296 
7297 .seealso `DMPlexRestoreClosureIndices()`, `DMPlexVecGetClosure()`, `DMPlexMatSetClosure()`, `DMGetLocalSection()`, `DMGetGlobalSection()`
7298 @*/
7299 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7300                                        PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7301 {
7302   /* Closure ordering */
7303   PetscSection        clSection;
7304   IS                  clPoints;
7305   const PetscInt     *clp;
7306   PetscInt           *points;
7307   const PetscInt     *clperm = NULL;
7308   /* Dof permutation and sign flips */
7309   const PetscInt    **perms[32] = {NULL};
7310   const PetscScalar **flips[32] = {NULL};
7311   PetscScalar        *valCopy   = NULL;
7312   /* Hanging node constraints */
7313   PetscInt           *pointsC = NULL;
7314   PetscScalar        *valuesC = NULL;
7315   PetscInt            NclC, NiC;
7316 
7317   PetscInt           *idx;
7318   PetscInt            Nf, Ncl, Ni = 0, offsets[32], p, f;
7319   PetscBool           isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7320 
7321   PetscFunctionBeginHot;
7322   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7323   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7324   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7325   if (numIndices) PetscValidIntPointer(numIndices, 6);
7326   if (indices)    PetscValidPointer(indices, 7);
7327   if (outOffsets) PetscValidIntPointer(outOffsets, 8);
7328   if (values)     PetscValidPointer(values, 9);
7329   PetscCall(PetscSectionGetNumFields(section, &Nf));
7330   PetscCheck(Nf <= 31,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %" PetscInt_FMT " limited to 31", Nf);
7331   PetscCall(PetscArrayzero(offsets, 32));
7332   /* 1) Get points in closure */
7333   PetscCall(DMPlexGetCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp));
7334   if (useClPerm) {
7335     PetscInt depth, clsize;
7336     PetscCall(DMPlexGetPointDepth(dm, point, &depth));
7337     for (clsize=0,p=0; p<Ncl; p++) {
7338       PetscInt dof;
7339       PetscCall(PetscSectionGetDof(section, points[2*p], &dof));
7340       clsize += dof;
7341     }
7342     PetscCall(PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm));
7343   }
7344   /* 2) Get number of indices on these points and field offsets from section */
7345   for (p = 0; p < Ncl*2; p += 2) {
7346     PetscInt dof, fdof;
7347 
7348     PetscCall(PetscSectionGetDof(section, points[p], &dof));
7349     for (f = 0; f < Nf; ++f) {
7350       PetscCall(PetscSectionGetFieldDof(section, points[p], f, &fdof));
7351       offsets[f+1] += fdof;
7352     }
7353     Ni += dof;
7354   }
7355   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
7356   PetscCheck(!Nf || offsets[Nf] == Ni,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, offsets[Nf], Ni);
7357   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7358   for (f = 0; f < PetscMax(1, Nf); ++f) {
7359     if (Nf) PetscCall(PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]));
7360     else    PetscCall(PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]));
7361     /* may need to apply sign changes to the element matrix */
7362     if (values && flips[f]) {
7363       PetscInt foffset = offsets[f];
7364 
7365       for (p = 0; p < Ncl; ++p) {
7366         PetscInt           pnt  = points[2*p], fdof;
7367         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7368 
7369         if (!Nf) PetscCall(PetscSectionGetDof(section, pnt, &fdof));
7370         else     PetscCall(PetscSectionGetFieldDof(section, pnt, f, &fdof));
7371         if (flip) {
7372           PetscInt i, j, k;
7373 
7374           if (!valCopy) {
7375             PetscCall(DMGetWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy));
7376             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7377             *values = valCopy;
7378           }
7379           for (i = 0; i < fdof; ++i) {
7380             PetscScalar fval = flip[i];
7381 
7382             for (k = 0; k < Ni; ++k) {
7383               valCopy[Ni * (foffset + i) + k] *= fval;
7384               valCopy[Ni * k + (foffset + i)] *= fval;
7385             }
7386           }
7387         }
7388         foffset += fdof;
7389       }
7390     }
7391   }
7392   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7393   PetscCall(DMPlexAnchorsModifyMat(dm, section, Ncl, Ni, points, perms, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, PETSC_TRUE));
7394   if (NclC) {
7395     if (valCopy) PetscCall(DMRestoreWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy));
7396     for (f = 0; f < PetscMax(1, Nf); ++f) {
7397       if (Nf) PetscCall(PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]));
7398       else    PetscCall(PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]));
7399     }
7400     for (f = 0; f < PetscMax(1, Nf); ++f) {
7401       if (Nf) PetscCall(PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]));
7402       else    PetscCall(PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]));
7403     }
7404     PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp));
7405     Ncl     = NclC;
7406     Ni      = NiC;
7407     points  = pointsC;
7408     if (values) *values = valuesC;
7409   }
7410   /* 5) Calculate indices */
7411   PetscCall(DMGetWorkArray(dm, Ni, MPIU_INT, &idx));
7412   if (Nf) {
7413     PetscInt  idxOff;
7414     PetscBool useFieldOffsets;
7415 
7416     if (outOffsets) {for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];}
7417     PetscCall(PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets));
7418     if (useFieldOffsets) {
7419       for (p = 0; p < Ncl; ++p) {
7420         const PetscInt pnt = points[p*2];
7421 
7422         PetscCall(DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx));
7423       }
7424     } else {
7425       for (p = 0; p < Ncl; ++p) {
7426         const PetscInt pnt = points[p*2];
7427 
7428         PetscCall(PetscSectionGetOffset(idxSection, pnt, &idxOff));
7429         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7430          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
7431          * global section. */
7432         PetscCall(DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx));
7433       }
7434     }
7435   } else {
7436     PetscInt off = 0, idxOff;
7437 
7438     for (p = 0; p < Ncl; ++p) {
7439       const PetscInt  pnt  = points[p*2];
7440       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
7441 
7442       PetscCall(PetscSectionGetOffset(idxSection, pnt, &idxOff));
7443       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7444        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
7445       PetscCall(DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx));
7446     }
7447   }
7448   /* 6) Cleanup */
7449   for (f = 0; f < PetscMax(1, Nf); ++f) {
7450     if (Nf) PetscCall(PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]));
7451     else    PetscCall(PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]));
7452   }
7453   if (NclC) {
7454     PetscCall(DMRestoreWorkArray(dm, NclC*2, MPIU_INT, &pointsC));
7455   } else {
7456     PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp));
7457   }
7458 
7459   if (numIndices) *numIndices = Ni;
7460   if (indices)    *indices    = idx;
7461   PetscFunctionReturn(0);
7462 }
7463 
7464 /*@C
7465   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
7466 
7467   Not collective
7468 
7469   Input Parameters:
7470 + dm         - The DM
7471 . section    - The PetscSection describing the points (a local section)
7472 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7473 . point      - The point defining the closure
7474 - useClPerm  - Use the closure point permutation if available
7475 
7476   Output Parameters:
7477 + numIndices - The number of dof indices in the closure of point with the input sections
7478 . indices    - The dof indices
7479 . outOffsets - Array to write the field offsets into, or NULL
7480 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7481 
7482   Notes:
7483   If values were modified, the user is responsible for calling DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values).
7484 
7485   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7486   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7487   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7488   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7489   indices (with the above semantics) are implied.
7490 
7491   Level: advanced
7492 
7493 .seealso `DMPlexGetClosureIndices()`, `DMPlexVecGetClosure()`, `DMPlexMatSetClosure()`, `DMGetLocalSection()`, `DMGetGlobalSection()`
7494 @*/
7495 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7496                                            PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7497 {
7498   PetscFunctionBegin;
7499   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7500   PetscValidPointer(indices, 7);
7501   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_INT, indices));
7502   PetscFunctionReturn(0);
7503 }
7504 
7505 /*@C
7506   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7507 
7508   Not collective
7509 
7510   Input Parameters:
7511 + dm - The DM
7512 . section - The section describing the layout in v, or NULL to use the default section
7513 . globalSection - The section describing the layout in v, or NULL to use the default global section
7514 . A - The matrix
7515 . point - The point in the DM
7516 . values - The array of values
7517 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7518 
7519   Fortran Notes:
7520   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7521 
7522   Level: intermediate
7523 
7524 .seealso `DMPlexMatSetClosureGeneral()`, `DMPlexVecGetClosure()`, `DMPlexVecSetClosure()`
7525 @*/
7526 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7527 {
7528   DM_Plex           *mesh = (DM_Plex*) dm->data;
7529   PetscInt          *indices;
7530   PetscInt           numIndices;
7531   const PetscScalar *valuesOrig = values;
7532   PetscErrorCode     ierr;
7533 
7534   PetscFunctionBegin;
7535   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7536   if (!section) PetscCall(DMGetLocalSection(dm, &section));
7537   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7538   if (!globalSection) PetscCall(DMGetGlobalSection(dm, &globalSection));
7539   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7540   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7541 
7542   PetscCall(DMPlexGetClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values));
7543 
7544   if (mesh->printSetValues) PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values));
7545   /* TODO: fix this code to not use error codes as handle-able exceptions! */
7546   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7547   if (ierr) {
7548     PetscMPIInt    rank;
7549 
7550     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank));
7551     PetscCall((*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank));
7552     PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values));
7553     PetscCall(DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values));
7554     if (values != valuesOrig) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values));
7555     SETERRQ(PetscObjectComm((PetscObject)dm),ierr,"Not possible to set matrix values");
7556   }
7557   if (mesh->printFEM > 1) {
7558     PetscInt i;
7559     PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Indices:"));
7560     for (i = 0; i < numIndices; ++i) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, indices[i]));
7561     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
7562   }
7563 
7564   PetscCall(DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values));
7565   if (values != valuesOrig) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values));
7566   PetscFunctionReturn(0);
7567 }
7568 
7569 /*@C
7570   DMPlexMatSetClosure - Set an array of the values on the closure of 'point' using a different row and column section
7571 
7572   Not collective
7573 
7574   Input Parameters:
7575 + dmRow - The DM for the row fields
7576 . sectionRow - The section describing the layout, or NULL to use the default section in dmRow
7577 . globalSectionRow - The section describing the layout, or NULL to use the default global section in dmRow
7578 . dmCol - The DM for the column fields
7579 . sectionCol - The section describing the layout, or NULL to use the default section in dmCol
7580 . globalSectionCol - The section describing the layout, or NULL to use the default global section in dmCol
7581 . A - The matrix
7582 . point - The point in the DMs
7583 . values - The array of values
7584 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7585 
7586   Level: intermediate
7587 
7588 .seealso `DMPlexMatSetClosure()`, `DMPlexVecGetClosure()`, `DMPlexVecSetClosure()`
7589 @*/
7590 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7591 {
7592   DM_Plex           *mesh = (DM_Plex*) dmRow->data;
7593   PetscInt          *indicesRow, *indicesCol;
7594   PetscInt           numIndicesRow, numIndicesCol;
7595   const PetscScalar *valuesOrig = values;
7596   PetscErrorCode     ierr;
7597 
7598   PetscFunctionBegin;
7599   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
7600   if (!sectionRow) PetscCall(DMGetLocalSection(dmRow, &sectionRow));
7601   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
7602   if (!globalSectionRow) PetscCall(DMGetGlobalSection(dmRow, &globalSectionRow));
7603   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
7604   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 4);
7605   if (!sectionCol) PetscCall(DMGetLocalSection(dmCol, &sectionCol));
7606   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 5);
7607   if (!globalSectionCol) PetscCall(DMGetGlobalSection(dmCol, &globalSectionCol));
7608   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 6);
7609   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7610 
7611   PetscCall(DMPlexGetClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values));
7612   PetscCall(DMPlexGetClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values));
7613 
7614   if (mesh->printSetValues) PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values));
7615   /* TODO: fix this code to not use error codes as handle-able exceptions! */
7616   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values, mode);
7617   if (ierr) {
7618     PetscMPIInt    rank;
7619 
7620     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank));
7621     PetscCall((*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank));
7622     PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values));
7623     PetscCall(DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values));
7624     PetscCall(DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **) &values));
7625     if (values != valuesOrig) PetscCall(DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values));
7626   }
7627 
7628   PetscCall(DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values));
7629   PetscCall(DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values));
7630   if (values != valuesOrig) PetscCall(DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values));
7631   PetscFunctionReturn(0);
7632 }
7633 
7634 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7635 {
7636   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
7637   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
7638   PetscInt       *cpoints = NULL;
7639   PetscInt       *findices, *cindices;
7640   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7641   PetscInt        foffsets[32], coffsets[32];
7642   DMPolytopeType  ct;
7643   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7644   PetscErrorCode  ierr;
7645 
7646   PetscFunctionBegin;
7647   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7648   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7649   if (!fsection) PetscCall(DMGetLocalSection(dmf, &fsection));
7650   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7651   if (!csection) PetscCall(DMGetLocalSection(dmc, &csection));
7652   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7653   if (!globalFSection) PetscCall(DMGetGlobalSection(dmf, &globalFSection));
7654   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7655   if (!globalCSection) PetscCall(DMGetGlobalSection(dmc, &globalCSection));
7656   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7657   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7658   PetscCall(PetscSectionGetNumFields(fsection, &numFields));
7659   PetscCheck(numFields <= 31,PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %" PetscInt_FMT " limited to 31", numFields);
7660   PetscCall(PetscArrayzero(foffsets, 32));
7661   PetscCall(PetscArrayzero(coffsets, 32));
7662   /* Column indices */
7663   PetscCall(DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints));
7664   maxFPoints = numCPoints;
7665   /* Compress out points not in the section */
7666   /*   TODO: Squeeze out points with 0 dof as well */
7667   PetscCall(PetscSectionGetChart(csection, &pStart, &pEnd));
7668   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7669     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7670       cpoints[q*2]   = cpoints[p];
7671       cpoints[q*2+1] = cpoints[p+1];
7672       ++q;
7673     }
7674   }
7675   numCPoints = q;
7676   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7677     PetscInt fdof;
7678 
7679     PetscCall(PetscSectionGetDof(csection, cpoints[p], &dof));
7680     if (!dof) continue;
7681     for (f = 0; f < numFields; ++f) {
7682       PetscCall(PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof));
7683       coffsets[f+1] += fdof;
7684     }
7685     numCIndices += dof;
7686   }
7687   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7688   /* Row indices */
7689   PetscCall(DMPlexGetCellType(dmc, point, &ct));
7690   {
7691     DMPlexTransform tr;
7692     DMPolytopeType *rct;
7693     PetscInt       *rsize, *rcone, *rornt, Nt;
7694 
7695     PetscCall(DMPlexTransformCreate(PETSC_COMM_SELF, &tr));
7696     PetscCall(DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR));
7697     PetscCall(DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt));
7698     numSubcells = rsize[Nt-1];
7699     PetscCall(DMPlexTransformDestroy(&tr));
7700   }
7701   PetscCall(DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints));
7702   for (r = 0, q = 0; r < numSubcells; ++r) {
7703     /* TODO Map from coarse to fine cells */
7704     PetscCall(DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints));
7705     /* Compress out points not in the section */
7706     PetscCall(PetscSectionGetChart(fsection, &pStart, &pEnd));
7707     for (p = 0; p < numFPoints*2; p += 2) {
7708       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7709         PetscCall(PetscSectionGetDof(fsection, fpoints[p], &dof));
7710         if (!dof) continue;
7711         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7712         if (s < q) continue;
7713         ftotpoints[q*2]   = fpoints[p];
7714         ftotpoints[q*2+1] = fpoints[p+1];
7715         ++q;
7716       }
7717     }
7718     PetscCall(DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints));
7719   }
7720   numFPoints = q;
7721   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7722     PetscInt fdof;
7723 
7724     PetscCall(PetscSectionGetDof(fsection, ftotpoints[p], &dof));
7725     if (!dof) continue;
7726     for (f = 0; f < numFields; ++f) {
7727       PetscCall(PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof));
7728       foffsets[f+1] += fdof;
7729     }
7730     numFIndices += dof;
7731   }
7732   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7733 
7734   PetscCheck(!numFields || foffsets[numFields] == numFIndices,PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, foffsets[numFields], numFIndices);
7735   PetscCheck(!numFields || coffsets[numFields] == numCIndices,PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, coffsets[numFields], numCIndices);
7736   PetscCall(DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices));
7737   PetscCall(DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices));
7738   if (numFields) {
7739     const PetscInt **permsF[32] = {NULL};
7740     const PetscInt **permsC[32] = {NULL};
7741 
7742     for (f = 0; f < numFields; f++) {
7743       PetscCall(PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL));
7744       PetscCall(PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL));
7745     }
7746     for (p = 0; p < numFPoints; p++) {
7747       PetscCall(PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff));
7748       PetscCall(DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices));
7749     }
7750     for (p = 0; p < numCPoints; p++) {
7751       PetscCall(PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff));
7752       PetscCall(DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices));
7753     }
7754     for (f = 0; f < numFields; f++) {
7755       PetscCall(PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL));
7756       PetscCall(PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL));
7757     }
7758   } else {
7759     const PetscInt **permsF = NULL;
7760     const PetscInt **permsC = NULL;
7761 
7762     PetscCall(PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL));
7763     PetscCall(PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL));
7764     for (p = 0, off = 0; p < numFPoints; p++) {
7765       const PetscInt *perm = permsF ? permsF[p] : NULL;
7766 
7767       PetscCall(PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff));
7768       PetscCall(DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices));
7769     }
7770     for (p = 0, off = 0; p < numCPoints; p++) {
7771       const PetscInt *perm = permsC ? permsC[p] : NULL;
7772 
7773       PetscCall(PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff));
7774       PetscCall(DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices));
7775     }
7776     PetscCall(PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL));
7777     PetscCall(PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL));
7778   }
7779   if (mesh->printSetValues) PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values));
7780   /* TODO: flips */
7781   /* TODO: fix this code to not use error codes as handle-able exceptions! */
7782   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
7783   if (ierr) {
7784     PetscMPIInt    rank;
7785 
7786     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank));
7787     PetscCall((*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank));
7788     PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values));
7789     PetscCall(DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices));
7790     PetscCall(DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices));
7791   }
7792   PetscCall(DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints));
7793   PetscCall(DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints));
7794   PetscCall(DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices));
7795   PetscCall(DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices));
7796   PetscFunctionReturn(0);
7797 }
7798 
7799 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
7800 {
7801   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
7802   PetscInt      *cpoints = NULL;
7803   PetscInt       foffsets[32], coffsets[32];
7804   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7805   DMPolytopeType ct;
7806   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7807 
7808   PetscFunctionBegin;
7809   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7810   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7811   if (!fsection) PetscCall(DMGetLocalSection(dmf, &fsection));
7812   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7813   if (!csection) PetscCall(DMGetLocalSection(dmc, &csection));
7814   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7815   if (!globalFSection) PetscCall(DMGetGlobalSection(dmf, &globalFSection));
7816   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7817   if (!globalCSection) PetscCall(DMGetGlobalSection(dmc, &globalCSection));
7818   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7819   PetscCall(PetscSectionGetNumFields(fsection, &numFields));
7820   PetscCheck(numFields <= 31,PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %" PetscInt_FMT " limited to 31", numFields);
7821   PetscCall(PetscArrayzero(foffsets, 32));
7822   PetscCall(PetscArrayzero(coffsets, 32));
7823   /* Column indices */
7824   PetscCall(DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints));
7825   maxFPoints = numCPoints;
7826   /* Compress out points not in the section */
7827   /*   TODO: Squeeze out points with 0 dof as well */
7828   PetscCall(PetscSectionGetChart(csection, &pStart, &pEnd));
7829   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7830     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7831       cpoints[q*2]   = cpoints[p];
7832       cpoints[q*2+1] = cpoints[p+1];
7833       ++q;
7834     }
7835   }
7836   numCPoints = q;
7837   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7838     PetscInt fdof;
7839 
7840     PetscCall(PetscSectionGetDof(csection, cpoints[p], &dof));
7841     if (!dof) continue;
7842     for (f = 0; f < numFields; ++f) {
7843       PetscCall(PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof));
7844       coffsets[f+1] += fdof;
7845     }
7846     numCIndices += dof;
7847   }
7848   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7849   /* Row indices */
7850   PetscCall(DMPlexGetCellType(dmc, point, &ct));
7851   {
7852     DMPlexTransform tr;
7853     DMPolytopeType *rct;
7854     PetscInt       *rsize, *rcone, *rornt, Nt;
7855 
7856     PetscCall(DMPlexTransformCreate(PETSC_COMM_SELF, &tr));
7857     PetscCall(DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR));
7858     PetscCall(DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt));
7859     numSubcells = rsize[Nt-1];
7860     PetscCall(DMPlexTransformDestroy(&tr));
7861   }
7862   PetscCall(DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints));
7863   for (r = 0, q = 0; r < numSubcells; ++r) {
7864     /* TODO Map from coarse to fine cells */
7865     PetscCall(DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints));
7866     /* Compress out points not in the section */
7867     PetscCall(PetscSectionGetChart(fsection, &pStart, &pEnd));
7868     for (p = 0; p < numFPoints*2; p += 2) {
7869       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7870         PetscCall(PetscSectionGetDof(fsection, fpoints[p], &dof));
7871         if (!dof) continue;
7872         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7873         if (s < q) continue;
7874         ftotpoints[q*2]   = fpoints[p];
7875         ftotpoints[q*2+1] = fpoints[p+1];
7876         ++q;
7877       }
7878     }
7879     PetscCall(DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints));
7880   }
7881   numFPoints = q;
7882   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7883     PetscInt fdof;
7884 
7885     PetscCall(PetscSectionGetDof(fsection, ftotpoints[p], &dof));
7886     if (!dof) continue;
7887     for (f = 0; f < numFields; ++f) {
7888       PetscCall(PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof));
7889       foffsets[f+1] += fdof;
7890     }
7891     numFIndices += dof;
7892   }
7893   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7894 
7895   PetscCheck(!numFields || foffsets[numFields] == numFIndices,PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, foffsets[numFields], numFIndices);
7896   PetscCheck(!numFields || coffsets[numFields] == numCIndices,PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, coffsets[numFields], numCIndices);
7897   if (numFields) {
7898     const PetscInt **permsF[32] = {NULL};
7899     const PetscInt **permsC[32] = {NULL};
7900 
7901     for (f = 0; f < numFields; f++) {
7902       PetscCall(PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL));
7903       PetscCall(PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL));
7904     }
7905     for (p = 0; p < numFPoints; p++) {
7906       PetscCall(PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff));
7907       PetscCall(DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices));
7908     }
7909     for (p = 0; p < numCPoints; p++) {
7910       PetscCall(PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff));
7911       PetscCall(DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices));
7912     }
7913     for (f = 0; f < numFields; f++) {
7914       PetscCall(PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL));
7915       PetscCall(PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL));
7916     }
7917   } else {
7918     const PetscInt **permsF = NULL;
7919     const PetscInt **permsC = NULL;
7920 
7921     PetscCall(PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL));
7922     PetscCall(PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL));
7923     for (p = 0, off = 0; p < numFPoints; p++) {
7924       const PetscInt *perm = permsF ? permsF[p] : NULL;
7925 
7926       PetscCall(PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff));
7927       PetscCall(DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices));
7928     }
7929     for (p = 0, off = 0; p < numCPoints; p++) {
7930       const PetscInt *perm = permsC ? permsC[p] : NULL;
7931 
7932       PetscCall(PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff));
7933       PetscCall(DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices));
7934     }
7935     PetscCall(PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL));
7936     PetscCall(PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL));
7937   }
7938   PetscCall(DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints));
7939   PetscCall(DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints));
7940   PetscFunctionReturn(0);
7941 }
7942 
7943 /*@C
7944   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
7945 
7946   Input Parameter:
7947 . dm   - The DMPlex object
7948 
7949   Output Parameter:
7950 . cellHeight - The height of a cell
7951 
7952   Level: developer
7953 
7954 .seealso `DMPlexSetVTKCellHeight()`
7955 @*/
7956 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7957 {
7958   DM_Plex *mesh = (DM_Plex*) dm->data;
7959 
7960   PetscFunctionBegin;
7961   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7962   PetscValidIntPointer(cellHeight, 2);
7963   *cellHeight = mesh->vtkCellHeight;
7964   PetscFunctionReturn(0);
7965 }
7966 
7967 /*@C
7968   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
7969 
7970   Input Parameters:
7971 + dm   - The DMPlex object
7972 - cellHeight - The height of a cell
7973 
7974   Level: developer
7975 
7976 .seealso `DMPlexGetVTKCellHeight()`
7977 @*/
7978 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7979 {
7980   DM_Plex *mesh = (DM_Plex*) dm->data;
7981 
7982   PetscFunctionBegin;
7983   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7984   mesh->vtkCellHeight = cellHeight;
7985   PetscFunctionReturn(0);
7986 }
7987 
7988 /*@
7989   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
7990 
7991   Input Parameter:
7992 . dm - The DMPlex object
7993 
7994   Output Parameters:
7995 + gcStart - The first ghost cell, or NULL
7996 - gcEnd   - The upper bound on ghost cells, or NULL
7997 
7998   Level: advanced
7999 
8000 .seealso `DMPlexConstructGhostCells()`, `DMPlexGetGhostCellStratum()`
8001 @*/
8002 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
8003 {
8004   DMLabel        ctLabel;
8005 
8006   PetscFunctionBegin;
8007   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8008   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
8009   PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd));
8010   PetscFunctionReturn(0);
8011 }
8012 
8013 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
8014 {
8015   PetscSection   section, globalSection;
8016   PetscInt      *numbers, p;
8017 
8018   PetscFunctionBegin;
8019   if (PetscDefined(USE_DEBUG)) PetscCall(DMPlexCheckPointSF(dm, sf));
8020   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section));
8021   PetscCall(PetscSectionSetChart(section, pStart, pEnd));
8022   for (p = pStart; p < pEnd; ++p) {
8023     PetscCall(PetscSectionSetDof(section, p, 1));
8024   }
8025   PetscCall(PetscSectionSetUp(section));
8026   PetscCall(PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection));
8027   PetscCall(PetscMalloc1(pEnd - pStart, &numbers));
8028   for (p = pStart; p < pEnd; ++p) {
8029     PetscCall(PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]));
8030     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
8031     else                       numbers[p-pStart] += shift;
8032   }
8033   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering));
8034   if (globalSize) {
8035     PetscLayout layout;
8036     PetscCall(PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout));
8037     PetscCall(PetscLayoutGetSize(layout, globalSize));
8038     PetscCall(PetscLayoutDestroy(&layout));
8039   }
8040   PetscCall(PetscSectionDestroy(&section));
8041   PetscCall(PetscSectionDestroy(&globalSection));
8042   PetscFunctionReturn(0);
8043 }
8044 
8045 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
8046 {
8047   PetscInt       cellHeight, cStart, cEnd;
8048 
8049   PetscFunctionBegin;
8050   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
8051   if (includeHybrid) PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
8052   else               PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
8053   PetscCall(DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers));
8054   PetscFunctionReturn(0);
8055 }
8056 
8057 /*@
8058   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
8059 
8060   Input Parameter:
8061 . dm   - The DMPlex object
8062 
8063   Output Parameter:
8064 . globalCellNumbers - Global cell numbers for all cells on this process
8065 
8066   Level: developer
8067 
8068 .seealso `DMPlexGetVertexNumbering()`
8069 @*/
8070 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
8071 {
8072   DM_Plex       *mesh = (DM_Plex*) dm->data;
8073 
8074   PetscFunctionBegin;
8075   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8076   if (!mesh->globalCellNumbers) PetscCall(DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers));
8077   *globalCellNumbers = mesh->globalCellNumbers;
8078   PetscFunctionReturn(0);
8079 }
8080 
8081 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
8082 {
8083   PetscInt       vStart, vEnd;
8084 
8085   PetscFunctionBegin;
8086   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8087   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
8088   PetscCall(DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers));
8089   PetscFunctionReturn(0);
8090 }
8091 
8092 /*@
8093   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
8094 
8095   Input Parameter:
8096 . dm   - The DMPlex object
8097 
8098   Output Parameter:
8099 . globalVertexNumbers - Global vertex numbers for all vertices on this process
8100 
8101   Level: developer
8102 
8103 .seealso `DMPlexGetCellNumbering()`
8104 @*/
8105 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
8106 {
8107   DM_Plex       *mesh = (DM_Plex*) dm->data;
8108 
8109   PetscFunctionBegin;
8110   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8111   if (!mesh->globalVertexNumbers) PetscCall(DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers));
8112   *globalVertexNumbers = mesh->globalVertexNumbers;
8113   PetscFunctionReturn(0);
8114 }
8115 
8116 /*@
8117   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
8118 
8119   Input Parameter:
8120 . dm   - The DMPlex object
8121 
8122   Output Parameter:
8123 . globalPointNumbers - Global numbers for all points on this process
8124 
8125   Level: developer
8126 
8127 .seealso `DMPlexGetCellNumbering()`
8128 @*/
8129 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
8130 {
8131   IS             nums[4];
8132   PetscInt       depths[4], gdepths[4], starts[4];
8133   PetscInt       depth, d, shift = 0;
8134 
8135   PetscFunctionBegin;
8136   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8137   PetscCall(DMPlexGetDepth(dm, &depth));
8138   /* For unstratified meshes use dim instead of depth */
8139   if (depth < 0) PetscCall(DMGetDimension(dm, &depth));
8140   for (d = 0; d <= depth; ++d) {
8141     PetscInt end;
8142 
8143     depths[d] = depth-d;
8144     PetscCall(DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end));
8145     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
8146   }
8147   PetscCall(PetscSortIntWithArray(depth+1, starts, depths));
8148   PetscCall(MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm)));
8149   for (d = 0; d <= depth; ++d) {
8150     PetscCheck(starts[d] < 0 || depths[d] == gdepths[d],PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %" PetscInt_FMT ", found %" PetscInt_FMT,depths[d],gdepths[d]);
8151   }
8152   for (d = 0; d <= depth; ++d) {
8153     PetscInt pStart, pEnd, gsize;
8154 
8155     PetscCall(DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd));
8156     PetscCall(DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]));
8157     shift += gsize;
8158   }
8159   PetscCall(ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers));
8160   for (d = 0; d <= depth; ++d) PetscCall(ISDestroy(&nums[d]));
8161   PetscFunctionReturn(0);
8162 }
8163 
8164 /*@
8165   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
8166 
8167   Input Parameter:
8168 . dm - The DMPlex object
8169 
8170   Output Parameter:
8171 . ranks - The rank field
8172 
8173   Options Database Keys:
8174 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
8175 
8176   Level: intermediate
8177 
8178 .seealso: `DMView()`
8179 @*/
8180 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
8181 {
8182   DM             rdm;
8183   PetscFE        fe;
8184   PetscScalar   *r;
8185   PetscMPIInt    rank;
8186   DMPolytopeType ct;
8187   PetscInt       dim, cStart, cEnd, c;
8188   PetscBool      simplex;
8189 
8190   PetscFunctionBeginUser;
8191   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8192   PetscValidPointer(ranks, 2);
8193   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
8194   PetscCall(DMClone(dm, &rdm));
8195   PetscCall(DMGetDimension(rdm, &dim));
8196   PetscCall(DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd));
8197   PetscCall(DMPlexGetCellType(dm, cStart, &ct));
8198   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
8199   PetscCall(PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe));
8200   PetscCall(PetscObjectSetName((PetscObject) fe, "rank"));
8201   PetscCall(DMSetField(rdm, 0, NULL, (PetscObject) fe));
8202   PetscCall(PetscFEDestroy(&fe));
8203   PetscCall(DMCreateDS(rdm));
8204   PetscCall(DMCreateGlobalVector(rdm, ranks));
8205   PetscCall(PetscObjectSetName((PetscObject) *ranks, "partition"));
8206   PetscCall(VecGetArray(*ranks, &r));
8207   for (c = cStart; c < cEnd; ++c) {
8208     PetscScalar *lr;
8209 
8210     PetscCall(DMPlexPointGlobalRef(rdm, c, r, &lr));
8211     if (lr) *lr = rank;
8212   }
8213   PetscCall(VecRestoreArray(*ranks, &r));
8214   PetscCall(DMDestroy(&rdm));
8215   PetscFunctionReturn(0);
8216 }
8217 
8218 /*@
8219   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
8220 
8221   Input Parameters:
8222 + dm    - The DMPlex
8223 - label - The DMLabel
8224 
8225   Output Parameter:
8226 . val - The label value field
8227 
8228   Options Database Keys:
8229 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
8230 
8231   Level: intermediate
8232 
8233 .seealso: `DMView()`
8234 @*/
8235 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
8236 {
8237   DM             rdm;
8238   PetscFE        fe;
8239   PetscScalar   *v;
8240   PetscInt       dim, cStart, cEnd, c;
8241 
8242   PetscFunctionBeginUser;
8243   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8244   PetscValidPointer(label, 2);
8245   PetscValidPointer(val, 3);
8246   PetscCall(DMClone(dm, &rdm));
8247   PetscCall(DMGetDimension(rdm, &dim));
8248   PetscCall(PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe));
8249   PetscCall(PetscObjectSetName((PetscObject) fe, "label_value"));
8250   PetscCall(DMSetField(rdm, 0, NULL, (PetscObject) fe));
8251   PetscCall(PetscFEDestroy(&fe));
8252   PetscCall(DMCreateDS(rdm));
8253   PetscCall(DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd));
8254   PetscCall(DMCreateGlobalVector(rdm, val));
8255   PetscCall(PetscObjectSetName((PetscObject) *val, "label_value"));
8256   PetscCall(VecGetArray(*val, &v));
8257   for (c = cStart; c < cEnd; ++c) {
8258     PetscScalar *lv;
8259     PetscInt     cval;
8260 
8261     PetscCall(DMPlexPointGlobalRef(rdm, c, v, &lv));
8262     PetscCall(DMLabelGetValue(label, c, &cval));
8263     *lv = cval;
8264   }
8265   PetscCall(VecRestoreArray(*val, &v));
8266   PetscCall(DMDestroy(&rdm));
8267   PetscFunctionReturn(0);
8268 }
8269 
8270 /*@
8271   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
8272 
8273   Input Parameter:
8274 . dm - The DMPlex object
8275 
8276   Notes:
8277   This is a useful diagnostic when creating meshes programmatically.
8278 
8279   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8280 
8281   Level: developer
8282 
8283 .seealso: `DMCreate()`, `DMSetFromOptions()`
8284 @*/
8285 PetscErrorCode DMPlexCheckSymmetry(DM dm)
8286 {
8287   PetscSection    coneSection, supportSection;
8288   const PetscInt *cone, *support;
8289   PetscInt        coneSize, c, supportSize, s;
8290   PetscInt        pStart, pEnd, p, pp, csize, ssize;
8291   PetscBool       storagecheck = PETSC_TRUE;
8292 
8293   PetscFunctionBegin;
8294   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8295   PetscCall(DMViewFromOptions(dm, NULL, "-sym_dm_view"));
8296   PetscCall(DMPlexGetConeSection(dm, &coneSection));
8297   PetscCall(DMPlexGetSupportSection(dm, &supportSection));
8298   /* Check that point p is found in the support of its cone points, and vice versa */
8299   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
8300   for (p = pStart; p < pEnd; ++p) {
8301     PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
8302     PetscCall(DMPlexGetCone(dm, p, &cone));
8303     for (c = 0; c < coneSize; ++c) {
8304       PetscBool dup = PETSC_FALSE;
8305       PetscInt  d;
8306       for (d = c-1; d >= 0; --d) {
8307         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
8308       }
8309       PetscCall(DMPlexGetSupportSize(dm, cone[c], &supportSize));
8310       PetscCall(DMPlexGetSupport(dm, cone[c], &support));
8311       for (s = 0; s < supportSize; ++s) {
8312         if (support[s] == p) break;
8313       }
8314       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
8315         PetscCall(PetscPrintf(PETSC_COMM_SELF, "p: %" PetscInt_FMT " cone: ", p));
8316         for (s = 0; s < coneSize; ++s) {
8317           PetscCall(PetscPrintf(PETSC_COMM_SELF, "%" PetscInt_FMT ", ", cone[s]));
8318         }
8319         PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
8320         PetscCall(PetscPrintf(PETSC_COMM_SELF, "p: %" PetscInt_FMT " support: ", cone[c]));
8321         for (s = 0; s < supportSize; ++s) {
8322           PetscCall(PetscPrintf(PETSC_COMM_SELF, "%" PetscInt_FMT ", ", support[s]));
8323         }
8324         PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
8325         PetscCheck(!dup,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " not repeatedly found in support of repeated cone point %" PetscInt_FMT, p, cone[c]);
8326         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " not found in support of cone point %" PetscInt_FMT, p, cone[c]);
8327       }
8328     }
8329     PetscCall(DMPlexGetTreeParent(dm, p, &pp, NULL));
8330     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
8331     PetscCall(DMPlexGetSupportSize(dm, p, &supportSize));
8332     PetscCall(DMPlexGetSupport(dm, p, &support));
8333     for (s = 0; s < supportSize; ++s) {
8334       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
8335       PetscCall(DMPlexGetCone(dm, support[s], &cone));
8336       for (c = 0; c < coneSize; ++c) {
8337         PetscCall(DMPlexGetTreeParent(dm, cone[c], &pp, NULL));
8338         if (cone[c] != pp) { c = 0; break; }
8339         if (cone[c] == p) break;
8340       }
8341       if (c >= coneSize) {
8342         PetscCall(PetscPrintf(PETSC_COMM_SELF, "p: %" PetscInt_FMT " support: ", p));
8343         for (c = 0; c < supportSize; ++c) {
8344           PetscCall(PetscPrintf(PETSC_COMM_SELF, "%" PetscInt_FMT ", ", support[c]));
8345         }
8346         PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
8347         PetscCall(PetscPrintf(PETSC_COMM_SELF, "p: %" PetscInt_FMT " cone: ", support[s]));
8348         for (c = 0; c < coneSize; ++c) {
8349           PetscCall(PetscPrintf(PETSC_COMM_SELF, "%" PetscInt_FMT ", ", cone[c]));
8350         }
8351         PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
8352         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " not found in cone of support point %" PetscInt_FMT, p, support[s]);
8353       }
8354     }
8355   }
8356   if (storagecheck) {
8357     PetscCall(PetscSectionGetStorageSize(coneSection, &csize));
8358     PetscCall(PetscSectionGetStorageSize(supportSection, &ssize));
8359     PetscCheck(csize == ssize,PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %" PetscInt_FMT " != Total support size %" PetscInt_FMT, csize, ssize);
8360   }
8361   PetscFunctionReturn(0);
8362 }
8363 
8364 /*
8365   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
8366 */
8367 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
8368 {
8369   DMPolytopeType  cct;
8370   PetscInt        ptpoints[4];
8371   const PetscInt *cone, *ccone, *ptcone;
8372   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
8373 
8374   PetscFunctionBegin;
8375   *unsplit = 0;
8376   switch (ct) {
8377     case DM_POLYTOPE_POINT_PRISM_TENSOR:
8378       ptpoints[npt++] = c;
8379       break;
8380     case DM_POLYTOPE_SEG_PRISM_TENSOR:
8381       PetscCall(DMPlexGetCone(dm, c, &cone));
8382       PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
8383       for (cp = 0; cp < coneSize; ++cp) {
8384         PetscCall(DMPlexGetCellType(dm, cone[cp], &cct));
8385         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
8386       }
8387       break;
8388     case DM_POLYTOPE_TRI_PRISM_TENSOR:
8389     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8390       PetscCall(DMPlexGetCone(dm, c, &cone));
8391       PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
8392       for (cp = 0; cp < coneSize; ++cp) {
8393         PetscCall(DMPlexGetCone(dm, cone[cp], &ccone));
8394         PetscCall(DMPlexGetConeSize(dm, cone[cp], &cconeSize));
8395         for (ccp = 0; ccp < cconeSize; ++ccp) {
8396           PetscCall(DMPlexGetCellType(dm, ccone[ccp], &cct));
8397           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
8398             PetscInt p;
8399             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
8400             if (p == npt) ptpoints[npt++] = ccone[ccp];
8401           }
8402         }
8403       }
8404       break;
8405     default: break;
8406   }
8407   for (pt = 0; pt < npt; ++pt) {
8408     PetscCall(DMPlexGetCone(dm, ptpoints[pt], &ptcone));
8409     if (ptcone[0] == ptcone[1]) ++(*unsplit);
8410   }
8411   PetscFunctionReturn(0);
8412 }
8413 
8414 /*@
8415   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
8416 
8417   Input Parameters:
8418 + dm - The DMPlex object
8419 - cellHeight - Normally 0
8420 
8421   Notes:
8422   This is a useful diagnostic when creating meshes programmatically.
8423   Currently applicable only to homogeneous simplex or tensor meshes.
8424 
8425   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8426 
8427   Level: developer
8428 
8429 .seealso: `DMCreate()`, `DMSetFromOptions()`
8430 @*/
8431 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
8432 {
8433   DMPlexInterpolatedFlag interp;
8434   DMPolytopeType         ct;
8435   PetscInt               vStart, vEnd, cStart, cEnd, c;
8436 
8437   PetscFunctionBegin;
8438   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8439   PetscCall(DMPlexIsInterpolated(dm, &interp));
8440   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
8441   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
8442   for (c = cStart; c < cEnd; ++c) {
8443     PetscInt *closure = NULL;
8444     PetscInt  coneSize, closureSize, cl, Nv = 0;
8445 
8446     PetscCall(DMPlexGetCellType(dm, c, &ct));
8447     PetscCheck((PetscInt) ct >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " has no cell type", c);
8448     if (ct == DM_POLYTOPE_UNKNOWN) continue;
8449     if (interp == DMPLEX_INTERPOLATED_FULL) {
8450       PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
8451       PetscCheck(coneSize == DMPolytopeTypeGetConeSize(ct),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " of type %s has cone size %" PetscInt_FMT " != %" PetscInt_FMT, c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
8452     }
8453     PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
8454     for (cl = 0; cl < closureSize*2; cl += 2) {
8455       const PetscInt p = closure[cl];
8456       if ((p >= vStart) && (p < vEnd)) ++Nv;
8457     }
8458     PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
8459     /* Special Case: Tensor faces with identified vertices */
8460     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
8461       PetscInt unsplit;
8462 
8463       PetscCall(DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit));
8464       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
8465     }
8466     PetscCheck(Nv == DMPolytopeTypeGetNumVertices(ct),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " of type %s has %" PetscInt_FMT " vertices != %" PetscInt_FMT, c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
8467   }
8468   PetscFunctionReturn(0);
8469 }
8470 
8471 /*@
8472   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
8473 
8474   Collective
8475 
8476   Input Parameters:
8477 + dm - The DMPlex object
8478 - cellHeight - Normally 0
8479 
8480   Notes:
8481   This is a useful diagnostic when creating meshes programmatically.
8482   This routine is only relevant for meshes that are fully interpolated across all ranks.
8483   It will error out if a partially interpolated mesh is given on some rank.
8484   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
8485 
8486   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8487 
8488   Level: developer
8489 
8490 .seealso: `DMCreate()`, `DMPlexGetVTKCellHeight()`, `DMSetFromOptions()`
8491 @*/
8492 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
8493 {
8494   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
8495   DMPlexInterpolatedFlag interpEnum;
8496 
8497   PetscFunctionBegin;
8498   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8499   PetscCall(DMPlexIsInterpolatedCollective(dm, &interpEnum));
8500   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
8501   if (interpEnum != DMPLEX_INTERPOLATED_FULL) {
8502     PetscPrintf(PetscObjectComm((PetscObject)dm), "DMPlexCheckFaces() warning: Mesh is only partially interpolated, this is currently not supported");
8503     PetscFunctionReturn(0);
8504   }
8505 
8506   PetscCall(DMGetDimension(dm, &dim));
8507   PetscCall(DMPlexGetDepth(dm, &depth));
8508   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
8509   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
8510     PetscCall(DMPlexGetHeightStratum(dm, h, &cStart, &cEnd));
8511     for (c = cStart; c < cEnd; ++c) {
8512       const PetscInt      *cone, *ornt, *faceSizes, *faces;
8513       const DMPolytopeType *faceTypes;
8514       DMPolytopeType        ct;
8515       PetscInt              numFaces, coneSize, f;
8516       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
8517 
8518       PetscCall(DMPlexGetCellType(dm, c, &ct));
8519       PetscCall(DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit));
8520       if (unsplit) continue;
8521       PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
8522       PetscCall(DMPlexGetCone(dm, c, &cone));
8523       PetscCall(DMPlexGetConeOrientation(dm, c, &ornt));
8524       PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
8525       for (cl = 0; cl < closureSize*2; cl += 2) {
8526         const PetscInt p = closure[cl];
8527         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
8528       }
8529       PetscCall(DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces));
8530       PetscCheck(coneSize == numFaces,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " of type %s has %" PetscInt_FMT " faces but should have %" PetscInt_FMT, c, DMPolytopeTypes[ct], coneSize, numFaces);
8531       for (f = 0; f < numFaces; ++f) {
8532         DMPolytopeType fct;
8533         PetscInt       *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
8534 
8535         PetscCall(DMPlexGetCellType(dm, cone[f], &fct));
8536         PetscCall(DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure));
8537         for (cl = 0; cl < fclosureSize*2; cl += 2) {
8538           const PetscInt p = fclosure[cl];
8539           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
8540         }
8541         PetscCheck(fnumCorners == faceSizes[f],PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " of type %s (cone idx %" PetscInt_FMT ") of cell %" PetscInt_FMT " of type %s has %" PetscInt_FMT " vertices but should have %" PetscInt_FMT, cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
8542         for (v = 0; v < fnumCorners; ++v) {
8543           if (fclosure[v] != faces[fOff+v]) {
8544             PetscInt v1;
8545 
8546             PetscCall(PetscPrintf(PETSC_COMM_SELF, "face closure:"));
8547             for (v1 = 0; v1 < fnumCorners; ++v1) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, fclosure[v1]));
8548             PetscCall(PetscPrintf(PETSC_COMM_SELF, "\ncell face:"));
8549             for (v1 = 0; v1 < fnumCorners; ++v1) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, faces[fOff+v1]));
8550             PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
8551             SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " of type %s (cone idx %" PetscInt_FMT ", ornt %" PetscInt_FMT ") of cell %" PetscInt_FMT " of type %s vertex %" PetscInt_FMT ", %" PetscInt_FMT " != %" PetscInt_FMT, cone[f], DMPolytopeTypes[fct], f, ornt[f], c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff+v]);
8552           }
8553         }
8554         PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure));
8555         fOff += faceSizes[f];
8556       }
8557       PetscCall(DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces));
8558       PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
8559     }
8560   }
8561   PetscFunctionReturn(0);
8562 }
8563 
8564 /*@
8565   DMPlexCheckGeometry - Check the geometry of mesh cells
8566 
8567   Input Parameter:
8568 . dm - The DMPlex object
8569 
8570   Notes:
8571   This is a useful diagnostic when creating meshes programmatically.
8572 
8573   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8574 
8575   Level: developer
8576 
8577 .seealso: `DMCreate()`, `DMSetFromOptions()`
8578 @*/
8579 PetscErrorCode DMPlexCheckGeometry(DM dm)
8580 {
8581   Vec            coordinates;
8582   PetscReal      detJ, J[9], refVol = 1.0;
8583   PetscReal      vol;
8584   PetscBool      periodic;
8585   PetscInt       dim, depth, dE, d, cStart, cEnd, c;
8586 
8587   PetscFunctionBegin;
8588   PetscCall(DMGetDimension(dm, &dim));
8589   PetscCall(DMGetCoordinateDim(dm, &dE));
8590   if (dim != dE) PetscFunctionReturn(0);
8591   PetscCall(DMPlexGetDepth(dm, &depth));
8592   PetscCall(DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL));
8593   for (d = 0; d < dim; ++d) refVol *= 2.0;
8594   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
8595   /* Make sure local coordinates are created, because that step is collective */
8596   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
8597   for (c = cStart; c < cEnd; ++c) {
8598     DMPolytopeType ct;
8599     PetscInt       unsplit;
8600     PetscBool      ignoreZeroVol = PETSC_FALSE;
8601 
8602     PetscCall(DMPlexGetCellType(dm, c, &ct));
8603     switch (ct) {
8604       case DM_POLYTOPE_SEG_PRISM_TENSOR:
8605       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8606       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8607         ignoreZeroVol = PETSC_TRUE; break;
8608       default: break;
8609     }
8610     switch (ct) {
8611       case DM_POLYTOPE_TRI_PRISM:
8612       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8613       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8614       case DM_POLYTOPE_PYRAMID:
8615         continue;
8616       default: break;
8617     }
8618     PetscCall(DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit));
8619     if (unsplit) continue;
8620     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ));
8621     PetscCheck(detJ >= -PETSC_SMALL && (detJ > 0.0 || ignoreZeroVol),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %" PetscInt_FMT " of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double) detJ);
8622     PetscCall(PetscInfo(dm, "Cell %" PetscInt_FMT " FEM Volume %g\n", c, (double)(detJ*refVol)));
8623     if (depth > 1 && !periodic) {
8624       PetscCall(DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL));
8625       PetscCheck(vol >= -PETSC_SMALL && (vol > 0.0 || ignoreZeroVol),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %" PetscInt_FMT " of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double) vol);
8626       PetscCall(PetscInfo(dm, "Cell %" PetscInt_FMT " FVM Volume %g\n", c, (double) vol));
8627     }
8628   }
8629   PetscFunctionReturn(0);
8630 }
8631 
8632 /*@
8633   DMPlexCheckPointSF - Check that several necessary conditions are met for the Point SF of this plex.
8634 
8635   Collective
8636 
8637   Input Parameters:
8638 + dm - The DMPlex object
8639 - pointSF - The Point SF, or NULL for Point SF attached to DM
8640 
8641   Notes:
8642   This is mainly intended for debugging/testing purposes.
8643 
8644   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8645 
8646   Level: developer
8647 
8648 .seealso: `DMGetPointSF()`, `DMSetFromOptions()`
8649 @*/
8650 PetscErrorCode DMPlexCheckPointSF(DM dm, PetscSF pointSF)
8651 {
8652   PetscInt        l, nleaves, nroots, overlap;
8653   const PetscInt *locals;
8654   const PetscSFNode *remotes;
8655   PetscBool       distributed;
8656   MPI_Comm        comm;
8657   PetscMPIInt     rank;
8658 
8659   PetscFunctionBegin;
8660   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8661   if (pointSF) PetscValidHeaderSpecific(pointSF, PETSCSF_CLASSID, 2);
8662   else         pointSF = dm->sf;
8663   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
8664   PetscCheck(pointSF, comm, PETSC_ERR_ARG_WRONGSTATE, "DMPlex must have Point SF attached");
8665   PetscCallMPI(MPI_Comm_rank(comm, &rank));
8666   {
8667     PetscMPIInt    mpiFlag;
8668 
8669     PetscCallMPI(MPI_Comm_compare(comm, PetscObjectComm((PetscObject)pointSF),&mpiFlag));
8670     PetscCheck(mpiFlag == MPI_CONGRUENT || mpiFlag == MPI_IDENT, PETSC_COMM_SELF, PETSC_ERR_ARG_NOTSAMECOMM, "DM and Point SF have different communicators (flag %d)",mpiFlag);
8671   }
8672   PetscCall(PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, &remotes));
8673   PetscCall(DMPlexIsDistributed(dm, &distributed));
8674   if (!distributed) {
8675     PetscCheck(nroots < 0 || nleaves == 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Undistributed DMPlex cannot have non-empty PointSF (has %" PetscInt_FMT " roots, %" PetscInt_FMT " leaves)", nroots, nleaves);
8676     PetscFunctionReturn(0);
8677   }
8678   PetscCheck(nroots >= 0, comm, PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set (has %" PetscInt_FMT " roots, %" PetscInt_FMT " leaves)", nroots, nleaves);
8679   PetscCall(DMPlexGetOverlap(dm, &overlap));
8680 
8681   /* Check SF graph is compatible with DMPlex chart */
8682   {
8683     PetscInt pStart, pEnd, maxLeaf;
8684 
8685     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
8686     PetscCall(PetscSFGetLeafRange(pointSF, NULL, &maxLeaf));
8687     PetscCheck(pEnd - pStart == nroots, PETSC_COMM_SELF, PETSC_ERR_PLIB, "pEnd - pStart = %" PetscInt_FMT " != nroots = %" PetscInt_FMT, pEnd-pStart, nroots);
8688     PetscCheck(maxLeaf < pEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "maxLeaf = %" PetscInt_FMT " >= pEnd = %" PetscInt_FMT, maxLeaf, pEnd);
8689   }
8690 
8691   /* Check Point SF has no local points referenced */
8692   for (l = 0; l < nleaves; l++) {
8693     PetscAssert(remotes[l].rank != (PetscInt) rank, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains local point %" PetscInt_FMT " <- (%" PetscInt_FMT ",%" PetscInt_FMT ")", locals ? locals[l] : l, remotes[l].rank, remotes[l].index);
8694   }
8695 
8696   /* Check there are no cells in interface */
8697   if (!overlap) {
8698     PetscInt cellHeight, cStart, cEnd;
8699 
8700     PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
8701     PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
8702     for (l = 0; l < nleaves; ++l) {
8703       const PetscInt point = locals ? locals[l] : l;
8704 
8705       PetscCheck(point < cStart || point >= cEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %" PetscInt_FMT " which is a cell", point);
8706     }
8707   }
8708 
8709   /* If some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
8710   {
8711     const PetscInt *rootdegree;
8712 
8713     PetscCall(PetscSFComputeDegreeBegin(pointSF, &rootdegree));
8714     PetscCall(PetscSFComputeDegreeEnd(pointSF, &rootdegree));
8715     for (l = 0; l < nleaves; ++l) {
8716       const PetscInt  point = locals ? locals[l] : l;
8717       const PetscInt *cone;
8718       PetscInt        coneSize, c, idx;
8719 
8720       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
8721       PetscCall(DMPlexGetCone(dm, point, &cone));
8722       for (c = 0; c < coneSize; ++c) {
8723         if (!rootdegree[cone[c]]) {
8724           if (locals) {
8725             PetscCall(PetscFindInt(cone[c], nleaves, locals, &idx));
8726           } else {
8727             idx = (cone[c] < nleaves) ? cone[c] : -1;
8728           }
8729           PetscCheck(idx >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %" PetscInt_FMT " but not %" PetscInt_FMT " from its cone", point, cone[c]);
8730         }
8731       }
8732     }
8733   }
8734   PetscFunctionReturn(0);
8735 }
8736 
8737 /*@
8738   DMPlexCheck - Perform various checks of Plex sanity
8739 
8740   Input Parameter:
8741 . dm - The DMPlex object
8742 
8743   Notes:
8744   This is a useful diagnostic when creating meshes programmatically.
8745 
8746   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8747 
8748   Currently does not include DMPlexCheckCellShape().
8749 
8750   Level: developer
8751 
8752 .seealso: DMCreate(), DMSetFromOptions()
8753 @*/
8754 PetscErrorCode DMPlexCheck(DM dm)
8755 {
8756   PetscInt cellHeight;
8757 
8758   PetscFunctionBegin;
8759   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
8760   PetscCall(DMPlexCheckSymmetry(dm));
8761   PetscCall(DMPlexCheckSkeleton(dm, cellHeight));
8762   PetscCall(DMPlexCheckFaces(dm, cellHeight));
8763   PetscCall(DMPlexCheckGeometry(dm));
8764   PetscCall(DMPlexCheckPointSF(dm, NULL));
8765   PetscCall(DMPlexCheckInterfaceCones(dm));
8766   PetscFunctionReturn(0);
8767 }
8768 
8769 typedef struct cell_stats
8770 {
8771   PetscReal min, max, sum, squaresum;
8772   PetscInt  count;
8773 } cell_stats_t;
8774 
8775 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
8776 {
8777   PetscInt i, N = *len;
8778 
8779   for (i = 0; i < N; i++) {
8780     cell_stats_t *A = (cell_stats_t *) a;
8781     cell_stats_t *B = (cell_stats_t *) b;
8782 
8783     B->min = PetscMin(A->min,B->min);
8784     B->max = PetscMax(A->max,B->max);
8785     B->sum += A->sum;
8786     B->squaresum += A->squaresum;
8787     B->count += A->count;
8788   }
8789 }
8790 
8791 /*@
8792   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
8793 
8794   Collective on dm
8795 
8796   Input Parameters:
8797 + dm        - The DMPlex object
8798 . output    - If true, statistics will be displayed on stdout
8799 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
8800 
8801   Notes:
8802   This is mainly intended for debugging/testing purposes.
8803 
8804   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8805 
8806   Level: developer
8807 
8808 .seealso: `DMSetFromOptions()`, `DMPlexComputeOrthogonalQuality()`
8809 @*/
8810 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
8811 {
8812   DM             dmCoarse;
8813   cell_stats_t   stats, globalStats;
8814   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
8815   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
8816   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
8817   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
8818   PetscMPIInt    rank,size;
8819 
8820   PetscFunctionBegin;
8821   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8822   stats.min   = PETSC_MAX_REAL;
8823   stats.max   = PETSC_MIN_REAL;
8824   stats.sum   = stats.squaresum = 0.;
8825   stats.count = 0;
8826 
8827   PetscCallMPI(MPI_Comm_size(comm, &size));
8828   PetscCallMPI(MPI_Comm_rank(comm, &rank));
8829   PetscCall(DMGetCoordinateDim(dm,&cdim));
8830   PetscCall(PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ));
8831   PetscCall(DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd));
8832   PetscCall(DMPlexGetDepthStratum(dm,1,&eStart,&eEnd));
8833   for (c = cStart; c < cEnd; c++) {
8834     PetscInt  i;
8835     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
8836 
8837     PetscCall(DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ));
8838     PetscCheck(detJ >= 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %" PetscInt_FMT " is inverted", c);
8839     for (i = 0; i < PetscSqr(cdim); ++i) {
8840       frobJ    += J[i] * J[i];
8841       frobInvJ += invJ[i] * invJ[i];
8842     }
8843     cond2 = frobJ * frobInvJ;
8844     cond  = PetscSqrtReal(cond2);
8845 
8846     stats.min        = PetscMin(stats.min,cond);
8847     stats.max        = PetscMax(stats.max,cond);
8848     stats.sum       += cond;
8849     stats.squaresum += cond2;
8850     stats.count++;
8851     if (output && cond > limit) {
8852       PetscSection coordSection;
8853       Vec          coordsLocal;
8854       PetscScalar *coords = NULL;
8855       PetscInt     Nv, d, clSize, cl, *closure = NULL;
8856 
8857       PetscCall(DMGetCoordinatesLocal(dm, &coordsLocal));
8858       PetscCall(DMGetCoordinateSection(dm, &coordSection));
8859       PetscCall(DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords));
8860       PetscCall(PetscSynchronizedPrintf(comm, "[%d] Cell %" PetscInt_FMT " cond %g\n", rank, c, (double) cond));
8861       for (i = 0; i < Nv/cdim; ++i) {
8862         PetscCall(PetscSynchronizedPrintf(comm, "  Vertex %" PetscInt_FMT ": (", i));
8863         for (d = 0; d < cdim; ++d) {
8864           if (d > 0) PetscCall(PetscSynchronizedPrintf(comm, ", "));
8865           PetscCall(PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d])));
8866         }
8867         PetscCall(PetscSynchronizedPrintf(comm, ")\n"));
8868       }
8869       PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure));
8870       for (cl = 0; cl < clSize*2; cl += 2) {
8871         const PetscInt edge = closure[cl];
8872 
8873         if ((edge >= eStart) && (edge < eEnd)) {
8874           PetscReal len;
8875 
8876           PetscCall(DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL));
8877           PetscCall(PetscSynchronizedPrintf(comm, "  Edge %" PetscInt_FMT ": length %g\n", edge, (double) len));
8878         }
8879       }
8880       PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure));
8881       PetscCall(DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords));
8882     }
8883   }
8884   if (output) PetscCall(PetscSynchronizedFlush(comm, NULL));
8885 
8886   if (size > 1) {
8887     PetscMPIInt   blockLengths[2] = {4,1};
8888     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
8889     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
8890     MPI_Op        statReduce;
8891 
8892     PetscCallMPI(MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType));
8893     PetscCallMPI(MPI_Type_commit(&statType));
8894     PetscCallMPI(MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce));
8895     PetscCallMPI(MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm));
8896     PetscCallMPI(MPI_Op_free(&statReduce));
8897     PetscCallMPI(MPI_Type_free(&statType));
8898   } else {
8899     PetscCall(PetscArraycpy(&globalStats,&stats,1));
8900   }
8901   if (rank == 0) {
8902     count = globalStats.count;
8903     min   = globalStats.min;
8904     max   = globalStats.max;
8905     mean  = globalStats.sum / globalStats.count;
8906     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
8907   }
8908 
8909   if (output) {
8910     PetscCall(PetscPrintf(comm,"Mesh with %" PetscInt_FMT " cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev));
8911   }
8912   PetscCall(PetscFree2(J,invJ));
8913 
8914   PetscCall(DMGetCoarseDM(dm,&dmCoarse));
8915   if (dmCoarse) {
8916     PetscBool isplex;
8917 
8918     PetscCall(PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex));
8919     if (isplex) {
8920       PetscCall(DMPlexCheckCellShape(dmCoarse,output,condLimit));
8921     }
8922   }
8923   PetscFunctionReturn(0);
8924 }
8925 
8926 /*@
8927   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
8928   orthogonal quality below given tolerance.
8929 
8930   Collective on dm
8931 
8932   Input Parameters:
8933 + dm   - The DMPlex object
8934 . fv   - Optional PetscFV object for pre-computed cell/face centroid information
8935 - atol - [0, 1] Absolute tolerance for tagging cells.
8936 
8937   Output Parameters:
8938 + OrthQual      - Vec containing orthogonal quality per cell
8939 - OrthQualLabel - DMLabel tagging cells below atol with DM_ADAPT_REFINE
8940 
8941   Options Database Keys:
8942 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only PETSCVIEWERASCII is
8943 supported.
8944 - -dm_plex_orthogonal_quality_vec_view - view OrthQual vector.
8945 
8946   Notes:
8947   Orthogonal quality is given by the following formula:
8948 
8949   \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]
8950 
8951   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
8952   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
8953   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
8954   calculating the cosine of the angle between these vectors.
8955 
8956   Orthogonal quality ranges from 1 (best) to 0 (worst).
8957 
8958   This routine is mainly useful for FVM, however is not restricted to only FVM. The PetscFV object is optionally used to check for
8959   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
8960 
8961   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
8962 
8963   Level: intermediate
8964 
8965 .seealso: `DMPlexCheckCellShape()`, `DMCreateLabel()`
8966 @*/
8967 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
8968 {
8969   PetscInt                nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
8970   PetscInt                *idx;
8971   PetscScalar             *oqVals;
8972   const PetscScalar       *cellGeomArr, *faceGeomArr;
8973   PetscReal               *ci, *fi, *Ai;
8974   MPI_Comm                comm;
8975   Vec                     cellgeom, facegeom;
8976   DM                      dmFace, dmCell;
8977   IS                      glob;
8978   ISLocalToGlobalMapping  ltog;
8979   PetscViewer             vwr;
8980 
8981   PetscFunctionBegin;
8982   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8983   if (fv) {PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);}
8984   PetscValidPointer(OrthQual, 4);
8985   PetscCheck(atol >= 0.0 && atol <= 1.0,PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Absolute tolerance %g not in [0,1]",(double)atol);
8986   PetscCall(PetscObjectGetComm((PetscObject) dm, &comm));
8987   PetscCall(DMGetDimension(dm, &nc));
8988   PetscCheck(nc >= 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %" PetscInt_FMT ")", nc);
8989   {
8990     DMPlexInterpolatedFlag interpFlag;
8991 
8992     PetscCall(DMPlexIsInterpolated(dm, &interpFlag));
8993     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
8994       PetscMPIInt rank;
8995 
8996       PetscCallMPI(MPI_Comm_rank(comm, &rank));
8997       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
8998     }
8999   }
9000   if (OrthQualLabel) {
9001     PetscValidPointer(OrthQualLabel, 5);
9002     PetscCall(DMCreateLabel(dm, "Orthogonal_Quality"));
9003     PetscCall(DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel));
9004   } else {*OrthQualLabel = NULL;}
9005   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
9006   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
9007   PetscCall(DMPlexCreateCellNumbering_Internal(dm, PETSC_TRUE, &glob));
9008   PetscCall(ISLocalToGlobalMappingCreateIS(glob, &ltog));
9009   PetscCall(ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH));
9010   PetscCall(VecCreate(comm, OrthQual));
9011   PetscCall(VecSetType(*OrthQual, VECSTANDARD));
9012   PetscCall(VecSetSizes(*OrthQual, cEnd-cStart, PETSC_DETERMINE));
9013   PetscCall(VecSetLocalToGlobalMapping(*OrthQual, ltog));
9014   PetscCall(VecSetUp(*OrthQual));
9015   PetscCall(ISDestroy(&glob));
9016   PetscCall(ISLocalToGlobalMappingDestroy(&ltog));
9017   PetscCall(DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL));
9018   PetscCall(VecGetArrayRead(cellgeom, &cellGeomArr));
9019   PetscCall(VecGetArrayRead(facegeom, &faceGeomArr));
9020   PetscCall(VecGetDM(cellgeom, &dmCell));
9021   PetscCall(VecGetDM(facegeom, &dmFace));
9022   PetscCall(PetscMalloc5(cEnd-cStart, &idx, cEnd-cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai));
9023   for (cell = cStart; cell < cEnd; cellIter++,cell++) {
9024     PetscInt           cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
9025     PetscInt           cellarr[2], *adj = NULL;
9026     PetscScalar        *cArr, *fArr;
9027     PetscReal          minvalc = 1.0, minvalf = 1.0;
9028     PetscFVCellGeom    *cg;
9029 
9030     idx[cellIter] = cell-cStart;
9031     cellarr[0] = cell;
9032     /* Make indexing into cellGeom easier */
9033     PetscCall(DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg));
9034     PetscCall(DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj));
9035     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
9036     PetscCall(PetscCalloc2(adjSize, &cArr, adjSize, &fArr));
9037     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++,cellneigh++) {
9038       PetscInt         i;
9039       const PetscInt   neigh = adj[cellneigh];
9040       PetscReal        normci = 0, normfi = 0, normai = 0;
9041       PetscFVCellGeom  *cgneigh;
9042       PetscFVFaceGeom  *fg;
9043 
9044       /* Don't count ourselves in the neighbor list */
9045       if (neigh == cell) continue;
9046       PetscCall(DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh));
9047       cellarr[1] = neigh;
9048       {
9049         PetscInt       numcovpts;
9050         const PetscInt *covpts;
9051 
9052         PetscCall(DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts));
9053         PetscCall(DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg));
9054         PetscCall(DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts));
9055       }
9056 
9057       /* Compute c_i, f_i and their norms */
9058       for (i = 0; i < nc; i++) {
9059         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
9060         fi[i] = fg->centroid[i] - cg->centroid[i];
9061         Ai[i] = fg->normal[i];
9062         normci += PetscPowReal(ci[i], 2);
9063         normfi += PetscPowReal(fi[i], 2);
9064         normai += PetscPowReal(Ai[i], 2);
9065       }
9066       normci = PetscSqrtReal(normci);
9067       normfi = PetscSqrtReal(normfi);
9068       normai = PetscSqrtReal(normai);
9069 
9070       /* Normalize and compute for each face-cell-normal pair */
9071       for (i = 0; i < nc; i++) {
9072         ci[i] = ci[i]/normci;
9073         fi[i] = fi[i]/normfi;
9074         Ai[i] = Ai[i]/normai;
9075         /* PetscAbs because I don't know if normals are guaranteed to point out */
9076         cArr[cellneighiter] += PetscAbs(Ai[i]*ci[i]);
9077         fArr[cellneighiter] += PetscAbs(Ai[i]*fi[i]);
9078       }
9079       if (PetscRealPart(cArr[cellneighiter]) < minvalc) {
9080         minvalc = PetscRealPart(cArr[cellneighiter]);
9081       }
9082       if (PetscRealPart(fArr[cellneighiter]) < minvalf) {
9083         minvalf = PetscRealPart(fArr[cellneighiter]);
9084       }
9085     }
9086     PetscCall(PetscFree(adj));
9087     PetscCall(PetscFree2(cArr, fArr));
9088     /* Defer to cell if they're equal */
9089     oqVals[cellIter] = PetscMin(minvalf, minvalc);
9090     if (OrthQualLabel) {
9091       if (PetscRealPart(oqVals[cellIter]) <= atol) PetscCall(DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE));
9092     }
9093   }
9094   PetscCall(VecSetValuesLocal(*OrthQual, cEnd-cStart, idx, oqVals, INSERT_VALUES));
9095   PetscCall(VecAssemblyBegin(*OrthQual));
9096   PetscCall(VecAssemblyEnd(*OrthQual));
9097   PetscCall(VecRestoreArrayRead(cellgeom, &cellGeomArr));
9098   PetscCall(VecRestoreArrayRead(facegeom, &faceGeomArr));
9099   PetscCall(PetscOptionsGetViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL));
9100   if (OrthQualLabel) {
9101     if (vwr) PetscCall(DMLabelView(*OrthQualLabel, vwr));
9102   }
9103   PetscCall(PetscFree5(idx, oqVals, ci, fi, Ai));
9104   PetscCall(PetscViewerDestroy(&vwr));
9105   PetscCall(VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view"));
9106   PetscFunctionReturn(0);
9107 }
9108 
9109 /* this is here insead of DMGetOutputDM because output DM still has constraints in the local indices that affect
9110  * interpolator construction */
9111 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
9112 {
9113   PetscSection   section, newSection, gsection;
9114   PetscSF        sf;
9115   PetscBool      hasConstraints, ghasConstraints;
9116 
9117   PetscFunctionBegin;
9118   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
9119   PetscValidPointer(odm,2);
9120   PetscCall(DMGetLocalSection(dm, &section));
9121   PetscCall(PetscSectionHasConstraints(section, &hasConstraints));
9122   PetscCallMPI(MPI_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject) dm)));
9123   if (!ghasConstraints) {
9124     PetscCall(PetscObjectReference((PetscObject)dm));
9125     *odm = dm;
9126     PetscFunctionReturn(0);
9127   }
9128   PetscCall(DMClone(dm, odm));
9129   PetscCall(DMCopyFields(dm, *odm));
9130   PetscCall(DMGetLocalSection(*odm, &newSection));
9131   PetscCall(DMGetPointSF(*odm, &sf));
9132   PetscCall(PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_FALSE, &gsection));
9133   PetscCall(DMSetGlobalSection(*odm, gsection));
9134   PetscCall(PetscSectionDestroy(&gsection));
9135   PetscFunctionReturn(0);
9136 }
9137 
9138 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
9139 {
9140   DM             dmco, dmfo;
9141   Mat            interpo;
9142   Vec            rscale;
9143   Vec            cglobalo, clocal;
9144   Vec            fglobal, fglobalo, flocal;
9145   PetscBool      regular;
9146 
9147   PetscFunctionBegin;
9148   PetscCall(DMGetFullDM(dmc, &dmco));
9149   PetscCall(DMGetFullDM(dmf, &dmfo));
9150   PetscCall(DMSetCoarseDM(dmfo, dmco));
9151   PetscCall(DMPlexGetRegularRefinement(dmf, &regular));
9152   PetscCall(DMPlexSetRegularRefinement(dmfo, regular));
9153   PetscCall(DMCreateInterpolation(dmco, dmfo, &interpo, &rscale));
9154   PetscCall(DMCreateGlobalVector(dmco, &cglobalo));
9155   PetscCall(DMCreateLocalVector(dmc, &clocal));
9156   PetscCall(VecSet(cglobalo, 0.));
9157   PetscCall(VecSet(clocal, 0.));
9158   PetscCall(DMCreateGlobalVector(dmf, &fglobal));
9159   PetscCall(DMCreateGlobalVector(dmfo, &fglobalo));
9160   PetscCall(DMCreateLocalVector(dmf, &flocal));
9161   PetscCall(VecSet(fglobal, 0.));
9162   PetscCall(VecSet(fglobalo, 0.));
9163   PetscCall(VecSet(flocal, 0.));
9164   PetscCall(DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL));
9165   PetscCall(DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo));
9166   PetscCall(DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo));
9167   PetscCall(MatMult(interpo, cglobalo, fglobalo));
9168   PetscCall(DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal));
9169   PetscCall(DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal));
9170   PetscCall(DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal));
9171   PetscCall(DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal));
9172   *shift = fglobal;
9173   PetscCall(VecDestroy(&flocal));
9174   PetscCall(VecDestroy(&fglobalo));
9175   PetscCall(VecDestroy(&clocal));
9176   PetscCall(VecDestroy(&cglobalo));
9177   PetscCall(VecDestroy(&rscale));
9178   PetscCall(MatDestroy(&interpo));
9179   PetscCall(DMDestroy(&dmfo));
9180   PetscCall(DMDestroy(&dmco));
9181   PetscFunctionReturn(0);
9182 }
9183 
9184 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
9185 {
9186   PetscObject    shifto;
9187   Vec            shift;
9188 
9189   PetscFunctionBegin;
9190   if (!interp) {
9191     Vec rscale;
9192 
9193     PetscCall(DMCreateInterpolation(coarse, fine, &interp, &rscale));
9194     PetscCall(VecDestroy(&rscale));
9195   } else {
9196     PetscCall(PetscObjectReference((PetscObject)interp));
9197   }
9198   PetscCall(PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto));
9199   if (!shifto) {
9200     PetscCall(DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift));
9201     PetscCall(PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject) shift));
9202     shifto = (PetscObject) shift;
9203     PetscCall(VecDestroy(&shift));
9204   }
9205   shift = (Vec) shifto;
9206   PetscCall(MatInterpolate(interp, coarseSol, fineSol));
9207   PetscCall(VecAXPY(fineSol, 1.0, shift));
9208   PetscCall(MatDestroy(&interp));
9209   PetscFunctionReturn(0);
9210 }
9211 
9212 /* Pointwise interpolation
9213      Just code FEM for now
9214      u^f = I u^c
9215      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
9216      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
9217      I_{ij} = psi^f_i phi^c_j
9218 */
9219 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
9220 {
9221   PetscSection   gsc, gsf;
9222   PetscInt       m, n;
9223   void          *ctx;
9224   DM             cdm;
9225   PetscBool      regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
9226 
9227   PetscFunctionBegin;
9228   PetscCall(DMGetGlobalSection(dmFine, &gsf));
9229   PetscCall(PetscSectionGetConstrainedStorageSize(gsf, &m));
9230   PetscCall(DMGetGlobalSection(dmCoarse, &gsc));
9231   PetscCall(PetscSectionGetConstrainedStorageSize(gsc, &n));
9232 
9233   PetscCall(PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis));
9234   PetscCall(MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation));
9235   PetscCall(MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE));
9236   PetscCall(MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype));
9237   PetscCall(DMGetApplicationContext(dmFine, &ctx));
9238 
9239   PetscCall(DMGetCoarseDM(dmFine, &cdm));
9240   PetscCall(DMPlexGetRegularRefinement(dmFine, &regular));
9241   if (!isRefined || (regular && cdm == dmCoarse)) PetscCall(DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx));
9242   else                                            PetscCall(DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx));
9243   PetscCall(MatViewFromOptions(*interpolation, NULL, "-interp_mat_view"));
9244   if (scaling) {
9245     /* Use naive scaling */
9246     PetscCall(DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling));
9247   }
9248   PetscFunctionReturn(0);
9249 }
9250 
9251 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
9252 {
9253   VecScatter     ctx;
9254 
9255   PetscFunctionBegin;
9256   PetscCall(DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL));
9257   PetscCall(MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat));
9258   PetscCall(VecScatterDestroy(&ctx));
9259   PetscFunctionReturn(0);
9260 }
9261 
9262 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux,
9263                                 const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
9264                                 const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
9265                                 PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
9266 {
9267   const PetscInt Nc = uOff[1] - uOff[0];
9268   PetscInt       c;
9269   for (c = 0; c < Nc; ++c) g0[c*Nc+c] = 1.0;
9270 }
9271 
9272 PetscErrorCode DMCreateMassMatrixLumped_Plex(DM dm, Vec *mass)
9273 {
9274   DM             dmc;
9275   PetscDS        ds;
9276   Vec            ones, locmass;
9277   IS             cellIS;
9278   PetscFormKey   key;
9279   PetscInt       depth;
9280 
9281   PetscFunctionBegin;
9282   PetscCall(DMClone(dm, &dmc));
9283   PetscCall(DMCopyDisc(dm, dmc));
9284   PetscCall(DMGetDS(dmc, &ds));
9285   PetscCall(PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL));
9286   PetscCall(DMCreateGlobalVector(dmc, mass));
9287   PetscCall(DMGetLocalVector(dmc, &ones));
9288   PetscCall(DMGetLocalVector(dmc, &locmass));
9289   PetscCall(DMPlexGetDepth(dmc, &depth));
9290   PetscCall(DMGetStratumIS(dmc, "depth", depth, &cellIS));
9291   PetscCall(VecSet(locmass, 0.0));
9292   PetscCall(VecSet(ones, 1.0));
9293   key.label = NULL;
9294   key.value = 0;
9295   key.field = 0;
9296   key.part  = 0;
9297   PetscCall(DMPlexComputeJacobian_Action_Internal(dmc, key, cellIS, 0.0, 0.0, ones, NULL, ones, locmass, NULL));
9298   PetscCall(ISDestroy(&cellIS));
9299   PetscCall(VecSet(*mass, 0.0));
9300   PetscCall(DMLocalToGlobalBegin(dmc, locmass, ADD_VALUES, *mass));
9301   PetscCall(DMLocalToGlobalEnd(dmc, locmass, ADD_VALUES, *mass));
9302   PetscCall(DMRestoreLocalVector(dmc, &ones));
9303   PetscCall(DMRestoreLocalVector(dmc, &locmass));
9304   PetscCall(DMDestroy(&dmc));
9305   PetscFunctionReturn(0);
9306 }
9307 
9308 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
9309 {
9310   PetscSection   gsc, gsf;
9311   PetscInt       m, n;
9312   void          *ctx;
9313   DM             cdm;
9314   PetscBool      regular;
9315 
9316   PetscFunctionBegin;
9317   if (dmFine == dmCoarse) {
9318     DM            dmc;
9319     PetscDS       ds;
9320     PetscWeakForm wf;
9321     Vec           u;
9322     IS            cellIS;
9323     PetscFormKey  key;
9324     PetscInt      depth;
9325 
9326     PetscCall(DMClone(dmFine, &dmc));
9327     PetscCall(DMCopyDisc(dmFine, dmc));
9328     PetscCall(DMGetDS(dmc, &ds));
9329     PetscCall(PetscDSGetWeakForm(ds, &wf));
9330     PetscCall(PetscWeakFormClear(wf));
9331     PetscCall(PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL));
9332     PetscCall(DMCreateMatrix(dmc, mass));
9333     PetscCall(DMGetGlobalVector(dmc, &u));
9334     PetscCall(DMPlexGetDepth(dmc, &depth));
9335     PetscCall(DMGetStratumIS(dmc, "depth", depth, &cellIS));
9336     PetscCall(MatZeroEntries(*mass));
9337     key.label = NULL;
9338     key.value = 0;
9339     key.field = 0;
9340     key.part  = 0;
9341     PetscCall(DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL));
9342     PetscCall(ISDestroy(&cellIS));
9343     PetscCall(DMRestoreGlobalVector(dmc, &u));
9344     PetscCall(DMDestroy(&dmc));
9345   } else {
9346     PetscCall(DMGetGlobalSection(dmFine, &gsf));
9347     PetscCall(PetscSectionGetConstrainedStorageSize(gsf, &m));
9348     PetscCall(DMGetGlobalSection(dmCoarse, &gsc));
9349     PetscCall(PetscSectionGetConstrainedStorageSize(gsc, &n));
9350 
9351     PetscCall(MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass));
9352     PetscCall(MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE));
9353     PetscCall(MatSetType(*mass, dmCoarse->mattype));
9354     PetscCall(DMGetApplicationContext(dmFine, &ctx));
9355 
9356     PetscCall(DMGetCoarseDM(dmFine, &cdm));
9357     PetscCall(DMPlexGetRegularRefinement(dmFine, &regular));
9358     if (regular && cdm == dmCoarse) PetscCall(DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx));
9359     else                            PetscCall(DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx));
9360   }
9361   PetscCall(MatViewFromOptions(*mass, NULL, "-mass_mat_view"));
9362   PetscFunctionReturn(0);
9363 }
9364 
9365 /*@
9366   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9367 
9368   Input Parameter:
9369 . dm - The DMPlex object
9370 
9371   Output Parameter:
9372 . regular - The flag
9373 
9374   Level: intermediate
9375 
9376 .seealso: `DMPlexSetRegularRefinement()`
9377 @*/
9378 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
9379 {
9380   PetscFunctionBegin;
9381   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9382   PetscValidBoolPointer(regular, 2);
9383   *regular = ((DM_Plex *) dm->data)->regularRefinement;
9384   PetscFunctionReturn(0);
9385 }
9386 
9387 /*@
9388   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9389 
9390   Input Parameters:
9391 + dm - The DMPlex object
9392 - regular - The flag
9393 
9394   Level: intermediate
9395 
9396 .seealso: `DMPlexGetRegularRefinement()`
9397 @*/
9398 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
9399 {
9400   PetscFunctionBegin;
9401   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9402   ((DM_Plex *) dm->data)->regularRefinement = regular;
9403   PetscFunctionReturn(0);
9404 }
9405 
9406 /* anchors */
9407 /*@
9408   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
9409   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetDefaultConstraints().
9410 
9411   not collective
9412 
9413   Input Parameter:
9414 . dm - The DMPlex object
9415 
9416   Output Parameters:
9417 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
9418 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
9419 
9420   Level: intermediate
9421 
9422 .seealso: `DMPlexSetAnchors()`, `DMGetDefaultConstraints()`, `DMSetDefaultConstraints()`
9423 @*/
9424 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
9425 {
9426   DM_Plex *plex = (DM_Plex *)dm->data;
9427 
9428   PetscFunctionBegin;
9429   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9430   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) PetscCall((*plex->createanchors)(dm));
9431   if (anchorSection) *anchorSection = plex->anchorSection;
9432   if (anchorIS) *anchorIS = plex->anchorIS;
9433   PetscFunctionReturn(0);
9434 }
9435 
9436 /*@
9437   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
9438   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
9439   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
9440 
9441   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
9442   DMGetDefaultConstraints() and filling in the entries in the constraint matrix.
9443 
9444   collective on dm
9445 
9446   Input Parameters:
9447 + dm - The DMPlex object
9448 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9449 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9450 
9451   The reference counts of anchorSection and anchorIS are incremented.
9452 
9453   Level: intermediate
9454 
9455 .seealso: `DMPlexGetAnchors()`, `DMGetDefaultConstraints()`, `DMSetDefaultConstraints()`
9456 @*/
9457 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
9458 {
9459   DM_Plex        *plex = (DM_Plex *)dm->data;
9460   PetscMPIInt    result;
9461 
9462   PetscFunctionBegin;
9463   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9464   if (anchorSection) {
9465     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
9466     PetscCallMPI(MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result));
9467     PetscCheck(result == MPI_CONGRUENT || result == MPI_IDENT,PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
9468   }
9469   if (anchorIS) {
9470     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
9471     PetscCallMPI(MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result));
9472     PetscCheck(result == MPI_CONGRUENT || result == MPI_IDENT,PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
9473   }
9474 
9475   PetscCall(PetscObjectReference((PetscObject)anchorSection));
9476   PetscCall(PetscSectionDestroy(&plex->anchorSection));
9477   plex->anchorSection = anchorSection;
9478 
9479   PetscCall(PetscObjectReference((PetscObject)anchorIS));
9480   PetscCall(ISDestroy(&plex->anchorIS));
9481   plex->anchorIS = anchorIS;
9482 
9483   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
9484     PetscInt size, a, pStart, pEnd;
9485     const PetscInt *anchors;
9486 
9487     PetscCall(PetscSectionGetChart(anchorSection,&pStart,&pEnd));
9488     PetscCall(ISGetLocalSize(anchorIS,&size));
9489     PetscCall(ISGetIndices(anchorIS,&anchors));
9490     for (a = 0; a < size; a++) {
9491       PetscInt p;
9492 
9493       p = anchors[a];
9494       if (p >= pStart && p < pEnd) {
9495         PetscInt dof;
9496 
9497         PetscCall(PetscSectionGetDof(anchorSection,p,&dof));
9498         if (dof) {
9499 
9500           PetscCall(ISRestoreIndices(anchorIS,&anchors));
9501           SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %" PetscInt_FMT " cannot be constrained and an anchor",p);
9502         }
9503       }
9504     }
9505     PetscCall(ISRestoreIndices(anchorIS,&anchors));
9506   }
9507   /* reset the generic constraints */
9508   PetscCall(DMSetDefaultConstraints(dm,NULL,NULL,NULL));
9509   PetscFunctionReturn(0);
9510 }
9511 
9512 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
9513 {
9514   PetscSection anchorSection;
9515   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
9516 
9517   PetscFunctionBegin;
9518   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9519   PetscCall(DMPlexGetAnchors(dm,&anchorSection,NULL));
9520   PetscCall(PetscSectionCreate(PETSC_COMM_SELF,cSec));
9521   PetscCall(PetscSectionGetNumFields(section,&numFields));
9522   if (numFields) {
9523     PetscInt f;
9524     PetscCall(PetscSectionSetNumFields(*cSec,numFields));
9525 
9526     for (f = 0; f < numFields; f++) {
9527       PetscInt numComp;
9528 
9529       PetscCall(PetscSectionGetFieldComponents(section,f,&numComp));
9530       PetscCall(PetscSectionSetFieldComponents(*cSec,f,numComp));
9531     }
9532   }
9533   PetscCall(PetscSectionGetChart(anchorSection,&pStart,&pEnd));
9534   PetscCall(PetscSectionGetChart(section,&sStart,&sEnd));
9535   pStart = PetscMax(pStart,sStart);
9536   pEnd   = PetscMin(pEnd,sEnd);
9537   pEnd   = PetscMax(pStart,pEnd);
9538   PetscCall(PetscSectionSetChart(*cSec,pStart,pEnd));
9539   for (p = pStart; p < pEnd; p++) {
9540     PetscCall(PetscSectionGetDof(anchorSection,p,&dof));
9541     if (dof) {
9542       PetscCall(PetscSectionGetDof(section,p,&dof));
9543       PetscCall(PetscSectionSetDof(*cSec,p,dof));
9544       for (f = 0; f < numFields; f++) {
9545         PetscCall(PetscSectionGetFieldDof(section,p,f,&dof));
9546         PetscCall(PetscSectionSetFieldDof(*cSec,p,f,dof));
9547       }
9548     }
9549   }
9550   PetscCall(PetscSectionSetUp(*cSec));
9551   PetscCall(PetscObjectSetName((PetscObject) *cSec, "Constraint Section"));
9552   PetscFunctionReturn(0);
9553 }
9554 
9555 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
9556 {
9557   PetscSection   aSec;
9558   PetscInt       pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
9559   const PetscInt *anchors;
9560   PetscInt       numFields, f;
9561   IS             aIS;
9562   MatType        mtype;
9563   PetscBool      iscuda,iskokkos;
9564 
9565   PetscFunctionBegin;
9566   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9567   PetscCall(PetscSectionGetStorageSize(cSec, &m));
9568   PetscCall(PetscSectionGetStorageSize(section, &n));
9569   PetscCall(MatCreate(PETSC_COMM_SELF,cMat));
9570   PetscCall(MatSetSizes(*cMat,m,n,m,n));
9571   PetscCall(PetscStrcmp(dm->mattype,MATSEQAIJCUSPARSE,&iscuda));
9572   if (!iscuda) PetscCall(PetscStrcmp(dm->mattype,MATMPIAIJCUSPARSE,&iscuda));
9573   PetscCall(PetscStrcmp(dm->mattype,MATSEQAIJKOKKOS,&iskokkos));
9574   if (!iskokkos) PetscCall(PetscStrcmp(dm->mattype,MATMPIAIJKOKKOS,&iskokkos));
9575   if (iscuda) mtype = MATSEQAIJCUSPARSE;
9576   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
9577   else mtype = MATSEQAIJ;
9578   PetscCall(MatSetType(*cMat,mtype));
9579   PetscCall(DMPlexGetAnchors(dm,&aSec,&aIS));
9580   PetscCall(ISGetIndices(aIS,&anchors));
9581   /* cSec will be a subset of aSec and section */
9582   PetscCall(PetscSectionGetChart(cSec,&pStart,&pEnd));
9583   PetscCall(PetscSectionGetChart(section,&sStart,&sEnd));
9584   PetscCall(PetscMalloc1(m+1,&i));
9585   i[0] = 0;
9586   PetscCall(PetscSectionGetNumFields(section,&numFields));
9587   for (p = pStart; p < pEnd; p++) {
9588     PetscInt rDof, rOff, r;
9589 
9590     PetscCall(PetscSectionGetDof(aSec,p,&rDof));
9591     if (!rDof) continue;
9592     PetscCall(PetscSectionGetOffset(aSec,p,&rOff));
9593     if (numFields) {
9594       for (f = 0; f < numFields; f++) {
9595         annz = 0;
9596         for (r = 0; r < rDof; r++) {
9597           a = anchors[rOff + r];
9598           if (a < sStart || a >= sEnd) continue;
9599           PetscCall(PetscSectionGetFieldDof(section,a,f,&aDof));
9600           annz += aDof;
9601         }
9602         PetscCall(PetscSectionGetFieldDof(cSec,p,f,&dof));
9603         PetscCall(PetscSectionGetFieldOffset(cSec,p,f,&off));
9604         for (q = 0; q < dof; q++) {
9605           i[off + q + 1] = i[off + q] + annz;
9606         }
9607       }
9608     } else {
9609       annz = 0;
9610       PetscCall(PetscSectionGetDof(cSec,p,&dof));
9611       for (q = 0; q < dof; q++) {
9612         a = anchors[rOff + q];
9613         if (a < sStart || a >= sEnd) continue;
9614         PetscCall(PetscSectionGetDof(section,a,&aDof));
9615         annz += aDof;
9616       }
9617       PetscCall(PetscSectionGetDof(cSec,p,&dof));
9618       PetscCall(PetscSectionGetOffset(cSec,p,&off));
9619       for (q = 0; q < dof; q++) {
9620         i[off + q + 1] = i[off + q] + annz;
9621       }
9622     }
9623   }
9624   nnz = i[m];
9625   PetscCall(PetscMalloc1(nnz,&j));
9626   offset = 0;
9627   for (p = pStart; p < pEnd; p++) {
9628     if (numFields) {
9629       for (f = 0; f < numFields; f++) {
9630         PetscCall(PetscSectionGetFieldDof(cSec,p,f,&dof));
9631         for (q = 0; q < dof; q++) {
9632           PetscInt rDof, rOff, r;
9633           PetscCall(PetscSectionGetDof(aSec,p,&rDof));
9634           PetscCall(PetscSectionGetOffset(aSec,p,&rOff));
9635           for (r = 0; r < rDof; r++) {
9636             PetscInt s;
9637 
9638             a = anchors[rOff + r];
9639             if (a < sStart || a >= sEnd) continue;
9640             PetscCall(PetscSectionGetFieldDof(section,a,f,&aDof));
9641             PetscCall(PetscSectionGetFieldOffset(section,a,f,&aOff));
9642             for (s = 0; s < aDof; s++) {
9643               j[offset++] = aOff + s;
9644             }
9645           }
9646         }
9647       }
9648     } else {
9649       PetscCall(PetscSectionGetDof(cSec,p,&dof));
9650       for (q = 0; q < dof; q++) {
9651         PetscInt rDof, rOff, r;
9652         PetscCall(PetscSectionGetDof(aSec,p,&rDof));
9653         PetscCall(PetscSectionGetOffset(aSec,p,&rOff));
9654         for (r = 0; r < rDof; r++) {
9655           PetscInt s;
9656 
9657           a = anchors[rOff + r];
9658           if (a < sStart || a >= sEnd) continue;
9659           PetscCall(PetscSectionGetDof(section,a,&aDof));
9660           PetscCall(PetscSectionGetOffset(section,a,&aOff));
9661           for (s = 0; s < aDof; s++) {
9662             j[offset++] = aOff + s;
9663           }
9664         }
9665       }
9666     }
9667   }
9668   PetscCall(MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL));
9669   PetscCall(PetscFree(i));
9670   PetscCall(PetscFree(j));
9671   PetscCall(ISRestoreIndices(aIS,&anchors));
9672   PetscFunctionReturn(0);
9673 }
9674 
9675 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
9676 {
9677   DM_Plex        *plex = (DM_Plex *)dm->data;
9678   PetscSection   anchorSection, section, cSec;
9679   Mat            cMat;
9680 
9681   PetscFunctionBegin;
9682   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9683   PetscCall(DMPlexGetAnchors(dm,&anchorSection,NULL));
9684   if (anchorSection) {
9685     PetscInt Nf;
9686 
9687     PetscCall(DMGetLocalSection(dm,&section));
9688     PetscCall(DMPlexCreateConstraintSection_Anchors(dm,section,&cSec));
9689     PetscCall(DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat));
9690     PetscCall(DMGetNumFields(dm,&Nf));
9691     if (Nf && plex->computeanchormatrix) PetscCall((*plex->computeanchormatrix)(dm,section,cSec,cMat));
9692     PetscCall(DMSetDefaultConstraints(dm,cSec,cMat,NULL));
9693     PetscCall(PetscSectionDestroy(&cSec));
9694     PetscCall(MatDestroy(&cMat));
9695   }
9696   PetscFunctionReturn(0);
9697 }
9698 
9699 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
9700 {
9701   IS             subis;
9702   PetscSection   section, subsection;
9703 
9704   PetscFunctionBegin;
9705   PetscCall(DMGetLocalSection(dm, &section));
9706   PetscCheck(section,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
9707   PetscCheck(subdm,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
9708   /* Create subdomain */
9709   PetscCall(DMPlexFilter(dm, label, value, subdm));
9710   /* Create submodel */
9711   PetscCall(DMPlexGetSubpointIS(*subdm, &subis));
9712   PetscCall(PetscSectionCreateSubmeshSection(section, subis, &subsection));
9713   PetscCall(DMSetLocalSection(*subdm, subsection));
9714   PetscCall(PetscSectionDestroy(&subsection));
9715   PetscCall(DMCopyDisc(dm, *subdm));
9716   /* Create map from submodel to global model */
9717   if (is) {
9718     PetscSection    sectionGlobal, subsectionGlobal;
9719     IS              spIS;
9720     const PetscInt *spmap;
9721     PetscInt       *subIndices;
9722     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
9723     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
9724 
9725     PetscCall(DMPlexGetSubpointIS(*subdm, &spIS));
9726     PetscCall(ISGetIndices(spIS, &spmap));
9727     PetscCall(PetscSectionGetNumFields(section, &Nf));
9728     PetscCall(DMGetGlobalSection(dm, &sectionGlobal));
9729     PetscCall(DMGetGlobalSection(*subdm, &subsectionGlobal));
9730     PetscCall(PetscSectionGetChart(subsection, &pStart, &pEnd));
9731     for (p = pStart; p < pEnd; ++p) {
9732       PetscInt gdof, pSubSize  = 0;
9733 
9734       PetscCall(PetscSectionGetDof(sectionGlobal, p, &gdof));
9735       if (gdof > 0) {
9736         for (f = 0; f < Nf; ++f) {
9737           PetscInt fdof, fcdof;
9738 
9739           PetscCall(PetscSectionGetFieldDof(subsection, p, f, &fdof));
9740           PetscCall(PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof));
9741           pSubSize += fdof-fcdof;
9742         }
9743         subSize += pSubSize;
9744         if (pSubSize) {
9745           if (bs < 0) {
9746             bs = pSubSize;
9747           } else if (bs != pSubSize) {
9748             /* Layout does not admit a pointwise block size */
9749             bs = 1;
9750           }
9751         }
9752       }
9753     }
9754     /* Must have same blocksize on all procs (some might have no points) */
9755     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
9756     PetscCall(PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax));
9757     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
9758     else                            {bs = bsMinMax[0];}
9759     PetscCall(PetscMalloc1(subSize, &subIndices));
9760     for (p = pStart; p < pEnd; ++p) {
9761       PetscInt gdof, goff;
9762 
9763       PetscCall(PetscSectionGetDof(subsectionGlobal, p, &gdof));
9764       if (gdof > 0) {
9765         const PetscInt point = spmap[p];
9766 
9767         PetscCall(PetscSectionGetOffset(sectionGlobal, point, &goff));
9768         for (f = 0; f < Nf; ++f) {
9769           PetscInt fdof, fcdof, fc, f2, poff = 0;
9770 
9771           /* Can get rid of this loop by storing field information in the global section */
9772           for (f2 = 0; f2 < f; ++f2) {
9773             PetscCall(PetscSectionGetFieldDof(section, p, f2, &fdof));
9774             PetscCall(PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof));
9775             poff += fdof-fcdof;
9776           }
9777           PetscCall(PetscSectionGetFieldDof(section, p, f, &fdof));
9778           PetscCall(PetscSectionGetFieldConstraintDof(section, p, f, &fcdof));
9779           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
9780             subIndices[subOff] = goff+poff+fc;
9781           }
9782         }
9783       }
9784     }
9785     PetscCall(ISRestoreIndices(spIS, &spmap));
9786     PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is));
9787     if (bs > 1) {
9788       /* We need to check that the block size does not come from non-contiguous fields */
9789       PetscInt i, j, set = 1;
9790       for (i = 0; i < subSize; i += bs) {
9791         for (j = 0; j < bs; ++j) {
9792           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
9793         }
9794       }
9795       if (set) PetscCall(ISSetBlockSize(*is, bs));
9796     }
9797     /* Attach nullspace */
9798     for (f = 0; f < Nf; ++f) {
9799       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
9800       if ((*subdm)->nullspaceConstructors[f]) break;
9801     }
9802     if (f < Nf) {
9803       MatNullSpace nullSpace;
9804       PetscCall((*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace));
9805 
9806       PetscCall(PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace));
9807       PetscCall(MatNullSpaceDestroy(&nullSpace));
9808     }
9809   }
9810   PetscFunctionReturn(0);
9811 }
9812 
9813 /*@
9814   DMPlexMonitorThroughput - Report the cell throughput of FE integration
9815 
9816   Input Parameter:
9817 - dm - The DM
9818 
9819   Level: developer
9820 
9821   Options Database Keys:
9822 . -dm_plex_monitor_throughput - Activate the monitor
9823 
9824 .seealso: `DMSetFromOptions()`, `DMPlexCreate()`
9825 @*/
9826 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
9827 {
9828 #if defined(PETSC_USE_LOG)
9829   PetscStageLog      stageLog;
9830   PetscLogEvent      event;
9831   PetscLogStage      stage;
9832   PetscEventPerfInfo eventInfo;
9833   PetscReal          cellRate, flopRate;
9834   PetscInt           cStart, cEnd, Nf, N;
9835   const char        *name;
9836 #endif
9837 
9838   PetscFunctionBegin;
9839   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9840 #if defined(PETSC_USE_LOG)
9841   PetscCall(PetscObjectGetName((PetscObject) dm, &name));
9842   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
9843   PetscCall(DMGetNumFields(dm, &Nf));
9844   PetscCall(PetscLogGetStageLog(&stageLog));
9845   PetscCall(PetscStageLogGetCurrent(stageLog, &stage));
9846   PetscCall(PetscLogEventGetId("DMPlexResidualFE", &event));
9847   PetscCall(PetscLogEventGetPerfInfo(stage, event, &eventInfo));
9848   N        = (cEnd - cStart)*Nf*eventInfo.count;
9849   flopRate = eventInfo.flops/eventInfo.time;
9850   cellRate = N/eventInfo.time;
9851   PetscCall(PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %" PetscInt_FMT " integrals %d reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6)));
9852 #else
9853   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
9854 #endif
9855   PetscFunctionReturn(0);
9856 }
9857