xref: /petsc/src/dm/impls/plex/plex.c (revision c0420575261eeb98391c41a3432bc0520f74555f)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 
6 /* Logging support */
7 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM;
8 
9 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
10 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
11 PETSC_EXTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);
12 
13 #undef __FUNCT__
14 #define __FUNCT__ "DMPlexGetFieldType_Internal"
15 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
16 {
17   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, vdof = 0, cdof = 0;
18   PetscErrorCode ierr;
19 
20   PetscFunctionBegin;
21   *ft  = PETSC_VTK_POINT_FIELD;
22   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
23   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
24   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
25   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
26   if (field >= 0) {
27     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vdof);CHKERRQ(ierr);}
28     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &cdof);CHKERRQ(ierr);}
29   } else {
30     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
31     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
32   }
33   if (vdof) {
34     *sStart = vStart;
35     *sEnd   = vEnd;
36     if (vdof == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
37     else             *ft = PETSC_VTK_POINT_FIELD;
38   } else if (cdof) {
39     *sStart = cStart;
40     *sEnd   = cEnd;
41     if (cdof == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
42     else             *ft = PETSC_VTK_CELL_FIELD;
43   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
44   PetscFunctionReturn(0);
45 }
46 
47 #undef __FUNCT__
48 #define __FUNCT__ "VecView_Plex_Local"
49 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
50 {
51   DM             dm;
52   PetscBool      isvtk, ishdf5, isseq;
53   PetscErrorCode ierr;
54 
55   PetscFunctionBegin;
56   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
57   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
58   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
59   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
60   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
61   if (isvtk || ishdf5) {
62     PetscInt  numFields;
63     PetscBool fem = PETSC_FALSE;
64 
65     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
66     if (numFields) {
67       PetscObject fe;
68 
69       ierr = DMGetField(dm, 0, &fe);CHKERRQ(ierr);
70       if (fe->classid == PETSCFE_CLASSID) fem = PETSC_TRUE;
71     }
72     if (fem) {
73       ierr = DMPlexInsertBoundaryValuesFEM(dm, v);CHKERRQ(ierr);
74     } else {
75       PetscObject faceGeometry;
76 
77       /* TODO Fix the time, and add FVM objects */
78       ierr = PetscObjectQuery((PetscObject) dm, "FaceGeometry", &faceGeometry);CHKERRQ(ierr);
79       if (faceGeometry) {ierr = DMPlexInsertBoundaryValuesFVM(dm, 0.0, v);CHKERRQ(ierr);}
80     }
81   }
82   if (isvtk) {
83     PetscSection            section;
84     PetscViewerVTKFieldType ft;
85     PetscInt                pStart, pEnd;
86 
87     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
88     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
89     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
90     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
91     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
92   } else if (ishdf5) {
93 #if defined(PETSC_HAVE_HDF5)
94     ierr = VecView_Plex_Local_HDF5(v, viewer);CHKERRQ(ierr);
95 #else
96     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
97 #endif
98   } else {
99     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
100     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
101   }
102   PetscFunctionReturn(0);
103 }
104 
105 #undef __FUNCT__
106 #define __FUNCT__ "VecView_Plex"
107 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
108 {
109   DM             dm;
110   PetscBool      isvtk, ishdf5, isseq;
111   PetscErrorCode ierr;
112 
113   PetscFunctionBegin;
114   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
115   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
116   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
117   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
118   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
119   if (isvtk) {
120     Vec         locv;
121     const char *name;
122 
123     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
124     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
125     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
126     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
127     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
128     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
129     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
130   } else if (ishdf5) {
131 #if defined(PETSC_HAVE_HDF5)
132     ierr = VecView_Plex_HDF5(v, viewer);CHKERRQ(ierr);
133 #else
134     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
135 #endif
136   } else {
137     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
138     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
139   }
140   PetscFunctionReturn(0);
141 }
142 
143 #undef __FUNCT__
144 #define __FUNCT__ "VecLoad_Plex_Local"
145 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
146 {
147   DM             dm;
148   PetscBool      ishdf5;
149   PetscErrorCode ierr;
150 
151   PetscFunctionBegin;
152   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
153   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
154   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
155   if (ishdf5) {
156     DM          dmBC;
157     Vec         gv;
158     const char *name;
159 
160     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
161     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
162     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
163     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
164     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
165     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
166     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
167     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
168   } else {
169     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
170   }
171   PetscFunctionReturn(0);
172 }
173 
174 #undef __FUNCT__
175 #define __FUNCT__ "VecLoad_Plex"
176 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
177 {
178   DM             dm;
179   PetscBool      ishdf5;
180   PetscErrorCode ierr;
181 
182   PetscFunctionBegin;
183   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
184   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
185   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
186   if (ishdf5) {
187 #if defined(PETSC_HAVE_HDF5)
188     ierr = VecLoad_Plex_HDF5(v, viewer);CHKERRQ(ierr);
189 #else
190     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
191 #endif
192   } else {
193     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
194   }
195   PetscFunctionReturn(0);
196 }
197 
198 #undef __FUNCT__
199 #define __FUNCT__ "DMPlexView_Ascii"
200 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
201 {
202   DM_Plex          *mesh = (DM_Plex*) dm->data;
203   DM                cdm;
204   DMLabel           markers;
205   PetscSection      coordSection;
206   Vec               coordinates;
207   PetscViewerFormat format;
208   PetscErrorCode    ierr;
209 
210   PetscFunctionBegin;
211   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
212   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
213   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
214   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
215   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
216     const char *name;
217     PetscInt    maxConeSize, maxSupportSize;
218     PetscInt    pStart, pEnd, p;
219     PetscMPIInt rank, size;
220 
221     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
222     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
223     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
224     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
225     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
226     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
227     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
228     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
229     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
230     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
231     for (p = pStart; p < pEnd; ++p) {
232       PetscInt dof, off, s;
233 
234       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
235       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
236       for (s = off; s < off+dof; ++s) {
237         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
238       }
239     }
240     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
241     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
242     for (p = pStart; p < pEnd; ++p) {
243       PetscInt dof, off, c;
244 
245       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
246       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
247       for (c = off; c < off+dof; ++c) {
248         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
249       }
250     }
251     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
252     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
253     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
254     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
255     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
256     if (size > 1) {
257       PetscSF sf;
258 
259       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
260       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
261     }
262     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
263   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
264     const char  *name;
265     const char  *colors[3] = {"red", "blue", "green"};
266     const int    numColors  = 3;
267     PetscReal    scale      = 2.0;
268     PetscScalar *coords;
269     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
270     PetscMPIInt  rank, size;
271 
272     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
273     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
274     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
275     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
276     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
277     ierr = PetscViewerASCIIPrintf(viewer, "\
278 \\documentclass[crop,multi=false]{standalone}\n\n\
279 \\usepackage{tikz}\n\
280 \\usepackage{pgflibraryshapes}\n\
281 \\usetikzlibrary{backgrounds}\n\
282 \\usetikzlibrary{arrows}\n\
283 \\begin{document}\n\
284 \\section{%s}\n\
285 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
286     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
287     for (p = 0; p < size; ++p) {
288       if (p > 0 && p == size-1) {
289         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
290       } else if (p > 0) {
291         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
292       }
293       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
294     }
295     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
296 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
297     /* Plot vertices */
298     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
299     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
300     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
301     for (v = vStart; v < vEnd; ++v) {
302       PetscInt off, dof, d;
303 
304       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
305       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
306       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
307       for (d = 0; d < dof; ++d) {
308         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
309         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*PetscRealPart(coords[off+d])));CHKERRQ(ierr);
310       }
311       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
312     }
313     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
314     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
315     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
316     /* Plot edges */
317     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
318     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
319     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
320     for (e = eStart; e < eEnd; ++e) {
321       const PetscInt *cone;
322       PetscInt        coneSize, offA, offB, dof, d;
323 
324       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
325       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
326       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
327       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
328       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
329       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
330       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
331       for (d = 0; d < dof; ++d) {
332         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
333         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d])));CHKERRQ(ierr);
334       }
335       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
336     }
337     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
338     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
339     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
340     /* Plot cells */
341     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
342     for (c = cStart; c < cEnd; ++c) {
343       PetscInt *closure = NULL;
344       PetscInt  closureSize, firstPoint = -1;
345 
346       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
347       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
348       for (p = 0; p < closureSize*2; p += 2) {
349         const PetscInt point = closure[p];
350 
351         if ((point < vStart) || (point >= vEnd)) continue;
352         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
353         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
354         if (firstPoint < 0) firstPoint = point;
355       }
356       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
357       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
358       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
359     }
360     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
361     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
362     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
363   } else {
364     MPI_Comm    comm;
365     PetscInt   *sizes, *hybsizes;
366     PetscInt    locDepth, depth, dim, d, pMax[4];
367     PetscInt    pStart, pEnd, p;
368     PetscInt    numLabels, l;
369     const char *name;
370     PetscMPIInt size;
371 
372     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
373     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
374     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
375     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
376     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dim);CHKERRQ(ierr);}
377     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);}
378     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
379     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
380     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], &pMax[depth-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
381     ierr = PetscMalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
382     if (depth == 1) {
383       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
384       pEnd = pEnd - pStart;
385       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
386       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
387       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
388       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
389       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
390       pEnd = pEnd - pStart;
391       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
392       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
393       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
394       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
395     } else {
396       for (d = 0; d <= dim; d++) {
397         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
398         pEnd    -= pStart;
399         pMax[d] -= pStart;
400         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
401         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
402         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
403         for (p = 0; p < size; ++p) {
404           if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
405           else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
406         }
407         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
408       }
409     }
410     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
411     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
412     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
413     for (l = 0; l < numLabels; ++l) {
414       DMLabel         label;
415       const char     *name;
416       IS              valueIS;
417       const PetscInt *values;
418       PetscInt        numValues, v;
419 
420       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
421       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
422       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
423       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
424       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
425       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
426       for (v = 0; v < numValues; ++v) {
427         PetscInt size;
428 
429         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
430         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
431         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
432       }
433       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
434       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
435       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
436     }
437   }
438   PetscFunctionReturn(0);
439 }
440 
441 #undef __FUNCT__
442 #define __FUNCT__ "DMView_Plex"
443 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
444 {
445   PetscBool      iascii, ishdf5;
446   PetscErrorCode ierr;
447 
448   PetscFunctionBegin;
449   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
450   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
451   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
452   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
453   if (iascii) {
454     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
455   } else if (ishdf5) {
456 #if defined(PETSC_HAVE_HDF5)
457     ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);CHKERRQ(ierr);
458     ierr = DMPlexView_HDF5(dm, viewer);CHKERRQ(ierr);
459     ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
460 #else
461     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
462 #endif
463   }
464   PetscFunctionReturn(0);
465 }
466 
467 #undef __FUNCT__
468 #define __FUNCT__ "DMLoad_Plex"
469 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
470 {
471   PetscBool      isbinary, ishdf5;
472   PetscErrorCode ierr;
473 
474   PetscFunctionBegin;
475   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
476   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
477   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
478   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
479   if (isbinary) {SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Do not yet support binary viewers");}
480   else if (ishdf5) {
481 #if defined(PETSC_HAVE_HDF5)
482     ierr = DMPlexLoad_HDF5(dm, viewer);CHKERRQ(ierr);
483 #else
484     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
485 #endif
486   }
487   PetscFunctionReturn(0);
488 }
489 
490 #undef __FUNCT__
491 #define __FUNCT__ "BoundaryDestroy"
492 static PetscErrorCode BoundaryDestroy(DMBoundary *boundary)
493 {
494   DMBoundary     b, next;
495   PetscErrorCode ierr;
496 
497   PetscFunctionBeginUser;
498   if (!boundary) PetscFunctionReturn(0);
499   b = *boundary;
500   *boundary = NULL;
501   for (; b; b = next) {
502     next = b->next;
503     ierr = PetscFree(b->ids);CHKERRQ(ierr);
504     ierr = PetscFree(b->name);CHKERRQ(ierr);
505     ierr = PetscFree(b);CHKERRQ(ierr);
506   }
507   PetscFunctionReturn(0);
508 }
509 
510 #undef __FUNCT__
511 #define __FUNCT__ "DMDestroy_Plex"
512 PetscErrorCode DMDestroy_Plex(DM dm)
513 {
514   DM_Plex       *mesh = (DM_Plex*) dm->data;
515   DMLabel        next  = mesh->labels;
516   PetscErrorCode ierr;
517 
518   PetscFunctionBegin;
519   if (--mesh->refct > 0) PetscFunctionReturn(0);
520   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
521   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
522   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
523   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
524   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
525   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
526   while (next) {
527     DMLabel tmp = next->next;
528 
529     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
530     next = tmp;
531   }
532   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
533   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
534   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
535   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
536   ierr = BoundaryDestroy(&mesh->boundary);CHKERRQ(ierr);
537   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
538   ierr = PetscFree(mesh);CHKERRQ(ierr);
539   PetscFunctionReturn(0);
540 }
541 
542 #undef __FUNCT__
543 #define __FUNCT__ "DMCreateMatrix_Plex"
544 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
545 {
546   PetscSection   section, sectionGlobal;
547   PetscInt       bs = -1;
548   PetscInt       localSize;
549   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
550   PetscErrorCode ierr;
551   MatType        mtype;
552 
553   PetscFunctionBegin;
554   ierr = MatInitializePackage();CHKERRQ(ierr);
555   mtype = dm->mattype;
556   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
557   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
558   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
559   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
560   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
561   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
562   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
563   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
564   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
565   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
566   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
567   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
568   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
569   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
570   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
571   if (!isShell) {
572     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
573     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
574 
575     if (bs < 0) {
576       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
577         PetscInt pStart, pEnd, p, dof, cdof;
578 
579         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
580         for (p = pStart; p < pEnd; ++p) {
581           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
582           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
583           if (dof-cdof) {
584             if (bs < 0) {
585               bs = dof-cdof;
586             } else if (bs != dof-cdof) {
587               /* Layout does not admit a pointwise block size */
588               bs = 1;
589               break;
590             }
591           }
592         }
593         /* Must have same blocksize on all procs (some might have no points) */
594         bsLocal = bs;
595         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
596         bsLocal = bs < 0 ? bsMax : bs;
597         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
598         if (bsMin != bsMax) {
599           bs = 1;
600         } else {
601           bs = bsMax;
602         }
603       } else {
604         bs = 1;
605       }
606     }
607     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
608     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
609     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
610   }
611   PetscFunctionReturn(0);
612 }
613 
614 #undef __FUNCT__
615 #define __FUNCT__ "DMPlexGetDimension"
616 /*@
617   DMPlexGetDimension - Return the topological mesh dimension
618 
619   Not collective
620 
621   Input Parameter:
622 . mesh - The DMPlex
623 
624   Output Parameter:
625 . dim - The topological mesh dimension
626 
627   Level: beginner
628 
629 .seealso: DMPlexCreate()
630 @*/
631 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
632 {
633   DM_Plex *mesh = (DM_Plex*) dm->data;
634 
635   PetscFunctionBegin;
636   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
637   PetscValidPointer(dim, 2);
638   *dim = mesh->dim;
639   PetscFunctionReturn(0);
640 }
641 
642 #undef __FUNCT__
643 #define __FUNCT__ "DMPlexSetDimension"
644 /*@
645   DMPlexSetDimension - Set the topological mesh dimension
646 
647   Collective on mesh
648 
649   Input Parameters:
650 + mesh - The DMPlex
651 - dim - The topological mesh dimension
652 
653   Level: beginner
654 
655 .seealso: DMPlexCreate()
656 @*/
657 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
658 {
659   DM_Plex *mesh = (DM_Plex*) dm->data;
660 
661   PetscFunctionBegin;
662   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
663   PetscValidLogicalCollectiveInt(dm, dim, 2);
664   mesh->dim = dim;
665   PetscFunctionReturn(0);
666 }
667 
668 #undef __FUNCT__
669 #define __FUNCT__ "DMPlexGetChart"
670 /*@
671   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
672 
673   Not collective
674 
675   Input Parameter:
676 . mesh - The DMPlex
677 
678   Output Parameters:
679 + pStart - The first mesh point
680 - pEnd   - The upper bound for mesh points
681 
682   Level: beginner
683 
684 .seealso: DMPlexCreate(), DMPlexSetChart()
685 @*/
686 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
687 {
688   DM_Plex       *mesh = (DM_Plex*) dm->data;
689   PetscErrorCode ierr;
690 
691   PetscFunctionBegin;
692   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
693   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
694   PetscFunctionReturn(0);
695 }
696 
697 #undef __FUNCT__
698 #define __FUNCT__ "DMPlexSetChart"
699 /*@
700   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
701 
702   Not collective
703 
704   Input Parameters:
705 + mesh - The DMPlex
706 . pStart - The first mesh point
707 - pEnd   - The upper bound for mesh points
708 
709   Output Parameters:
710 
711   Level: beginner
712 
713 .seealso: DMPlexCreate(), DMPlexGetChart()
714 @*/
715 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
716 {
717   DM_Plex       *mesh = (DM_Plex*) dm->data;
718   PetscErrorCode ierr;
719 
720   PetscFunctionBegin;
721   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
722   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
723   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
724   PetscFunctionReturn(0);
725 }
726 
727 #undef __FUNCT__
728 #define __FUNCT__ "DMPlexGetConeSize"
729 /*@
730   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
731 
732   Not collective
733 
734   Input Parameters:
735 + mesh - The DMPlex
736 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
737 
738   Output Parameter:
739 . size - The cone size for point p
740 
741   Level: beginner
742 
743 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
744 @*/
745 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
746 {
747   DM_Plex       *mesh = (DM_Plex*) dm->data;
748   PetscErrorCode ierr;
749 
750   PetscFunctionBegin;
751   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
752   PetscValidPointer(size, 3);
753   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
754   PetscFunctionReturn(0);
755 }
756 
757 #undef __FUNCT__
758 #define __FUNCT__ "DMPlexSetConeSize"
759 /*@
760   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
761 
762   Not collective
763 
764   Input Parameters:
765 + mesh - The DMPlex
766 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
767 - size - The cone size for point p
768 
769   Output Parameter:
770 
771   Note:
772   This should be called after DMPlexSetChart().
773 
774   Level: beginner
775 
776 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
777 @*/
778 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
779 {
780   DM_Plex       *mesh = (DM_Plex*) dm->data;
781   PetscErrorCode ierr;
782 
783   PetscFunctionBegin;
784   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
785   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
786 
787   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
788   PetscFunctionReturn(0);
789 }
790 
791 #undef __FUNCT__
792 #define __FUNCT__ "DMPlexGetCone"
793 /*@C
794   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
795 
796   Not collective
797 
798   Input Parameters:
799 + mesh - The DMPlex
800 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
801 
802   Output Parameter:
803 . cone - An array of points which are on the in-edges for point p
804 
805   Level: beginner
806 
807   Fortran Notes:
808   Since it returns an array, this routine is only available in Fortran 90, and you must
809   include petsc.h90 in your code.
810 
811   You must also call DMPlexRestoreCone() after you finish using the returned array.
812 
813 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
814 @*/
815 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
816 {
817   DM_Plex       *mesh = (DM_Plex*) dm->data;
818   PetscInt       off;
819   PetscErrorCode ierr;
820 
821   PetscFunctionBegin;
822   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
823   PetscValidPointer(cone, 3);
824   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
825   *cone = &mesh->cones[off];
826   PetscFunctionReturn(0);
827 }
828 
829 #undef __FUNCT__
830 #define __FUNCT__ "DMPlexSetCone"
831 /*@
832   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
833 
834   Not collective
835 
836   Input Parameters:
837 + mesh - The DMPlex
838 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
839 - cone - An array of points which are on the in-edges for point p
840 
841   Output Parameter:
842 
843   Note:
844   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
845 
846   Level: beginner
847 
848 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
849 @*/
850 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
851 {
852   DM_Plex       *mesh = (DM_Plex*) dm->data;
853   PetscInt       pStart, pEnd;
854   PetscInt       dof, off, c;
855   PetscErrorCode ierr;
856 
857   PetscFunctionBegin;
858   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
859   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
860   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
861   if (dof) PetscValidPointer(cone, 3);
862   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
863   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
864   for (c = 0; c < dof; ++c) {
865     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
866     mesh->cones[off+c] = cone[c];
867   }
868   PetscFunctionReturn(0);
869 }
870 
871 #undef __FUNCT__
872 #define __FUNCT__ "DMPlexGetConeOrientation"
873 /*@C
874   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
875 
876   Not collective
877 
878   Input Parameters:
879 + mesh - The DMPlex
880 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
881 
882   Output Parameter:
883 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
884                     integer giving the prescription for cone traversal. If it is negative, the cone is
885                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
886                     the index of the cone point on which to start.
887 
888   Level: beginner
889 
890   Fortran Notes:
891   Since it returns an array, this routine is only available in Fortran 90, and you must
892   include petsc.h90 in your code.
893 
894   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
895 
896 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
897 @*/
898 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
899 {
900   DM_Plex       *mesh = (DM_Plex*) dm->data;
901   PetscInt       off;
902   PetscErrorCode ierr;
903 
904   PetscFunctionBegin;
905   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
906 #if defined(PETSC_USE_DEBUG)
907   {
908     PetscInt dof;
909     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
910     if (dof) PetscValidPointer(coneOrientation, 3);
911   }
912 #endif
913   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
914 
915   *coneOrientation = &mesh->coneOrientations[off];
916   PetscFunctionReturn(0);
917 }
918 
919 #undef __FUNCT__
920 #define __FUNCT__ "DMPlexSetConeOrientation"
921 /*@
922   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
923 
924   Not collective
925 
926   Input Parameters:
927 + mesh - The DMPlex
928 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
929 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
930                     integer giving the prescription for cone traversal. If it is negative, the cone is
931                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
932                     the index of the cone point on which to start.
933 
934   Output Parameter:
935 
936   Note:
937   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
938 
939   Level: beginner
940 
941 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
942 @*/
943 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
944 {
945   DM_Plex       *mesh = (DM_Plex*) dm->data;
946   PetscInt       pStart, pEnd;
947   PetscInt       dof, off, c;
948   PetscErrorCode ierr;
949 
950   PetscFunctionBegin;
951   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
952   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
953   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
954   if (dof) PetscValidPointer(coneOrientation, 3);
955   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
956   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
957   for (c = 0; c < dof; ++c) {
958     PetscInt cdof, o = coneOrientation[c];
959 
960     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
961     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
962     mesh->coneOrientations[off+c] = o;
963   }
964   PetscFunctionReturn(0);
965 }
966 
967 #undef __FUNCT__
968 #define __FUNCT__ "DMPlexInsertCone"
969 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
970 {
971   DM_Plex       *mesh = (DM_Plex*) dm->data;
972   PetscInt       pStart, pEnd;
973   PetscInt       dof, off;
974   PetscErrorCode ierr;
975 
976   PetscFunctionBegin;
977   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
978   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
979   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
980   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
981   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
982   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
983   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
984   mesh->cones[off+conePos] = conePoint;
985   PetscFunctionReturn(0);
986 }
987 
988 #undef __FUNCT__
989 #define __FUNCT__ "DMPlexInsertConeOrientation"
990 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
991 {
992   DM_Plex       *mesh = (DM_Plex*) dm->data;
993   PetscInt       pStart, pEnd;
994   PetscInt       dof, off;
995   PetscErrorCode ierr;
996 
997   PetscFunctionBegin;
998   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
999   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1000   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1001   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1002   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1003   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1004   mesh->coneOrientations[off+conePos] = coneOrientation;
1005   PetscFunctionReturn(0);
1006 }
1007 
1008 #undef __FUNCT__
1009 #define __FUNCT__ "DMPlexGetSupportSize"
1010 /*@
1011   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
1012 
1013   Not collective
1014 
1015   Input Parameters:
1016 + mesh - The DMPlex
1017 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1018 
1019   Output Parameter:
1020 . size - The support size for point p
1021 
1022   Level: beginner
1023 
1024 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1025 @*/
1026 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1027 {
1028   DM_Plex       *mesh = (DM_Plex*) dm->data;
1029   PetscErrorCode ierr;
1030 
1031   PetscFunctionBegin;
1032   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1033   PetscValidPointer(size, 3);
1034   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1035   PetscFunctionReturn(0);
1036 }
1037 
1038 #undef __FUNCT__
1039 #define __FUNCT__ "DMPlexSetSupportSize"
1040 /*@
1041   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
1042 
1043   Not collective
1044 
1045   Input Parameters:
1046 + mesh - The DMPlex
1047 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1048 - size - The support size for point p
1049 
1050   Output Parameter:
1051 
1052   Note:
1053   This should be called after DMPlexSetChart().
1054 
1055   Level: beginner
1056 
1057 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1058 @*/
1059 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1060 {
1061   DM_Plex       *mesh = (DM_Plex*) dm->data;
1062   PetscErrorCode ierr;
1063 
1064   PetscFunctionBegin;
1065   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1066   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1067 
1068   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
1069   PetscFunctionReturn(0);
1070 }
1071 
1072 #undef __FUNCT__
1073 #define __FUNCT__ "DMPlexGetSupport"
1074 /*@C
1075   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
1076 
1077   Not collective
1078 
1079   Input Parameters:
1080 + mesh - The DMPlex
1081 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1082 
1083   Output Parameter:
1084 . support - An array of points which are on the out-edges for point p
1085 
1086   Level: beginner
1087 
1088   Fortran Notes:
1089   Since it returns an array, this routine is only available in Fortran 90, and you must
1090   include petsc.h90 in your code.
1091 
1092   You must also call DMPlexRestoreSupport() after you finish using the returned array.
1093 
1094 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1095 @*/
1096 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
1097 {
1098   DM_Plex       *mesh = (DM_Plex*) dm->data;
1099   PetscInt       off;
1100   PetscErrorCode ierr;
1101 
1102   PetscFunctionBegin;
1103   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1104   PetscValidPointer(support, 3);
1105   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1106   *support = &mesh->supports[off];
1107   PetscFunctionReturn(0);
1108 }
1109 
1110 #undef __FUNCT__
1111 #define __FUNCT__ "DMPlexSetSupport"
1112 /*@
1113   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
1114 
1115   Not collective
1116 
1117   Input Parameters:
1118 + mesh - The DMPlex
1119 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1120 - support - An array of points which are on the in-edges for point p
1121 
1122   Output Parameter:
1123 
1124   Note:
1125   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
1126 
1127   Level: beginner
1128 
1129 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
1130 @*/
1131 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
1132 {
1133   DM_Plex       *mesh = (DM_Plex*) dm->data;
1134   PetscInt       pStart, pEnd;
1135   PetscInt       dof, off, c;
1136   PetscErrorCode ierr;
1137 
1138   PetscFunctionBegin;
1139   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1140   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1141   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1142   if (dof) PetscValidPointer(support, 3);
1143   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1144   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1145   for (c = 0; c < dof; ++c) {
1146     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1147     mesh->supports[off+c] = support[c];
1148   }
1149   PetscFunctionReturn(0);
1150 }
1151 
1152 #undef __FUNCT__
1153 #define __FUNCT__ "DMPlexInsertSupport"
1154 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1155 {
1156   DM_Plex       *mesh = (DM_Plex*) dm->data;
1157   PetscInt       pStart, pEnd;
1158   PetscInt       dof, off;
1159   PetscErrorCode ierr;
1160 
1161   PetscFunctionBegin;
1162   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1163   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1164   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1165   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1166   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1167   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1168   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1169   mesh->supports[off+supportPos] = supportPoint;
1170   PetscFunctionReturn(0);
1171 }
1172 
1173 #undef __FUNCT__
1174 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1175 /*@C
1176   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1177 
1178   Not collective
1179 
1180   Input Parameters:
1181 + mesh - The DMPlex
1182 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1183 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1184 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1185 
1186   Output Parameters:
1187 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1188 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1189 
1190   Note:
1191   If using internal storage (points is NULL on input), each call overwrites the last output.
1192 
1193   Fortran Notes:
1194   Since it returns an array, this routine is only available in Fortran 90, and you must
1195   include petsc.h90 in your code.
1196 
1197   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1198 
1199   Level: beginner
1200 
1201 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1202 @*/
1203 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1204 {
1205   DM_Plex        *mesh = (DM_Plex*) dm->data;
1206   PetscInt       *closure, *fifo;
1207   const PetscInt *tmp = NULL, *tmpO = NULL;
1208   PetscInt        tmpSize, t;
1209   PetscInt        depth       = 0, maxSize;
1210   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1211   PetscErrorCode  ierr;
1212 
1213   PetscFunctionBegin;
1214   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1215   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1216   /* This is only 1-level */
1217   if (useCone) {
1218     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1219     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1220     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1221   } else {
1222     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1223     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1224   }
1225   if (depth == 1) {
1226     if (*points) {
1227       closure = *points;
1228     } else {
1229       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1230       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1231     }
1232     closure[0] = p; closure[1] = 0;
1233     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1234       closure[closureSize]   = tmp[t];
1235       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1236     }
1237     if (numPoints) *numPoints = closureSize/2;
1238     if (points)    *points    = closure;
1239     PetscFunctionReturn(0);
1240   }
1241   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1242   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1243   if (*points) {
1244     closure = *points;
1245   } else {
1246     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1247   }
1248   closure[0] = p; closure[1] = 0;
1249   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1250     const PetscInt cp = tmp[t];
1251     const PetscInt co = tmpO ? tmpO[t] : 0;
1252 
1253     closure[closureSize]   = cp;
1254     closure[closureSize+1] = co;
1255     fifo[fifoSize]         = cp;
1256     fifo[fifoSize+1]       = co;
1257   }
1258   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1259   while (fifoSize - fifoStart) {
1260     const PetscInt q   = fifo[fifoStart];
1261     const PetscInt o   = fifo[fifoStart+1];
1262     const PetscInt rev = o >= 0 ? 0 : 1;
1263     const PetscInt off = rev ? -(o+1) : o;
1264 
1265     if (useCone) {
1266       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1267       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1268       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1269     } else {
1270       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1271       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1272       tmpO = NULL;
1273     }
1274     for (t = 0; t < tmpSize; ++t) {
1275       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1276       const PetscInt cp = tmp[i];
1277       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1278       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1279        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1280       PetscInt       co = tmpO ? tmpO[i] : 0;
1281       PetscInt       c;
1282 
1283       if (rev) {
1284         PetscInt childSize, coff;
1285         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1286         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1287         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1288       }
1289       /* Check for duplicate */
1290       for (c = 0; c < closureSize; c += 2) {
1291         if (closure[c] == cp) break;
1292       }
1293       if (c == closureSize) {
1294         closure[closureSize]   = cp;
1295         closure[closureSize+1] = co;
1296         fifo[fifoSize]         = cp;
1297         fifo[fifoSize+1]       = co;
1298         closureSize           += 2;
1299         fifoSize              += 2;
1300       }
1301     }
1302     fifoStart += 2;
1303   }
1304   if (numPoints) *numPoints = closureSize/2;
1305   if (points)    *points    = closure;
1306   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1307   PetscFunctionReturn(0);
1308 }
1309 
1310 #undef __FUNCT__
1311 #define __FUNCT__ "DMPlexGetTransitiveClosure_Internal"
1312 /*@C
1313   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG with a specified initial orientation
1314 
1315   Not collective
1316 
1317   Input Parameters:
1318 + mesh - The DMPlex
1319 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1320 . orientation - The orientation of the point
1321 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1322 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1323 
1324   Output Parameters:
1325 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1326 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1327 
1328   Note:
1329   If using internal storage (points is NULL on input), each call overwrites the last output.
1330 
1331   Fortran Notes:
1332   Since it returns an array, this routine is only available in Fortran 90, and you must
1333   include petsc.h90 in your code.
1334 
1335   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1336 
1337   Level: beginner
1338 
1339 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1340 @*/
1341 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1342 {
1343   DM_Plex        *mesh = (DM_Plex*) dm->data;
1344   PetscInt       *closure, *fifo;
1345   const PetscInt *tmp = NULL, *tmpO = NULL;
1346   PetscInt        tmpSize, t;
1347   PetscInt        depth       = 0, maxSize;
1348   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1349   PetscErrorCode  ierr;
1350 
1351   PetscFunctionBegin;
1352   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1353   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1354   /* This is only 1-level */
1355   if (useCone) {
1356     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1357     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1358     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1359   } else {
1360     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1361     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1362   }
1363   if (depth == 1) {
1364     if (*points) {
1365       closure = *points;
1366     } else {
1367       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1368       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1369     }
1370     closure[0] = p; closure[1] = ornt;
1371     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1372       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1373       closure[closureSize]   = tmp[i];
1374       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
1375     }
1376     if (numPoints) *numPoints = closureSize/2;
1377     if (points)    *points    = closure;
1378     PetscFunctionReturn(0);
1379   }
1380   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1381   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1382   if (*points) {
1383     closure = *points;
1384   } else {
1385     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1386   }
1387   closure[0] = p; closure[1] = ornt;
1388   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1389     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1390     const PetscInt cp = tmp[i];
1391     PetscInt       co = tmpO ? tmpO[i] : 0;
1392 
1393     if (ornt < 0) {
1394       PetscInt childSize, coff;
1395       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1396       coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1397       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1398     }
1399     closure[closureSize]   = cp;
1400     closure[closureSize+1] = co;
1401     fifo[fifoSize]         = cp;
1402     fifo[fifoSize+1]       = co;
1403   }
1404   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1405   while (fifoSize - fifoStart) {
1406     const PetscInt q   = fifo[fifoStart];
1407     const PetscInt o   = fifo[fifoStart+1];
1408     const PetscInt rev = o >= 0 ? 0 : 1;
1409     const PetscInt off = rev ? -(o+1) : o;
1410 
1411     if (useCone) {
1412       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1413       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1414       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1415     } else {
1416       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1417       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1418       tmpO = NULL;
1419     }
1420     for (t = 0; t < tmpSize; ++t) {
1421       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1422       const PetscInt cp = tmp[i];
1423       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1424       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1425        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1426       PetscInt       co = tmpO ? tmpO[i] : 0;
1427       PetscInt       c;
1428 
1429       if (rev) {
1430         PetscInt childSize, coff;
1431         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1432         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1433         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1434       }
1435       /* Check for duplicate */
1436       for (c = 0; c < closureSize; c += 2) {
1437         if (closure[c] == cp) break;
1438       }
1439       if (c == closureSize) {
1440         closure[closureSize]   = cp;
1441         closure[closureSize+1] = co;
1442         fifo[fifoSize]         = cp;
1443         fifo[fifoSize+1]       = co;
1444         closureSize           += 2;
1445         fifoSize              += 2;
1446       }
1447     }
1448     fifoStart += 2;
1449   }
1450   if (numPoints) *numPoints = closureSize/2;
1451   if (points)    *points    = closure;
1452   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1453   PetscFunctionReturn(0);
1454 }
1455 
1456 #undef __FUNCT__
1457 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1458 /*@C
1459   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1460 
1461   Not collective
1462 
1463   Input Parameters:
1464 + mesh - The DMPlex
1465 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1466 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1467 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
1468 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
1469 
1470   Note:
1471   If not using internal storage (points is not NULL on input), this call is unnecessary
1472 
1473   Fortran Notes:
1474   Since it returns an array, this routine is only available in Fortran 90, and you must
1475   include petsc.h90 in your code.
1476 
1477   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1478 
1479   Level: beginner
1480 
1481 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1482 @*/
1483 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1484 {
1485   PetscErrorCode ierr;
1486 
1487   PetscFunctionBegin;
1488   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1489   if (numPoints) PetscValidIntPointer(numPoints,4);
1490   if (points) PetscValidPointer(points,5);
1491   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
1492   if (numPoints) *numPoints = 0;
1493   PetscFunctionReturn(0);
1494 }
1495 
1496 #undef __FUNCT__
1497 #define __FUNCT__ "DMPlexGetMaxSizes"
1498 /*@
1499   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
1500 
1501   Not collective
1502 
1503   Input Parameter:
1504 . mesh - The DMPlex
1505 
1506   Output Parameters:
1507 + maxConeSize - The maximum number of in-edges
1508 - maxSupportSize - The maximum number of out-edges
1509 
1510   Level: beginner
1511 
1512 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1513 @*/
1514 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
1515 {
1516   DM_Plex *mesh = (DM_Plex*) dm->data;
1517 
1518   PetscFunctionBegin;
1519   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1520   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
1521   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
1522   PetscFunctionReturn(0);
1523 }
1524 
1525 #undef __FUNCT__
1526 #define __FUNCT__ "DMSetUp_Plex"
1527 PetscErrorCode DMSetUp_Plex(DM dm)
1528 {
1529   DM_Plex       *mesh = (DM_Plex*) dm->data;
1530   PetscInt       size;
1531   PetscErrorCode ierr;
1532 
1533   PetscFunctionBegin;
1534   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1535   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
1536   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
1537   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
1538   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
1539   if (mesh->maxSupportSize) {
1540     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
1541     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
1542     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
1543   }
1544   PetscFunctionReturn(0);
1545 }
1546 
1547 #undef __FUNCT__
1548 #define __FUNCT__ "DMCreateSubDM_Plex"
1549 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
1550 {
1551   PetscErrorCode ierr;
1552 
1553   PetscFunctionBegin;
1554   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
1555   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
1556   PetscFunctionReturn(0);
1557 }
1558 
1559 #undef __FUNCT__
1560 #define __FUNCT__ "DMPlexSymmetrize"
1561 /*@
1562   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
1563 
1564   Not collective
1565 
1566   Input Parameter:
1567 . mesh - The DMPlex
1568 
1569   Output Parameter:
1570 
1571   Note:
1572   This should be called after all calls to DMPlexSetCone()
1573 
1574   Level: beginner
1575 
1576 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
1577 @*/
1578 PetscErrorCode DMPlexSymmetrize(DM dm)
1579 {
1580   DM_Plex       *mesh = (DM_Plex*) dm->data;
1581   PetscInt      *offsets;
1582   PetscInt       supportSize;
1583   PetscInt       pStart, pEnd, p;
1584   PetscErrorCode ierr;
1585 
1586   PetscFunctionBegin;
1587   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1588   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
1589   /* Calculate support sizes */
1590   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
1591   for (p = pStart; p < pEnd; ++p) {
1592     PetscInt dof, off, c;
1593 
1594     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1595     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1596     for (c = off; c < off+dof; ++c) {
1597       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
1598     }
1599   }
1600   for (p = pStart; p < pEnd; ++p) {
1601     PetscInt dof;
1602 
1603     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1604 
1605     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
1606   }
1607   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
1608   /* Calculate supports */
1609   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
1610   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
1611   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
1612   for (p = pStart; p < pEnd; ++p) {
1613     PetscInt dof, off, c;
1614 
1615     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1616     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1617     for (c = off; c < off+dof; ++c) {
1618       const PetscInt q = mesh->cones[c];
1619       PetscInt       offS;
1620 
1621       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
1622 
1623       mesh->supports[offS+offsets[q]] = p;
1624       ++offsets[q];
1625     }
1626   }
1627   ierr = PetscFree(offsets);CHKERRQ(ierr);
1628   PetscFunctionReturn(0);
1629 }
1630 
1631 #undef __FUNCT__
1632 #define __FUNCT__ "DMPlexStratify"
1633 /*@
1634   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
1635   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
1636   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
1637   the DAG.
1638 
1639   Not collective
1640 
1641   Input Parameter:
1642 . mesh - The DMPlex
1643 
1644   Output Parameter:
1645 
1646   Notes:
1647   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
1648   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
1649 
1650   This should be called after all calls to DMPlexSymmetrize()
1651 
1652   Level: beginner
1653 
1654 .seealso: DMPlexCreate(), DMPlexSymmetrize()
1655 @*/
1656 PetscErrorCode DMPlexStratify(DM dm)
1657 {
1658   DMLabel        label;
1659   PetscInt       pStart, pEnd, p;
1660   PetscInt       numRoots = 0, numLeaves = 0;
1661   PetscErrorCode ierr;
1662 
1663   PetscFunctionBegin;
1664   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1665   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
1666   /* Calculate depth */
1667   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
1668   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
1669   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
1670   /* Initialize roots and count leaves */
1671   for (p = pStart; p < pEnd; ++p) {
1672     PetscInt coneSize, supportSize;
1673 
1674     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1675     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
1676     if (!coneSize && supportSize) {
1677       ++numRoots;
1678       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
1679     } else if (!supportSize && coneSize) {
1680       ++numLeaves;
1681     } else if (!supportSize && !coneSize) {
1682       /* Isolated points */
1683       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
1684     }
1685   }
1686   if (numRoots + numLeaves == (pEnd - pStart)) {
1687     for (p = pStart; p < pEnd; ++p) {
1688       PetscInt coneSize, supportSize;
1689 
1690       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1691       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
1692       if (!supportSize && coneSize) {
1693         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
1694       }
1695     }
1696   } else {
1697     IS       pointIS;
1698     PetscInt numPoints = 0, level = 0;
1699 
1700     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
1701     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
1702     while (numPoints) {
1703       const PetscInt *points;
1704       const PetscInt  newLevel = level+1;
1705 
1706       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
1707       for (p = 0; p < numPoints; ++p) {
1708         const PetscInt  point = points[p];
1709         const PetscInt *support;
1710         PetscInt        supportSize, s;
1711 
1712         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1713         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1714         for (s = 0; s < supportSize; ++s) {
1715           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
1716         }
1717       }
1718       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
1719       ++level;
1720       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
1721       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
1722       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
1723       else         {numPoints = 0;}
1724     }
1725     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
1726   }
1727   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
1728   PetscFunctionReturn(0);
1729 }
1730 
1731 #undef __FUNCT__
1732 #define __FUNCT__ "DMPlexGetJoin"
1733 /*@C
1734   DMPlexGetJoin - Get an array for the join of the set of points
1735 
1736   Not Collective
1737 
1738   Input Parameters:
1739 + dm - The DMPlex object
1740 . numPoints - The number of input points for the join
1741 - points - The input points
1742 
1743   Output Parameters:
1744 + numCoveredPoints - The number of points in the join
1745 - coveredPoints - The points in the join
1746 
1747   Level: intermediate
1748 
1749   Note: Currently, this is restricted to a single level join
1750 
1751   Fortran Notes:
1752   Since it returns an array, this routine is only available in Fortran 90, and you must
1753   include petsc.h90 in your code.
1754 
1755   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1756 
1757 .keywords: mesh
1758 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
1759 @*/
1760 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1761 {
1762   DM_Plex       *mesh = (DM_Plex*) dm->data;
1763   PetscInt      *join[2];
1764   PetscInt       joinSize, i = 0;
1765   PetscInt       dof, off, p, c, m;
1766   PetscErrorCode ierr;
1767 
1768   PetscFunctionBegin;
1769   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1770   PetscValidPointer(points, 2);
1771   PetscValidPointer(numCoveredPoints, 3);
1772   PetscValidPointer(coveredPoints, 4);
1773   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
1774   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
1775   /* Copy in support of first point */
1776   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
1777   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
1778   for (joinSize = 0; joinSize < dof; ++joinSize) {
1779     join[i][joinSize] = mesh->supports[off+joinSize];
1780   }
1781   /* Check each successive support */
1782   for (p = 1; p < numPoints; ++p) {
1783     PetscInt newJoinSize = 0;
1784 
1785     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
1786     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
1787     for (c = 0; c < dof; ++c) {
1788       const PetscInt point = mesh->supports[off+c];
1789 
1790       for (m = 0; m < joinSize; ++m) {
1791         if (point == join[i][m]) {
1792           join[1-i][newJoinSize++] = point;
1793           break;
1794         }
1795       }
1796     }
1797     joinSize = newJoinSize;
1798     i        = 1-i;
1799   }
1800   *numCoveredPoints = joinSize;
1801   *coveredPoints    = join[i];
1802   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
1803   PetscFunctionReturn(0);
1804 }
1805 
1806 #undef __FUNCT__
1807 #define __FUNCT__ "DMPlexRestoreJoin"
1808 /*@C
1809   DMPlexRestoreJoin - Restore an array for the join of the set of points
1810 
1811   Not Collective
1812 
1813   Input Parameters:
1814 + dm - The DMPlex object
1815 . numPoints - The number of input points for the join
1816 - points - The input points
1817 
1818   Output Parameters:
1819 + numCoveredPoints - The number of points in the join
1820 - coveredPoints - The points in the join
1821 
1822   Fortran Notes:
1823   Since it returns an array, this routine is only available in Fortran 90, and you must
1824   include petsc.h90 in your code.
1825 
1826   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1827 
1828   Level: intermediate
1829 
1830 .keywords: mesh
1831 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
1832 @*/
1833 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1834 {
1835   PetscErrorCode ierr;
1836 
1837   PetscFunctionBegin;
1838   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1839   if (points) PetscValidIntPointer(points,3);
1840   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
1841   PetscValidPointer(coveredPoints, 5);
1842   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
1843   if (numCoveredPoints) *numCoveredPoints = 0;
1844   PetscFunctionReturn(0);
1845 }
1846 
1847 #undef __FUNCT__
1848 #define __FUNCT__ "DMPlexGetFullJoin"
1849 /*@C
1850   DMPlexGetFullJoin - Get an array for the join of the set of points
1851 
1852   Not Collective
1853 
1854   Input Parameters:
1855 + dm - The DMPlex object
1856 . numPoints - The number of input points for the join
1857 - points - The input points
1858 
1859   Output Parameters:
1860 + numCoveredPoints - The number of points in the join
1861 - coveredPoints - The points in the join
1862 
1863   Fortran Notes:
1864   Since it returns an array, this routine is only available in Fortran 90, and you must
1865   include petsc.h90 in your code.
1866 
1867   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1868 
1869   Level: intermediate
1870 
1871 .keywords: mesh
1872 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
1873 @*/
1874 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1875 {
1876   DM_Plex       *mesh = (DM_Plex*) dm->data;
1877   PetscInt      *offsets, **closures;
1878   PetscInt      *join[2];
1879   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
1880   PetscInt       p, d, c, m;
1881   PetscErrorCode ierr;
1882 
1883   PetscFunctionBegin;
1884   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1885   PetscValidPointer(points, 2);
1886   PetscValidPointer(numCoveredPoints, 3);
1887   PetscValidPointer(coveredPoints, 4);
1888 
1889   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1890   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
1891   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1892   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
1893   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
1894   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
1895 
1896   for (p = 0; p < numPoints; ++p) {
1897     PetscInt closureSize;
1898 
1899     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
1900 
1901     offsets[p*(depth+2)+0] = 0;
1902     for (d = 0; d < depth+1; ++d) {
1903       PetscInt pStart, pEnd, i;
1904 
1905       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1906       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
1907         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
1908           offsets[p*(depth+2)+d+1] = i;
1909           break;
1910         }
1911       }
1912       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
1913     }
1914     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
1915   }
1916   for (d = 0; d < depth+1; ++d) {
1917     PetscInt dof;
1918 
1919     /* Copy in support of first point */
1920     dof = offsets[d+1] - offsets[d];
1921     for (joinSize = 0; joinSize < dof; ++joinSize) {
1922       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
1923     }
1924     /* Check each successive cone */
1925     for (p = 1; p < numPoints && joinSize; ++p) {
1926       PetscInt newJoinSize = 0;
1927 
1928       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
1929       for (c = 0; c < dof; ++c) {
1930         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
1931 
1932         for (m = 0; m < joinSize; ++m) {
1933           if (point == join[i][m]) {
1934             join[1-i][newJoinSize++] = point;
1935             break;
1936           }
1937         }
1938       }
1939       joinSize = newJoinSize;
1940       i        = 1-i;
1941     }
1942     if (joinSize) break;
1943   }
1944   *numCoveredPoints = joinSize;
1945   *coveredPoints    = join[i];
1946   for (p = 0; p < numPoints; ++p) {
1947     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
1948   }
1949   ierr = PetscFree(closures);CHKERRQ(ierr);
1950   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1951   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
1952   PetscFunctionReturn(0);
1953 }
1954 
1955 #undef __FUNCT__
1956 #define __FUNCT__ "DMPlexGetMeet"
1957 /*@C
1958   DMPlexGetMeet - Get an array for the meet of the set of points
1959 
1960   Not Collective
1961 
1962   Input Parameters:
1963 + dm - The DMPlex object
1964 . numPoints - The number of input points for the meet
1965 - points - The input points
1966 
1967   Output Parameters:
1968 + numCoveredPoints - The number of points in the meet
1969 - coveredPoints - The points in the meet
1970 
1971   Level: intermediate
1972 
1973   Note: Currently, this is restricted to a single level meet
1974 
1975   Fortran Notes:
1976   Since it returns an array, this routine is only available in Fortran 90, and you must
1977   include petsc.h90 in your code.
1978 
1979   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1980 
1981 .keywords: mesh
1982 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
1983 @*/
1984 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
1985 {
1986   DM_Plex       *mesh = (DM_Plex*) dm->data;
1987   PetscInt      *meet[2];
1988   PetscInt       meetSize, i = 0;
1989   PetscInt       dof, off, p, c, m;
1990   PetscErrorCode ierr;
1991 
1992   PetscFunctionBegin;
1993   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1994   PetscValidPointer(points, 2);
1995   PetscValidPointer(numCoveringPoints, 3);
1996   PetscValidPointer(coveringPoints, 4);
1997   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
1998   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
1999   /* Copy in cone of first point */
2000   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
2001   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
2002   for (meetSize = 0; meetSize < dof; ++meetSize) {
2003     meet[i][meetSize] = mesh->cones[off+meetSize];
2004   }
2005   /* Check each successive cone */
2006   for (p = 1; p < numPoints; ++p) {
2007     PetscInt newMeetSize = 0;
2008 
2009     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
2010     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
2011     for (c = 0; c < dof; ++c) {
2012       const PetscInt point = mesh->cones[off+c];
2013 
2014       for (m = 0; m < meetSize; ++m) {
2015         if (point == meet[i][m]) {
2016           meet[1-i][newMeetSize++] = point;
2017           break;
2018         }
2019       }
2020     }
2021     meetSize = newMeetSize;
2022     i        = 1-i;
2023   }
2024   *numCoveringPoints = meetSize;
2025   *coveringPoints    = meet[i];
2026   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2027   PetscFunctionReturn(0);
2028 }
2029 
2030 #undef __FUNCT__
2031 #define __FUNCT__ "DMPlexRestoreMeet"
2032 /*@C
2033   DMPlexRestoreMeet - Restore an array for the meet of the set of points
2034 
2035   Not Collective
2036 
2037   Input Parameters:
2038 + dm - The DMPlex object
2039 . numPoints - The number of input points for the meet
2040 - points - The input points
2041 
2042   Output Parameters:
2043 + numCoveredPoints - The number of points in the meet
2044 - coveredPoints - The points in the meet
2045 
2046   Level: intermediate
2047 
2048   Fortran Notes:
2049   Since it returns an array, this routine is only available in Fortran 90, and you must
2050   include petsc.h90 in your code.
2051 
2052   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2053 
2054 .keywords: mesh
2055 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
2056 @*/
2057 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2058 {
2059   PetscErrorCode ierr;
2060 
2061   PetscFunctionBegin;
2062   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2063   if (points) PetscValidIntPointer(points,3);
2064   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2065   PetscValidPointer(coveredPoints,5);
2066   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2067   if (numCoveredPoints) *numCoveredPoints = 0;
2068   PetscFunctionReturn(0);
2069 }
2070 
2071 #undef __FUNCT__
2072 #define __FUNCT__ "DMPlexGetFullMeet"
2073 /*@C
2074   DMPlexGetFullMeet - Get an array for the meet of the set of points
2075 
2076   Not Collective
2077 
2078   Input Parameters:
2079 + dm - The DMPlex object
2080 . numPoints - The number of input points for the meet
2081 - points - The input points
2082 
2083   Output Parameters:
2084 + numCoveredPoints - The number of points in the meet
2085 - coveredPoints - The points in the meet
2086 
2087   Level: intermediate
2088 
2089   Fortran Notes:
2090   Since it returns an array, this routine is only available in Fortran 90, and you must
2091   include petsc.h90 in your code.
2092 
2093   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2094 
2095 .keywords: mesh
2096 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
2097 @*/
2098 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2099 {
2100   DM_Plex       *mesh = (DM_Plex*) dm->data;
2101   PetscInt      *offsets, **closures;
2102   PetscInt      *meet[2];
2103   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
2104   PetscInt       p, h, c, m;
2105   PetscErrorCode ierr;
2106 
2107   PetscFunctionBegin;
2108   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2109   PetscValidPointer(points, 2);
2110   PetscValidPointer(numCoveredPoints, 3);
2111   PetscValidPointer(coveredPoints, 4);
2112 
2113   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
2114   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
2115   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2116   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
2117   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2118   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2119 
2120   for (p = 0; p < numPoints; ++p) {
2121     PetscInt closureSize;
2122 
2123     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
2124 
2125     offsets[p*(height+2)+0] = 0;
2126     for (h = 0; h < height+1; ++h) {
2127       PetscInt pStart, pEnd, i;
2128 
2129       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
2130       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
2131         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2132           offsets[p*(height+2)+h+1] = i;
2133           break;
2134         }
2135       }
2136       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
2137     }
2138     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
2139   }
2140   for (h = 0; h < height+1; ++h) {
2141     PetscInt dof;
2142 
2143     /* Copy in cone of first point */
2144     dof = offsets[h+1] - offsets[h];
2145     for (meetSize = 0; meetSize < dof; ++meetSize) {
2146       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
2147     }
2148     /* Check each successive cone */
2149     for (p = 1; p < numPoints && meetSize; ++p) {
2150       PetscInt newMeetSize = 0;
2151 
2152       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
2153       for (c = 0; c < dof; ++c) {
2154         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
2155 
2156         for (m = 0; m < meetSize; ++m) {
2157           if (point == meet[i][m]) {
2158             meet[1-i][newMeetSize++] = point;
2159             break;
2160           }
2161         }
2162       }
2163       meetSize = newMeetSize;
2164       i        = 1-i;
2165     }
2166     if (meetSize) break;
2167   }
2168   *numCoveredPoints = meetSize;
2169   *coveredPoints    = meet[i];
2170   for (p = 0; p < numPoints; ++p) {
2171     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
2172   }
2173   ierr = PetscFree(closures);CHKERRQ(ierr);
2174   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2175   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2176   PetscFunctionReturn(0);
2177 }
2178 
2179 #undef __FUNCT__
2180 #define __FUNCT__ "DMPlexEqual"
2181 /*@C
2182   DMPlexEqual - Determine if two DMs have the same topology
2183 
2184   Not Collective
2185 
2186   Input Parameters:
2187 + dmA - A DMPlex object
2188 - dmB - A DMPlex object
2189 
2190   Output Parameters:
2191 . equal - PETSC_TRUE if the topologies are identical
2192 
2193   Level: intermediate
2194 
2195   Notes:
2196   We are not solving graph isomorphism, so we do not permutation.
2197 
2198 .keywords: mesh
2199 .seealso: DMPlexGetCone()
2200 @*/
2201 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
2202 {
2203   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
2204   PetscErrorCode ierr;
2205 
2206   PetscFunctionBegin;
2207   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
2208   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
2209   PetscValidPointer(equal, 3);
2210 
2211   *equal = PETSC_FALSE;
2212   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
2213   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
2214   if (depth != depthB) PetscFunctionReturn(0);
2215   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
2216   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
2217   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
2218   for (p = pStart; p < pEnd; ++p) {
2219     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
2220     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
2221 
2222     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
2223     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
2224     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
2225     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
2226     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
2227     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
2228     if (coneSize != coneSizeB) PetscFunctionReturn(0);
2229     for (c = 0; c < coneSize; ++c) {
2230       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
2231       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
2232     }
2233     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
2234     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
2235     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
2236     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
2237     if (supportSize != supportSizeB) PetscFunctionReturn(0);
2238     for (s = 0; s < supportSize; ++s) {
2239       if (support[s] != supportB[s]) PetscFunctionReturn(0);
2240     }
2241   }
2242   *equal = PETSC_TRUE;
2243   PetscFunctionReturn(0);
2244 }
2245 
2246 #undef __FUNCT__
2247 #define __FUNCT__ "DMPlexGetNumFaceVertices"
2248 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
2249 {
2250   MPI_Comm       comm;
2251   PetscErrorCode ierr;
2252 
2253   PetscFunctionBegin;
2254   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2255   PetscValidPointer(numFaceVertices,3);
2256   switch (cellDim) {
2257   case 0:
2258     *numFaceVertices = 0;
2259     break;
2260   case 1:
2261     *numFaceVertices = 1;
2262     break;
2263   case 2:
2264     switch (numCorners) {
2265     case 3: /* triangle */
2266       *numFaceVertices = 2; /* Edge has 2 vertices */
2267       break;
2268     case 4: /* quadrilateral */
2269       *numFaceVertices = 2; /* Edge has 2 vertices */
2270       break;
2271     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
2272       *numFaceVertices = 3; /* Edge has 3 vertices */
2273       break;
2274     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
2275       *numFaceVertices = 3; /* Edge has 3 vertices */
2276       break;
2277     default:
2278       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2279     }
2280     break;
2281   case 3:
2282     switch (numCorners) {
2283     case 4: /* tetradehdron */
2284       *numFaceVertices = 3; /* Face has 3 vertices */
2285       break;
2286     case 6: /* tet cohesive cells */
2287       *numFaceVertices = 4; /* Face has 4 vertices */
2288       break;
2289     case 8: /* hexahedron */
2290       *numFaceVertices = 4; /* Face has 4 vertices */
2291       break;
2292     case 9: /* tet cohesive Lagrange cells */
2293       *numFaceVertices = 6; /* Face has 6 vertices */
2294       break;
2295     case 10: /* quadratic tetrahedron */
2296       *numFaceVertices = 6; /* Face has 6 vertices */
2297       break;
2298     case 12: /* hex cohesive Lagrange cells */
2299       *numFaceVertices = 6; /* Face has 6 vertices */
2300       break;
2301     case 18: /* quadratic tet cohesive Lagrange cells */
2302       *numFaceVertices = 6; /* Face has 6 vertices */
2303       break;
2304     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
2305       *numFaceVertices = 9; /* Face has 9 vertices */
2306       break;
2307     default:
2308       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2309     }
2310     break;
2311   default:
2312     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
2313   }
2314   PetscFunctionReturn(0);
2315 }
2316 
2317 #undef __FUNCT__
2318 #define __FUNCT__ "DMPlexOrient"
2319 /* Trys to give the mesh a consistent orientation */
2320 PetscErrorCode DMPlexOrient(DM dm)
2321 {
2322   PetscBT        seenCells, flippedCells, seenFaces;
2323   PetscInt      *faceFIFO, fTop, fBottom;
2324   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
2325   PetscErrorCode ierr;
2326 
2327   PetscFunctionBegin;
2328   /* Truth Table
2329      mismatch    flips   do action   mismatch   flipA ^ flipB   action
2330          F       0 flips     no         F             F           F
2331          F       1 flip      yes        F             T           T
2332          F       2 flips     no         T             F           T
2333          T       0 flips     yes        T             T           F
2334          T       1 flip      no
2335          T       2 flips     yes
2336   */
2337   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2338   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
2339   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
2340   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
2341   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
2342   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
2343   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
2344   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
2345   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
2346   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
2347   ierr = PetscMalloc1((fEnd - fStart), &faceFIFO);CHKERRQ(ierr);
2348   fTop = fBottom = 0;
2349   /* Initialize FIFO with first cell */
2350   if (cEnd > cStart) {
2351     const PetscInt *cone;
2352     PetscInt        coneSize;
2353 
2354     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
2355     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2356     for (c = 0; c < coneSize; ++c) {
2357       faceFIFO[fBottom++] = cone[c];
2358       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2359     }
2360   }
2361   /* Consider each face in FIFO */
2362   while (fTop < fBottom) {
2363     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2364     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2365     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2366 
2367     face = faceFIFO[fTop++];
2368     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2369     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2370     if (supportSize < 2) continue;
2371     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2372     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2373     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2374     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2375     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2376 
2377     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2378     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2379     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2380     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2381     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2382     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2383     for (c = 0; c < coneSizeA; ++c) {
2384       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2385         faceFIFO[fBottom++] = coneA[c];
2386         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2387       }
2388       if (coneA[c] == face) posA = c;
2389       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2390     }
2391     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2392     for (c = 0; c < coneSizeB; ++c) {
2393       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2394         faceFIFO[fBottom++] = coneB[c];
2395         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2396       }
2397       if (coneB[c] == face) posB = c;
2398       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2399     }
2400     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2401 
2402     if (dim == 1) {
2403       mismatch = posA == posB;
2404     } else {
2405       mismatch = coneOA[posA] == coneOB[posB];
2406     }
2407 
2408     if (mismatch ^ (flippedA ^ flippedB)) {
2409       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2410       if (!seenA && !flippedA) {
2411         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2412       } else if (!seenB && !flippedB) {
2413         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2414       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2415     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2416     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2417     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2418   }
2419   /* Now all subdomains are oriented, but we need a consistent parallel orientation */
2420   {
2421     /* Find a representative face (edge) separating pairs of procs */
2422     PetscSF            sf;
2423     const PetscInt    *lpoints;
2424     const PetscSFNode *rpoints;
2425     PetscInt          *neighbors, *nranks;
2426     PetscInt           numLeaves, numRoots, numNeighbors = 0, l, n;
2427 
2428     ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2429     ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
2430     if (numLeaves >= 0) {
2431       const PetscInt *cone, *ornt, *support;
2432       PetscInt        coneSize, supportSize;
2433       int            *rornt, *lornt; /* PetscSF cannot handle smaller than int */
2434       PetscBool      *match, flipped = PETSC_FALSE;
2435 
2436       ierr = PetscMalloc1(numLeaves,&neighbors);CHKERRQ(ierr);
2437       /* I know this is p^2 time in general, but for bounded degree its alright */
2438       for (l = 0; l < numLeaves; ++l) {
2439         const PetscInt face = lpoints[l];
2440         if ((face >= fStart) && (face < fEnd)) {
2441           const PetscInt rank = rpoints[l].rank;
2442           for (n = 0; n < numNeighbors; ++n) if (rank == rpoints[neighbors[n]].rank) break;
2443           if (n >= numNeighbors) {
2444             PetscInt supportSize;
2445             ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2446             if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
2447             neighbors[numNeighbors++] = l;
2448           }
2449         }
2450       }
2451       ierr = PetscCalloc4(numNeighbors,&match,numNeighbors,&nranks,numRoots,&rornt,numRoots,&lornt);CHKERRQ(ierr);
2452       for (face = fStart; face < fEnd; ++face) {
2453         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2454         if (supportSize != 1) continue;
2455         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2456 
2457         ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
2458         ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
2459         ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
2460         for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
2461         if (dim == 1) {
2462           /* Use cone position instead, shifted to -1 or 1 */
2463           rornt[face] = c*2-1;
2464         } else {
2465           if (PetscBTLookup(flippedCells, support[0]-cStart)) rornt[face] = ornt[c] < 0 ? -1 :  1;
2466           else                                                rornt[face] = ornt[c] < 0 ?  1 : -1;
2467         }
2468       }
2469       /* Mark each edge with match or nomatch */
2470       ierr = PetscSFBcastBegin(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2471       ierr = PetscSFBcastEnd(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2472       for (n = 0; n < numNeighbors; ++n) {
2473         const PetscInt face = lpoints[neighbors[n]];
2474 
2475         if (rornt[face]*lornt[face] < 0) match[n] = PETSC_TRUE;
2476         else                             match[n] = PETSC_FALSE;
2477         nranks[n] = rpoints[neighbors[n]].rank;
2478       }
2479       /* Collect the graph on 0 */
2480       {
2481         MPI_Comm     comm = PetscObjectComm((PetscObject) sf);
2482         PetscBT      seenProcs, flippedProcs;
2483         PetscInt    *procFIFO, pTop, pBottom;
2484         PetscInt    *adj = NULL;
2485         PetscBool   *val = NULL;
2486         PetscMPIInt *recvcounts = NULL, *displs = NULL, p;
2487         PetscMPIInt  N = numNeighbors, numProcs = 0, rank;
2488         PetscInt     debug = 0;
2489 
2490         ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2491         if (!rank) {ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);}
2492         ierr = PetscCalloc2(numProcs,&recvcounts,numProcs+1,&displs);CHKERRQ(ierr);
2493         ierr = MPI_Gather(&N, 1, MPI_INT, recvcounts, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
2494         for (p = 0; p < numProcs; ++p) {
2495           displs[p+1] = displs[p] + recvcounts[p];
2496         }
2497         if (!rank) {ierr = PetscMalloc2(displs[numProcs],&adj,displs[numProcs],&val);CHKERRQ(ierr);}
2498         ierr = MPI_Gatherv(nranks, numNeighbors, MPIU_INT, adj, recvcounts, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
2499         ierr = MPI_Gatherv(match, numNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
2500         if (debug) {
2501           for (p = 0; p < numProcs; ++p) {
2502             ierr = PetscPrintf(comm, "Proc %d:\n", p);
2503             for (n = 0; n < recvcounts[p]; ++n) {
2504               ierr = PetscPrintf(comm, "  edge %d (%d):\n", adj[displs[p]+n], val[displs[p]+n]);
2505             }
2506           }
2507         }
2508         ierr = PetscBTCreate(numProcs, &seenProcs);CHKERRQ(ierr);
2509         ierr = PetscBTMemzero(numProcs, seenProcs);CHKERRQ(ierr);
2510         ierr = PetscBTCreate(numProcs, &flippedProcs);CHKERRQ(ierr);
2511         ierr = PetscBTMemzero(numProcs, flippedProcs);CHKERRQ(ierr);
2512         ierr = PetscMalloc1(numProcs,&procFIFO);CHKERRQ(ierr);
2513         pTop = pBottom = 0;
2514         for (p = 0; p < numProcs; ++p) {
2515           if (PetscBTLookup(seenProcs, p)) continue;
2516           /* Initialize FIFO with next proc */
2517           procFIFO[pBottom++] = p;
2518           ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
2519           /* Consider each proc in FIFO */
2520           while (pTop < pBottom) {
2521             PetscInt proc, nproc, seen, flippedA, flippedB, mismatch;
2522 
2523             proc     = procFIFO[pTop++];
2524             flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
2525             /* Loop over neighboring procs */
2526             for (n = 0; n < recvcounts[proc]; ++n) {
2527               nproc    = adj[displs[proc]+n];
2528               mismatch = val[displs[proc]+n] ? 0 : 1;
2529               seen     = PetscBTLookup(seenProcs, nproc);
2530               flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
2531 
2532               if (mismatch ^ (flippedA ^ flippedB)) {
2533                 if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
2534                 if (!flippedB) {
2535                   ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
2536               } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2537               } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2538               if (!seen) {
2539                 procFIFO[pBottom++] = nproc;
2540                 ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
2541               }
2542             }
2543           }
2544         }
2545         ierr = PetscFree(procFIFO);CHKERRQ(ierr);
2546 
2547         ierr = PetscFree2(recvcounts,displs);CHKERRQ(ierr);
2548         ierr = PetscFree2(adj,val);CHKERRQ(ierr);
2549         {
2550           PetscBool *flips;
2551 
2552           ierr = PetscMalloc1(numProcs,&flips);CHKERRQ(ierr);
2553           for (p = 0; p < numProcs; ++p) {
2554             flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
2555             if (debug && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc %d:\n", p);}
2556           }
2557           ierr = MPI_Scatter(flips, 1, MPIU_BOOL, &flipped, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
2558           ierr = PetscFree(flips);CHKERRQ(ierr);
2559         }
2560         ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
2561         ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);
2562       }
2563       ierr = PetscFree4(match,nranks,rornt,lornt);CHKERRQ(ierr);
2564       ierr = PetscFree(neighbors);CHKERRQ(ierr);
2565       if (flipped) {for (c = cStart; c < cEnd; ++c) {ierr = PetscBTNegate(flippedCells, c-cStart);CHKERRQ(ierr);}}
2566     }
2567   }
2568   /* Reverse flipped cells in the mesh */
2569   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2570   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
2571   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
2572   for (c = cStart; c < cEnd; ++c) {
2573     const PetscInt *cone, *coneO, *support;
2574     PetscInt        coneSize, supportSize, faceSize, cp, sp;
2575 
2576     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
2577     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2578     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2579     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
2580     for (cp = 0; cp < coneSize; ++cp) {
2581       const PetscInt rcp = coneSize-cp-1;
2582 
2583       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
2584       revcone[cp]  = cone[rcp];
2585       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
2586     }
2587     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
2588     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
2589     /* Reverse orientations of support */
2590     faceSize = coneSize;
2591     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
2592     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
2593     for (sp = 0; sp < supportSize; ++sp) {
2594       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
2595       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
2596       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
2597       for (cp = 0; cp < coneSize; ++cp) {
2598         if (cone[cp] != c) continue;
2599         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
2600       }
2601     }
2602   }
2603   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
2604   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
2605   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
2606   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
2607   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
2608   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
2609   PetscFunctionReturn(0);
2610 }
2611 
2612 #undef __FUNCT__
2613 #define __FUNCT__ "DMPlexInvertCell_Internal"
2614 PetscErrorCode DMPlexInvertCell_Internal(PetscInt dim, PetscInt numCorners, PetscInt cone[])
2615 {
2616   int tmpc;
2617 
2618   PetscFunctionBegin;
2619   if (dim != 3) PetscFunctionReturn(0);
2620   switch (numCorners) {
2621   case 4:
2622     tmpc    = cone[0];
2623     cone[0] = cone[1];
2624     cone[1] = tmpc;
2625     break;
2626   case 8:
2627     tmpc    = cone[1];
2628     cone[1] = cone[3];
2629     cone[3] = tmpc;
2630     break;
2631   default: break;
2632   }
2633   PetscFunctionReturn(0);
2634 }
2635 
2636 #undef __FUNCT__
2637 #define __FUNCT__ "DMPlexInvertCell"
2638 /*@C
2639   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
2640 
2641   Input Parameters:
2642 + numCorners - The number of vertices in a cell
2643 - cone - The incoming cone
2644 
2645   Output Parameter:
2646 . cone - The inverted cone (in-place)
2647 
2648   Level: developer
2649 
2650 .seealso: DMPlexGenerate()
2651 @*/
2652 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
2653 {
2654   int tmpc;
2655 
2656   PetscFunctionBegin;
2657   if (dim != 3) PetscFunctionReturn(0);
2658   switch (numCorners) {
2659   case 4:
2660     tmpc    = cone[0];
2661     cone[0] = cone[1];
2662     cone[1] = tmpc;
2663     break;
2664   case 8:
2665     tmpc    = cone[1];
2666     cone[1] = cone[3];
2667     cone[3] = tmpc;
2668     break;
2669   default: break;
2670   }
2671   PetscFunctionReturn(0);
2672 }
2673 
2674 #undef __FUNCT__
2675 #define __FUNCT__ "DMPlexInvertCells_Internal"
2676 /* This is to fix the tetrahedron orientation from TetGen */
2677 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
2678 {
2679   PetscInt       bound = numCells*numCorners, coff;
2680   PetscErrorCode ierr;
2681 
2682   PetscFunctionBegin;
2683   for (coff = 0; coff < bound; coff += numCorners) {
2684     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
2685   }
2686   PetscFunctionReturn(0);
2687 }
2688 
2689 #if defined(PETSC_HAVE_TRIANGLE)
2690 #include <triangle.h>
2691 
2692 #undef __FUNCT__
2693 #define __FUNCT__ "InitInput_Triangle"
2694 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
2695 {
2696   PetscFunctionBegin;
2697   inputCtx->numberofpoints             = 0;
2698   inputCtx->numberofpointattributes    = 0;
2699   inputCtx->pointlist                  = NULL;
2700   inputCtx->pointattributelist         = NULL;
2701   inputCtx->pointmarkerlist            = NULL;
2702   inputCtx->numberofsegments           = 0;
2703   inputCtx->segmentlist                = NULL;
2704   inputCtx->segmentmarkerlist          = NULL;
2705   inputCtx->numberoftriangleattributes = 0;
2706   inputCtx->trianglelist               = NULL;
2707   inputCtx->numberofholes              = 0;
2708   inputCtx->holelist                   = NULL;
2709   inputCtx->numberofregions            = 0;
2710   inputCtx->regionlist                 = NULL;
2711   PetscFunctionReturn(0);
2712 }
2713 
2714 #undef __FUNCT__
2715 #define __FUNCT__ "InitOutput_Triangle"
2716 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
2717 {
2718   PetscFunctionBegin;
2719   outputCtx->numberofpoints        = 0;
2720   outputCtx->pointlist             = NULL;
2721   outputCtx->pointattributelist    = NULL;
2722   outputCtx->pointmarkerlist       = NULL;
2723   outputCtx->numberoftriangles     = 0;
2724   outputCtx->trianglelist          = NULL;
2725   outputCtx->triangleattributelist = NULL;
2726   outputCtx->neighborlist          = NULL;
2727   outputCtx->segmentlist           = NULL;
2728   outputCtx->segmentmarkerlist     = NULL;
2729   outputCtx->numberofedges         = 0;
2730   outputCtx->edgelist              = NULL;
2731   outputCtx->edgemarkerlist        = NULL;
2732   PetscFunctionReturn(0);
2733 }
2734 
2735 #undef __FUNCT__
2736 #define __FUNCT__ "FiniOutput_Triangle"
2737 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
2738 {
2739   PetscFunctionBegin;
2740   free(outputCtx->pointlist);
2741   free(outputCtx->pointmarkerlist);
2742   free(outputCtx->segmentlist);
2743   free(outputCtx->segmentmarkerlist);
2744   free(outputCtx->edgelist);
2745   free(outputCtx->edgemarkerlist);
2746   free(outputCtx->trianglelist);
2747   free(outputCtx->neighborlist);
2748   PetscFunctionReturn(0);
2749 }
2750 
2751 #undef __FUNCT__
2752 #define __FUNCT__ "DMPlexGenerate_Triangle"
2753 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
2754 {
2755   MPI_Comm             comm;
2756   PetscInt             dim              = 2;
2757   const PetscBool      createConvexHull = PETSC_FALSE;
2758   const PetscBool      constrained      = PETSC_FALSE;
2759   struct triangulateio in;
2760   struct triangulateio out;
2761   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
2762   PetscMPIInt          rank;
2763   PetscErrorCode       ierr;
2764 
2765   PetscFunctionBegin;
2766   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
2767   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2768   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
2769   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
2770   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
2771 
2772   in.numberofpoints = vEnd - vStart;
2773   if (in.numberofpoints > 0) {
2774     PetscSection coordSection;
2775     Vec          coordinates;
2776     PetscScalar *array;
2777 
2778     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
2779     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
2780     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
2781     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
2782     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
2783     for (v = vStart; v < vEnd; ++v) {
2784       const PetscInt idx = v - vStart;
2785       PetscInt       off, d;
2786 
2787       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
2788       for (d = 0; d < dim; ++d) {
2789         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
2790       }
2791       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
2792     }
2793     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
2794   }
2795   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
2796   in.numberofsegments = eEnd - eStart;
2797   if (in.numberofsegments > 0) {
2798     ierr = PetscMalloc1(in.numberofsegments*2, &in.segmentlist);CHKERRQ(ierr);
2799     ierr = PetscMalloc1(in.numberofsegments, &in.segmentmarkerlist);CHKERRQ(ierr);
2800     for (e = eStart; e < eEnd; ++e) {
2801       const PetscInt  idx = e - eStart;
2802       const PetscInt *cone;
2803 
2804       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
2805 
2806       in.segmentlist[idx*2+0] = cone[0] - vStart;
2807       in.segmentlist[idx*2+1] = cone[1] - vStart;
2808 
2809       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
2810     }
2811   }
2812 #if 0 /* Do not currently support holes */
2813   PetscReal *holeCoords;
2814   PetscInt   h, d;
2815 
2816   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
2817   if (in.numberofholes > 0) {
2818     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
2819     for (h = 0; h < in.numberofholes; ++h) {
2820       for (d = 0; d < dim; ++d) {
2821         in.holelist[h*dim+d] = holeCoords[h*dim+d];
2822       }
2823     }
2824   }
2825 #endif
2826   if (!rank) {
2827     char args[32];
2828 
2829     /* Take away 'Q' for verbose output */
2830     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
2831     if (createConvexHull) {
2832       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
2833     }
2834     if (constrained) {
2835       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
2836     }
2837     triangulate(args, &in, &out, NULL);
2838   }
2839   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
2840   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
2841   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
2842   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
2843   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
2844 
2845   {
2846     const PetscInt numCorners  = 3;
2847     const PetscInt numCells    = out.numberoftriangles;
2848     const PetscInt numVertices = out.numberofpoints;
2849     const int     *cells      = out.trianglelist;
2850     const double  *meshCoords = out.pointlist;
2851 
2852     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
2853     /* Set labels */
2854     for (v = 0; v < numVertices; ++v) {
2855       if (out.pointmarkerlist[v]) {
2856         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
2857       }
2858     }
2859     if (interpolate) {
2860       for (e = 0; e < out.numberofedges; e++) {
2861         if (out.edgemarkerlist[e]) {
2862           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
2863           const PetscInt *edges;
2864           PetscInt        numEdges;
2865 
2866           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
2867           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
2868           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
2869           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
2870         }
2871       }
2872     }
2873     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
2874   }
2875 #if 0 /* Do not currently support holes */
2876   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
2877 #endif
2878   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
2879   PetscFunctionReturn(0);
2880 }
2881 
2882 #undef __FUNCT__
2883 #define __FUNCT__ "DMPlexRefine_Triangle"
2884 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
2885 {
2886   MPI_Comm             comm;
2887   PetscInt             dim  = 2;
2888   struct triangulateio in;
2889   struct triangulateio out;
2890   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
2891   PetscMPIInt          rank;
2892   PetscErrorCode       ierr;
2893 
2894   PetscFunctionBegin;
2895   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2896   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2897   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
2898   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
2899   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2900   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
2901   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2902 
2903   in.numberofpoints = vEnd - vStart;
2904   if (in.numberofpoints > 0) {
2905     PetscSection coordSection;
2906     Vec          coordinates;
2907     PetscScalar *array;
2908 
2909     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
2910     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
2911     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2912     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2913     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
2914     for (v = vStart; v < vEnd; ++v) {
2915       const PetscInt idx = v - vStart;
2916       PetscInt       off, d;
2917 
2918       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
2919       for (d = 0; d < dim; ++d) {
2920         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
2921       }
2922       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
2923     }
2924     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
2925   }
2926   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
2927 
2928   in.numberofcorners   = 3;
2929   in.numberoftriangles = cEnd - cStart;
2930 
2931   in.trianglearealist  = (double*) maxVolumes;
2932   if (in.numberoftriangles > 0) {
2933     ierr = PetscMalloc1(in.numberoftriangles*in.numberofcorners, &in.trianglelist);CHKERRQ(ierr);
2934     for (c = cStart; c < cEnd; ++c) {
2935       const PetscInt idx      = c - cStart;
2936       PetscInt      *closure = NULL;
2937       PetscInt       closureSize;
2938 
2939       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2940       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
2941       for (v = 0; v < 3; ++v) {
2942         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
2943       }
2944       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2945     }
2946   }
2947   /* TODO: Segment markers are missing on input */
2948 #if 0 /* Do not currently support holes */
2949   PetscReal *holeCoords;
2950   PetscInt   h, d;
2951 
2952   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
2953   if (in.numberofholes > 0) {
2954     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
2955     for (h = 0; h < in.numberofholes; ++h) {
2956       for (d = 0; d < dim; ++d) {
2957         in.holelist[h*dim+d] = holeCoords[h*dim+d];
2958       }
2959     }
2960   }
2961 #endif
2962   if (!rank) {
2963     char args[32];
2964 
2965     /* Take away 'Q' for verbose output */
2966     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
2967     triangulate(args, &in, &out, NULL);
2968   }
2969   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
2970   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
2971   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
2972   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
2973   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
2974 
2975   {
2976     const PetscInt numCorners  = 3;
2977     const PetscInt numCells    = out.numberoftriangles;
2978     const PetscInt numVertices = out.numberofpoints;
2979     const int     *cells      = out.trianglelist;
2980     const double  *meshCoords = out.pointlist;
2981     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
2982 
2983     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
2984     /* Set labels */
2985     for (v = 0; v < numVertices; ++v) {
2986       if (out.pointmarkerlist[v]) {
2987         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
2988       }
2989     }
2990     if (interpolate) {
2991       PetscInt e;
2992 
2993       for (e = 0; e < out.numberofedges; e++) {
2994         if (out.edgemarkerlist[e]) {
2995           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
2996           const PetscInt *edges;
2997           PetscInt        numEdges;
2998 
2999           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3000           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3001           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3002           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3003         }
3004       }
3005     }
3006     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3007   }
3008 #if 0 /* Do not currently support holes */
3009   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3010 #endif
3011   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3012   PetscFunctionReturn(0);
3013 }
3014 #endif
3015 
3016 #if defined(PETSC_HAVE_TETGEN)
3017 #include <tetgen.h>
3018 #undef __FUNCT__
3019 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3020 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3021 {
3022   MPI_Comm       comm;
3023   const PetscInt dim  = 3;
3024   ::tetgenio     in;
3025   ::tetgenio     out;
3026   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3027   PetscMPIInt    rank;
3028   PetscErrorCode ierr;
3029 
3030   PetscFunctionBegin;
3031   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3032   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3033   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3034   in.numberofpoints = vEnd - vStart;
3035   if (in.numberofpoints > 0) {
3036     PetscSection coordSection;
3037     Vec          coordinates;
3038     PetscScalar *array;
3039 
3040     in.pointlist       = new double[in.numberofpoints*dim];
3041     in.pointmarkerlist = new int[in.numberofpoints];
3042 
3043     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3044     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3045     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3046     for (v = vStart; v < vEnd; ++v) {
3047       const PetscInt idx = v - vStart;
3048       PetscInt       off, d;
3049 
3050       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3051       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3052       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3053     }
3054     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3055   }
3056   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3057 
3058   in.numberoffacets = fEnd - fStart;
3059   if (in.numberoffacets > 0) {
3060     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3061     in.facetmarkerlist = new int[in.numberoffacets];
3062     for (f = fStart; f < fEnd; ++f) {
3063       const PetscInt idx     = f - fStart;
3064       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3065 
3066       in.facetlist[idx].numberofpolygons = 1;
3067       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3068       in.facetlist[idx].numberofholes    = 0;
3069       in.facetlist[idx].holelist         = NULL;
3070 
3071       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3072       for (p = 0; p < numPoints*2; p += 2) {
3073         const PetscInt point = points[p];
3074         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3075       }
3076 
3077       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3078       poly->numberofvertices = numVertices;
3079       poly->vertexlist       = new int[poly->numberofvertices];
3080       for (v = 0; v < numVertices; ++v) {
3081         const PetscInt vIdx = points[v] - vStart;
3082         poly->vertexlist[v] = vIdx;
3083       }
3084       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3085       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3086     }
3087   }
3088   if (!rank) {
3089     char args[32];
3090 
3091     /* Take away 'Q' for verbose output */
3092     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3093     ::tetrahedralize(args, &in, &out);
3094   }
3095   {
3096     const PetscInt numCorners  = 4;
3097     const PetscInt numCells    = out.numberoftetrahedra;
3098     const PetscInt numVertices = out.numberofpoints;
3099     const double   *meshCoords = out.pointlist;
3100     int            *cells      = out.tetrahedronlist;
3101 
3102     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3103     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3104     /* Set labels */
3105     for (v = 0; v < numVertices; ++v) {
3106       if (out.pointmarkerlist[v]) {
3107         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3108       }
3109     }
3110     if (interpolate) {
3111       PetscInt e;
3112 
3113       for (e = 0; e < out.numberofedges; e++) {
3114         if (out.edgemarkerlist[e]) {
3115           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3116           const PetscInt *edges;
3117           PetscInt        numEdges;
3118 
3119           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3120           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3121           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3122           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3123         }
3124       }
3125       for (f = 0; f < out.numberoftrifaces; f++) {
3126         if (out.trifacemarkerlist[f]) {
3127           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3128           const PetscInt *faces;
3129           PetscInt        numFaces;
3130 
3131           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3132           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3133           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3134           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3135         }
3136       }
3137     }
3138     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3139   }
3140   PetscFunctionReturn(0);
3141 }
3142 
3143 #undef __FUNCT__
3144 #define __FUNCT__ "DMPlexRefine_Tetgen"
3145 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3146 {
3147   MPI_Comm       comm;
3148   const PetscInt dim  = 3;
3149   ::tetgenio     in;
3150   ::tetgenio     out;
3151   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3152   PetscMPIInt    rank;
3153   PetscErrorCode ierr;
3154 
3155   PetscFunctionBegin;
3156   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3157   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3158   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3159   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3160   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3161 
3162   in.numberofpoints = vEnd - vStart;
3163   if (in.numberofpoints > 0) {
3164     PetscSection coordSection;
3165     Vec          coordinates;
3166     PetscScalar *array;
3167 
3168     in.pointlist       = new double[in.numberofpoints*dim];
3169     in.pointmarkerlist = new int[in.numberofpoints];
3170 
3171     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3172     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3173     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3174     for (v = vStart; v < vEnd; ++v) {
3175       const PetscInt idx = v - vStart;
3176       PetscInt       off, d;
3177 
3178       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3179       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3180       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3181     }
3182     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3183   }
3184   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3185 
3186   in.numberofcorners       = 4;
3187   in.numberoftetrahedra    = cEnd - cStart;
3188   in.tetrahedronvolumelist = (double*) maxVolumes;
3189   if (in.numberoftetrahedra > 0) {
3190     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3191     for (c = cStart; c < cEnd; ++c) {
3192       const PetscInt idx      = c - cStart;
3193       PetscInt      *closure = NULL;
3194       PetscInt       closureSize;
3195 
3196       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3197       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3198       for (v = 0; v < 4; ++v) {
3199         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3200       }
3201       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3202     }
3203   }
3204   /* TODO: Put in boundary faces with markers */
3205   if (!rank) {
3206     char args[32];
3207 
3208     /* Take away 'Q' for verbose output */
3209     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3210     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3211     ::tetrahedralize(args, &in, &out);
3212   }
3213   in.tetrahedronvolumelist = NULL;
3214 
3215   {
3216     const PetscInt numCorners  = 4;
3217     const PetscInt numCells    = out.numberoftetrahedra;
3218     const PetscInt numVertices = out.numberofpoints;
3219     const double   *meshCoords = out.pointlist;
3220     int            *cells      = out.tetrahedronlist;
3221 
3222     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3223 
3224     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3225     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3226     /* Set labels */
3227     for (v = 0; v < numVertices; ++v) {
3228       if (out.pointmarkerlist[v]) {
3229         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3230       }
3231     }
3232     if (interpolate) {
3233       PetscInt e, f;
3234 
3235       for (e = 0; e < out.numberofedges; e++) {
3236         if (out.edgemarkerlist[e]) {
3237           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3238           const PetscInt *edges;
3239           PetscInt        numEdges;
3240 
3241           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3242           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3243           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3244           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3245         }
3246       }
3247       for (f = 0; f < out.numberoftrifaces; f++) {
3248         if (out.trifacemarkerlist[f]) {
3249           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3250           const PetscInt *faces;
3251           PetscInt        numFaces;
3252 
3253           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3254           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3255           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3256           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3257         }
3258       }
3259     }
3260     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3261   }
3262   PetscFunctionReturn(0);
3263 }
3264 #endif
3265 
3266 #if defined(PETSC_HAVE_CTETGEN)
3267 #include <ctetgen.h>
3268 
3269 #undef __FUNCT__
3270 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3271 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3272 {
3273   MPI_Comm       comm;
3274   const PetscInt dim  = 3;
3275   PLC           *in, *out;
3276   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3277   PetscMPIInt    rank;
3278   PetscErrorCode ierr;
3279 
3280   PetscFunctionBegin;
3281   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3282   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3283   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3284   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3285   ierr = PLCCreate(&in);CHKERRQ(ierr);
3286   ierr = PLCCreate(&out);CHKERRQ(ierr);
3287 
3288   in->numberofpoints = vEnd - vStart;
3289   if (in->numberofpoints > 0) {
3290     PetscSection coordSection;
3291     Vec          coordinates;
3292     PetscScalar *array;
3293 
3294     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3295     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3296     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3297     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3298     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3299     for (v = vStart; v < vEnd; ++v) {
3300       const PetscInt idx = v - vStart;
3301       PetscInt       off, d, m;
3302 
3303       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3304       for (d = 0; d < dim; ++d) {
3305         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3306       }
3307       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3308 
3309       in->pointmarkerlist[idx] = (int) m;
3310     }
3311     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3312   }
3313   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3314 
3315   in->numberoffacets = fEnd - fStart;
3316   if (in->numberoffacets > 0) {
3317     ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
3318     ierr = PetscMalloc1(in->numberoffacets,   &in->facetmarkerlist);CHKERRQ(ierr);
3319     for (f = fStart; f < fEnd; ++f) {
3320       const PetscInt idx     = f - fStart;
3321       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3322       polygon       *poly;
3323 
3324       in->facetlist[idx].numberofpolygons = 1;
3325 
3326       ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3327 
3328       in->facetlist[idx].numberofholes    = 0;
3329       in->facetlist[idx].holelist         = NULL;
3330 
3331       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3332       for (p = 0; p < numPoints*2; p += 2) {
3333         const PetscInt point = points[p];
3334         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3335       }
3336 
3337       poly                   = in->facetlist[idx].polygonlist;
3338       poly->numberofvertices = numVertices;
3339       ierr                   = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
3340       for (v = 0; v < numVertices; ++v) {
3341         const PetscInt vIdx = points[v] - vStart;
3342         poly->vertexlist[v] = vIdx;
3343       }
3344       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3345       in->facetmarkerlist[idx] = (int) m;
3346       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3347     }
3348   }
3349   if (!rank) {
3350     TetGenOpts t;
3351 
3352     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3353     t.in        = boundary; /* Should go away */
3354     t.plc       = 1;
3355     t.quality   = 1;
3356     t.edgesout  = 1;
3357     t.zeroindex = 1;
3358     t.quiet     = 1;
3359     t.verbose   = verbose;
3360     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3361     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3362   }
3363   {
3364     const PetscInt numCorners  = 4;
3365     const PetscInt numCells    = out->numberoftetrahedra;
3366     const PetscInt numVertices = out->numberofpoints;
3367     const double   *meshCoords = out->pointlist;
3368     int            *cells      = out->tetrahedronlist;
3369 
3370     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3371     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3372     /* Set labels */
3373     for (v = 0; v < numVertices; ++v) {
3374       if (out->pointmarkerlist[v]) {
3375         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3376       }
3377     }
3378     if (interpolate) {
3379       PetscInt e;
3380 
3381       for (e = 0; e < out->numberofedges; e++) {
3382         if (out->edgemarkerlist[e]) {
3383           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3384           const PetscInt *edges;
3385           PetscInt        numEdges;
3386 
3387           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3388           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3389           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3390           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3391         }
3392       }
3393       for (f = 0; f < out->numberoftrifaces; f++) {
3394         if (out->trifacemarkerlist[f]) {
3395           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3396           const PetscInt *faces;
3397           PetscInt        numFaces;
3398 
3399           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3400           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3401           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3402           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3403         }
3404       }
3405     }
3406     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3407   }
3408 
3409   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3410   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3411   PetscFunctionReturn(0);
3412 }
3413 
3414 #undef __FUNCT__
3415 #define __FUNCT__ "DMPlexRefine_CTetgen"
3416 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3417 {
3418   MPI_Comm       comm;
3419   const PetscInt dim  = 3;
3420   PLC           *in, *out;
3421   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3422   PetscMPIInt    rank;
3423   PetscErrorCode ierr;
3424 
3425   PetscFunctionBegin;
3426   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3427   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3428   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3429   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3430   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3431   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3432   ierr = PLCCreate(&in);CHKERRQ(ierr);
3433   ierr = PLCCreate(&out);CHKERRQ(ierr);
3434 
3435   in->numberofpoints = vEnd - vStart;
3436   if (in->numberofpoints > 0) {
3437     PetscSection coordSection;
3438     Vec          coordinates;
3439     PetscScalar *array;
3440 
3441     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3442     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3443     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3444     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3445     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3446     for (v = vStart; v < vEnd; ++v) {
3447       const PetscInt idx = v - vStart;
3448       PetscInt       off, d, m;
3449 
3450       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3451       for (d = 0; d < dim; ++d) {
3452         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3453       }
3454       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3455 
3456       in->pointmarkerlist[idx] = (int) m;
3457     }
3458     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3459   }
3460   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3461 
3462   in->numberofcorners       = 4;
3463   in->numberoftetrahedra    = cEnd - cStart;
3464   in->tetrahedronvolumelist = maxVolumes;
3465   if (in->numberoftetrahedra > 0) {
3466     ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
3467     for (c = cStart; c < cEnd; ++c) {
3468       const PetscInt idx      = c - cStart;
3469       PetscInt      *closure = NULL;
3470       PetscInt       closureSize;
3471 
3472       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3473       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3474       for (v = 0; v < 4; ++v) {
3475         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3476       }
3477       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3478     }
3479   }
3480   if (!rank) {
3481     TetGenOpts t;
3482 
3483     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3484 
3485     t.in        = dm; /* Should go away */
3486     t.refine    = 1;
3487     t.varvolume = 1;
3488     t.quality   = 1;
3489     t.edgesout  = 1;
3490     t.zeroindex = 1;
3491     t.quiet     = 1;
3492     t.verbose   = verbose; /* Change this */
3493 
3494     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3495     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3496   }
3497   {
3498     const PetscInt numCorners  = 4;
3499     const PetscInt numCells    = out->numberoftetrahedra;
3500     const PetscInt numVertices = out->numberofpoints;
3501     const double   *meshCoords = out->pointlist;
3502     int            *cells      = out->tetrahedronlist;
3503     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3504 
3505     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3506     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3507     /* Set labels */
3508     for (v = 0; v < numVertices; ++v) {
3509       if (out->pointmarkerlist[v]) {
3510         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3511       }
3512     }
3513     if (interpolate) {
3514       PetscInt e, f;
3515 
3516       for (e = 0; e < out->numberofedges; e++) {
3517         if (out->edgemarkerlist[e]) {
3518           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3519           const PetscInt *edges;
3520           PetscInt        numEdges;
3521 
3522           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3523           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3524           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3525           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3526         }
3527       }
3528       for (f = 0; f < out->numberoftrifaces; f++) {
3529         if (out->trifacemarkerlist[f]) {
3530           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3531           const PetscInt *faces;
3532           PetscInt        numFaces;
3533 
3534           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3535           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3536           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3537           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3538         }
3539       }
3540     }
3541     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3542   }
3543   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3544   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3545   PetscFunctionReturn(0);
3546 }
3547 #endif
3548 
3549 #undef __FUNCT__
3550 #define __FUNCT__ "DMPlexGenerate"
3551 /*@C
3552   DMPlexGenerate - Generates a mesh.
3553 
3554   Not Collective
3555 
3556   Input Parameters:
3557 + boundary - The DMPlex boundary object
3558 . name - The mesh generation package name
3559 - interpolate - Flag to create intermediate mesh elements
3560 
3561   Output Parameter:
3562 . mesh - The DMPlex object
3563 
3564   Level: intermediate
3565 
3566 .keywords: mesh, elements
3567 .seealso: DMPlexCreate(), DMRefine()
3568 @*/
3569 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
3570 {
3571   PetscInt       dim;
3572   char           genname[1024];
3573   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
3574   PetscErrorCode ierr;
3575 
3576   PetscFunctionBegin;
3577   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
3578   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
3579   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
3580   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
3581   if (flg) name = genname;
3582   if (name) {
3583     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
3584     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
3585     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
3586   }
3587   switch (dim) {
3588   case 1:
3589     if (!name || isTriangle) {
3590 #if defined(PETSC_HAVE_TRIANGLE)
3591       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
3592 #else
3593       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
3594 #endif
3595     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
3596     break;
3597   case 2:
3598     if (!name || isCTetgen) {
3599 #if defined(PETSC_HAVE_CTETGEN)
3600       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
3601 #else
3602       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
3603 #endif
3604     } else if (isTetgen) {
3605 #if defined(PETSC_HAVE_TETGEN)
3606       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
3607 #else
3608       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
3609 #endif
3610     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
3611     break;
3612   default:
3613     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
3614   }
3615   PetscFunctionReturn(0);
3616 }
3617 
3618 #undef __FUNCT__
3619 #define __FUNCT__ "DMRefine_Plex"
3620 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
3621 {
3622   PetscReal      refinementLimit;
3623   PetscInt       dim, cStart, cEnd;
3624   char           genname[1024], *name = NULL;
3625   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
3626   PetscErrorCode ierr;
3627 
3628   PetscFunctionBegin;
3629   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
3630   if (isUniform) {
3631     CellRefiner cellRefiner;
3632 
3633     ierr = DMPlexGetCellRefiner_Internal(dm, &cellRefiner);CHKERRQ(ierr);
3634     ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
3635     PetscFunctionReturn(0);
3636   }
3637   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
3638   if (refinementLimit == 0.0) PetscFunctionReturn(0);
3639   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
3640   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3641   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
3642   if (flg) name = genname;
3643   if (name) {
3644     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
3645     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
3646     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
3647   }
3648   switch (dim) {
3649   case 2:
3650     if (!name || isTriangle) {
3651 #if defined(PETSC_HAVE_TRIANGLE)
3652       double  *maxVolumes;
3653       PetscInt c;
3654 
3655       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
3656       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
3657       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
3658       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
3659 #else
3660       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
3661 #endif
3662     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
3663     break;
3664   case 3:
3665     if (!name || isCTetgen) {
3666 #if defined(PETSC_HAVE_CTETGEN)
3667       PetscReal *maxVolumes;
3668       PetscInt   c;
3669 
3670       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
3671       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
3672       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
3673 #else
3674       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
3675 #endif
3676     } else if (isTetgen) {
3677 #if defined(PETSC_HAVE_TETGEN)
3678       double  *maxVolumes;
3679       PetscInt c;
3680 
3681       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
3682       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
3683       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
3684 #else
3685       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
3686 #endif
3687     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
3688     break;
3689   default:
3690     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
3691   }
3692   PetscFunctionReturn(0);
3693 }
3694 
3695 #undef __FUNCT__
3696 #define __FUNCT__ "DMRefineHierarchy_Plex"
3697 PetscErrorCode DMRefineHierarchy_Plex(DM dm, PetscInt nlevels, DM dmRefined[])
3698 {
3699   DM             cdm = dm;
3700   PetscInt       r;
3701   PetscBool      isUniform;
3702   PetscErrorCode ierr;
3703 
3704   PetscFunctionBegin;
3705   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
3706   if (!isUniform) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Non-uniform refinement is incompatible with the hierarchy");
3707   for (r = 0; r < nlevels; ++r) {
3708     CellRefiner cellRefiner;
3709 
3710     ierr = DMPlexGetCellRefiner_Internal(cdm, &cellRefiner);CHKERRQ(ierr);
3711     ierr = DMPlexRefineUniform_Internal(cdm, cellRefiner, &dmRefined[r]);CHKERRQ(ierr);
3712     ierr = DMPlexSetCoarseDM(dmRefined[r], cdm);CHKERRQ(ierr);
3713     cdm  = dmRefined[r];
3714   }
3715   PetscFunctionReturn(0);
3716 }
3717 
3718 #undef __FUNCT__
3719 #define __FUNCT__ "DMCoarsen_Plex"
3720 PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
3721 {
3722   DM_Plex       *mesh = (DM_Plex*) dm->data;
3723   PetscErrorCode ierr;
3724 
3725   PetscFunctionBegin;
3726   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
3727   *dmCoarsened = mesh->coarseMesh;
3728   PetscFunctionReturn(0);
3729 }
3730 
3731 #undef __FUNCT__
3732 #define __FUNCT__ "DMPlexLocalizeCoordinate_Internal"
3733 PetscErrorCode DMPlexLocalizeCoordinate_Internal(DM dm, PetscInt dim, const PetscScalar anchor[], const PetscScalar in[], PetscScalar out[])
3734 {
3735   PetscInt d;
3736 
3737   PetscFunctionBegin;
3738   if (!dm->maxCell) {
3739     for (d = 0; d < dim; ++d) out[d] = in[d];
3740   } else {
3741     for (d = 0; d < dim; ++d) {
3742       if (PetscAbsScalar(anchor[d] - in[d]) > dm->maxCell[d]) {
3743         out[d] = PetscRealPart(anchor[d]) > PetscRealPart(in[d]) ? dm->L[d] + in[d] : in[d] - dm->L[d];
3744       } else {
3745         out[d] = in[d];
3746       }
3747     }
3748   }
3749   PetscFunctionReturn(0);
3750 }
3751 
3752 #undef __FUNCT__
3753 #define __FUNCT__ "DMPlexLocalizeAddCoordinate_Internal"
3754 PetscErrorCode DMPlexLocalizeAddCoordinate_Internal(DM dm, PetscInt dim, const PetscScalar anchor[], const PetscScalar in[], PetscScalar out[])
3755 {
3756   PetscInt d;
3757 
3758   PetscFunctionBegin;
3759   if (!dm->maxCell) {
3760     for (d = 0; d < dim; ++d) out[d] += in[d];
3761   } else {
3762     for (d = 0; d < dim; ++d) {
3763       if (PetscAbsScalar(anchor[d] - in[d]) > dm->maxCell[d]) {
3764         out[d] += PetscRealPart(anchor[d]) > PetscRealPart(in[d]) ? dm->L[d] + in[d] : in[d] - dm->L[d];
3765       } else {
3766         out[d] += in[d];
3767       }
3768     }
3769   }
3770   PetscFunctionReturn(0);
3771 }
3772 
3773 #undef __FUNCT__
3774 #define __FUNCT__ "DMPlexLocalizeCoordinates"
3775 PetscErrorCode DMPlexLocalizeCoordinates(DM dm)
3776 {
3777   PetscSection   coordSection, cSection;
3778   Vec            coordinates,  cVec;
3779   PetscScalar   *coords, *coords2, *anchor;
3780   PetscInt       cStart, cEnd, c, vStart, vEnd, v, dof, d, off, off2, bs, coordSize;
3781   PetscErrorCode ierr;
3782 
3783   PetscFunctionBegin;
3784   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3785   if (!dm->maxCell) PetscFunctionReturn(0);
3786   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3787   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3788   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3789   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3790   ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &cSection);CHKERRQ(ierr);
3791   ierr = PetscSectionSetChart(cSection, cStart, vEnd);CHKERRQ(ierr);
3792   for (v = vStart; v < vEnd; ++v) {
3793     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
3794     ierr = PetscSectionSetDof(cSection,     v,  dof);CHKERRQ(ierr);
3795   }
3796   for (c = cStart; c < cEnd; ++c) {
3797     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &dof, NULL);CHKERRQ(ierr);
3798     ierr = PetscSectionSetDof(cSection, c, dof);CHKERRQ(ierr);
3799   }
3800   ierr = PetscSectionSetUp(cSection);CHKERRQ(ierr);
3801   ierr = PetscSectionGetStorageSize(cSection, &coordSize);CHKERRQ(ierr);
3802   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cVec);CHKERRQ(ierr);
3803   ierr = VecGetBlockSize(coordinates, &bs);CHKERRQ(ierr);
3804   ierr = VecSetBlockSize(cVec,         bs);CHKERRQ(ierr);
3805   ierr = VecSetSizes(cVec, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3806   ierr = VecSetType(cVec,VECSTANDARD);CHKERRQ(ierr);
3807   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3808   ierr = VecGetArray(cVec,        &coords2);CHKERRQ(ierr);
3809   for (v = vStart; v < vEnd; ++v) {
3810     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
3811     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3812     ierr = PetscSectionGetOffset(cSection,     v, &off2);CHKERRQ(ierr);
3813     for (d = 0; d < dof; ++d) coords2[off2+d] = coords[off+d];
3814   }
3815   ierr = DMGetWorkArray(dm, 3, PETSC_SCALAR, &anchor);CHKERRQ(ierr);
3816   for (c = cStart; c < cEnd; ++c) {
3817     PetscScalar *cellCoords = NULL;
3818     PetscInt     b;
3819 
3820     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &dof, &cellCoords);CHKERRQ(ierr);
3821     ierr = PetscSectionGetOffset(cSection, c, &off2);CHKERRQ(ierr);
3822     for (b = 0; b < bs; ++b) anchor[b] = cellCoords[b];
3823     for (d = 0; d < dof/bs; ++d) {ierr = DMPlexLocalizeCoordinate_Internal(dm, bs, anchor, &cellCoords[d*bs], &coords2[off2+d*bs]);CHKERRQ(ierr);}
3824     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &dof, &cellCoords);CHKERRQ(ierr);
3825   }
3826   ierr = DMRestoreWorkArray(dm, 3, PETSC_SCALAR, &anchor);CHKERRQ(ierr);
3827   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3828   ierr = VecRestoreArray(cVec,        &coords2);CHKERRQ(ierr);
3829   ierr = DMSetCoordinateSection(dm, cSection);CHKERRQ(ierr);
3830   ierr = DMSetCoordinatesLocal(dm, cVec);CHKERRQ(ierr);
3831   ierr = VecDestroy(&cVec);CHKERRQ(ierr);
3832   ierr = PetscSectionDestroy(&cSection);CHKERRQ(ierr);
3833   PetscFunctionReturn(0);
3834 }
3835 
3836 #undef __FUNCT__
3837 #define __FUNCT__ "DMPlexGetDepthLabel"
3838 /*@
3839   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3840 
3841   Not Collective
3842 
3843   Input Parameter:
3844 . dm    - The DMPlex object
3845 
3846   Output Parameter:
3847 . depthLabel - The DMLabel recording point depth
3848 
3849   Level: developer
3850 
3851 .keywords: mesh, points
3852 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3853 @*/
3854 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3855 {
3856   DM_Plex       *mesh = (DM_Plex*) dm->data;
3857   PetscErrorCode ierr;
3858 
3859   PetscFunctionBegin;
3860   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3861   PetscValidPointer(depthLabel, 2);
3862   if (!mesh->depthLabel) {ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);}
3863   *depthLabel = mesh->depthLabel;
3864   PetscFunctionReturn(0);
3865 }
3866 
3867 #undef __FUNCT__
3868 #define __FUNCT__ "DMPlexGetDepth"
3869 /*@
3870   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3871 
3872   Not Collective
3873 
3874   Input Parameter:
3875 . dm    - The DMPlex object
3876 
3877   Output Parameter:
3878 . depth - The number of strata (breadth first levels) in the DAG
3879 
3880   Level: developer
3881 
3882 .keywords: mesh, points
3883 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3884 @*/
3885 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3886 {
3887   DMLabel        label;
3888   PetscInt       d = 0;
3889   PetscErrorCode ierr;
3890 
3891   PetscFunctionBegin;
3892   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3893   PetscValidPointer(depth, 2);
3894   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3895   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3896   *depth = d-1;
3897   PetscFunctionReturn(0);
3898 }
3899 
3900 #undef __FUNCT__
3901 #define __FUNCT__ "DMPlexGetDepthStratum"
3902 /*@
3903   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3904 
3905   Not Collective
3906 
3907   Input Parameters:
3908 + dm           - The DMPlex object
3909 - stratumValue - The requested depth
3910 
3911   Output Parameters:
3912 + start - The first point at this depth
3913 - end   - One beyond the last point at this depth
3914 
3915   Level: developer
3916 
3917 .keywords: mesh, points
3918 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3919 @*/
3920 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3921 {
3922   DMLabel        label;
3923   PetscInt       pStart, pEnd;
3924   PetscErrorCode ierr;
3925 
3926   PetscFunctionBegin;
3927   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3928   if (start) {PetscValidPointer(start, 3); *start = 0;}
3929   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3930   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3931   if (pStart == pEnd) PetscFunctionReturn(0);
3932   if (stratumValue < 0) {
3933     if (start) *start = pStart;
3934     if (end)   *end   = pEnd;
3935     PetscFunctionReturn(0);
3936   }
3937   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3938   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3939   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3940   PetscFunctionReturn(0);
3941 }
3942 
3943 #undef __FUNCT__
3944 #define __FUNCT__ "DMPlexGetHeightStratum"
3945 /*@
3946   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3947 
3948   Not Collective
3949 
3950   Input Parameters:
3951 + dm           - The DMPlex object
3952 - stratumValue - The requested height
3953 
3954   Output Parameters:
3955 + start - The first point at this height
3956 - end   - One beyond the last point at this height
3957 
3958   Level: developer
3959 
3960 .keywords: mesh, points
3961 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3962 @*/
3963 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3964 {
3965   DMLabel        label;
3966   PetscInt       depth, pStart, pEnd;
3967   PetscErrorCode ierr;
3968 
3969   PetscFunctionBegin;
3970   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3971   if (start) {PetscValidPointer(start, 3); *start = 0;}
3972   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3973   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3974   if (pStart == pEnd) PetscFunctionReturn(0);
3975   if (stratumValue < 0) {
3976     if (start) *start = pStart;
3977     if (end)   *end   = pEnd;
3978     PetscFunctionReturn(0);
3979   }
3980   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3981   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
3982   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3983   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3984   PetscFunctionReturn(0);
3985 }
3986 
3987 #undef __FUNCT__
3988 #define __FUNCT__ "DMPlexCreateSectionInitial"
3989 /* Set the number of dof on each point and separate by fields */
3990 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
3991 {
3992   PetscInt      *numDofTot;
3993   PetscInt       depth, pStart = 0, pEnd = 0;
3994   PetscInt       p, d, dep, f;
3995   PetscErrorCode ierr;
3996 
3997   PetscFunctionBegin;
3998   ierr = PetscMalloc1((dim+1), &numDofTot);CHKERRQ(ierr);
3999   for (d = 0; d <= dim; ++d) {
4000     numDofTot[d] = 0;
4001     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
4002   }
4003   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
4004   if (numFields > 0) {
4005     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
4006     if (numComp) {
4007       for (f = 0; f < numFields; ++f) {
4008         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
4009       }
4010     }
4011   }
4012   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4013   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
4014   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4015   for (dep = 0; dep <= depth; ++dep) {
4016     d    = dim == depth ? dep : (!dep ? 0 : dim);
4017     ierr = DMPlexGetDepthStratum(dm, dep, &pStart, &pEnd);CHKERRQ(ierr);
4018     for (p = pStart; p < pEnd; ++p) {
4019       for (f = 0; f < numFields; ++f) {
4020         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
4021       }
4022       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
4023     }
4024   }
4025   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
4026   PetscFunctionReturn(0);
4027 }
4028 
4029 #undef __FUNCT__
4030 #define __FUNCT__ "DMPlexCreateSectionBCDof"
4031 /* Set the number of dof on each point and separate by fields
4032    If constDof is PETSC_DETERMINE, constrain every dof on the point
4033 */
4034 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
4035 {
4036   PetscInt       numFields;
4037   PetscInt       bc;
4038   PetscErrorCode ierr;
4039 
4040   PetscFunctionBegin;
4041   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4042   for (bc = 0; bc < numBC; ++bc) {
4043     PetscInt        field = 0;
4044     const PetscInt *idx;
4045     PetscInt        n, i;
4046 
4047     if (numFields) field = bcField[bc];
4048     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
4049     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4050     for (i = 0; i < n; ++i) {
4051       const PetscInt p        = idx[i];
4052       PetscInt       numConst = constDof;
4053 
4054       /* Constrain every dof on the point */
4055       if (numConst < 0) {
4056         if (numFields) {
4057           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
4058         } else {
4059           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
4060         }
4061       }
4062       if (numFields) {
4063         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
4064       }
4065       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
4066     }
4067     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4068   }
4069   PetscFunctionReturn(0);
4070 }
4071 
4072 #undef __FUNCT__
4073 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
4074 /* Set the constrained indices on each point and separate by fields */
4075 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
4076 {
4077   PetscInt      *maxConstraints;
4078   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
4079   PetscErrorCode ierr;
4080 
4081   PetscFunctionBegin;
4082   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4083   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4084   ierr = PetscMalloc1((numFields+1), &maxConstraints);CHKERRQ(ierr);
4085   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
4086   for (p = pStart; p < pEnd; ++p) {
4087     PetscInt cdof;
4088 
4089     if (numFields) {
4090       for (f = 0; f < numFields; ++f) {
4091         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
4092         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
4093       }
4094     } else {
4095       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4096       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
4097     }
4098   }
4099   for (f = 0; f < numFields; ++f) {
4100     maxConstraints[numFields] += maxConstraints[f];
4101   }
4102   if (maxConstraints[numFields]) {
4103     PetscInt *indices;
4104 
4105     ierr = PetscMalloc1(maxConstraints[numFields], &indices);CHKERRQ(ierr);
4106     for (p = pStart; p < pEnd; ++p) {
4107       PetscInt cdof, d;
4108 
4109       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4110       if (cdof) {
4111         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
4112         if (numFields) {
4113           PetscInt numConst = 0, foff = 0;
4114 
4115           for (f = 0; f < numFields; ++f) {
4116             PetscInt cfdof, fdof;
4117 
4118             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4119             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
4120             /* Change constraint numbering from absolute local dof number to field relative local dof number */
4121             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
4122             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
4123             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
4124             numConst += cfdof;
4125             foff     += fdof;
4126           }
4127           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4128         } else {
4129           for (d = 0; d < cdof; ++d) indices[d] = d;
4130         }
4131         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4132       }
4133     }
4134     ierr = PetscFree(indices);CHKERRQ(ierr);
4135   }
4136   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
4137   PetscFunctionReturn(0);
4138 }
4139 
4140 #undef __FUNCT__
4141 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
4142 /* Set the constrained field indices on each point */
4143 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
4144 {
4145   const PetscInt *points, *indices;
4146   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
4147   PetscErrorCode  ierr;
4148 
4149   PetscFunctionBegin;
4150   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4151   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
4152 
4153   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
4154   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
4155   if (!constraintIndices) {
4156     PetscInt *idx, i;
4157 
4158     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4159     ierr = PetscMalloc1(maxDof, &idx);CHKERRQ(ierr);
4160     for (i = 0; i < maxDof; ++i) idx[i] = i;
4161     for (p = 0; p < numPoints; ++p) {
4162       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
4163     }
4164     ierr = PetscFree(idx);CHKERRQ(ierr);
4165   } else {
4166     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
4167     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
4168     for (p = 0; p < numPoints; ++p) {
4169       PetscInt fcdof;
4170 
4171       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
4172       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
4173       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
4174     }
4175     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
4176   }
4177   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
4178   PetscFunctionReturn(0);
4179 }
4180 
4181 #undef __FUNCT__
4182 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
4183 /* Set the constrained indices on each point and separate by fields */
4184 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
4185 {
4186   PetscInt      *indices;
4187   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
4188   PetscErrorCode ierr;
4189 
4190   PetscFunctionBegin;
4191   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4192   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
4193   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4194   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
4195   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4196   for (p = pStart; p < pEnd; ++p) {
4197     PetscInt cdof, d;
4198 
4199     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4200     if (cdof) {
4201       PetscInt numConst = 0, foff = 0;
4202 
4203       for (f = 0; f < numFields; ++f) {
4204         const PetscInt *fcind;
4205         PetscInt        fdof, fcdof;
4206 
4207         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4208         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
4209         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
4210         /* Change constraint numbering from field relative local dof number to absolute local dof number */
4211         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
4212         foff     += fdof;
4213         numConst += fcdof;
4214       }
4215       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4216       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4217     }
4218   }
4219   ierr = PetscFree(indices);CHKERRQ(ierr);
4220   PetscFunctionReturn(0);
4221 }
4222 
4223 #undef __FUNCT__
4224 #define __FUNCT__ "DMPlexCreateSection"
4225 /*@C
4226   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
4227 
4228   Not Collective
4229 
4230   Input Parameters:
4231 + dm        - The DMPlex object
4232 . dim       - The spatial dimension of the problem
4233 . numFields - The number of fields in the problem
4234 . numComp   - An array of size numFields that holds the number of components for each field
4235 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
4236 . numBC     - The number of boundary conditions
4237 . bcField   - An array of size numBC giving the field number for each boundry condition
4238 . bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
4239 - perm      - Optional permutation of the chart, or NULL
4240 
4241   Output Parameter:
4242 . section - The PetscSection object
4243 
4244   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
4245   number of dof for field 0 on each edge.
4246 
4247   The chart permutation is the same one set using PetscSectionSetPermutation()
4248 
4249   Level: developer
4250 
4251   Fortran Notes:
4252   A Fortran 90 version is available as DMPlexCreateSectionF90()
4253 
4254 .keywords: mesh, elements
4255 .seealso: DMPlexCreate(), PetscSectionCreate(), PetscSectionSetPermutation()
4256 @*/
4257 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], IS perm, PetscSection *section)
4258 {
4259   PetscErrorCode ierr;
4260 
4261   PetscFunctionBegin;
4262   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
4263   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
4264   if (perm) {ierr = PetscSectionSetPermutation(*section, perm);CHKERRQ(ierr);}
4265   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
4266   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
4267   ierr = PetscSectionViewFromOptions(*section,NULL,"-section_view");CHKERRQ(ierr);
4268   PetscFunctionReturn(0);
4269 }
4270 
4271 #undef __FUNCT__
4272 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
4273 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4274 {
4275   PetscSection   section;
4276   PetscErrorCode ierr;
4277 
4278   PetscFunctionBegin;
4279   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4280   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4281   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
4282   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4283   PetscFunctionReturn(0);
4284 }
4285 
4286 #undef __FUNCT__
4287 #define __FUNCT__ "DMPlexGetConeSection"
4288 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4289 {
4290   DM_Plex *mesh = (DM_Plex*) dm->data;
4291 
4292   PetscFunctionBegin;
4293   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4294   if (section) *section = mesh->coneSection;
4295   PetscFunctionReturn(0);
4296 }
4297 
4298 #undef __FUNCT__
4299 #define __FUNCT__ "DMPlexGetSupportSection"
4300 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4301 {
4302   DM_Plex *mesh = (DM_Plex*) dm->data;
4303 
4304   PetscFunctionBegin;
4305   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4306   if (section) *section = mesh->supportSection;
4307   PetscFunctionReturn(0);
4308 }
4309 
4310 #undef __FUNCT__
4311 #define __FUNCT__ "DMPlexGetCones"
4312 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4313 {
4314   DM_Plex *mesh = (DM_Plex*) dm->data;
4315 
4316   PetscFunctionBegin;
4317   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4318   if (cones) *cones = mesh->cones;
4319   PetscFunctionReturn(0);
4320 }
4321 
4322 #undef __FUNCT__
4323 #define __FUNCT__ "DMPlexGetConeOrientations"
4324 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4325 {
4326   DM_Plex *mesh = (DM_Plex*) dm->data;
4327 
4328   PetscFunctionBegin;
4329   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4330   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4331   PetscFunctionReturn(0);
4332 }
4333 
4334 /******************************** FEM Support **********************************/
4335 
4336 #undef __FUNCT__
4337 #define __FUNCT__ "DMPlexVecGetClosure_Depth1_Static"
4338 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4339 {
4340   PetscScalar    *array, *vArray;
4341   const PetscInt *cone, *coneO;
4342   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4343   PetscErrorCode  ierr;
4344 
4345   PetscFunctionBeginHot;
4346   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4347   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4348   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4349   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4350   if (!values || !*values) {
4351     if ((point >= pStart) && (point < pEnd)) {
4352       PetscInt dof;
4353 
4354       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4355       size += dof;
4356     }
4357     for (p = 0; p < numPoints; ++p) {
4358       const PetscInt cp = cone[p];
4359       PetscInt       dof;
4360 
4361       if ((cp < pStart) || (cp >= pEnd)) continue;
4362       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4363       size += dof;
4364     }
4365     if (!values) {
4366       if (csize) *csize = size;
4367       PetscFunctionReturn(0);
4368     }
4369     ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
4370   } else {
4371     array = *values;
4372   }
4373   size = 0;
4374   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4375   if ((point >= pStart) && (point < pEnd)) {
4376     PetscInt     dof, off, d;
4377     PetscScalar *varr;
4378 
4379     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4380     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4381     varr = &vArray[off];
4382     for (d = 0; d < dof; ++d, ++offset) {
4383       array[offset] = varr[d];
4384     }
4385     size += dof;
4386   }
4387   for (p = 0; p < numPoints; ++p) {
4388     const PetscInt cp = cone[p];
4389     PetscInt       o  = coneO[p];
4390     PetscInt       dof, off, d;
4391     PetscScalar   *varr;
4392 
4393     if ((cp < pStart) || (cp >= pEnd)) continue;
4394     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4395     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4396     varr = &vArray[off];
4397     if (o >= 0) {
4398       for (d = 0; d < dof; ++d, ++offset) {
4399         array[offset] = varr[d];
4400       }
4401     } else {
4402       for (d = dof-1; d >= 0; --d, ++offset) {
4403         array[offset] = varr[d];
4404       }
4405     }
4406     size += dof;
4407   }
4408   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4409   if (!*values) {
4410     if (csize) *csize = size;
4411     *values = array;
4412   } else {
4413     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4414     *csize = size;
4415   }
4416   PetscFunctionReturn(0);
4417 }
4418 
4419 #undef __FUNCT__
4420 #define __FUNCT__ "DMPlexVecGetClosure_Static"
4421 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4422 {
4423   PetscInt       offset = 0, p;
4424   PetscErrorCode ierr;
4425 
4426   PetscFunctionBeginHot;
4427   *size = 0;
4428   for (p = 0; p < numPoints*2; p += 2) {
4429     const PetscInt point = points[p];
4430     const PetscInt o     = points[p+1];
4431     PetscInt       dof, off, d;
4432     const PetscScalar *varr;
4433 
4434     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4435     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4436     varr = &vArray[off];
4437     if (o >= 0) {
4438       for (d = 0; d < dof; ++d, ++offset)    array[offset] = varr[d];
4439     } else {
4440       for (d = dof-1; d >= 0; --d, ++offset) array[offset] = varr[d];
4441     }
4442   }
4443   *size = offset;
4444   PetscFunctionReturn(0);
4445 }
4446 
4447 #undef __FUNCT__
4448 #define __FUNCT__ "DMPlexVecGetClosure_Fields_Static"
4449 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4450 {
4451   PetscInt       offset = 0, f;
4452   PetscErrorCode ierr;
4453 
4454   PetscFunctionBeginHot;
4455   *size = 0;
4456   for (f = 0; f < numFields; ++f) {
4457     PetscInt fcomp, p;
4458 
4459     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4460     for (p = 0; p < numPoints*2; p += 2) {
4461       const PetscInt point = points[p];
4462       const PetscInt o     = points[p+1];
4463       PetscInt       fdof, foff, d, c;
4464       const PetscScalar *varr;
4465 
4466       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4467       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4468       varr = &vArray[foff];
4469       if (o >= 0) {
4470         for (d = 0; d < fdof; ++d, ++offset) array[offset] = varr[d];
4471       } else {
4472         for (d = fdof/fcomp-1; d >= 0; --d) {
4473           for (c = 0; c < fcomp; ++c, ++offset) {
4474             array[offset] = varr[d*fcomp+c];
4475           }
4476         }
4477       }
4478     }
4479   }
4480   *size = offset;
4481   PetscFunctionReturn(0);
4482 }
4483 
4484 #undef __FUNCT__
4485 #define __FUNCT__ "DMPlexVecGetClosure"
4486 /*@C
4487   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4488 
4489   Not collective
4490 
4491   Input Parameters:
4492 + dm - The DM
4493 . section - The section describing the layout in v, or NULL to use the default section
4494 . v - The local vector
4495 - point - The sieve point in the DM
4496 
4497   Output Parameters:
4498 + csize - The number of values in the closure, or NULL
4499 - values - The array of values, which is a borrowed array and should not be freed
4500 
4501   Fortran Notes:
4502   Since it returns an array, this routine is only available in Fortran 90, and you must
4503   include petsc.h90 in your code.
4504 
4505   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4506 
4507   Level: intermediate
4508 
4509 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4510 @*/
4511 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4512 {
4513   PetscSection    clSection;
4514   IS              clPoints;
4515   PetscScalar    *array, *vArray;
4516   PetscInt       *points = NULL;
4517   const PetscInt *clp;
4518   PetscInt        depth, numFields, numPoints, size;
4519   PetscErrorCode  ierr;
4520 
4521   PetscFunctionBeginHot;
4522   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4523   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4524   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4525   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4526   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4527   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4528   if (depth == 1 && numFields < 2) {
4529     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4530     PetscFunctionReturn(0);
4531   }
4532   /* Get points */
4533   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
4534   if (!clPoints) {
4535     PetscInt pStart, pEnd, p, q;
4536 
4537     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4538     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
4539     /* Compress out points not in the section */
4540     for (p = 0, q = 0; p < numPoints*2; p += 2) {
4541       if ((points[p] >= pStart) && (points[p] < pEnd)) {
4542         points[q*2]   = points[p];
4543         points[q*2+1] = points[p+1];
4544         ++q;
4545       }
4546     }
4547     numPoints = q;
4548   } else {
4549     PetscInt dof, off;
4550 
4551     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
4552     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
4553     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
4554     numPoints = dof/2;
4555     points    = (PetscInt *) &clp[off];
4556   }
4557   /* Get array */
4558   if (!values || !*values) {
4559     PetscInt asize = 0, dof, p;
4560 
4561     for (p = 0; p < numPoints*2; p += 2) {
4562       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4563       asize += dof;
4564     }
4565     if (!values) {
4566       if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4567       else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4568       if (csize) *csize = asize;
4569       PetscFunctionReturn(0);
4570     }
4571     ierr = DMGetWorkArray(dm, asize, PETSC_SCALAR, &array);CHKERRQ(ierr);
4572   } else {
4573     array = *values;
4574   }
4575   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4576   /* Get values */
4577   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(section, numPoints, points, numFields, vArray, &size, array);CHKERRQ(ierr);}
4578   else               {ierr = DMPlexVecGetClosure_Static(section, numPoints, points, vArray, &size, array);CHKERRQ(ierr);}
4579   /* Cleanup points */
4580   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4581   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4582   /* Cleanup array */
4583   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4584   if (!*values) {
4585     if (csize) *csize = size;
4586     *values = array;
4587   } else {
4588     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4589     *csize = size;
4590   }
4591   PetscFunctionReturn(0);
4592 }
4593 
4594 #undef __FUNCT__
4595 #define __FUNCT__ "DMPlexVecRestoreClosure"
4596 /*@C
4597   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4598 
4599   Not collective
4600 
4601   Input Parameters:
4602 + dm - The DM
4603 . section - The section describing the layout in v, or NULL to use the default section
4604 . v - The local vector
4605 . point - The sieve point in the DM
4606 . csize - The number of values in the closure, or NULL
4607 - values - The array of values, which is a borrowed array and should not be freed
4608 
4609   Fortran Notes:
4610   Since it returns an array, this routine is only available in Fortran 90, and you must
4611   include petsc.h90 in your code.
4612 
4613   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4614 
4615   Level: intermediate
4616 
4617 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4618 @*/
4619 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4620 {
4621   PetscInt       size = 0;
4622   PetscErrorCode ierr;
4623 
4624   PetscFunctionBegin;
4625   /* Should work without recalculating size */
4626   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
4627   PetscFunctionReturn(0);
4628 }
4629 
4630 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4631 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4632 
4633 #undef __FUNCT__
4634 #define __FUNCT__ "updatePoint_private"
4635 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
4636 {
4637   PetscInt        cdof;   /* The number of constraints on this point */
4638   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4639   PetscScalar    *a;
4640   PetscInt        off, cind = 0, k;
4641   PetscErrorCode  ierr;
4642 
4643   PetscFunctionBegin;
4644   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4645   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4646   a    = &array[off];
4647   if (!cdof || setBC) {
4648     if (orientation >= 0) {
4649       for (k = 0; k < dof; ++k) {
4650         fuse(&a[k], values[k]);
4651       }
4652     } else {
4653       for (k = 0; k < dof; ++k) {
4654         fuse(&a[k], values[dof-k-1]);
4655       }
4656     }
4657   } else {
4658     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4659     if (orientation >= 0) {
4660       for (k = 0; k < dof; ++k) {
4661         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4662         fuse(&a[k], values[k]);
4663       }
4664     } else {
4665       for (k = 0; k < dof; ++k) {
4666         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4667         fuse(&a[k], values[dof-k-1]);
4668       }
4669     }
4670   }
4671   PetscFunctionReturn(0);
4672 }
4673 
4674 #undef __FUNCT__
4675 #define __FUNCT__ "updatePointBC_private"
4676 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscInt orientation, const PetscScalar values[], PetscScalar array[])
4677 {
4678   PetscInt        cdof;   /* The number of constraints on this point */
4679   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4680   PetscScalar    *a;
4681   PetscInt        off, cind = 0, k;
4682   PetscErrorCode  ierr;
4683 
4684   PetscFunctionBegin;
4685   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4686   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4687   a    = &array[off];
4688   if (cdof) {
4689     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4690     if (orientation >= 0) {
4691       for (k = 0; k < dof; ++k) {
4692         if ((cind < cdof) && (k == cdofs[cind])) {
4693           fuse(&a[k], values[k]);
4694           ++cind;
4695         }
4696       }
4697     } else {
4698       for (k = 0; k < dof; ++k) {
4699         if ((cind < cdof) && (k == cdofs[cind])) {
4700           fuse(&a[k], values[dof-k-1]);
4701           ++cind;
4702         }
4703       }
4704     }
4705   }
4706   PetscFunctionReturn(0);
4707 }
4708 
4709 #undef __FUNCT__
4710 #define __FUNCT__ "updatePointFields_private"
4711 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4712 {
4713   PetscScalar    *a;
4714   PetscInt        fdof, foff, fcdof, foffset = *offset;
4715   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4716   PetscInt        cind = 0, k, c;
4717   PetscErrorCode  ierr;
4718 
4719   PetscFunctionBegin;
4720   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4721   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4722   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4723   a    = &array[foff];
4724   if (!fcdof || setBC) {
4725     if (o >= 0) {
4726       for (k = 0; k < fdof; ++k) fuse(&a[k], values[foffset+k]);
4727     } else {
4728       for (k = fdof/fcomp-1; k >= 0; --k) {
4729         for (c = 0; c < fcomp; ++c) {
4730           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
4731         }
4732       }
4733     }
4734   } else {
4735     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4736     if (o >= 0) {
4737       for (k = 0; k < fdof; ++k) {
4738         if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
4739         fuse(&a[k], values[foffset+k]);
4740       }
4741     } else {
4742       for (k = fdof/fcomp-1; k >= 0; --k) {
4743         for (c = 0; c < fcomp; ++c) {
4744           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
4745           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
4746         }
4747       }
4748     }
4749   }
4750   *offset += fdof;
4751   PetscFunctionReturn(0);
4752 }
4753 
4754 #undef __FUNCT__
4755 #define __FUNCT__ "updatePointFieldsBC_private"
4756 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4757 {
4758   PetscScalar    *a;
4759   PetscInt        fdof, foff, fcdof, foffset = *offset;
4760   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4761   PetscInt        cind = 0, k, c;
4762   PetscErrorCode  ierr;
4763 
4764   PetscFunctionBegin;
4765   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4766   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4767   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4768   a    = &array[foff];
4769   if (fcdof) {
4770     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4771     if (o >= 0) {
4772       for (k = 0; k < fdof; ++k) {
4773         if ((cind < fcdof) && (k == fcdofs[cind])) {
4774           fuse(&a[k], values[foffset+k]);
4775           ++cind;
4776         }
4777       }
4778     } else {
4779       for (k = fdof/fcomp-1; k >= 0; --k) {
4780         for (c = 0; c < fcomp; ++c) {
4781           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {
4782             fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
4783             ++cind;
4784           }
4785         }
4786       }
4787     }
4788   }
4789   *offset += fdof;
4790   PetscFunctionReturn(0);
4791 }
4792 
4793 #undef __FUNCT__
4794 #define __FUNCT__ "DMPlexVecSetClosure_Static"
4795 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4796 {
4797   PetscScalar    *array;
4798   const PetscInt *cone, *coneO;
4799   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4800   PetscErrorCode  ierr;
4801 
4802   PetscFunctionBeginHot;
4803   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4804   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4805   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4806   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4807   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4808   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4809     const PetscInt cp = !p ? point : cone[p-1];
4810     const PetscInt o  = !p ? 0     : coneO[p-1];
4811 
4812     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4813     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4814     /* ADD_VALUES */
4815     {
4816       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4817       PetscScalar    *a;
4818       PetscInt        cdof, coff, cind = 0, k;
4819 
4820       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4821       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4822       a    = &array[coff];
4823       if (!cdof) {
4824         if (o >= 0) {
4825           for (k = 0; k < dof; ++k) {
4826             a[k] += values[off+k];
4827           }
4828         } else {
4829           for (k = 0; k < dof; ++k) {
4830             a[k] += values[off+dof-k-1];
4831           }
4832         }
4833       } else {
4834         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4835         if (o >= 0) {
4836           for (k = 0; k < dof; ++k) {
4837             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4838             a[k] += values[off+k];
4839           }
4840         } else {
4841           for (k = 0; k < dof; ++k) {
4842             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4843             a[k] += values[off+dof-k-1];
4844           }
4845         }
4846       }
4847     }
4848   }
4849   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4850   PetscFunctionReturn(0);
4851 }
4852 
4853 #undef __FUNCT__
4854 #define __FUNCT__ "DMPlexVecSetClosure"
4855 /*@C
4856   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4857 
4858   Not collective
4859 
4860   Input Parameters:
4861 + dm - The DM
4862 . section - The section describing the layout in v, or NULL to use the default section
4863 . v - The local vector
4864 . point - The sieve point in the DM
4865 . values - The array of values
4866 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
4867 
4868   Fortran Notes:
4869   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4870 
4871   Level: intermediate
4872 
4873 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4874 @*/
4875 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4876 {
4877   PetscSection    clSection;
4878   IS              clPoints;
4879   PetscScalar    *array;
4880   PetscInt       *points = NULL;
4881   const PetscInt *clp;
4882   PetscInt        depth, numFields, numPoints, p;
4883   PetscErrorCode  ierr;
4884 
4885   PetscFunctionBeginHot;
4886   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4887   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4888   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4889   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4890   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4891   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4892   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4893     ierr = DMPlexVecSetClosure_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4894     PetscFunctionReturn(0);
4895   }
4896   /* Get points */
4897   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
4898   if (!clPoints) {
4899     PetscInt pStart, pEnd, q;
4900 
4901     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4902     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
4903     /* Compress out points not in the section */
4904     for (p = 0, q = 0; p < numPoints*2; p += 2) {
4905       if ((points[p] >= pStart) && (points[p] < pEnd)) {
4906         points[q*2]   = points[p];
4907         points[q*2+1] = points[p+1];
4908         ++q;
4909       }
4910     }
4911     numPoints = q;
4912   } else {
4913     PetscInt dof, off;
4914 
4915     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
4916     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
4917     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
4918     numPoints = dof/2;
4919     points    = (PetscInt *) &clp[off];
4920   }
4921   /* Get array */
4922   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4923   /* Get values */
4924   if (numFields > 0) {
4925     PetscInt offset = 0, fcomp, f;
4926     for (f = 0; f < numFields; ++f) {
4927       ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4928       switch (mode) {
4929       case INSERT_VALUES:
4930         for (p = 0; p < numPoints*2; p += 2) {
4931           const PetscInt point = points[p];
4932           const PetscInt o     = points[p+1];
4933           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
4934         } break;
4935       case INSERT_ALL_VALUES:
4936         for (p = 0; p < numPoints*2; p += 2) {
4937           const PetscInt point = points[p];
4938           const PetscInt o     = points[p+1];
4939           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
4940         } break;
4941       case INSERT_BC_VALUES:
4942         for (p = 0; p < numPoints*2; p += 2) {
4943           const PetscInt point = points[p];
4944           const PetscInt o     = points[p+1];
4945           updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
4946         } break;
4947       case ADD_VALUES:
4948         for (p = 0; p < numPoints*2; p += 2) {
4949           const PetscInt point = points[p];
4950           const PetscInt o     = points[p+1];
4951           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
4952         } break;
4953       case ADD_ALL_VALUES:
4954         for (p = 0; p < numPoints*2; p += 2) {
4955           const PetscInt point = points[p];
4956           const PetscInt o     = points[p+1];
4957           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
4958         } break;
4959       default:
4960         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
4961       }
4962     }
4963   } else {
4964     PetscInt dof, off;
4965 
4966     switch (mode) {
4967     case INSERT_VALUES:
4968       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
4969         PetscInt o = points[p+1];
4970         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4971         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
4972       } break;
4973     case INSERT_ALL_VALUES:
4974       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
4975         PetscInt o = points[p+1];
4976         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4977         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
4978       } break;
4979     case INSERT_BC_VALUES:
4980       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
4981         PetscInt o = points[p+1];
4982         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4983         updatePointBC_private(section, points[p], dof, insert,  o, &values[off], array);
4984       } break;
4985     case ADD_VALUES:
4986       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
4987         PetscInt o = points[p+1];
4988         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4989         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
4990       } break;
4991     case ADD_ALL_VALUES:
4992       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
4993         PetscInt o = points[p+1];
4994         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4995         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
4996       } break;
4997     default:
4998       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
4999     }
5000   }
5001   /* Cleanup points */
5002   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5003   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5004   /* Cleanup array */
5005   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5006   PetscFunctionReturn(0);
5007 }
5008 
5009 #undef __FUNCT__
5010 #define __FUNCT__ "DMPlexVecSetFieldClosure_Internal"
5011 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, const PetscScalar values[], InsertMode mode)
5012 {
5013   PetscSection    clSection;
5014   IS              clPoints;
5015   PetscScalar    *array;
5016   PetscInt       *points = NULL;
5017   const PetscInt *clp;
5018   PetscInt        numFields, numPoints, p;
5019   PetscInt        offset = 0, fcomp, f;
5020   PetscErrorCode  ierr;
5021 
5022   PetscFunctionBeginHot;
5023   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5024   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5025   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5026   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5027   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5028   /* Get points */
5029   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5030   if (!clPoints) {
5031     PetscInt pStart, pEnd, q;
5032 
5033     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5034     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5035     /* Compress out points not in the section */
5036     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5037       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5038         points[q*2]   = points[p];
5039         points[q*2+1] = points[p+1];
5040         ++q;
5041       }
5042     }
5043     numPoints = q;
5044   } else {
5045     PetscInt dof, off;
5046 
5047     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5048     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5049     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5050     numPoints = dof/2;
5051     points    = (PetscInt *) &clp[off];
5052   }
5053   /* Get array */
5054   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5055   /* Get values */
5056   for (f = 0; f < numFields; ++f) {
5057     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5058     if (!fieldActive[f]) {
5059       for (p = 0; p < numPoints*2; p += 2) {
5060         PetscInt fdof;
5061         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5062         offset += fdof;
5063       }
5064       continue;
5065     }
5066     switch (mode) {
5067     case INSERT_VALUES:
5068       for (p = 0; p < numPoints*2; p += 2) {
5069         const PetscInt point = points[p];
5070         const PetscInt o     = points[p+1];
5071         updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
5072       } break;
5073     case INSERT_ALL_VALUES:
5074       for (p = 0; p < numPoints*2; p += 2) {
5075         const PetscInt point = points[p];
5076         const PetscInt o     = points[p+1];
5077         updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
5078         } break;
5079     case INSERT_BC_VALUES:
5080       for (p = 0; p < numPoints*2; p += 2) {
5081         const PetscInt point = points[p];
5082         const PetscInt o     = points[p+1];
5083         updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
5084       } break;
5085     case ADD_VALUES:
5086       for (p = 0; p < numPoints*2; p += 2) {
5087         const PetscInt point = points[p];
5088         const PetscInt o     = points[p+1];
5089         updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
5090       } break;
5091     case ADD_ALL_VALUES:
5092       for (p = 0; p < numPoints*2; p += 2) {
5093         const PetscInt point = points[p];
5094         const PetscInt o     = points[p+1];
5095         updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
5096       } break;
5097     default:
5098       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5099     }
5100   }
5101   /* Cleanup points */
5102   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5103   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5104   /* Cleanup array */
5105   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5106   PetscFunctionReturn(0);
5107 }
5108 
5109 #undef __FUNCT__
5110 #define __FUNCT__ "DMPlexPrintMatSetValues"
5111 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5112 {
5113   PetscMPIInt    rank;
5114   PetscInt       i, j;
5115   PetscErrorCode ierr;
5116 
5117   PetscFunctionBegin;
5118   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5119   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
5120   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5121   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5122   numCIndices = numCIndices ? numCIndices : numRIndices;
5123   for (i = 0; i < numRIndices; i++) {
5124     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
5125     for (j = 0; j < numCIndices; j++) {
5126 #if defined(PETSC_USE_COMPLEX)
5127       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5128 #else
5129       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5130 #endif
5131     }
5132     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5133   }
5134   PetscFunctionReturn(0);
5135 }
5136 
5137 #undef __FUNCT__
5138 #define __FUNCT__ "indicesPoint_private"
5139 /* . off - The global offset of this point */
5140 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
5141 {
5142   PetscInt        dof;    /* The number of unknowns on this point */
5143   PetscInt        cdof;   /* The number of constraints on this point */
5144   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5145   PetscInt        cind = 0, k;
5146   PetscErrorCode  ierr;
5147 
5148   PetscFunctionBegin;
5149   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5150   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5151   if (!cdof || setBC) {
5152     if (orientation >= 0) {
5153       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
5154     } else {
5155       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
5156     }
5157   } else {
5158     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5159     if (orientation >= 0) {
5160       for (k = 0; k < dof; ++k) {
5161         if ((cind < cdof) && (k == cdofs[cind])) {
5162           /* Insert check for returning constrained indices */
5163           indices[*loff+k] = -(off+k+1);
5164           ++cind;
5165         } else {
5166           indices[*loff+k] = off+k-cind;
5167         }
5168       }
5169     } else {
5170       for (k = 0; k < dof; ++k) {
5171         if ((cind < cdof) && (k == cdofs[cind])) {
5172           /* Insert check for returning constrained indices */
5173           indices[*loff+dof-k-1] = -(off+k+1);
5174           ++cind;
5175         } else {
5176           indices[*loff+dof-k-1] = off+k-cind;
5177         }
5178       }
5179     }
5180   }
5181   *loff += dof;
5182   PetscFunctionReturn(0);
5183 }
5184 
5185 #undef __FUNCT__
5186 #define __FUNCT__ "indicesPointFields_private"
5187 /* . off - The global offset of this point */
5188 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
5189 {
5190   PetscInt       numFields, foff, f;
5191   PetscErrorCode ierr;
5192 
5193   PetscFunctionBegin;
5194   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5195   for (f = 0, foff = 0; f < numFields; ++f) {
5196     PetscInt        fdof, fcomp, cfdof;
5197     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5198     PetscInt        cind = 0, k, c;
5199 
5200     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5201     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5202     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5203     if (!cfdof || setBC) {
5204       if (orientation >= 0) {
5205         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
5206       } else {
5207         for (k = fdof/fcomp-1; k >= 0; --k) {
5208           for (c = 0; c < fcomp; ++c) {
5209             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
5210           }
5211         }
5212       }
5213     } else {
5214       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5215       if (orientation >= 0) {
5216         for (k = 0; k < fdof; ++k) {
5217           if ((cind < cfdof) && (k == fcdofs[cind])) {
5218             indices[foffs[f]+k] = -(off+foff+k+1);
5219             ++cind;
5220           } else {
5221             indices[foffs[f]+k] = off+foff+k-cind;
5222           }
5223         }
5224       } else {
5225         for (k = fdof/fcomp-1; k >= 0; --k) {
5226           for (c = 0; c < fcomp; ++c) {
5227             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
5228               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
5229               ++cind;
5230             } else {
5231               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
5232             }
5233           }
5234         }
5235       }
5236     }
5237     foff     += fdof - cfdof;
5238     foffs[f] += fdof;
5239   }
5240   PetscFunctionReturn(0);
5241 }
5242 
5243 #undef __FUNCT__
5244 #define __FUNCT__ "DMPlexMatSetClosure"
5245 /*@C
5246   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5247 
5248   Not collective
5249 
5250   Input Parameters:
5251 + dm - The DM
5252 . section - The section describing the layout in v, or NULL to use the default section
5253 . globalSection - The section describing the layout in v, or NULL to use the default global section
5254 . A - The matrix
5255 . point - The sieve point in the DM
5256 . values - The array of values
5257 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5258 
5259   Fortran Notes:
5260   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5261 
5262   Level: intermediate
5263 
5264 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5265 @*/
5266 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5267 {
5268   DM_Plex        *mesh   = (DM_Plex*) dm->data;
5269   PetscSection    clSection;
5270   IS              clPoints;
5271   PetscInt       *points = NULL;
5272   const PetscInt *clp;
5273   PetscInt       *indices;
5274   PetscInt        offsets[32];
5275   PetscInt        numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
5276   PetscErrorCode  ierr;
5277 
5278   PetscFunctionBegin;
5279   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5280   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5281   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5282   if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5283   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5284   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5285   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5286   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5287   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5288   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5289   if (!clPoints) {
5290     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5291     /* Compress out points not in the section */
5292     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5293     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5294       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5295         points[q*2]   = points[p];
5296         points[q*2+1] = points[p+1];
5297         ++q;
5298       }
5299     }
5300     numPoints = q;
5301   } else {
5302     PetscInt dof, off;
5303 
5304     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5305     numPoints = dof/2;
5306     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5307     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5308     points = (PetscInt *) &clp[off];
5309   }
5310   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5311     PetscInt fdof;
5312 
5313     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5314     for (f = 0; f < numFields; ++f) {
5315       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5316       offsets[f+1] += fdof;
5317     }
5318     numIndices += dof;
5319   }
5320   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5321 
5322   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
5323   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5324   if (numFields) {
5325     for (p = 0; p < numPoints*2; p += 2) {
5326       PetscInt o = points[p+1];
5327       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5328       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
5329     }
5330   } else {
5331     for (p = 0, off = 0; p < numPoints*2; p += 2) {
5332       PetscInt o = points[p+1];
5333       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5334       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
5335     }
5336   }
5337   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5338   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5339   if (ierr) {
5340     PetscMPIInt    rank;
5341     PetscErrorCode ierr2;
5342 
5343     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5344     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5345     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5346     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
5347     CHKERRQ(ierr);
5348   }
5349   if (!clPoints) {
5350     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5351   } else {
5352     ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
5353   }
5354   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5355   PetscFunctionReturn(0);
5356 }
5357 
5358 #undef __FUNCT__
5359 #define __FUNCT__ "DMPlexMatSetClosureRefined"
5360 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5361 {
5362   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5363   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5364   PetscInt       *cpoints = NULL;
5365   PetscInt       *findices, *cindices;
5366   PetscInt        foffsets[32], coffsets[32];
5367   CellRefiner     cellRefiner;
5368   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5369   PetscErrorCode  ierr;
5370 
5371   PetscFunctionBegin;
5372   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5373   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5374   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5375   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5376   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5377   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5378   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5379   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5380   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5381   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5382   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5383   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5384   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5385   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5386   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5387   /* Column indices */
5388   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5389   maxFPoints = numCPoints;
5390   /* Compress out points not in the section */
5391   /*   TODO: Squeeze out points with 0 dof as well */
5392   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5393   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5394     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5395       cpoints[q*2]   = cpoints[p];
5396       cpoints[q*2+1] = cpoints[p+1];
5397       ++q;
5398     }
5399   }
5400   numCPoints = q;
5401   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5402     PetscInt fdof;
5403 
5404     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5405     if (!dof) continue;
5406     for (f = 0; f < numFields; ++f) {
5407       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5408       coffsets[f+1] += fdof;
5409     }
5410     numCIndices += dof;
5411   }
5412   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5413   /* Row indices */
5414   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5415   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5416   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5417   for (r = 0, q = 0; r < numSubcells; ++r) {
5418     /* TODO Map from coarse to fine cells */
5419     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5420     /* Compress out points not in the section */
5421     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5422     for (p = 0; p < numFPoints*2; p += 2) {
5423       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5424         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5425         if (!dof) continue;
5426         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5427         if (s < q) continue;
5428         ftotpoints[q*2]   = fpoints[p];
5429         ftotpoints[q*2+1] = fpoints[p+1];
5430         ++q;
5431       }
5432     }
5433     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5434   }
5435   numFPoints = q;
5436   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5437     PetscInt fdof;
5438 
5439     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5440     if (!dof) continue;
5441     for (f = 0; f < numFields; ++f) {
5442       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5443       foffsets[f+1] += fdof;
5444     }
5445     numFIndices += dof;
5446   }
5447   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5448 
5449   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", foffsets[numFields], numFIndices);
5450   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", coffsets[numFields], numCIndices);
5451   ierr = DMGetWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5452   ierr = DMGetWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5453   if (numFields) {
5454     for (p = 0; p < numFPoints*2; p += 2) {
5455       PetscInt o = ftotpoints[p+1];
5456       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5457       indicesPointFields_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, o, findices);
5458     }
5459     for (p = 0; p < numCPoints*2; p += 2) {
5460       PetscInt o = cpoints[p+1];
5461       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5462       indicesPointFields_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, o, cindices);
5463     }
5464   } else {
5465     for (p = 0, off = 0; p < numFPoints*2; p += 2) {
5466       PetscInt o = ftotpoints[p+1];
5467       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5468       indicesPoint_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, findices);
5469     }
5470     for (p = 0, off = 0; p < numCPoints*2; p += 2) {
5471       PetscInt o = cpoints[p+1];
5472       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5473       indicesPoint_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, cindices);
5474     }
5475   }
5476   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
5477   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
5478   if (ierr) {
5479     PetscMPIInt    rank;
5480     PetscErrorCode ierr2;
5481 
5482     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5483     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5484     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
5485     ierr2 = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr2);
5486     ierr2 = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr2);
5487     CHKERRQ(ierr);
5488   }
5489   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5490   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5491   ierr = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5492   ierr = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5493   PetscFunctionReturn(0);
5494 }
5495 
5496 #undef __FUNCT__
5497 #define __FUNCT__ "DMPlexGetHybridBounds"
5498 /*@
5499   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
5500 
5501   Input Parameter:
5502 . dm - The DMPlex object
5503 
5504   Output Parameters:
5505 + cMax - The first hybrid cell
5506 . fMax - The first hybrid face
5507 . eMax - The first hybrid edge
5508 - vMax - The first hybrid vertex
5509 
5510   Level: developer
5511 
5512 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
5513 @*/
5514 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
5515 {
5516   DM_Plex       *mesh = (DM_Plex*) dm->data;
5517   PetscInt       dim;
5518   PetscErrorCode ierr;
5519 
5520   PetscFunctionBegin;
5521   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5522   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5523   if (cMax) *cMax = mesh->hybridPointMax[dim];
5524   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
5525   if (eMax) *eMax = mesh->hybridPointMax[1];
5526   if (vMax) *vMax = mesh->hybridPointMax[0];
5527   PetscFunctionReturn(0);
5528 }
5529 
5530 #undef __FUNCT__
5531 #define __FUNCT__ "DMPlexSetHybridBounds"
5532 /*@
5533   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
5534 
5535   Input Parameters:
5536 . dm   - The DMPlex object
5537 . cMax - The first hybrid cell
5538 . fMax - The first hybrid face
5539 . eMax - The first hybrid edge
5540 - vMax - The first hybrid vertex
5541 
5542   Level: developer
5543 
5544 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
5545 @*/
5546 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
5547 {
5548   DM_Plex       *mesh = (DM_Plex*) dm->data;
5549   PetscInt       dim;
5550   PetscErrorCode ierr;
5551 
5552   PetscFunctionBegin;
5553   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5554   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5555   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
5556   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
5557   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
5558   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
5559   PetscFunctionReturn(0);
5560 }
5561 
5562 #undef __FUNCT__
5563 #define __FUNCT__ "DMPlexGetVTKCellHeight"
5564 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
5565 {
5566   DM_Plex *mesh = (DM_Plex*) dm->data;
5567 
5568   PetscFunctionBegin;
5569   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5570   PetscValidPointer(cellHeight, 2);
5571   *cellHeight = mesh->vtkCellHeight;
5572   PetscFunctionReturn(0);
5573 }
5574 
5575 #undef __FUNCT__
5576 #define __FUNCT__ "DMPlexSetVTKCellHeight"
5577 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
5578 {
5579   DM_Plex *mesh = (DM_Plex*) dm->data;
5580 
5581   PetscFunctionBegin;
5582   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5583   mesh->vtkCellHeight = cellHeight;
5584   PetscFunctionReturn(0);
5585 }
5586 
5587 #undef __FUNCT__
5588 #define __FUNCT__ "DMPlexCreateNumbering_Private"
5589 /* We can easily have a form that takes an IS instead */
5590 static PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
5591 {
5592   PetscSection   section, globalSection;
5593   PetscInt      *numbers, p;
5594   PetscErrorCode ierr;
5595 
5596   PetscFunctionBegin;
5597   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5598   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
5599   for (p = pStart; p < pEnd; ++p) {
5600     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
5601   }
5602   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
5603   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
5604   ierr = PetscMalloc1((pEnd - pStart), &numbers);CHKERRQ(ierr);
5605   for (p = pStart; p < pEnd; ++p) {
5606     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
5607     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
5608     else                       numbers[p-pStart] += shift;
5609   }
5610   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
5611   if (globalSize) {
5612     PetscLayout layout;
5613     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
5614     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
5615     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
5616   }
5617   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5618   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
5619   PetscFunctionReturn(0);
5620 }
5621 
5622 #undef __FUNCT__
5623 #define __FUNCT__ "DMPlexGetCellNumbering"
5624 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
5625 {
5626   DM_Plex       *mesh = (DM_Plex*) dm->data;
5627   PetscInt       cellHeight, cStart, cEnd, cMax;
5628   PetscErrorCode ierr;
5629 
5630   PetscFunctionBegin;
5631   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5632   if (!mesh->globalCellNumbers) {
5633     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
5634     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5635     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5636     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
5637     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, 0, NULL, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
5638   }
5639   *globalCellNumbers = mesh->globalCellNumbers;
5640   PetscFunctionReturn(0);
5641 }
5642 
5643 #undef __FUNCT__
5644 #define __FUNCT__ "DMPlexGetVertexNumbering"
5645 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
5646 {
5647   DM_Plex       *mesh = (DM_Plex*) dm->data;
5648   PetscInt       vStart, vEnd, vMax;
5649   PetscErrorCode ierr;
5650 
5651   PetscFunctionBegin;
5652   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5653   if (!mesh->globalVertexNumbers) {
5654     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5655     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
5656     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
5657     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, 0, NULL, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
5658   }
5659   *globalVertexNumbers = mesh->globalVertexNumbers;
5660   PetscFunctionReturn(0);
5661 }
5662 
5663 #undef __FUNCT__
5664 #define __FUNCT__ "DMPlexCreatePointNumbering"
5665 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
5666 {
5667   IS             nums[4];
5668   PetscInt       depths[4];
5669   PetscInt       depth, d, shift = 0;
5670   PetscErrorCode ierr;
5671 
5672   PetscFunctionBegin;
5673   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5674   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5675   depths[0] = depth; depths[1] = 0;
5676   for (d = 2; d <= depth; ++d) depths[d] = depth-d+1;
5677   for (d = 0; d <= depth; ++d) {
5678     PetscInt pStart, pEnd, gsize;
5679 
5680     ierr = DMPlexGetDepthStratum(dm, depths[d], &pStart, &pEnd);CHKERRQ(ierr);
5681     ierr = DMPlexCreateNumbering_Private(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
5682     shift += gsize;
5683   }
5684   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);
5685   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
5686   PetscFunctionReturn(0);
5687 }
5688 
5689 
5690 #undef __FUNCT__
5691 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
5692 /*@C
5693   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
5694   the local section and an SF describing the section point overlap.
5695 
5696   Input Parameters:
5697   + s - The PetscSection for the local field layout
5698   . sf - The SF describing parallel layout of the section points
5699   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
5700   . label - The label specifying the points
5701   - labelValue - The label stratum specifying the points
5702 
5703   Output Parameter:
5704   . gsection - The PetscSection for the global field layout
5705 
5706   Note: This gives negative sizes and offsets to points not owned by this process
5707 
5708   Level: developer
5709 
5710 .seealso: PetscSectionCreate()
5711 @*/
5712 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
5713 {
5714   PetscInt      *neg = NULL, *tmpOff = NULL;
5715   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
5716   PetscErrorCode ierr;
5717 
5718   PetscFunctionBegin;
5719   ierr = PetscSectionCreate(PetscObjectComm((PetscObject) s), gsection);CHKERRQ(ierr);
5720   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
5721   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
5722   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
5723   if (nroots >= 0) {
5724     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
5725     ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
5726     if (nroots > pEnd-pStart) {
5727       ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
5728     } else {
5729       tmpOff = &(*gsection)->atlasDof[-pStart];
5730     }
5731   }
5732   /* Mark ghost points with negative dof */
5733   for (p = pStart; p < pEnd; ++p) {
5734     PetscInt value;
5735 
5736     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
5737     if (value != labelValue) continue;
5738     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
5739     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
5740     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
5741     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
5742     if (neg) neg[p] = -(dof+1);
5743   }
5744   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
5745   if (nroots >= 0) {
5746     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5747     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5748     if (nroots > pEnd-pStart) {
5749       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
5750     }
5751   }
5752   /* Calculate new sizes, get proccess offset, and calculate point offsets */
5753   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5754     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
5755     (*gsection)->atlasOff[p] = off;
5756     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
5757   }
5758   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) s));CHKERRQ(ierr);
5759   globalOff -= off;
5760   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5761     (*gsection)->atlasOff[p] += globalOff;
5762     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
5763   }
5764   /* Put in negative offsets for ghost points */
5765   if (nroots >= 0) {
5766     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5767     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5768     if (nroots > pEnd-pStart) {
5769       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
5770     }
5771   }
5772   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
5773   ierr = PetscFree(neg);CHKERRQ(ierr);
5774   PetscFunctionReturn(0);
5775 }
5776 
5777 #undef __FUNCT__
5778 #define __FUNCT__ "DMPlexCheckSymmetry"
5779 /*@
5780   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
5781 
5782   Input Parameters:
5783   + dm - The DMPlex object
5784 
5785   Note: This is a useful diagnostic when creating meshes programmatically.
5786 
5787   Level: developer
5788 
5789 .seealso: DMCreate(), DMCheckSkeleton(), DMCheckFaces()
5790 @*/
5791 PetscErrorCode DMPlexCheckSymmetry(DM dm)
5792 {
5793   PetscSection    coneSection, supportSection;
5794   const PetscInt *cone, *support;
5795   PetscInt        coneSize, c, supportSize, s;
5796   PetscInt        pStart, pEnd, p, csize, ssize;
5797   PetscErrorCode  ierr;
5798 
5799   PetscFunctionBegin;
5800   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5801   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
5802   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
5803   /* Check that point p is found in the support of its cone points, and vice versa */
5804   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
5805   for (p = pStart; p < pEnd; ++p) {
5806     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
5807     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
5808     for (c = 0; c < coneSize; ++c) {
5809       PetscBool dup = PETSC_FALSE;
5810       PetscInt  d;
5811       for (d = c-1; d >= 0; --d) {
5812         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
5813       }
5814       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
5815       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
5816       for (s = 0; s < supportSize; ++s) {
5817         if (support[s] == p) break;
5818       }
5819       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
5820         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", p);
5821         for (s = 0; s < coneSize; ++s) {
5822           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[s]);
5823         }
5824         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
5825         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", cone[c]);
5826         for (s = 0; s < supportSize; ++s) {
5827           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[s]);
5828         }
5829         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
5830         if (dup) {
5831           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not repeatedly found in support of repeated cone point %d", p, cone[c]);
5832         } else {
5833           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in support of cone point %d", p, cone[c]);
5834         }
5835       }
5836     }
5837     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
5838     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
5839     for (s = 0; s < supportSize; ++s) {
5840       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
5841       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
5842       for (c = 0; c < coneSize; ++c) {
5843         if (cone[c] == p) break;
5844       }
5845       if (c >= coneSize) {
5846         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", p);
5847         for (c = 0; c < supportSize; ++c) {
5848           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[c]);
5849         }
5850         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
5851         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", support[s]);
5852         for (c = 0; c < coneSize; ++c) {
5853           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[c]);
5854         }
5855         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
5856         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in cone of support point %d", p, support[s]);
5857       }
5858     }
5859   }
5860   ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
5861   ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
5862   if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %d != Total support size %d", csize, ssize);
5863   PetscFunctionReturn(0);
5864 }
5865 
5866 #undef __FUNCT__
5867 #define __FUNCT__ "DMPlexCheckSkeleton"
5868 /*@
5869   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
5870 
5871   Input Parameters:
5872 + dm - The DMPlex object
5873 . isSimplex - Are the cells simplices or tensor products
5874 - cellHeight - Normally 0
5875 
5876   Note: This is a useful diagnostic when creating meshes programmatically.
5877 
5878   Level: developer
5879 
5880 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckFaces()
5881 @*/
5882 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscBool isSimplex, PetscInt cellHeight)
5883 {
5884   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
5885   PetscErrorCode ierr;
5886 
5887   PetscFunctionBegin;
5888   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5889   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5890   switch (dim) {
5891   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
5892   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
5893   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
5894   default:
5895     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %d", dim);
5896   }
5897   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5898   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5899   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5900   cMax = cMax >= 0 ? cMax : cEnd;
5901   for (c = cStart; c < cMax; ++c) {
5902     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
5903 
5904     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5905     for (cl = 0; cl < closureSize*2; cl += 2) {
5906       const PetscInt p = closure[cl];
5907       if ((p >= vStart) && (p < vEnd)) ++coneSize;
5908     }
5909     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5910     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has  %d vertices != %d", c, coneSize, numCorners);
5911   }
5912   for (c = cMax; c < cEnd; ++c) {
5913     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
5914 
5915     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5916     for (cl = 0; cl < closureSize*2; cl += 2) {
5917       const PetscInt p = closure[cl];
5918       if ((p >= vStart) && (p < vEnd)) ++coneSize;
5919     }
5920     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5921     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %d has  %d vertices > %d", c, coneSize, numHybridCorners);
5922   }
5923   PetscFunctionReturn(0);
5924 }
5925 
5926 #undef __FUNCT__
5927 #define __FUNCT__ "DMPlexCheckFaces"
5928 /*@
5929   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
5930 
5931   Input Parameters:
5932 + dm - The DMPlex object
5933 . isSimplex - Are the cells simplices or tensor products
5934 - cellHeight - Normally 0
5935 
5936   Note: This is a useful diagnostic when creating meshes programmatically.
5937 
5938   Level: developer
5939 
5940 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton()
5941 @*/
5942 PetscErrorCode DMPlexCheckFaces(DM dm, PetscBool isSimplex, PetscInt cellHeight)
5943 {
5944   PetscInt       pMax[4];
5945   PetscInt       dim, vStart, vEnd, cStart, cEnd, c, h;
5946   PetscErrorCode ierr;
5947 
5948   PetscFunctionBegin;
5949   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5950   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5951   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5952   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
5953   for (h = cellHeight; h < dim; ++h) {
5954     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
5955     for (c = cStart; c < cEnd; ++c) {
5956       const PetscInt *cone, *ornt, *faces;
5957       PetscInt        numFaces, faceSize, coneSize,f;
5958       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
5959 
5960       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
5961       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
5962       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
5963       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
5964       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5965       for (cl = 0; cl < closureSize*2; cl += 2) {
5966         const PetscInt p = closure[cl];
5967         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
5968       }
5969       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
5970       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has %d faces but should have %d", c, coneSize, numFaces);
5971       for (f = 0; f < numFaces; ++f) {
5972         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
5973 
5974         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
5975         for (cl = 0; cl < fclosureSize*2; cl += 2) {
5976           const PetscInt p = fclosure[cl];
5977           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
5978         }
5979         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d has %d vertices but should have %d", cone[f], f, c, fnumCorners, faceSize);
5980         for (v = 0; v < fnumCorners; ++v) {
5981           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d vertex %d, %d != %d", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
5982         }
5983         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
5984       }
5985       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
5986       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5987     }
5988   }
5989   PetscFunctionReturn(0);
5990 }
5991 
5992 #undef __FUNCT__
5993 #define __FUNCT__ "DMCreateInterpolation_Plex"
5994 /* Pointwise interpolation
5995      Just code FEM for now
5996      u^f = I u^c
5997      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
5998      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
5999      I_{ij} = psi^f_i phi^c_j
6000 */
6001 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
6002 {
6003   PetscSection   gsc, gsf;
6004   PetscInt       m, n;
6005   void          *ctx;
6006   PetscErrorCode ierr;
6007 
6008   PetscFunctionBegin;
6009   /*
6010   Loop over coarse cells
6011     Loop over coarse basis functions
6012       Loop over fine cells in coarse cell
6013         Loop over fine dual basis functions
6014           Evaluate coarse basis on fine dual basis quad points
6015           Sum
6016           Update local element matrix
6017     Accumulate to interpolation matrix
6018 
6019    Can extend PetscFEIntegrateJacobian_Basic() to do a specialized cell loop
6020   */
6021   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6022   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6023   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6024   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6025   /* We need to preallocate properly */
6026   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
6027   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6028   ierr = MatSetType(*interpolation, dmCoarse->mattype);CHKERRQ(ierr);
6029   ierr = MatSetUp(*interpolation);CHKERRQ(ierr);
6030   ierr = MatSetFromOptions(*interpolation);CHKERRQ(ierr);
6031   ierr = MatSetOption(*interpolation, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);CHKERRQ(ierr);
6032   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6033   ierr = DMPlexComputeInterpolatorFEM(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);
6034   /* Use naive scaling */
6035   ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
6036   PetscFunctionReturn(0);
6037 }
6038 
6039 #undef __FUNCT__
6040 #define __FUNCT__ "DMCreateInjection_Plex"
6041 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, VecScatter *ctx)
6042 {
6043   Vec             cv,  fv;
6044   IS              cis, fis, fpointIS;
6045   PetscSection    sc, gsc, gsf;
6046   const PetscInt *fpoints;
6047   PetscInt       *cindices, *findices;
6048   PetscInt        cpStart, cpEnd, m, off, cp;
6049   PetscErrorCode  ierr;
6050 
6051   PetscFunctionBegin;
6052   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6053   ierr = DMGetGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6054   ierr = DMGetDefaultSection(dmCoarse, &sc);CHKERRQ(ierr);
6055   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6056   ierr = DMGetGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6057   ierr = DMPlexCreateCoarsePointIS(dmCoarse, &fpointIS);CHKERRQ(ierr);
6058   ierr = PetscSectionGetConstrainedStorageSize(gsc, &m);CHKERRQ(ierr);
6059   ierr = PetscMalloc2(m,&cindices,m,&findices);CHKERRQ(ierr);
6060   ierr = PetscSectionGetChart(gsc, &cpStart, &cpEnd);CHKERRQ(ierr);
6061   ierr = ISGetIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6062   for (cp = cpStart, off = 0; cp < cpEnd; ++cp) {
6063     const PetscInt *cdofsC = NULL;
6064     PetscInt        fp     = fpoints[cp-cpStart], dofC, cdofC, dofF, offC, offF, d, e;
6065 
6066     ierr = PetscSectionGetDof(gsc, cp, &dofC);CHKERRQ(ierr);
6067     if (dofC <= 0) continue;
6068     ierr = PetscSectionGetConstraintDof(sc, cp, &cdofC);CHKERRQ(ierr);
6069     ierr = PetscSectionGetDof(gsf, fp, &dofF);CHKERRQ(ierr);
6070     ierr = PetscSectionGetOffset(gsc, cp, &offC);CHKERRQ(ierr);
6071     ierr = PetscSectionGetOffset(gsf, fp, &offF);CHKERRQ(ierr);
6072     if (cdofC) {ierr = PetscSectionGetConstraintIndices(sc, cp, &cdofsC);CHKERRQ(ierr);}
6073     if (dofC != dofF) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has %d coarse dofs != %d fine dofs", cp, fp, dofC, dofF);
6074     if (offC < 0 || offF < 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Coarse point %d has invalid offset %d (%d)", cp, offC, offF);
6075     for (d = 0, e = 0; d < dofC; ++d) {
6076       if (cdofsC && cdofsC[e] == d) {++e; continue;}
6077       cindices[off+d-e] = offC+d; findices[off+d-e] = offF+d;
6078     }
6079     if (e != cdofC) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has invalid number of constraints %d != %d", cp, fp, e, cdofC);
6080     off += dofC-cdofC;
6081   }
6082   ierr = ISRestoreIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6083   if (off != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coarse dofs %d != %d", off, m);
6084   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);CHKERRQ(ierr);
6085   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);CHKERRQ(ierr);
6086   ierr = VecScatterCreate(cv, cis, fv, fis, ctx);CHKERRQ(ierr);
6087   ierr = ISDestroy(&cis);CHKERRQ(ierr);
6088   ierr = ISDestroy(&fis);CHKERRQ(ierr);
6089   ierr = DMRestoreGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6090   ierr = DMRestoreGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6091   ierr = ISDestroy(&fpointIS);CHKERRQ(ierr);
6092   PetscFunctionReturn(0);
6093 }
6094 
6095 #undef __FUNCT__
6096 #define __FUNCT__ "DMCreateDefaultSection_Plex"
6097 /* Pointwise interpolation
6098      Just code FEM for now
6099      u^f = I u^c
6100      sum_k u^f_k phi^f_k = I sum_l u^c_l phi^c_l
6101      u^f_i = sum_l int psi^f_i I phi^c_l u^c_l
6102      I_{ij} = int psi^f_i phi^c_j
6103 */
6104 PetscErrorCode DMCreateDefaultSection_Plex(DM dm)
6105 {
6106   PetscSection   section;
6107   IS            *bcPoints;
6108   PetscInt      *bcFields, *numComp, *numDof;
6109   PetscInt       depth, dim, numBd, numBC = 0, numFields, bd, bc, f;
6110   PetscErrorCode ierr;
6111 
6112   PetscFunctionBegin;
6113   /* Handle boundary conditions */
6114   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6115   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6116   ierr = DMPlexGetNumBoundary(dm, &numBd);CHKERRQ(ierr);
6117   for (bd = 0; bd < numBd; ++bd) {
6118     PetscBool isEssential;
6119     ierr = DMPlexGetBoundary(dm, bd, &isEssential, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6120     if (isEssential) ++numBC;
6121   }
6122   ierr = PetscMalloc2(numBC,&bcFields,numBC,&bcPoints);CHKERRQ(ierr);
6123   for (bd = 0, bc = 0; bd < numBd; ++bd) {
6124     const char     *bdLabel;
6125     DMLabel         label;
6126     const PetscInt *values;
6127     PetscInt        bd2, field, numValues;
6128     PetscBool       isEssential, has, duplicate = PETSC_FALSE;
6129 
6130     ierr = DMPlexGetBoundary(dm, bd, &isEssential, &bdLabel, &field, NULL, &numValues, &values, NULL);CHKERRQ(ierr);
6131     if (numValues != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Bug me and I will fix this");
6132     ierr = DMPlexHasLabel(dm, bdLabel, &has);CHKERRQ(ierr);
6133     if (!has) {
6134       ierr = DMPlexCreateLabel(dm, bdLabel);CHKERRQ(ierr);
6135       ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6136       ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
6137     }
6138     ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6139     /* Only want to do this for FEM, and only once */
6140     for (bd2 = 0; bd2 < bd; ++bd2) {
6141       const char *bdname;
6142       ierr = DMPlexGetBoundary(dm, bd2, NULL, NULL, &bdname, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6143       ierr = PetscStrcmp(bdname, bdLabel, &duplicate);CHKERRQ(ierr);
6144       if (duplicate) break;
6145     }
6146     if (!duplicate) {
6147       ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
6148       ierr = DMPlexLabelAddCells(dm, label);CHKERRQ(ierr);
6149     }
6150     /* Filter out cells, if you actually want to constraint cells you need to do things by hand right now */
6151     if (isEssential) {
6152       IS              tmp;
6153       PetscInt       *newidx;
6154       const PetscInt *idx;
6155       PetscInt        cStart, cEnd, n, p, newn = 0;
6156 
6157       bcFields[bc] = field;
6158       ierr = DMPlexGetStratumIS(dm, bdLabel, values[0], &tmp);CHKERRQ(ierr);
6159       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6160       ierr = ISGetLocalSize(tmp, &n);CHKERRQ(ierr);
6161       ierr = ISGetIndices(tmp, &idx);CHKERRQ(ierr);
6162       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) ++newn;
6163       ierr = PetscMalloc1(newn,&newidx);CHKERRQ(ierr);
6164       newn = 0;
6165       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) newidx[newn++] = idx[p];
6166       ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), newn, newidx, PETSC_OWN_POINTER, &bcPoints[bc++]);CHKERRQ(ierr);
6167       ierr = ISRestoreIndices(tmp, &idx);CHKERRQ(ierr);
6168       ierr = ISDestroy(&tmp);CHKERRQ(ierr);
6169     }
6170   }
6171   /* Handle discretization */
6172   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
6173   ierr = PetscMalloc2(numFields,&numComp,numFields*(dim+1),&numDof);CHKERRQ(ierr);
6174   for (f = 0; f < numFields; ++f) {
6175     PetscFE         fe;
6176     const PetscInt *numFieldDof;
6177     PetscInt        d;
6178 
6179     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6180     ierr = PetscFEGetNumComponents(fe, &numComp[f]);CHKERRQ(ierr);
6181     ierr = PetscFEGetNumDof(fe, &numFieldDof);CHKERRQ(ierr);
6182     for (d = 0; d < dim+1; ++d) numDof[f*(dim+1)+d] = numFieldDof[d];
6183   }
6184   for (f = 0; f < numFields; ++f) {
6185     PetscInt d;
6186     for (d = 1; d < dim; ++d) {
6187       if ((numDof[f*(dim+1)+d] > 0) && (depth < dim)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mesh must be interpolated when unknowns are specified on edges or faces.");
6188     }
6189   }
6190   ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcPoints, NULL, &section);CHKERRQ(ierr);
6191   for (f = 0; f < numFields; ++f) {
6192     PetscFE     fe;
6193     const char *name;
6194 
6195     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6196     ierr = PetscObjectGetName((PetscObject) fe, &name);CHKERRQ(ierr);
6197     ierr = PetscSectionSetFieldName(section, f, name);CHKERRQ(ierr);
6198   }
6199   ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
6200   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6201   for (bc = 0; bc < numBC; ++bc) {ierr = ISDestroy(&bcPoints[bc]);CHKERRQ(ierr);}
6202   ierr = PetscFree2(bcFields,bcPoints);CHKERRQ(ierr);
6203   ierr = PetscFree2(numComp,numDof);CHKERRQ(ierr);
6204   PetscFunctionReturn(0);
6205 }
6206 
6207 #undef __FUNCT__
6208 #define __FUNCT__ "DMPlexGetCoarseDM"
6209 /*@
6210   DMPlexGetCoarseDM - Get the coarse mesh from which this was obtained by refinement
6211 
6212   Input Parameter:
6213 . dm - The DMPlex object
6214 
6215   Output Parameter:
6216 . cdm - The coarse DM
6217 
6218   Level: intermediate
6219 
6220 .seealso: DMPlexSetCoarseDM()
6221 @*/
6222 PetscErrorCode DMPlexGetCoarseDM(DM dm, DM *cdm)
6223 {
6224   PetscFunctionBegin;
6225   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6226   PetscValidPointer(cdm, 2);
6227   *cdm = ((DM_Plex *) dm->data)->coarseMesh;
6228   PetscFunctionReturn(0);
6229 }
6230 
6231 #undef __FUNCT__
6232 #define __FUNCT__ "DMPlexSetCoarseDM"
6233 /*@
6234   DMPlexSetCoarseDM - Set the coarse mesh from which this was obtained by refinement
6235 
6236   Input Parameters:
6237 + dm - The DMPlex object
6238 - cdm - The coarse DM
6239 
6240   Level: intermediate
6241 
6242 .seealso: DMPlexGetCoarseDM()
6243 @*/
6244 PetscErrorCode DMPlexSetCoarseDM(DM dm, DM cdm)
6245 {
6246   DM_Plex       *mesh;
6247   PetscErrorCode ierr;
6248 
6249   PetscFunctionBegin;
6250   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6251   if (cdm) PetscValidHeaderSpecific(cdm, DM_CLASSID, 2);
6252   mesh = (DM_Plex *) dm->data;
6253   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
6254   mesh->coarseMesh = cdm;
6255   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
6256   PetscFunctionReturn(0);
6257 }
6258