xref: /petsc/src/dm/impls/plex/plex.c (revision ef31f6718bb23ecc2f0a39e2dc66660d9174051d)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 #include <petscviewerhdf5.h>
6 
7 /* Logging support */
8 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM;
9 
10 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
11 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
12 PETSC_EXTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);
13 PETSC_EXTERN PetscErrorCode DMTSGetTimeStepNumber(DM,PetscInt*);
14 
15 #undef __FUNCT__
16 #define __FUNCT__ "GetFieldType_Static"
17 static PetscErrorCode GetFieldType_Static(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
18 {
19   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, vdof = 0, cdof = 0;
20   PetscErrorCode ierr;
21 
22   PetscFunctionBegin;
23   *ft  = PETSC_VTK_POINT_FIELD;
24   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
25   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
26   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
27   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
28   if (field >= 0) {
29     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vdof);CHKERRQ(ierr);}
30     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &cdof);CHKERRQ(ierr);}
31   } else {
32     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
33     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
34   }
35   if (vdof) {
36     *sStart = vStart;
37     *sEnd   = vEnd;
38     if (vdof == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
39     else             *ft = PETSC_VTK_POINT_FIELD;
40   } else if (cdof) {
41     *sStart = cStart;
42     *sEnd   = cEnd;
43     if (cdof == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
44     else             *ft = PETSC_VTK_CELL_FIELD;
45   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
46   PetscFunctionReturn(0);
47 }
48 
49 #if defined(PETSC_HAVE_HDF5)
50 #undef __FUNCT__
51 #define __FUNCT__ "GetField_Static"
52 static PetscErrorCode GetField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
53 {
54   PetscInt      *subIndices;
55   PetscInt       Nc, subSize = 0, subOff = 0, p;
56   PetscErrorCode ierr;
57 
58   PetscFunctionBegin;
59   ierr = PetscSectionGetFieldComponents(section, field, &Nc);CHKERRQ(ierr);
60   for (p = pStart; p < pEnd; ++p) {
61     PetscInt gdof, fdof = 0;
62 
63     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
64     if (gdof > 0) {ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);}
65     subSize += fdof;
66   }
67   ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
68   for (p = pStart; p < pEnd; ++p) {
69     PetscInt gdof, goff;
70 
71     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
72     if (gdof > 0) {
73       PetscInt fdof, fc, f2, poff = 0;
74 
75       ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
76       /* Can get rid of this loop by storing field information in the global section */
77       for (f2 = 0; f2 < field; ++f2) {
78         ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
79         poff += fdof;
80       }
81       ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);
82       for (fc = 0; fc < fdof; ++fc, ++subOff) subIndices[subOff] = goff+poff+fc;
83     }
84   }
85   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
86   ierr = VecGetSubVector(v, *is, subv);CHKERRQ(ierr);
87   ierr = VecSetBlockSize(*subv, Nc);CHKERRQ(ierr);
88   PetscFunctionReturn(0);
89 }
90 
91 #undef __FUNCT__
92 #define __FUNCT__ "RestoreField_Static"
93 static PetscErrorCode RestoreField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
94 {
95   PetscErrorCode ierr;
96 
97   PetscFunctionBegin;
98   ierr = VecRestoreSubVector(v, *is, subv);CHKERRQ(ierr);
99   ierr = ISDestroy(is);CHKERRQ(ierr);
100   PetscFunctionReturn(0);
101 }
102 
103 #undef __FUNCT__
104 #define __FUNCT__ "DMSequenceView_HDF5"
105 static PetscErrorCode DMSequenceView_HDF5(DM dm, const char *seqname, PetscInt seqnum, PetscScalar value, PetscViewer viewer)
106 {
107   Vec            stamp;
108   PetscMPIInt    rank;
109   PetscErrorCode ierr;
110 
111   PetscFunctionBegin;
112   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) viewer), &rank);CHKERRQ(ierr);
113   ierr = VecCreateMPI(PetscObjectComm((PetscObject) viewer), rank ? 0 : 1, 1, &stamp);CHKERRQ(ierr);
114   ierr = VecSetBlockSize(stamp, 1);CHKERRQ(ierr);
115   ierr = PetscObjectSetName((PetscObject) stamp, seqname);CHKERRQ(ierr);
116   if (!rank) {
117     PetscReal timeScale;
118     PetscBool istime;
119 
120     ierr = PetscStrncmp(seqname, "time", 5, &istime);CHKERRQ(ierr);
121     if (istime) {ierr = DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale);CHKERRQ(ierr); value *= timeScale;}
122     ierr = VecSetValue(stamp, 0, value, INSERT_VALUES);CHKERRQ(ierr);
123   }
124   ierr = VecAssemblyBegin(stamp);CHKERRQ(ierr);
125   ierr = VecAssemblyEnd(stamp);CHKERRQ(ierr);
126   ierr = PetscViewerHDF5PushGroup(viewer, "/");CHKERRQ(ierr);
127   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
128   ierr = VecView(stamp, viewer);CHKERRQ(ierr);
129   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
130   ierr = VecDestroy(&stamp);CHKERRQ(ierr);
131   PetscFunctionReturn(0);
132 }
133 
134 #undef __FUNCT__
135 #define __FUNCT__ "VecView_Plex_Local_HDF5"
136 static PetscErrorCode VecView_Plex_Local_HDF5(Vec v, PetscViewer viewer)
137 {
138   DM                      dm;
139   DM                      dmBC;
140   PetscSection            section, sectionGlobal;
141   Vec                     gv;
142   const char             *name;
143   PetscViewerVTKFieldType ft;
144   PetscViewerFormat       format;
145   PetscInt                seqnum;
146   PetscBool               isseq;
147   PetscErrorCode          ierr;
148 
149   PetscFunctionBegin;
150   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
151   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
152   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
153   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
154   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
155   ierr = DMSequenceView_HDF5(dm, "time", seqnum, (PetscScalar) seqnum, viewer);CHKERRQ(ierr);
156   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
157   ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
158   ierr = DMGetDefaultGlobalSection(dmBC, &sectionGlobal);CHKERRQ(ierr);
159   ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
160   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
161   ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
162   ierr = DMLocalToGlobalBegin(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
163   ierr = DMLocalToGlobalEnd(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
164   ierr = PetscObjectTypeCompare((PetscObject) gv, VECSEQ, &isseq);CHKERRQ(ierr);
165   if (isseq) {ierr = VecView_Seq(gv, viewer);CHKERRQ(ierr);}
166   else       {ierr = VecView_MPI(gv, viewer);CHKERRQ(ierr);}
167   if (format == PETSC_VIEWER_HDF5_VIZ) {
168     /* Output visualization representation */
169     PetscInt numFields, f;
170 
171     ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
172     for (f = 0; f < numFields; ++f) {
173       Vec         subv;
174       IS          is;
175       const char *fname, *fgroup;
176       char        group[PETSC_MAX_PATH_LEN];
177       PetscInt    pStart, pEnd;
178       PetscBool   flag;
179 
180       ierr = GetFieldType_Static(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
181       fgroup = (ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_POINT_FIELD) ? "/vertex_fields" : "/cell_fields";
182       ierr = PetscSectionGetFieldName(section, f, &fname);CHKERRQ(ierr);
183       ierr = PetscViewerHDF5PushGroup(viewer, fgroup);CHKERRQ(ierr);
184       ierr = GetField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
185       ierr = PetscObjectSetName((PetscObject) subv, fname);CHKERRQ(ierr);
186       if (isseq) {ierr = VecView_Seq(subv, viewer);CHKERRQ(ierr);}
187       else       {ierr = VecView_MPI(subv, viewer);CHKERRQ(ierr);}
188       ierr = RestoreField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
189       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
190       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "%s/%s", fgroup, fname);CHKERRQ(ierr);
191       ierr = PetscViewerHDF5HasAttribute(viewer, group, "vector_field_type", &flag);CHKERRQ(ierr);
192       if (!flag) {
193         if ((ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_CELL_VECTOR_FIELD)) {
194           ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "vector");CHKERRQ(ierr);
195         } else {
196           ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "scalar");CHKERRQ(ierr);
197         }
198       }
199     }
200   }
201   ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
202   PetscFunctionReturn(0);
203 }
204 #endif
205 
206 #undef __FUNCT__
207 #define __FUNCT__ "VecView_Plex_Local"
208 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
209 {
210   DM             dm;
211   PetscBool      isvtk, ishdf5, isseq;
212   PetscErrorCode ierr;
213 
214   PetscFunctionBegin;
215   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
216   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
217   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
218   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
219   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
220   if (isvtk || ishdf5) {
221     PetscInt  numFields;
222     PetscBool fem = PETSC_FALSE;
223 
224     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
225     if (numFields) {
226       PetscObject fe;
227 
228       ierr = DMGetField(dm, 0, &fe);CHKERRQ(ierr);
229       if (fe->classid == PETSCFE_CLASSID) fem = PETSC_TRUE;
230     }
231     if (fem) {
232       ierr = DMPlexInsertBoundaryValuesFEM(dm, v);CHKERRQ(ierr);
233     } else {
234       /* TODO Fix the time, and add FVM objects */
235       ierr = DMPlexInsertBoundaryValuesFVM(dm, 0.0, v);CHKERRQ(ierr);
236     }
237   }
238   if (isvtk) {
239     PetscSection            section;
240     PetscViewerVTKFieldType ft;
241     PetscInt                pStart, pEnd;
242 
243     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
244     ierr = GetFieldType_Static(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
245     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
246     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
247     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
248   } else if (ishdf5) {
249 #if defined(PETSC_HAVE_HDF5)
250     ierr = VecView_Plex_Local_HDF5(v, viewer);CHKERRQ(ierr);
251 #else
252     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
253 #endif
254   } else {
255     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
256     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
257   }
258   PetscFunctionReturn(0);
259 }
260 
261 #if defined(PETSC_HAVE_HDF5)
262 #undef __FUNCT__
263 #define __FUNCT__ "VecView_Plex_HDF5"
264 static PetscErrorCode VecView_Plex_HDF5(Vec v, PetscViewer viewer)
265 {
266   DM             dm;
267   Vec            locv;
268   const char    *name;
269   PetscErrorCode ierr;
270 
271   PetscFunctionBegin;
272   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
273   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
274   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
275   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
276   ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
277   ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
278   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
279   ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);CHKERRQ(ierr);
280   ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
281   ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
282   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
283   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
284   PetscFunctionReturn(0);
285 }
286 #endif
287 
288 #undef __FUNCT__
289 #define __FUNCT__ "VecView_Plex"
290 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
291 {
292   DM             dm;
293   PetscBool      isvtk, ishdf5, isseq;
294   PetscErrorCode ierr;
295 
296   PetscFunctionBegin;
297   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
298   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
299   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
300   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
301   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
302   if (isvtk) {
303     Vec         locv;
304     const char *name;
305 
306     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
307     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
308     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
309     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
310     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
311     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
312     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
313   } else if (ishdf5) {
314 #if defined(PETSC_HAVE_HDF5)
315     ierr = VecView_Plex_HDF5(v, viewer);CHKERRQ(ierr);
316 #else
317     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
318 #endif
319   } else {
320     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
321     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
322   }
323   PetscFunctionReturn(0);
324 }
325 
326 #undef __FUNCT__
327 #define __FUNCT__ "VecLoad_Plex_Local"
328 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
329 {
330   DM             dm;
331   PetscBool      ishdf5;
332   PetscErrorCode ierr;
333 
334   PetscFunctionBegin;
335   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
336   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
337   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
338   if (ishdf5) {
339     DM          dmBC;
340     Vec         gv;
341     const char *name;
342 
343     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
344     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
345     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
346     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
347     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
348     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
349     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
350     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
351   } else {
352     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
353   }
354   PetscFunctionReturn(0);
355 }
356 
357 #if defined(PETSC_HAVE_HDF5)
358 #undef __FUNCT__
359 #define __FUNCT__ "VecLoad_Plex_HDF5"
360 static PetscErrorCode VecLoad_Plex_HDF5(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   Vec            locv;
364   const char    *name;
365   PetscInt       seqnum;
366   PetscErrorCode ierr;
367 
368   PetscFunctionBegin;
369   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
370   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
371   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
372   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
373   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
374   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
375   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
376   ierr = VecLoad_Plex_Local(locv, viewer);CHKERRQ(ierr);
377   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
378   ierr = DMLocalToGlobalBegin(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
379   ierr = DMLocalToGlobalEnd(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
380   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
381   PetscFunctionReturn(0);
382 }
383 #endif
384 
385 #undef __FUNCT__
386 #define __FUNCT__ "VecLoad_Plex"
387 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
388 {
389   DM             dm;
390   PetscBool      ishdf5;
391   PetscErrorCode ierr;
392 
393   PetscFunctionBegin;
394   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
395   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
396   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
397   if (ishdf5) {
398 #if defined(PETSC_HAVE_HDF5)
399     ierr = VecLoad_Plex_HDF5(v, viewer);CHKERRQ(ierr);
400 #else
401     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
402 #endif
403   } else {
404     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
405   }
406   PetscFunctionReturn(0);
407 }
408 
409 #undef __FUNCT__
410 #define __FUNCT__ "DMPlexView_Ascii"
411 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
412 {
413   DM_Plex          *mesh = (DM_Plex*) dm->data;
414   DM                cdm;
415   DMLabel           markers;
416   PetscSection      coordSection;
417   Vec               coordinates;
418   PetscViewerFormat format;
419   PetscErrorCode    ierr;
420 
421   PetscFunctionBegin;
422   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
423   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
424   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
425   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
426   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
427     const char *name;
428     PetscInt    maxConeSize, maxSupportSize;
429     PetscInt    pStart, pEnd, p;
430     PetscMPIInt rank, size;
431 
432     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
433     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
434     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
435     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
436     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
437     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
438     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
439     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
440     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
441     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
442     for (p = pStart; p < pEnd; ++p) {
443       PetscInt dof, off, s;
444 
445       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
446       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
447       for (s = off; s < off+dof; ++s) {
448         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
449       }
450     }
451     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
452     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
453     for (p = pStart; p < pEnd; ++p) {
454       PetscInt dof, off, c;
455 
456       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
457       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
458       for (c = off; c < off+dof; ++c) {
459         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
460       }
461     }
462     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
463     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
464     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
465     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
466     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
467     if (size > 1) {
468       PetscSF sf;
469 
470       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
471       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
472     }
473     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
474   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
475     const char  *name;
476     const char  *colors[3] = {"red", "blue", "green"};
477     const int    numColors  = 3;
478     PetscReal    scale      = 2.0;
479     PetscScalar *coords;
480     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
481     PetscMPIInt  rank, size;
482 
483     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
484     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
485     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
486     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
487     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
488     ierr = PetscViewerASCIIPrintf(viewer, "\
489 \\documentclass[crop,multi=false]{standalone}\n\n\
490 \\usepackage{tikz}\n\
491 \\usepackage{pgflibraryshapes}\n\
492 \\usetikzlibrary{backgrounds}\n\
493 \\usetikzlibrary{arrows}\n\
494 \\begin{document}\n\
495 \\section{%s}\n\
496 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
497     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
498     for (p = 0; p < size; ++p) {
499       if (p > 0 && p == size-1) {
500         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
501       } else if (p > 0) {
502         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
503       }
504       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
505     }
506     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
507 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
508     /* Plot vertices */
509     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
510     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
511     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
512     for (v = vStart; v < vEnd; ++v) {
513       PetscInt off, dof, d;
514 
515       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
516       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
517       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
518       for (d = 0; d < dof; ++d) {
519         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
520         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*PetscRealPart(coords[off+d])));CHKERRQ(ierr);
521       }
522       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
523     }
524     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
525     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
526     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
527     /* Plot edges */
528     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
529     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
530     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
531     for (e = eStart; e < eEnd; ++e) {
532       const PetscInt *cone;
533       PetscInt        coneSize, offA, offB, dof, d;
534 
535       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
536       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
537       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
538       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
539       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
540       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
541       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
542       for (d = 0; d < dof; ++d) {
543         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
544         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d])));CHKERRQ(ierr);
545       }
546       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
547     }
548     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
549     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
550     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
551     /* Plot cells */
552     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
553     for (c = cStart; c < cEnd; ++c) {
554       PetscInt *closure = NULL;
555       PetscInt  closureSize, firstPoint = -1;
556 
557       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
558       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
559       for (p = 0; p < closureSize*2; p += 2) {
560         const PetscInt point = closure[p];
561 
562         if ((point < vStart) || (point >= vEnd)) continue;
563         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
564         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
565         if (firstPoint < 0) firstPoint = point;
566       }
567       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
568       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
569       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
570     }
571     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
572     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
573     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
574   } else {
575     MPI_Comm    comm;
576     PetscInt   *sizes, *hybsizes;
577     PetscInt    locDepth, depth, dim, d, pMax[4];
578     PetscInt    pStart, pEnd, p;
579     PetscInt    numLabels, l;
580     const char *name;
581     PetscMPIInt size;
582 
583     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
584     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
585     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
586     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
587     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dim);CHKERRQ(ierr);}
588     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);}
589     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
590     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
591     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], &pMax[depth-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
592     ierr = PetscMalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
593     if (depth == 1) {
594       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
595       pEnd = pEnd - pStart;
596       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
597       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
598       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
599       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
600       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
601       pEnd = pEnd - pStart;
602       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
603       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
604       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
605       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
606     } else {
607       for (d = 0; d <= dim; d++) {
608         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
609         pEnd    -= pStart;
610         pMax[d] -= pStart;
611         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
612         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
613         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
614         for (p = 0; p < size; ++p) {
615           if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
616           else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
617         }
618         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
619       }
620     }
621     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
622     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
623     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
624     for (l = 0; l < numLabels; ++l) {
625       DMLabel         label;
626       const char     *name;
627       IS              valueIS;
628       const PetscInt *values;
629       PetscInt        numValues, v;
630 
631       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
632       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
633       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
634       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
635       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
636       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
637       for (v = 0; v < numValues; ++v) {
638         PetscInt size;
639 
640         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
641         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
642         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
643       }
644       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
645       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
646       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
647     }
648   }
649   PetscFunctionReturn(0);
650 }
651 
652 #if defined(PETSC_HAVE_HDF5)
653 #undef __FUNCT__
654 #define __FUNCT__ "DMPlexView_HDF5"
655 /* We only write cells and vertices. Does this screw up parallel reading? */
656 static PetscErrorCode DMPlexView_HDF5(DM dm, PetscViewer viewer)
657 {
658   DM              cdm;
659   Vec             coordinates, newcoords;
660   Vec             coneVec, cellVec;
661   IS              globalVertexNumbers;
662   const PetscInt *gvertex;
663   PetscScalar    *sizes, *vertices;
664   PetscReal       lengthScale;
665   const char     *label   = NULL;
666   PetscInt        labelId = 0, dim;
667   char            group[PETSC_MAX_PATH_LEN];
668   PetscInt        vStart, vEnd, v, cellHeight, cStart, cEnd, cMax, cell, conesSize = 0, numCornersLocal = 0, numCorners, numLabels, l;
669   hid_t           fileId, groupId;
670   herr_t          status;
671   PetscErrorCode  ierr;
672 
673   PetscFunctionBegin;
674   /* Write coordinates */
675   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
676   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
677   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
678   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
679   ierr = VecDuplicate(coordinates, &newcoords);CHKERRQ(ierr);
680   ierr = PetscObjectSetName((PetscObject) newcoords, "vertices");CHKERRQ(ierr);
681   ierr = VecCopy(coordinates, newcoords);CHKERRQ(ierr);
682   ierr = VecScale(newcoords, lengthScale);CHKERRQ(ierr);
683   /* Use the local version to bypass the default group setting */
684   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
685   ierr = VecView(newcoords, viewer);CHKERRQ(ierr);
686   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
687   ierr = VecDestroy(&newcoords);CHKERRQ(ierr);
688   /* Write toplogy */
689   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
690   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
691   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
692   ierr = DMPlexGetHybridBounds(dm, &cMax, PETSC_NULL, PETSC_NULL, PETSC_NULL);CHKERRQ(ierr);
693   if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
694 
695   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coneVec);CHKERRQ(ierr);
696   ierr = VecSetSizes(coneVec, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
697   ierr = VecSetBlockSize(coneVec, 1);CHKERRQ(ierr);
698   ierr = VecSetFromOptions(coneVec);CHKERRQ(ierr);
699   ierr = PetscObjectSetName((PetscObject) coneVec, "coneSize");CHKERRQ(ierr);
700   ierr = VecGetArray(coneVec, &sizes);CHKERRQ(ierr);
701   for (cell = cStart; cell < cEnd; ++cell) {
702     PetscInt *closure = NULL;
703     PetscInt  closureSize, v, Nc = 0;
704 
705     if (label) {
706       PetscInt value;
707       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
708       if (value == labelId) continue;
709     }
710     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
711     for (v = 0; v < closureSize*2; v += 2) {
712       if ((closure[v] >= vStart) && (closure[v] < vEnd)) ++Nc;
713     }
714     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
715     conesSize += Nc;
716     if (!numCornersLocal)           numCornersLocal = Nc;
717     else if (numCornersLocal != Nc) numCornersLocal = 1;
718   }
719   ierr = VecRestoreArray(coneVec, &sizes);CHKERRQ(ierr);
720   ierr = MPI_Allreduce(&numCornersLocal, &numCorners, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
721   if (numCornersLocal && numCornersLocal != numCorners) numCorners = 1;
722 
723   ierr = DMPlexGetVertexNumbering(dm, &globalVertexNumbers);CHKERRQ(ierr);
724   ierr = ISGetIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
725   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
726   ierr = VecSetSizes(cellVec, conesSize, PETSC_DETERMINE);CHKERRQ(ierr);
727   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
728   ierr = VecSetFromOptions(cellVec);CHKERRQ(ierr);
729   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
730   ierr = VecGetArray(cellVec, &vertices);CHKERRQ(ierr);
731   for (cell = cStart, v = 0; cell < cEnd; ++cell) {
732     PetscInt *closure = NULL;
733     PetscInt  closureSize, Nc = 0, p;
734 
735     if (label) {
736       PetscInt value;
737       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
738       if (value == labelId) continue;
739     }
740     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
741     for (p = 0; p < closureSize*2; p += 2) {
742       if ((closure[p] >= vStart) && (closure[p] < vEnd)) {
743         closure[Nc++] = closure[p];
744         }
745     }
746     ierr = DMPlexInvertCell(dim, Nc, closure);CHKERRQ(ierr);
747     for (p = 0; p < Nc; ++p) {
748       const PetscInt gv = gvertex[closure[p] - vStart];
749       vertices[v++] = gv < 0 ? -(gv+1) : gv;
750     }
751     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
752   }
753   if (v != conesSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cell vertices %d != %d", v, conesSize);
754   ierr = VecRestoreArray(cellVec, &vertices);CHKERRQ(ierr);
755   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
756   ierr = VecView(cellVec, viewer);CHKERRQ(ierr);
757   if (numCorners == 1) {
758     ierr = VecView(coneVec, viewer);CHKERRQ(ierr);
759   } else {
760     ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
761   }
762   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
763   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
764   ierr = VecDestroy(&coneVec);CHKERRQ(ierr);
765   ierr = ISRestoreIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
766 
767   ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
768   /* Write Labels*/
769   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
770   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
771   if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
772   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
773   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
774   for (l = 0; l < numLabels; ++l) {
775     DMLabel         label;
776     const char     *name;
777     IS              valueIS;
778     const PetscInt *values;
779     PetscInt        numValues, v;
780     PetscBool       isDepth;
781 
782     ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
783     ierr = PetscStrncmp(name, "depth", 10, &isDepth);CHKERRQ(ierr);
784     if (isDepth) continue;
785     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
786     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
787     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
788     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
789     ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s", name);CHKERRQ(ierr);
790     ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
791     ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
792     if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
793     ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
794     /* TODO: Need to actually loop over the union of label values, ISAllGather() */
795     for (v = 0; v < numValues; ++v) {
796       IS   stratumIS;
797 
798       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%d", name, values[v]);CHKERRQ(ierr);
799       ierr = DMLabelGetStratumIS(label, values[v], &stratumIS);CHKERRQ(ierr);
800       /* TODO: Need to globalize point names and remove unowned points */
801       ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
802       ierr = ISView(stratumIS, viewer);CHKERRQ(ierr);
803       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
804       ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
805     }
806     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
807     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
808   }
809   PetscFunctionReturn(0);
810 }
811 #endif
812 
813 #undef __FUNCT__
814 #define __FUNCT__ "DMView_Plex"
815 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
816 {
817   PetscBool      iascii, ishdf5;
818   PetscErrorCode ierr;
819 
820   PetscFunctionBegin;
821   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
822   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
823   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
824   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
825   if (iascii) {
826     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
827   } else if (ishdf5) {
828 #if defined(PETSC_HAVE_HDF5)
829     ierr = DMPlexView_HDF5(dm, viewer);CHKERRQ(ierr);
830 #else
831     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
832 #endif
833   }
834   PetscFunctionReturn(0);
835 }
836 
837 #if defined(PETSC_HAVE_HDF5)
838 typedef struct {
839   DM          dm;
840   PetscViewer viewer;
841   DMLabel     label;
842 } LabelCtx;
843 
844 static herr_t ReadLabelStratumHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
845 {
846   PetscViewer     viewer = ((LabelCtx *) op_data)->viewer;
847   DMLabel         label  = ((LabelCtx *) op_data)->label;
848   IS              stratumIS;
849   const PetscInt *ind;
850   PetscInt        value, N, i;
851   const char     *lname;
852   char            group[PETSC_MAX_PATH_LEN];
853   PetscErrorCode  ierr;
854 
855   ierr = PetscOptionsStringToInt(name, &value);
856   ierr = ISCreate(PetscObjectComm((PetscObject) viewer), &stratumIS);
857   ierr = PetscObjectSetName((PetscObject) stratumIS, "indices");
858   ierr = DMLabelGetName(label, &lname);
859   ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%s", lname, name);CHKERRQ(ierr);
860   ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
861   ierr = ISLoad(stratumIS, viewer);
862   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
863   ierr = ISGetSize(stratumIS, &N);
864   ierr = ISGetIndices(stratumIS, &ind);
865   for (i = 0; i < N; ++i) {ierr = DMLabelSetValue(label, ind[i], value);}
866   ierr = ISRestoreIndices(stratumIS, &ind);
867   ierr = ISDestroy(&stratumIS);
868   return 0;
869 }
870 
871 static herr_t ReadLabelHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
872 {
873   DM             dm = ((LabelCtx *) op_data)->dm;
874   hsize_t        idx;
875   herr_t         status;
876   PetscErrorCode ierr;
877 
878   ierr = DMPlexCreateLabel(dm, name); if (ierr) return (herr_t) ierr;
879   ierr = DMPlexGetLabel(dm, name, &((LabelCtx *) op_data)->label); if (ierr) return (herr_t) ierr;
880   status = H5Literate_by_name(g_id, name, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelStratumHDF5_Static, op_data, 0);
881   return status;
882 }
883 
884 #undef __FUNCT__
885 #define __FUNCT__ "DMPlexLoad_HDF5"
886 /* The first version will read everything onto proc 0, letting the user distribute
887    The next will create a naive partition, and then rebalance after reading
888 */
889 static PetscErrorCode DMPlexLoad_HDF5(DM dm, PetscViewer viewer)
890 {
891   LabelCtx        ctx;
892   PetscSection    coordSection;
893   Vec             coordinates;
894   Vec             cellVec;
895   PetscScalar    *cells;
896   PetscReal       lengthScale;
897   PetscInt       *cone;
898   PetscInt        dim, spatialDim, numVertices, v, numCorners, numCells, cell, c;
899   hid_t           fileId, groupId;
900   hsize_t         idx = 0;
901   herr_t          status;
902   PetscErrorCode  ierr;
903 
904   PetscFunctionBegin;
905   /* Read geometry */
906   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
907   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coordinates);CHKERRQ(ierr);
908   ierr = PetscObjectSetName((PetscObject) coordinates, "vertices");CHKERRQ(ierr);
909   ierr = VecLoad(coordinates, viewer);CHKERRQ(ierr);
910   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
911   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
912   ierr = VecScale(coordinates, 1.0/lengthScale);CHKERRQ(ierr);
913   ierr = VecGetSize(coordinates, &numVertices);CHKERRQ(ierr);
914   ierr = VecGetBlockSize(coordinates, &spatialDim);CHKERRQ(ierr);
915   numVertices /= spatialDim;
916   /* Read toplogy */
917   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
918   ierr = DMPlexSetDimension(dm, dim);CHKERRQ(ierr);
919   /*   TODO Check for coneSize vector rather than this attribute */
920   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
921   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
922   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
923   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
924   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
925   ierr = VecLoad(cellVec, viewer);CHKERRQ(ierr);
926   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
927   ierr = VecGetSize(cellVec, &numCells);CHKERRQ(ierr);
928   numCells /= numCorners;
929   /* Create Plex */
930   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
931   for (cell = 0; cell < numCells; ++cell) {ierr = DMPlexSetConeSize(dm, cell, numCorners);CHKERRQ(ierr);}
932   ierr = DMSetUp(dm);CHKERRQ(ierr);
933   ierr = PetscMalloc1(numCorners,&cone);CHKERRQ(ierr);
934   ierr = VecGetArray(cellVec, &cells);CHKERRQ(ierr);
935   for (cell = 0; cell < numCells; ++cell) {
936     for (c = 0; c < numCorners; ++c) {cone[c] = numCells + cells[cell*numCorners+c];}
937     ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
938   }
939   ierr = VecRestoreArray(cellVec, &cells);CHKERRQ(ierr);
940   ierr = PetscFree(cone);CHKERRQ(ierr);
941   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
942   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
943   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
944   /* Create coordinates */
945   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
946   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
947   ierr = PetscSectionSetFieldComponents(coordSection, 0, spatialDim);CHKERRQ(ierr);
948   ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
949   for (v = numCells; v < numCells+numVertices; ++v) {
950     ierr = PetscSectionSetDof(coordSection, v, spatialDim);CHKERRQ(ierr);
951     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spatialDim);CHKERRQ(ierr);
952   }
953   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
954   ierr = DMSetCoordinates(dm, coordinates);CHKERRQ(ierr);
955   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
956   /* Read Labels*/
957   ctx.dm     = dm;
958   ctx.viewer = viewer;
959   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
960   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
961   status = H5Literate(groupId, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelHDF5_Static, &ctx);CHKERRQ(status);
962   status = H5Gclose(groupId);CHKERRQ(status);
963   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
964   PetscFunctionReturn(0);
965 }
966 #endif
967 
968 #undef __FUNCT__
969 #define __FUNCT__ "DMLoad_Plex"
970 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
971 {
972   PetscBool      isbinary, ishdf5;
973   PetscErrorCode ierr;
974 
975   PetscFunctionBegin;
976   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
977   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
978   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
979   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
980   if (isbinary) {SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Do not yet support binary viewers");}
981   else if (ishdf5) {
982 #if defined(PETSC_HAVE_HDF5)
983     DM odm;
984     ierr = DMCreate(PetscObjectComm((PetscObject) dm), &odm);CHKERRQ(ierr);
985     ierr = DMSetType(odm, DMPLEX);CHKERRQ(ierr);
986     ierr = DMPlexLoad_HDF5(odm, viewer);CHKERRQ(ierr);
987     ierr = DMPlexInterpolate(odm, &dm);CHKERRQ(ierr);
988     ierr = DMPlexCopyCoordinates(odm, dm);CHKERRQ(ierr);
989     ierr = DMPlexCopyLabels(odm, dm);CHKERRQ(ierr);
990     ierr = DMDestroy(&odm);CHKERRQ(ierr);
991 #else
992     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
993 #endif
994   }
995   PetscFunctionReturn(0);
996 }
997 
998 #undef __FUNCT__
999 #define __FUNCT__ "BoundaryDestroy"
1000 static PetscErrorCode BoundaryDestroy(DMBoundary *boundary)
1001 {
1002   DMBoundary     b, next;
1003   PetscErrorCode ierr;
1004 
1005   PetscFunctionBeginUser;
1006   if (!boundary) PetscFunctionReturn(0);
1007   b = *boundary;
1008   *boundary = NULL;
1009   for (; b; b = next) {
1010     next = b->next;
1011     ierr = PetscFree(b->ids);CHKERRQ(ierr);
1012     ierr = PetscFree(b->name);CHKERRQ(ierr);
1013     ierr = PetscFree(b);CHKERRQ(ierr);
1014   }
1015   PetscFunctionReturn(0);
1016 }
1017 
1018 #undef __FUNCT__
1019 #define __FUNCT__ "DMDestroy_Plex"
1020 PetscErrorCode DMDestroy_Plex(DM dm)
1021 {
1022   DM_Plex       *mesh = (DM_Plex*) dm->data;
1023   DMLabel        next  = mesh->labels;
1024   PetscErrorCode ierr;
1025 
1026   PetscFunctionBegin;
1027   if (--mesh->refct > 0) PetscFunctionReturn(0);
1028   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1029   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1030   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1031   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1032   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1033   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1034   while (next) {
1035     DMLabel tmp = next->next;
1036 
1037     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
1038     next = tmp;
1039   }
1040   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
1041   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1042   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1043   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1044   ierr = BoundaryDestroy(&mesh->boundary);CHKERRQ(ierr);
1045   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1046   ierr = PetscFree(mesh);CHKERRQ(ierr);
1047   PetscFunctionReturn(0);
1048 }
1049 
1050 #undef __FUNCT__
1051 #define __FUNCT__ "DMCreateMatrix_Plex"
1052 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1053 {
1054   PetscSection   section, sectionGlobal;
1055   PetscInt       bs = -1;
1056   PetscInt       localSize;
1057   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
1058   PetscErrorCode ierr;
1059   MatType        mtype;
1060 
1061   PetscFunctionBegin;
1062   ierr = MatInitializePackage();CHKERRQ(ierr);
1063   mtype = dm->mattype;
1064   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
1065   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1066   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1067   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1068   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1069   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1070   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1071   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1072   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1073   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1074   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1075   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1076   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1077   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1078   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1079   if (!isShell) {
1080     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
1081     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
1082 
1083     if (bs < 0) {
1084       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
1085         PetscInt pStart, pEnd, p, dof, cdof;
1086 
1087         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1088         for (p = pStart; p < pEnd; ++p) {
1089           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1090           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1091           if (dof-cdof) {
1092             if (bs < 0) {
1093               bs = dof-cdof;
1094             } else if (bs != dof-cdof) {
1095               /* Layout does not admit a pointwise block size */
1096               bs = 1;
1097               break;
1098             }
1099           }
1100         }
1101         /* Must have same blocksize on all procs (some might have no points) */
1102         bsLocal = bs;
1103         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1104         bsLocal = bs < 0 ? bsMax : bs;
1105         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1106         if (bsMin != bsMax) {
1107           bs = 1;
1108         } else {
1109           bs = bsMax;
1110         }
1111       } else {
1112         bs = 1;
1113       }
1114     }
1115     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1116     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1117     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1118   }
1119   PetscFunctionReturn(0);
1120 }
1121 
1122 #undef __FUNCT__
1123 #define __FUNCT__ "DMPlexGetDimension"
1124 /*@
1125   DMPlexGetDimension - Return the topological mesh dimension
1126 
1127   Not collective
1128 
1129   Input Parameter:
1130 . mesh - The DMPlex
1131 
1132   Output Parameter:
1133 . dim - The topological mesh dimension
1134 
1135   Level: beginner
1136 
1137 .seealso: DMPlexCreate()
1138 @*/
1139 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
1140 {
1141   DM_Plex *mesh = (DM_Plex*) dm->data;
1142 
1143   PetscFunctionBegin;
1144   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1145   PetscValidPointer(dim, 2);
1146   *dim = mesh->dim;
1147   PetscFunctionReturn(0);
1148 }
1149 
1150 #undef __FUNCT__
1151 #define __FUNCT__ "DMPlexSetDimension"
1152 /*@
1153   DMPlexSetDimension - Set the topological mesh dimension
1154 
1155   Collective on mesh
1156 
1157   Input Parameters:
1158 + mesh - The DMPlex
1159 - dim - The topological mesh dimension
1160 
1161   Level: beginner
1162 
1163 .seealso: DMPlexCreate()
1164 @*/
1165 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
1166 {
1167   DM_Plex *mesh = (DM_Plex*) dm->data;
1168 
1169   PetscFunctionBegin;
1170   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1171   PetscValidLogicalCollectiveInt(dm, dim, 2);
1172   mesh->dim = dim;
1173   PetscFunctionReturn(0);
1174 }
1175 
1176 #undef __FUNCT__
1177 #define __FUNCT__ "DMPlexGetChart"
1178 /*@
1179   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1180 
1181   Not collective
1182 
1183   Input Parameter:
1184 . mesh - The DMPlex
1185 
1186   Output Parameters:
1187 + pStart - The first mesh point
1188 - pEnd   - The upper bound for mesh points
1189 
1190   Level: beginner
1191 
1192 .seealso: DMPlexCreate(), DMPlexSetChart()
1193 @*/
1194 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1195 {
1196   DM_Plex       *mesh = (DM_Plex*) dm->data;
1197   PetscErrorCode ierr;
1198 
1199   PetscFunctionBegin;
1200   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1201   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1202   PetscFunctionReturn(0);
1203 }
1204 
1205 #undef __FUNCT__
1206 #define __FUNCT__ "DMPlexSetChart"
1207 /*@
1208   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1209 
1210   Not collective
1211 
1212   Input Parameters:
1213 + mesh - The DMPlex
1214 . pStart - The first mesh point
1215 - pEnd   - The upper bound for mesh points
1216 
1217   Output Parameters:
1218 
1219   Level: beginner
1220 
1221 .seealso: DMPlexCreate(), DMPlexGetChart()
1222 @*/
1223 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1224 {
1225   DM_Plex       *mesh = (DM_Plex*) dm->data;
1226   PetscErrorCode ierr;
1227 
1228   PetscFunctionBegin;
1229   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1230   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1231   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1232   PetscFunctionReturn(0);
1233 }
1234 
1235 #undef __FUNCT__
1236 #define __FUNCT__ "DMPlexGetConeSize"
1237 /*@
1238   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
1239 
1240   Not collective
1241 
1242   Input Parameters:
1243 + mesh - The DMPlex
1244 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1245 
1246   Output Parameter:
1247 . size - The cone size for point p
1248 
1249   Level: beginner
1250 
1251 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1252 @*/
1253 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1254 {
1255   DM_Plex       *mesh = (DM_Plex*) dm->data;
1256   PetscErrorCode ierr;
1257 
1258   PetscFunctionBegin;
1259   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1260   PetscValidPointer(size, 3);
1261   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1262   PetscFunctionReturn(0);
1263 }
1264 
1265 #undef __FUNCT__
1266 #define __FUNCT__ "DMPlexSetConeSize"
1267 /*@
1268   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
1269 
1270   Not collective
1271 
1272   Input Parameters:
1273 + mesh - The DMPlex
1274 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1275 - size - The cone size for point p
1276 
1277   Output Parameter:
1278 
1279   Note:
1280   This should be called after DMPlexSetChart().
1281 
1282   Level: beginner
1283 
1284 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1285 @*/
1286 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1287 {
1288   DM_Plex       *mesh = (DM_Plex*) dm->data;
1289   PetscErrorCode ierr;
1290 
1291   PetscFunctionBegin;
1292   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1293   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1294 
1295   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1296   PetscFunctionReturn(0);
1297 }
1298 
1299 #undef __FUNCT__
1300 #define __FUNCT__ "DMPlexGetCone"
1301 /*@C
1302   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
1303 
1304   Not collective
1305 
1306   Input Parameters:
1307 + mesh - The DMPlex
1308 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1309 
1310   Output Parameter:
1311 . cone - An array of points which are on the in-edges for point p
1312 
1313   Level: beginner
1314 
1315   Fortran Notes:
1316   Since it returns an array, this routine is only available in Fortran 90, and you must
1317   include petsc.h90 in your code.
1318 
1319   You must also call DMPlexRestoreCone() after you finish using the returned array.
1320 
1321 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
1322 @*/
1323 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1324 {
1325   DM_Plex       *mesh = (DM_Plex*) dm->data;
1326   PetscInt       off;
1327   PetscErrorCode ierr;
1328 
1329   PetscFunctionBegin;
1330   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1331   PetscValidPointer(cone, 3);
1332   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1333   *cone = &mesh->cones[off];
1334   PetscFunctionReturn(0);
1335 }
1336 
1337 #undef __FUNCT__
1338 #define __FUNCT__ "DMPlexSetCone"
1339 /*@
1340   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
1341 
1342   Not collective
1343 
1344   Input Parameters:
1345 + mesh - The DMPlex
1346 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1347 - cone - An array of points which are on the in-edges for point p
1348 
1349   Output Parameter:
1350 
1351   Note:
1352   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1353 
1354   Level: beginner
1355 
1356 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1357 @*/
1358 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1359 {
1360   DM_Plex       *mesh = (DM_Plex*) dm->data;
1361   PetscInt       pStart, pEnd;
1362   PetscInt       dof, off, c;
1363   PetscErrorCode ierr;
1364 
1365   PetscFunctionBegin;
1366   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1367   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1368   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1369   if (dof) PetscValidPointer(cone, 3);
1370   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1371   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1372   for (c = 0; c < dof; ++c) {
1373     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1374     mesh->cones[off+c] = cone[c];
1375   }
1376   PetscFunctionReturn(0);
1377 }
1378 
1379 #undef __FUNCT__
1380 #define __FUNCT__ "DMPlexGetConeOrientation"
1381 /*@C
1382   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
1383 
1384   Not collective
1385 
1386   Input Parameters:
1387 + mesh - The DMPlex
1388 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1389 
1390   Output Parameter:
1391 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1392                     integer giving the prescription for cone traversal. If it is negative, the cone is
1393                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1394                     the index of the cone point on which to start.
1395 
1396   Level: beginner
1397 
1398   Fortran Notes:
1399   Since it returns an array, this routine is only available in Fortran 90, and you must
1400   include petsc.h90 in your code.
1401 
1402   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1403 
1404 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1405 @*/
1406 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1407 {
1408   DM_Plex       *mesh = (DM_Plex*) dm->data;
1409   PetscInt       off;
1410   PetscErrorCode ierr;
1411 
1412   PetscFunctionBegin;
1413   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1414 #if defined(PETSC_USE_DEBUG)
1415   {
1416     PetscInt dof;
1417     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1418     if (dof) PetscValidPointer(coneOrientation, 3);
1419   }
1420 #endif
1421   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1422 
1423   *coneOrientation = &mesh->coneOrientations[off];
1424   PetscFunctionReturn(0);
1425 }
1426 
1427 #undef __FUNCT__
1428 #define __FUNCT__ "DMPlexSetConeOrientation"
1429 /*@
1430   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
1431 
1432   Not collective
1433 
1434   Input Parameters:
1435 + mesh - The DMPlex
1436 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1437 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1438                     integer giving the prescription for cone traversal. If it is negative, the cone is
1439                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1440                     the index of the cone point on which to start.
1441 
1442   Output Parameter:
1443 
1444   Note:
1445   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1446 
1447   Level: beginner
1448 
1449 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1450 @*/
1451 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1452 {
1453   DM_Plex       *mesh = (DM_Plex*) dm->data;
1454   PetscInt       pStart, pEnd;
1455   PetscInt       dof, off, c;
1456   PetscErrorCode ierr;
1457 
1458   PetscFunctionBegin;
1459   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1460   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1461   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1462   if (dof) PetscValidPointer(coneOrientation, 3);
1463   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1464   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1465   for (c = 0; c < dof; ++c) {
1466     PetscInt cdof, o = coneOrientation[c];
1467 
1468     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1469     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1470     mesh->coneOrientations[off+c] = o;
1471   }
1472   PetscFunctionReturn(0);
1473 }
1474 
1475 #undef __FUNCT__
1476 #define __FUNCT__ "DMPlexInsertCone"
1477 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1478 {
1479   DM_Plex       *mesh = (DM_Plex*) dm->data;
1480   PetscInt       pStart, pEnd;
1481   PetscInt       dof, off;
1482   PetscErrorCode ierr;
1483 
1484   PetscFunctionBegin;
1485   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1486   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1487   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1488   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1489   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1490   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1491   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1492   mesh->cones[off+conePos] = conePoint;
1493   PetscFunctionReturn(0);
1494 }
1495 
1496 #undef __FUNCT__
1497 #define __FUNCT__ "DMPlexInsertConeOrientation"
1498 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1499 {
1500   DM_Plex       *mesh = (DM_Plex*) dm->data;
1501   PetscInt       pStart, pEnd;
1502   PetscInt       dof, off;
1503   PetscErrorCode ierr;
1504 
1505   PetscFunctionBegin;
1506   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1507   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1508   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1509   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1510   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1511   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1512   mesh->coneOrientations[off+conePos] = coneOrientation;
1513   PetscFunctionReturn(0);
1514 }
1515 
1516 #undef __FUNCT__
1517 #define __FUNCT__ "DMPlexGetSupportSize"
1518 /*@
1519   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
1520 
1521   Not collective
1522 
1523   Input Parameters:
1524 + mesh - The DMPlex
1525 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1526 
1527   Output Parameter:
1528 . size - The support size for point p
1529 
1530   Level: beginner
1531 
1532 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1533 @*/
1534 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1535 {
1536   DM_Plex       *mesh = (DM_Plex*) dm->data;
1537   PetscErrorCode ierr;
1538 
1539   PetscFunctionBegin;
1540   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1541   PetscValidPointer(size, 3);
1542   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1543   PetscFunctionReturn(0);
1544 }
1545 
1546 #undef __FUNCT__
1547 #define __FUNCT__ "DMPlexSetSupportSize"
1548 /*@
1549   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
1550 
1551   Not collective
1552 
1553   Input Parameters:
1554 + mesh - The DMPlex
1555 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1556 - size - The support size for point p
1557 
1558   Output Parameter:
1559 
1560   Note:
1561   This should be called after DMPlexSetChart().
1562 
1563   Level: beginner
1564 
1565 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1566 @*/
1567 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1568 {
1569   DM_Plex       *mesh = (DM_Plex*) dm->data;
1570   PetscErrorCode ierr;
1571 
1572   PetscFunctionBegin;
1573   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1574   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1575 
1576   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
1577   PetscFunctionReturn(0);
1578 }
1579 
1580 #undef __FUNCT__
1581 #define __FUNCT__ "DMPlexGetSupport"
1582 /*@C
1583   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
1584 
1585   Not collective
1586 
1587   Input Parameters:
1588 + mesh - The DMPlex
1589 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1590 
1591   Output Parameter:
1592 . support - An array of points which are on the out-edges for point p
1593 
1594   Level: beginner
1595 
1596   Fortran Notes:
1597   Since it returns an array, this routine is only available in Fortran 90, and you must
1598   include petsc.h90 in your code.
1599 
1600   You must also call DMPlexRestoreSupport() after you finish using the returned array.
1601 
1602 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1603 @*/
1604 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
1605 {
1606   DM_Plex       *mesh = (DM_Plex*) dm->data;
1607   PetscInt       off;
1608   PetscErrorCode ierr;
1609 
1610   PetscFunctionBegin;
1611   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1612   PetscValidPointer(support, 3);
1613   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1614   *support = &mesh->supports[off];
1615   PetscFunctionReturn(0);
1616 }
1617 
1618 #undef __FUNCT__
1619 #define __FUNCT__ "DMPlexSetSupport"
1620 /*@
1621   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
1622 
1623   Not collective
1624 
1625   Input Parameters:
1626 + mesh - The DMPlex
1627 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1628 - support - An array of points which are on the in-edges for point p
1629 
1630   Output Parameter:
1631 
1632   Note:
1633   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
1634 
1635   Level: beginner
1636 
1637 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
1638 @*/
1639 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
1640 {
1641   DM_Plex       *mesh = (DM_Plex*) dm->data;
1642   PetscInt       pStart, pEnd;
1643   PetscInt       dof, off, c;
1644   PetscErrorCode ierr;
1645 
1646   PetscFunctionBegin;
1647   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1648   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1649   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1650   if (dof) PetscValidPointer(support, 3);
1651   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1652   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1653   for (c = 0; c < dof; ++c) {
1654     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1655     mesh->supports[off+c] = support[c];
1656   }
1657   PetscFunctionReturn(0);
1658 }
1659 
1660 #undef __FUNCT__
1661 #define __FUNCT__ "DMPlexInsertSupport"
1662 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1663 {
1664   DM_Plex       *mesh = (DM_Plex*) dm->data;
1665   PetscInt       pStart, pEnd;
1666   PetscInt       dof, off;
1667   PetscErrorCode ierr;
1668 
1669   PetscFunctionBegin;
1670   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1671   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1672   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1673   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1674   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1675   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1676   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1677   mesh->supports[off+supportPos] = supportPoint;
1678   PetscFunctionReturn(0);
1679 }
1680 
1681 #undef __FUNCT__
1682 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1683 /*@C
1684   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1685 
1686   Not collective
1687 
1688   Input Parameters:
1689 + mesh - The DMPlex
1690 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1691 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1692 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1693 
1694   Output Parameters:
1695 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1696 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1697 
1698   Note:
1699   If using internal storage (points is NULL on input), each call overwrites the last output.
1700 
1701   Fortran Notes:
1702   Since it returns an array, this routine is only available in Fortran 90, and you must
1703   include petsc.h90 in your code.
1704 
1705   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1706 
1707   Level: beginner
1708 
1709 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1710 @*/
1711 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1712 {
1713   DM_Plex        *mesh = (DM_Plex*) dm->data;
1714   PetscInt       *closure, *fifo;
1715   const PetscInt *tmp = NULL, *tmpO = NULL;
1716   PetscInt        tmpSize, t;
1717   PetscInt        depth       = 0, maxSize;
1718   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1719   PetscErrorCode  ierr;
1720 
1721   PetscFunctionBegin;
1722   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1723   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1724   /* This is only 1-level */
1725   if (useCone) {
1726     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1727     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1728     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1729   } else {
1730     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1731     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1732   }
1733   if (depth == 1) {
1734     if (*points) {
1735       closure = *points;
1736     } else {
1737       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1738       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1739     }
1740     closure[0] = p; closure[1] = 0;
1741     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1742       closure[closureSize]   = tmp[t];
1743       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1744     }
1745     if (numPoints) *numPoints = closureSize/2;
1746     if (points)    *points    = closure;
1747     PetscFunctionReturn(0);
1748   }
1749   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1750   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1751   if (*points) {
1752     closure = *points;
1753   } else {
1754     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1755   }
1756   closure[0] = p; closure[1] = 0;
1757   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1758     const PetscInt cp = tmp[t];
1759     const PetscInt co = tmpO ? tmpO[t] : 0;
1760 
1761     closure[closureSize]   = cp;
1762     closure[closureSize+1] = co;
1763     fifo[fifoSize]         = cp;
1764     fifo[fifoSize+1]       = co;
1765   }
1766   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1767   while (fifoSize - fifoStart) {
1768     const PetscInt q   = fifo[fifoStart];
1769     const PetscInt o   = fifo[fifoStart+1];
1770     const PetscInt rev = o >= 0 ? 0 : 1;
1771     const PetscInt off = rev ? -(o+1) : o;
1772 
1773     if (useCone) {
1774       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1775       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1776       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1777     } else {
1778       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1779       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1780       tmpO = NULL;
1781     }
1782     for (t = 0; t < tmpSize; ++t) {
1783       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1784       const PetscInt cp = tmp[i];
1785       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1786       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1787        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1788       PetscInt       co = tmpO ? tmpO[i] : 0;
1789       PetscInt       c;
1790 
1791       if (rev) {
1792         PetscInt childSize, coff;
1793         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1794         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1795         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1796       }
1797       /* Check for duplicate */
1798       for (c = 0; c < closureSize; c += 2) {
1799         if (closure[c] == cp) break;
1800       }
1801       if (c == closureSize) {
1802         closure[closureSize]   = cp;
1803         closure[closureSize+1] = co;
1804         fifo[fifoSize]         = cp;
1805         fifo[fifoSize+1]       = co;
1806         closureSize           += 2;
1807         fifoSize              += 2;
1808       }
1809     }
1810     fifoStart += 2;
1811   }
1812   if (numPoints) *numPoints = closureSize/2;
1813   if (points)    *points    = closure;
1814   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1815   PetscFunctionReturn(0);
1816 }
1817 
1818 #undef __FUNCT__
1819 #define __FUNCT__ "DMPlexGetTransitiveClosure_Internal"
1820 /*@C
1821   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG with a specified initial orientation
1822 
1823   Not collective
1824 
1825   Input Parameters:
1826 + mesh - The DMPlex
1827 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1828 . orientation - The orientation of the point
1829 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1830 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1831 
1832   Output Parameters:
1833 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1834 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1835 
1836   Note:
1837   If using internal storage (points is NULL on input), each call overwrites the last output.
1838 
1839   Fortran Notes:
1840   Since it returns an array, this routine is only available in Fortran 90, and you must
1841   include petsc.h90 in your code.
1842 
1843   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1844 
1845   Level: beginner
1846 
1847 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1848 @*/
1849 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1850 {
1851   DM_Plex        *mesh = (DM_Plex*) dm->data;
1852   PetscInt       *closure, *fifo;
1853   const PetscInt *tmp = NULL, *tmpO = NULL;
1854   PetscInt        tmpSize, t;
1855   PetscInt        depth       = 0, maxSize;
1856   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1857   PetscErrorCode  ierr;
1858 
1859   PetscFunctionBegin;
1860   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1861   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1862   /* This is only 1-level */
1863   if (useCone) {
1864     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1865     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1866     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1867   } else {
1868     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1869     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1870   }
1871   if (depth == 1) {
1872     if (*points) {
1873       closure = *points;
1874     } else {
1875       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1876       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1877     }
1878     closure[0] = p; closure[1] = ornt;
1879     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1880       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1881       closure[closureSize]   = tmp[i];
1882       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
1883     }
1884     if (numPoints) *numPoints = closureSize/2;
1885     if (points)    *points    = closure;
1886     PetscFunctionReturn(0);
1887   }
1888   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1889   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1890   if (*points) {
1891     closure = *points;
1892   } else {
1893     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1894   }
1895   closure[0] = p; closure[1] = ornt;
1896   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1897     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1898     const PetscInt cp = tmp[i];
1899     PetscInt       co = tmpO ? tmpO[i] : 0;
1900 
1901     if (ornt < 0) {
1902       PetscInt childSize, coff;
1903       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1904       coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1905       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1906     }
1907     closure[closureSize]   = cp;
1908     closure[closureSize+1] = co;
1909     fifo[fifoSize]         = cp;
1910     fifo[fifoSize+1]       = co;
1911   }
1912   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1913   while (fifoSize - fifoStart) {
1914     const PetscInt q   = fifo[fifoStart];
1915     const PetscInt o   = fifo[fifoStart+1];
1916     const PetscInt rev = o >= 0 ? 0 : 1;
1917     const PetscInt off = rev ? -(o+1) : o;
1918 
1919     if (useCone) {
1920       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1921       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1922       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1923     } else {
1924       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1925       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1926       tmpO = NULL;
1927     }
1928     for (t = 0; t < tmpSize; ++t) {
1929       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1930       const PetscInt cp = tmp[i];
1931       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1932       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1933        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1934       PetscInt       co = tmpO ? tmpO[i] : 0;
1935       PetscInt       c;
1936 
1937       if (rev) {
1938         PetscInt childSize, coff;
1939         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1940         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1941         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1942       }
1943       /* Check for duplicate */
1944       for (c = 0; c < closureSize; c += 2) {
1945         if (closure[c] == cp) break;
1946       }
1947       if (c == closureSize) {
1948         closure[closureSize]   = cp;
1949         closure[closureSize+1] = co;
1950         fifo[fifoSize]         = cp;
1951         fifo[fifoSize+1]       = co;
1952         closureSize           += 2;
1953         fifoSize              += 2;
1954       }
1955     }
1956     fifoStart += 2;
1957   }
1958   if (numPoints) *numPoints = closureSize/2;
1959   if (points)    *points    = closure;
1960   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1961   PetscFunctionReturn(0);
1962 }
1963 
1964 #undef __FUNCT__
1965 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1966 /*@C
1967   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1968 
1969   Not collective
1970 
1971   Input Parameters:
1972 + mesh - The DMPlex
1973 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1974 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1975 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
1976 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
1977 
1978   Note:
1979   If not using internal storage (points is not NULL on input), this call is unnecessary
1980 
1981   Fortran Notes:
1982   Since it returns an array, this routine is only available in Fortran 90, and you must
1983   include petsc.h90 in your code.
1984 
1985   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1986 
1987   Level: beginner
1988 
1989 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1990 @*/
1991 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1992 {
1993   PetscErrorCode ierr;
1994 
1995   PetscFunctionBegin;
1996   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1997   if (numPoints) PetscValidIntPointer(numPoints,4);
1998   if (points) PetscValidPointer(points,5);
1999   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
2000   if (numPoints) *numPoints = 0;
2001   PetscFunctionReturn(0);
2002 }
2003 
2004 #undef __FUNCT__
2005 #define __FUNCT__ "DMPlexGetMaxSizes"
2006 /*@
2007   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
2008 
2009   Not collective
2010 
2011   Input Parameter:
2012 . mesh - The DMPlex
2013 
2014   Output Parameters:
2015 + maxConeSize - The maximum number of in-edges
2016 - maxSupportSize - The maximum number of out-edges
2017 
2018   Level: beginner
2019 
2020 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2021 @*/
2022 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2023 {
2024   DM_Plex *mesh = (DM_Plex*) dm->data;
2025 
2026   PetscFunctionBegin;
2027   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2028   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2029   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2030   PetscFunctionReturn(0);
2031 }
2032 
2033 #undef __FUNCT__
2034 #define __FUNCT__ "DMSetUp_Plex"
2035 PetscErrorCode DMSetUp_Plex(DM dm)
2036 {
2037   DM_Plex       *mesh = (DM_Plex*) dm->data;
2038   PetscInt       size;
2039   PetscErrorCode ierr;
2040 
2041   PetscFunctionBegin;
2042   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2043   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2044   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2045   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2046   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2047   if (mesh->maxSupportSize) {
2048     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2049     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2050     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2051   }
2052   PetscFunctionReturn(0);
2053 }
2054 
2055 #undef __FUNCT__
2056 #define __FUNCT__ "DMCreateSubDM_Plex"
2057 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
2058 {
2059   PetscErrorCode ierr;
2060 
2061   PetscFunctionBegin;
2062   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2063   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2064   PetscFunctionReturn(0);
2065 }
2066 
2067 #undef __FUNCT__
2068 #define __FUNCT__ "DMPlexSymmetrize"
2069 /*@
2070   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
2071 
2072   Not collective
2073 
2074   Input Parameter:
2075 . mesh - The DMPlex
2076 
2077   Output Parameter:
2078 
2079   Note:
2080   This should be called after all calls to DMPlexSetCone()
2081 
2082   Level: beginner
2083 
2084 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2085 @*/
2086 PetscErrorCode DMPlexSymmetrize(DM dm)
2087 {
2088   DM_Plex       *mesh = (DM_Plex*) dm->data;
2089   PetscInt      *offsets;
2090   PetscInt       supportSize;
2091   PetscInt       pStart, pEnd, p;
2092   PetscErrorCode ierr;
2093 
2094   PetscFunctionBegin;
2095   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2096   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2097   /* Calculate support sizes */
2098   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2099   for (p = pStart; p < pEnd; ++p) {
2100     PetscInt dof, off, c;
2101 
2102     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2103     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2104     for (c = off; c < off+dof; ++c) {
2105       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2106     }
2107   }
2108   for (p = pStart; p < pEnd; ++p) {
2109     PetscInt dof;
2110 
2111     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2112 
2113     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2114   }
2115   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2116   /* Calculate supports */
2117   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2118   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2119   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2120   for (p = pStart; p < pEnd; ++p) {
2121     PetscInt dof, off, c;
2122 
2123     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2124     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2125     for (c = off; c < off+dof; ++c) {
2126       const PetscInt q = mesh->cones[c];
2127       PetscInt       offS;
2128 
2129       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2130 
2131       mesh->supports[offS+offsets[q]] = p;
2132       ++offsets[q];
2133     }
2134   }
2135   ierr = PetscFree(offsets);CHKERRQ(ierr);
2136   PetscFunctionReturn(0);
2137 }
2138 
2139 #undef __FUNCT__
2140 #define __FUNCT__ "DMPlexStratify"
2141 /*@
2142   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
2143   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2144   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2145   the DAG.
2146 
2147   Not collective
2148 
2149   Input Parameter:
2150 . mesh - The DMPlex
2151 
2152   Output Parameter:
2153 
2154   Notes:
2155   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
2156   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
2157 
2158   This should be called after all calls to DMPlexSymmetrize()
2159 
2160   Level: beginner
2161 
2162 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2163 @*/
2164 PetscErrorCode DMPlexStratify(DM dm)
2165 {
2166   DMLabel        label;
2167   PetscInt       pStart, pEnd, p;
2168   PetscInt       numRoots = 0, numLeaves = 0;
2169   PetscErrorCode ierr;
2170 
2171   PetscFunctionBegin;
2172   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2173   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2174   /* Calculate depth */
2175   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2176   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
2177   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2178   /* Initialize roots and count leaves */
2179   for (p = pStart; p < pEnd; ++p) {
2180     PetscInt coneSize, supportSize;
2181 
2182     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2183     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2184     if (!coneSize && supportSize) {
2185       ++numRoots;
2186       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2187     } else if (!supportSize && coneSize) {
2188       ++numLeaves;
2189     } else if (!supportSize && !coneSize) {
2190       /* Isolated points */
2191       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2192     }
2193   }
2194   if (numRoots + numLeaves == (pEnd - pStart)) {
2195     for (p = pStart; p < pEnd; ++p) {
2196       PetscInt coneSize, supportSize;
2197 
2198       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2199       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2200       if (!supportSize && coneSize) {
2201         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
2202       }
2203     }
2204   } else {
2205     IS       pointIS;
2206     PetscInt numPoints = 0, level = 0;
2207 
2208     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2209     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2210     while (numPoints) {
2211       const PetscInt *points;
2212       const PetscInt  newLevel = level+1;
2213 
2214       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2215       for (p = 0; p < numPoints; ++p) {
2216         const PetscInt  point = points[p];
2217         const PetscInt *support;
2218         PetscInt        supportSize, s;
2219 
2220         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
2221         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
2222         for (s = 0; s < supportSize; ++s) {
2223           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
2224         }
2225       }
2226       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2227       ++level;
2228       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2229       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2230       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2231       else         {numPoints = 0;}
2232     }
2233     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2234   }
2235   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2236   PetscFunctionReturn(0);
2237 }
2238 
2239 #undef __FUNCT__
2240 #define __FUNCT__ "DMPlexGetJoin"
2241 /*@C
2242   DMPlexGetJoin - Get an array for the join of the set of points
2243 
2244   Not Collective
2245 
2246   Input Parameters:
2247 + dm - The DMPlex object
2248 . numPoints - The number of input points for the join
2249 - points - The input points
2250 
2251   Output Parameters:
2252 + numCoveredPoints - The number of points in the join
2253 - coveredPoints - The points in the join
2254 
2255   Level: intermediate
2256 
2257   Note: Currently, this is restricted to a single level join
2258 
2259   Fortran Notes:
2260   Since it returns an array, this routine is only available in Fortran 90, and you must
2261   include petsc.h90 in your code.
2262 
2263   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2264 
2265 .keywords: mesh
2266 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2267 @*/
2268 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2269 {
2270   DM_Plex       *mesh = (DM_Plex*) dm->data;
2271   PetscInt      *join[2];
2272   PetscInt       joinSize, i = 0;
2273   PetscInt       dof, off, p, c, m;
2274   PetscErrorCode ierr;
2275 
2276   PetscFunctionBegin;
2277   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2278   PetscValidPointer(points, 2);
2279   PetscValidPointer(numCoveredPoints, 3);
2280   PetscValidPointer(coveredPoints, 4);
2281   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2282   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2283   /* Copy in support of first point */
2284   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2285   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2286   for (joinSize = 0; joinSize < dof; ++joinSize) {
2287     join[i][joinSize] = mesh->supports[off+joinSize];
2288   }
2289   /* Check each successive support */
2290   for (p = 1; p < numPoints; ++p) {
2291     PetscInt newJoinSize = 0;
2292 
2293     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2294     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2295     for (c = 0; c < dof; ++c) {
2296       const PetscInt point = mesh->supports[off+c];
2297 
2298       for (m = 0; m < joinSize; ++m) {
2299         if (point == join[i][m]) {
2300           join[1-i][newJoinSize++] = point;
2301           break;
2302         }
2303       }
2304     }
2305     joinSize = newJoinSize;
2306     i        = 1-i;
2307   }
2308   *numCoveredPoints = joinSize;
2309   *coveredPoints    = join[i];
2310   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2311   PetscFunctionReturn(0);
2312 }
2313 
2314 #undef __FUNCT__
2315 #define __FUNCT__ "DMPlexRestoreJoin"
2316 /*@C
2317   DMPlexRestoreJoin - Restore an array for the join of the set of points
2318 
2319   Not Collective
2320 
2321   Input Parameters:
2322 + dm - The DMPlex object
2323 . numPoints - The number of input points for the join
2324 - points - The input points
2325 
2326   Output Parameters:
2327 + numCoveredPoints - The number of points in the join
2328 - coveredPoints - The points in the join
2329 
2330   Fortran Notes:
2331   Since it returns an array, this routine is only available in Fortran 90, and you must
2332   include petsc.h90 in your code.
2333 
2334   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2335 
2336   Level: intermediate
2337 
2338 .keywords: mesh
2339 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2340 @*/
2341 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2342 {
2343   PetscErrorCode ierr;
2344 
2345   PetscFunctionBegin;
2346   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2347   if (points) PetscValidIntPointer(points,3);
2348   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2349   PetscValidPointer(coveredPoints, 5);
2350   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2351   if (numCoveredPoints) *numCoveredPoints = 0;
2352   PetscFunctionReturn(0);
2353 }
2354 
2355 #undef __FUNCT__
2356 #define __FUNCT__ "DMPlexGetFullJoin"
2357 /*@C
2358   DMPlexGetFullJoin - Get an array for the join of the set of points
2359 
2360   Not Collective
2361 
2362   Input Parameters:
2363 + dm - The DMPlex object
2364 . numPoints - The number of input points for the join
2365 - points - The input points
2366 
2367   Output Parameters:
2368 + numCoveredPoints - The number of points in the join
2369 - coveredPoints - The points in the join
2370 
2371   Fortran Notes:
2372   Since it returns an array, this routine is only available in Fortran 90, and you must
2373   include petsc.h90 in your code.
2374 
2375   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2376 
2377   Level: intermediate
2378 
2379 .keywords: mesh
2380 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2381 @*/
2382 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2383 {
2384   DM_Plex       *mesh = (DM_Plex*) dm->data;
2385   PetscInt      *offsets, **closures;
2386   PetscInt      *join[2];
2387   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2388   PetscInt       p, d, c, m;
2389   PetscErrorCode ierr;
2390 
2391   PetscFunctionBegin;
2392   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2393   PetscValidPointer(points, 2);
2394   PetscValidPointer(numCoveredPoints, 3);
2395   PetscValidPointer(coveredPoints, 4);
2396 
2397   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2398   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2399   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2400   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
2401   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2402   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2403 
2404   for (p = 0; p < numPoints; ++p) {
2405     PetscInt closureSize;
2406 
2407     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2408 
2409     offsets[p*(depth+2)+0] = 0;
2410     for (d = 0; d < depth+1; ++d) {
2411       PetscInt pStart, pEnd, i;
2412 
2413       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2414       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2415         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2416           offsets[p*(depth+2)+d+1] = i;
2417           break;
2418         }
2419       }
2420       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2421     }
2422     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2423   }
2424   for (d = 0; d < depth+1; ++d) {
2425     PetscInt dof;
2426 
2427     /* Copy in support of first point */
2428     dof = offsets[d+1] - offsets[d];
2429     for (joinSize = 0; joinSize < dof; ++joinSize) {
2430       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2431     }
2432     /* Check each successive cone */
2433     for (p = 1; p < numPoints && joinSize; ++p) {
2434       PetscInt newJoinSize = 0;
2435 
2436       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2437       for (c = 0; c < dof; ++c) {
2438         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2439 
2440         for (m = 0; m < joinSize; ++m) {
2441           if (point == join[i][m]) {
2442             join[1-i][newJoinSize++] = point;
2443             break;
2444           }
2445         }
2446       }
2447       joinSize = newJoinSize;
2448       i        = 1-i;
2449     }
2450     if (joinSize) break;
2451   }
2452   *numCoveredPoints = joinSize;
2453   *coveredPoints    = join[i];
2454   for (p = 0; p < numPoints; ++p) {
2455     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2456   }
2457   ierr = PetscFree(closures);CHKERRQ(ierr);
2458   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2459   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2460   PetscFunctionReturn(0);
2461 }
2462 
2463 #undef __FUNCT__
2464 #define __FUNCT__ "DMPlexGetMeet"
2465 /*@C
2466   DMPlexGetMeet - Get an array for the meet of the set of points
2467 
2468   Not Collective
2469 
2470   Input Parameters:
2471 + dm - The DMPlex object
2472 . numPoints - The number of input points for the meet
2473 - points - The input points
2474 
2475   Output Parameters:
2476 + numCoveredPoints - The number of points in the meet
2477 - coveredPoints - The points in the meet
2478 
2479   Level: intermediate
2480 
2481   Note: Currently, this is restricted to a single level meet
2482 
2483   Fortran Notes:
2484   Since it returns an array, this routine is only available in Fortran 90, and you must
2485   include petsc.h90 in your code.
2486 
2487   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2488 
2489 .keywords: mesh
2490 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
2491 @*/
2492 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
2493 {
2494   DM_Plex       *mesh = (DM_Plex*) dm->data;
2495   PetscInt      *meet[2];
2496   PetscInt       meetSize, i = 0;
2497   PetscInt       dof, off, p, c, m;
2498   PetscErrorCode ierr;
2499 
2500   PetscFunctionBegin;
2501   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2502   PetscValidPointer(points, 2);
2503   PetscValidPointer(numCoveringPoints, 3);
2504   PetscValidPointer(coveringPoints, 4);
2505   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2506   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2507   /* Copy in cone of first point */
2508   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
2509   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
2510   for (meetSize = 0; meetSize < dof; ++meetSize) {
2511     meet[i][meetSize] = mesh->cones[off+meetSize];
2512   }
2513   /* Check each successive cone */
2514   for (p = 1; p < numPoints; ++p) {
2515     PetscInt newMeetSize = 0;
2516 
2517     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
2518     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
2519     for (c = 0; c < dof; ++c) {
2520       const PetscInt point = mesh->cones[off+c];
2521 
2522       for (m = 0; m < meetSize; ++m) {
2523         if (point == meet[i][m]) {
2524           meet[1-i][newMeetSize++] = point;
2525           break;
2526         }
2527       }
2528     }
2529     meetSize = newMeetSize;
2530     i        = 1-i;
2531   }
2532   *numCoveringPoints = meetSize;
2533   *coveringPoints    = meet[i];
2534   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2535   PetscFunctionReturn(0);
2536 }
2537 
2538 #undef __FUNCT__
2539 #define __FUNCT__ "DMPlexRestoreMeet"
2540 /*@C
2541   DMPlexRestoreMeet - Restore an array for the meet of the set of points
2542 
2543   Not Collective
2544 
2545   Input Parameters:
2546 + dm - The DMPlex object
2547 . numPoints - The number of input points for the meet
2548 - points - The input points
2549 
2550   Output Parameters:
2551 + numCoveredPoints - The number of points in the meet
2552 - coveredPoints - The points in the meet
2553 
2554   Level: intermediate
2555 
2556   Fortran Notes:
2557   Since it returns an array, this routine is only available in Fortran 90, and you must
2558   include petsc.h90 in your code.
2559 
2560   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2561 
2562 .keywords: mesh
2563 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
2564 @*/
2565 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2566 {
2567   PetscErrorCode ierr;
2568 
2569   PetscFunctionBegin;
2570   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2571   if (points) PetscValidIntPointer(points,3);
2572   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2573   PetscValidPointer(coveredPoints,5);
2574   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2575   if (numCoveredPoints) *numCoveredPoints = 0;
2576   PetscFunctionReturn(0);
2577 }
2578 
2579 #undef __FUNCT__
2580 #define __FUNCT__ "DMPlexGetFullMeet"
2581 /*@C
2582   DMPlexGetFullMeet - Get an array for the meet of the set of points
2583 
2584   Not Collective
2585 
2586   Input Parameters:
2587 + dm - The DMPlex object
2588 . numPoints - The number of input points for the meet
2589 - points - The input points
2590 
2591   Output Parameters:
2592 + numCoveredPoints - The number of points in the meet
2593 - coveredPoints - The points in the meet
2594 
2595   Level: intermediate
2596 
2597   Fortran Notes:
2598   Since it returns an array, this routine is only available in Fortran 90, and you must
2599   include petsc.h90 in your code.
2600 
2601   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2602 
2603 .keywords: mesh
2604 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
2605 @*/
2606 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2607 {
2608   DM_Plex       *mesh = (DM_Plex*) dm->data;
2609   PetscInt      *offsets, **closures;
2610   PetscInt      *meet[2];
2611   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
2612   PetscInt       p, h, c, m;
2613   PetscErrorCode ierr;
2614 
2615   PetscFunctionBegin;
2616   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2617   PetscValidPointer(points, 2);
2618   PetscValidPointer(numCoveredPoints, 3);
2619   PetscValidPointer(coveredPoints, 4);
2620 
2621   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
2622   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
2623   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2624   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
2625   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2626   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2627 
2628   for (p = 0; p < numPoints; ++p) {
2629     PetscInt closureSize;
2630 
2631     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
2632 
2633     offsets[p*(height+2)+0] = 0;
2634     for (h = 0; h < height+1; ++h) {
2635       PetscInt pStart, pEnd, i;
2636 
2637       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
2638       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
2639         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2640           offsets[p*(height+2)+h+1] = i;
2641           break;
2642         }
2643       }
2644       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
2645     }
2646     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
2647   }
2648   for (h = 0; h < height+1; ++h) {
2649     PetscInt dof;
2650 
2651     /* Copy in cone of first point */
2652     dof = offsets[h+1] - offsets[h];
2653     for (meetSize = 0; meetSize < dof; ++meetSize) {
2654       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
2655     }
2656     /* Check each successive cone */
2657     for (p = 1; p < numPoints && meetSize; ++p) {
2658       PetscInt newMeetSize = 0;
2659 
2660       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
2661       for (c = 0; c < dof; ++c) {
2662         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
2663 
2664         for (m = 0; m < meetSize; ++m) {
2665           if (point == meet[i][m]) {
2666             meet[1-i][newMeetSize++] = point;
2667             break;
2668           }
2669         }
2670       }
2671       meetSize = newMeetSize;
2672       i        = 1-i;
2673     }
2674     if (meetSize) break;
2675   }
2676   *numCoveredPoints = meetSize;
2677   *coveredPoints    = meet[i];
2678   for (p = 0; p < numPoints; ++p) {
2679     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
2680   }
2681   ierr = PetscFree(closures);CHKERRQ(ierr);
2682   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2683   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2684   PetscFunctionReturn(0);
2685 }
2686 
2687 #undef __FUNCT__
2688 #define __FUNCT__ "DMPlexEqual"
2689 /*@C
2690   DMPlexEqual - Determine if two DMs have the same topology
2691 
2692   Not Collective
2693 
2694   Input Parameters:
2695 + dmA - A DMPlex object
2696 - dmB - A DMPlex object
2697 
2698   Output Parameters:
2699 . equal - PETSC_TRUE if the topologies are identical
2700 
2701   Level: intermediate
2702 
2703   Notes:
2704   We are not solving graph isomorphism, so we do not permutation.
2705 
2706 .keywords: mesh
2707 .seealso: DMPlexGetCone()
2708 @*/
2709 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
2710 {
2711   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
2712   PetscErrorCode ierr;
2713 
2714   PetscFunctionBegin;
2715   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
2716   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
2717   PetscValidPointer(equal, 3);
2718 
2719   *equal = PETSC_FALSE;
2720   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
2721   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
2722   if (depth != depthB) PetscFunctionReturn(0);
2723   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
2724   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
2725   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
2726   for (p = pStart; p < pEnd; ++p) {
2727     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
2728     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
2729 
2730     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
2731     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
2732     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
2733     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
2734     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
2735     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
2736     if (coneSize != coneSizeB) PetscFunctionReturn(0);
2737     for (c = 0; c < coneSize; ++c) {
2738       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
2739       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
2740     }
2741     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
2742     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
2743     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
2744     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
2745     if (supportSize != supportSizeB) PetscFunctionReturn(0);
2746     for (s = 0; s < supportSize; ++s) {
2747       if (support[s] != supportB[s]) PetscFunctionReturn(0);
2748     }
2749   }
2750   *equal = PETSC_TRUE;
2751   PetscFunctionReturn(0);
2752 }
2753 
2754 #undef __FUNCT__
2755 #define __FUNCT__ "DMPlexGetNumFaceVertices"
2756 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
2757 {
2758   MPI_Comm       comm;
2759   PetscErrorCode ierr;
2760 
2761   PetscFunctionBegin;
2762   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2763   PetscValidPointer(numFaceVertices,3);
2764   switch (cellDim) {
2765   case 0:
2766     *numFaceVertices = 0;
2767     break;
2768   case 1:
2769     *numFaceVertices = 1;
2770     break;
2771   case 2:
2772     switch (numCorners) {
2773     case 3: /* triangle */
2774       *numFaceVertices = 2; /* Edge has 2 vertices */
2775       break;
2776     case 4: /* quadrilateral */
2777       *numFaceVertices = 2; /* Edge has 2 vertices */
2778       break;
2779     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
2780       *numFaceVertices = 3; /* Edge has 3 vertices */
2781       break;
2782     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
2783       *numFaceVertices = 3; /* Edge has 3 vertices */
2784       break;
2785     default:
2786       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2787     }
2788     break;
2789   case 3:
2790     switch (numCorners) {
2791     case 4: /* tetradehdron */
2792       *numFaceVertices = 3; /* Face has 3 vertices */
2793       break;
2794     case 6: /* tet cohesive cells */
2795       *numFaceVertices = 4; /* Face has 4 vertices */
2796       break;
2797     case 8: /* hexahedron */
2798       *numFaceVertices = 4; /* Face has 4 vertices */
2799       break;
2800     case 9: /* tet cohesive Lagrange cells */
2801       *numFaceVertices = 6; /* Face has 6 vertices */
2802       break;
2803     case 10: /* quadratic tetrahedron */
2804       *numFaceVertices = 6; /* Face has 6 vertices */
2805       break;
2806     case 12: /* hex cohesive Lagrange cells */
2807       *numFaceVertices = 6; /* Face has 6 vertices */
2808       break;
2809     case 18: /* quadratic tet cohesive Lagrange cells */
2810       *numFaceVertices = 6; /* Face has 6 vertices */
2811       break;
2812     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
2813       *numFaceVertices = 9; /* Face has 9 vertices */
2814       break;
2815     default:
2816       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2817     }
2818     break;
2819   default:
2820     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
2821   }
2822   PetscFunctionReturn(0);
2823 }
2824 
2825 #undef __FUNCT__
2826 #define __FUNCT__ "DMPlexOrient"
2827 /* Trys to give the mesh a consistent orientation */
2828 PetscErrorCode DMPlexOrient(DM dm)
2829 {
2830   PetscBT        seenCells, flippedCells, seenFaces;
2831   PetscInt      *faceFIFO, fTop, fBottom;
2832   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
2833   PetscErrorCode ierr;
2834 
2835   PetscFunctionBegin;
2836   /* Truth Table
2837      mismatch    flips   do action   mismatch   flipA ^ flipB   action
2838          F       0 flips     no         F             F           F
2839          F       1 flip      yes        F             T           T
2840          F       2 flips     no         T             F           T
2841          T       0 flips     yes        T             T           F
2842          T       1 flip      no
2843          T       2 flips     yes
2844   */
2845   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2846   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
2847   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
2848   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
2849   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
2850   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
2851   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
2852   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
2853   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
2854   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
2855   ierr = PetscMalloc1((fEnd - fStart), &faceFIFO);CHKERRQ(ierr);
2856   fTop = fBottom = 0;
2857   /* Initialize FIFO with first cell */
2858   if (cEnd > cStart) {
2859     const PetscInt *cone;
2860     PetscInt        coneSize;
2861 
2862     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
2863     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2864     for (c = 0; c < coneSize; ++c) {
2865       faceFIFO[fBottom++] = cone[c];
2866       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2867     }
2868   }
2869   /* Consider each face in FIFO */
2870   while (fTop < fBottom) {
2871     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2872     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2873     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2874 
2875     face = faceFIFO[fTop++];
2876     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2877     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2878     if (supportSize < 2) continue;
2879     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2880     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2881     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2882     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2883     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2884 
2885     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2886     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2887     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2888     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2889     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2890     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2891     for (c = 0; c < coneSizeA; ++c) {
2892       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2893         faceFIFO[fBottom++] = coneA[c];
2894         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2895       }
2896       if (coneA[c] == face) posA = c;
2897       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2898     }
2899     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2900     for (c = 0; c < coneSizeB; ++c) {
2901       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2902         faceFIFO[fBottom++] = coneB[c];
2903         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2904       }
2905       if (coneB[c] == face) posB = c;
2906       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2907     }
2908     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2909 
2910     if (dim == 1) {
2911       mismatch = posA == posB;
2912     } else {
2913       mismatch = coneOA[posA] == coneOB[posB];
2914     }
2915 
2916     if (mismatch ^ (flippedA ^ flippedB)) {
2917       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2918       if (!seenA && !flippedA) {
2919         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2920       } else if (!seenB && !flippedB) {
2921         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2922       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2923     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2924     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2925     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2926   }
2927   /* Now all subdomains are oriented, but we need a consistent parallel orientation */
2928   {
2929     /* Find a representative face (edge) separating pairs of procs */
2930     PetscSF            sf;
2931     const PetscInt    *lpoints;
2932     const PetscSFNode *rpoints;
2933     PetscInt          *neighbors, *nranks;
2934     PetscInt           numLeaves, numRoots, numNeighbors = 0, l, n;
2935 
2936     ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2937     ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
2938     if (numLeaves >= 0) {
2939       const PetscInt *cone, *ornt, *support;
2940       PetscInt        coneSize, supportSize;
2941       int            *rornt, *lornt; /* PetscSF cannot handle smaller than int */
2942       PetscBool      *match, flipped = PETSC_FALSE;
2943 
2944       ierr = PetscMalloc1(numLeaves,&neighbors);CHKERRQ(ierr);
2945       /* I know this is p^2 time in general, but for bounded degree its alright */
2946       for (l = 0; l < numLeaves; ++l) {
2947         const PetscInt face = lpoints[l];
2948         if ((face >= fStart) && (face < fEnd)) {
2949           const PetscInt rank = rpoints[l].rank;
2950           for (n = 0; n < numNeighbors; ++n) if (rank == rpoints[neighbors[n]].rank) break;
2951           if (n >= numNeighbors) {
2952             PetscInt supportSize;
2953             ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2954             if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
2955             neighbors[numNeighbors++] = l;
2956           }
2957         }
2958       }
2959       ierr = PetscCalloc4(numNeighbors,&match,numNeighbors,&nranks,numRoots,&rornt,numRoots,&lornt);CHKERRQ(ierr);
2960       for (face = fStart; face < fEnd; ++face) {
2961         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2962         if (supportSize != 1) continue;
2963         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2964 
2965         ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
2966         ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
2967         ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
2968         for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
2969         if (dim == 1) {
2970           /* Use cone position instead, shifted to -1 or 1 */
2971           rornt[face] = c*2-1;
2972         } else {
2973           if (PetscBTLookup(flippedCells, support[0]-cStart)) rornt[face] = ornt[c] < 0 ? -1 :  1;
2974           else                                                rornt[face] = ornt[c] < 0 ?  1 : -1;
2975         }
2976       }
2977       /* Mark each edge with match or nomatch */
2978       ierr = PetscSFBcastBegin(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2979       ierr = PetscSFBcastEnd(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2980       for (n = 0; n < numNeighbors; ++n) {
2981         const PetscInt face = lpoints[neighbors[n]];
2982 
2983         if (rornt[face]*lornt[face] < 0) match[n] = PETSC_TRUE;
2984         else                             match[n] = PETSC_FALSE;
2985         nranks[n] = rpoints[neighbors[n]].rank;
2986       }
2987       /* Collect the graph on 0 */
2988       {
2989         MPI_Comm     comm = PetscObjectComm((PetscObject) sf);
2990         PetscBT      seenProcs, flippedProcs;
2991         PetscInt    *procFIFO, pTop, pBottom;
2992         PetscInt    *adj = NULL;
2993         PetscBool   *val = NULL;
2994         PetscMPIInt *recvcounts = NULL, *displs = NULL, p;
2995         PetscMPIInt  N = numNeighbors, numProcs = 0, rank;
2996         PetscInt     debug = 0;
2997 
2998         ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2999         if (!rank) {ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);}
3000         ierr = PetscCalloc2(numProcs,&recvcounts,numProcs+1,&displs);CHKERRQ(ierr);
3001         ierr = MPI_Gather(&N, 1, MPI_INT, recvcounts, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
3002         for (p = 0; p < numProcs; ++p) {
3003           displs[p+1] = displs[p] + recvcounts[p];
3004         }
3005         if (!rank) {ierr = PetscMalloc2(displs[numProcs],&adj,displs[numProcs],&val);CHKERRQ(ierr);}
3006         ierr = MPI_Gatherv(nranks, numNeighbors, MPIU_INT, adj, recvcounts, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
3007         ierr = MPI_Gatherv(match, numNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
3008         if (debug) {
3009           for (p = 0; p < numProcs; ++p) {
3010             ierr = PetscPrintf(comm, "Proc %d:\n", p);
3011             for (n = 0; n < recvcounts[p]; ++n) {
3012               ierr = PetscPrintf(comm, "  edge %d (%d):\n", adj[displs[p]+n], val[displs[p]+n]);
3013             }
3014           }
3015         }
3016         ierr = PetscBTCreate(numProcs, &seenProcs);CHKERRQ(ierr);
3017         ierr = PetscBTMemzero(numProcs, seenProcs);CHKERRQ(ierr);
3018         ierr = PetscBTCreate(numProcs, &flippedProcs);CHKERRQ(ierr);
3019         ierr = PetscBTMemzero(numProcs, flippedProcs);CHKERRQ(ierr);
3020         ierr = PetscMalloc1(numProcs,&procFIFO);CHKERRQ(ierr);
3021         pTop = pBottom = 0;
3022         for (p = 0; p < numProcs; ++p) {
3023           if (PetscBTLookup(seenProcs, p)) continue;
3024           /* Initialize FIFO with next proc */
3025           procFIFO[pBottom++] = p;
3026           ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
3027           /* Consider each proc in FIFO */
3028           while (pTop < pBottom) {
3029             PetscInt proc, nproc, seen, flippedA, flippedB, mismatch;
3030 
3031             proc     = procFIFO[pTop++];
3032             flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
3033             /* Loop over neighboring procs */
3034             for (n = 0; n < recvcounts[proc]; ++n) {
3035               nproc    = adj[displs[proc]+n];
3036               mismatch = val[displs[proc]+n] ? 0 : 1;
3037               seen     = PetscBTLookup(seenProcs, nproc);
3038               flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
3039 
3040               if (mismatch ^ (flippedA ^ flippedB)) {
3041                 if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
3042                 if (!flippedB) {
3043                   ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
3044               } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
3045               } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
3046               if (!seen) {
3047                 procFIFO[pBottom++] = nproc;
3048                 ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
3049               }
3050             }
3051           }
3052         }
3053         ierr = PetscFree(procFIFO);CHKERRQ(ierr);
3054 
3055         ierr = PetscFree2(recvcounts,displs);CHKERRQ(ierr);
3056         ierr = PetscFree2(adj,val);CHKERRQ(ierr);
3057         {
3058           PetscBool *flips;
3059 
3060           ierr = PetscMalloc1(numProcs,&flips);CHKERRQ(ierr);
3061           for (p = 0; p < numProcs; ++p) {
3062             flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
3063             if (debug && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc %d:\n", p);}
3064           }
3065           ierr = MPI_Scatter(flips, 1, MPIU_BOOL, &flipped, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
3066           ierr = PetscFree(flips);CHKERRQ(ierr);
3067         }
3068         ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
3069         ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);
3070       }
3071       ierr = PetscFree4(match,nranks,rornt,lornt);CHKERRQ(ierr);
3072       ierr = PetscFree(neighbors);CHKERRQ(ierr);
3073       if (flipped) {for (c = cStart; c < cEnd; ++c) {ierr = PetscBTNegate(flippedCells, c-cStart);CHKERRQ(ierr);}}
3074     }
3075   }
3076   /* Reverse flipped cells in the mesh */
3077   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3078   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3079   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3080   for (c = cStart; c < cEnd; ++c) {
3081     const PetscInt *cone, *coneO, *support;
3082     PetscInt        coneSize, supportSize, faceSize, cp, sp;
3083 
3084     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
3085     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
3086     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
3087     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
3088     for (cp = 0; cp < coneSize; ++cp) {
3089       const PetscInt rcp = coneSize-cp-1;
3090 
3091       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
3092       revcone[cp]  = cone[rcp];
3093       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
3094     }
3095     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
3096     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
3097     /* Reverse orientations of support */
3098     faceSize = coneSize;
3099     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
3100     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
3101     for (sp = 0; sp < supportSize; ++sp) {
3102       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
3103       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
3104       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
3105       for (cp = 0; cp < coneSize; ++cp) {
3106         if (cone[cp] != c) continue;
3107         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
3108       }
3109     }
3110   }
3111   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3112   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3113   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
3114   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
3115   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
3116   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
3117   PetscFunctionReturn(0);
3118 }
3119 
3120 #undef __FUNCT__
3121 #define __FUNCT__ "DMPlexInvertCell"
3122 /*@C
3123   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
3124 
3125   Input Parameters:
3126 + numCorners - The number of vertices in a cell
3127 - cone - The incoming cone
3128 
3129   Output Parameter:
3130 . cone - The inverted cone (in-place)
3131 
3132   Level: developer
3133 
3134 .seealso: DMPlexGenerate()
3135 @*/
3136 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
3137 {
3138   int tmpc;
3139 
3140   PetscFunctionBegin;
3141   if (dim != 3) PetscFunctionReturn(0);
3142   switch (numCorners) {
3143   case 4:
3144     tmpc    = cone[0];
3145     cone[0] = cone[1];
3146     cone[1] = tmpc;
3147     break;
3148   case 8:
3149     tmpc    = cone[1];
3150     cone[1] = cone[3];
3151     cone[3] = tmpc;
3152     break;
3153   default: break;
3154   }
3155   PetscFunctionReturn(0);
3156 }
3157 
3158 #undef __FUNCT__
3159 #define __FUNCT__ "DMPlexInvertCells_Internal"
3160 /* This is to fix the tetrahedron orientation from TetGen */
3161 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
3162 {
3163   PetscInt       bound = numCells*numCorners, coff;
3164   PetscErrorCode ierr;
3165 
3166   PetscFunctionBegin;
3167   for (coff = 0; coff < bound; coff += numCorners) {
3168     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
3169   }
3170   PetscFunctionReturn(0);
3171 }
3172 
3173 #if defined(PETSC_HAVE_TRIANGLE)
3174 #include <triangle.h>
3175 
3176 #undef __FUNCT__
3177 #define __FUNCT__ "InitInput_Triangle"
3178 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
3179 {
3180   PetscFunctionBegin;
3181   inputCtx->numberofpoints             = 0;
3182   inputCtx->numberofpointattributes    = 0;
3183   inputCtx->pointlist                  = NULL;
3184   inputCtx->pointattributelist         = NULL;
3185   inputCtx->pointmarkerlist            = NULL;
3186   inputCtx->numberofsegments           = 0;
3187   inputCtx->segmentlist                = NULL;
3188   inputCtx->segmentmarkerlist          = NULL;
3189   inputCtx->numberoftriangleattributes = 0;
3190   inputCtx->trianglelist               = NULL;
3191   inputCtx->numberofholes              = 0;
3192   inputCtx->holelist                   = NULL;
3193   inputCtx->numberofregions            = 0;
3194   inputCtx->regionlist                 = NULL;
3195   PetscFunctionReturn(0);
3196 }
3197 
3198 #undef __FUNCT__
3199 #define __FUNCT__ "InitOutput_Triangle"
3200 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
3201 {
3202   PetscFunctionBegin;
3203   outputCtx->numberofpoints        = 0;
3204   outputCtx->pointlist             = NULL;
3205   outputCtx->pointattributelist    = NULL;
3206   outputCtx->pointmarkerlist       = NULL;
3207   outputCtx->numberoftriangles     = 0;
3208   outputCtx->trianglelist          = NULL;
3209   outputCtx->triangleattributelist = NULL;
3210   outputCtx->neighborlist          = NULL;
3211   outputCtx->segmentlist           = NULL;
3212   outputCtx->segmentmarkerlist     = NULL;
3213   outputCtx->numberofedges         = 0;
3214   outputCtx->edgelist              = NULL;
3215   outputCtx->edgemarkerlist        = NULL;
3216   PetscFunctionReturn(0);
3217 }
3218 
3219 #undef __FUNCT__
3220 #define __FUNCT__ "FiniOutput_Triangle"
3221 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
3222 {
3223   PetscFunctionBegin;
3224   free(outputCtx->pointlist);
3225   free(outputCtx->pointmarkerlist);
3226   free(outputCtx->segmentlist);
3227   free(outputCtx->segmentmarkerlist);
3228   free(outputCtx->edgelist);
3229   free(outputCtx->edgemarkerlist);
3230   free(outputCtx->trianglelist);
3231   free(outputCtx->neighborlist);
3232   PetscFunctionReturn(0);
3233 }
3234 
3235 #undef __FUNCT__
3236 #define __FUNCT__ "DMPlexGenerate_Triangle"
3237 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
3238 {
3239   MPI_Comm             comm;
3240   PetscInt             dim              = 2;
3241   const PetscBool      createConvexHull = PETSC_FALSE;
3242   const PetscBool      constrained      = PETSC_FALSE;
3243   struct triangulateio in;
3244   struct triangulateio out;
3245   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
3246   PetscMPIInt          rank;
3247   PetscErrorCode       ierr;
3248 
3249   PetscFunctionBegin;
3250   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3251   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3252   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3253   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3254   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3255 
3256   in.numberofpoints = vEnd - vStart;
3257   if (in.numberofpoints > 0) {
3258     PetscSection coordSection;
3259     Vec          coordinates;
3260     PetscScalar *array;
3261 
3262     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3263     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3264     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3265     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3266     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3267     for (v = vStart; v < vEnd; ++v) {
3268       const PetscInt idx = v - vStart;
3269       PetscInt       off, d;
3270 
3271       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3272       for (d = 0; d < dim; ++d) {
3273         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3274       }
3275       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3276     }
3277     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3278   }
3279   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
3280   in.numberofsegments = eEnd - eStart;
3281   if (in.numberofsegments > 0) {
3282     ierr = PetscMalloc1(in.numberofsegments*2, &in.segmentlist);CHKERRQ(ierr);
3283     ierr = PetscMalloc1(in.numberofsegments, &in.segmentmarkerlist);CHKERRQ(ierr);
3284     for (e = eStart; e < eEnd; ++e) {
3285       const PetscInt  idx = e - eStart;
3286       const PetscInt *cone;
3287 
3288       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
3289 
3290       in.segmentlist[idx*2+0] = cone[0] - vStart;
3291       in.segmentlist[idx*2+1] = cone[1] - vStart;
3292 
3293       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
3294     }
3295   }
3296 #if 0 /* Do not currently support holes */
3297   PetscReal *holeCoords;
3298   PetscInt   h, d;
3299 
3300   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3301   if (in.numberofholes > 0) {
3302     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3303     for (h = 0; h < in.numberofholes; ++h) {
3304       for (d = 0; d < dim; ++d) {
3305         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3306       }
3307     }
3308   }
3309 #endif
3310   if (!rank) {
3311     char args[32];
3312 
3313     /* Take away 'Q' for verbose output */
3314     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3315     if (createConvexHull) {
3316       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
3317     }
3318     if (constrained) {
3319       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
3320     }
3321     triangulate(args, &in, &out, NULL);
3322   }
3323   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3324   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3325   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3326   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3327   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
3328 
3329   {
3330     const PetscInt numCorners  = 3;
3331     const PetscInt numCells    = out.numberoftriangles;
3332     const PetscInt numVertices = out.numberofpoints;
3333     const int     *cells      = out.trianglelist;
3334     const double  *meshCoords = out.pointlist;
3335 
3336     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3337     /* Set labels */
3338     for (v = 0; v < numVertices; ++v) {
3339       if (out.pointmarkerlist[v]) {
3340         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3341       }
3342     }
3343     if (interpolate) {
3344       for (e = 0; e < out.numberofedges; e++) {
3345         if (out.edgemarkerlist[e]) {
3346           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3347           const PetscInt *edges;
3348           PetscInt        numEdges;
3349 
3350           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3351           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3352           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3353           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3354         }
3355       }
3356     }
3357     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3358   }
3359 #if 0 /* Do not currently support holes */
3360   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3361 #endif
3362   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3363   PetscFunctionReturn(0);
3364 }
3365 
3366 #undef __FUNCT__
3367 #define __FUNCT__ "DMPlexRefine_Triangle"
3368 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
3369 {
3370   MPI_Comm             comm;
3371   PetscInt             dim  = 2;
3372   struct triangulateio in;
3373   struct triangulateio out;
3374   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3375   PetscMPIInt          rank;
3376   PetscErrorCode       ierr;
3377 
3378   PetscFunctionBegin;
3379   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3380   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3381   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3382   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3383   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3384   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3385   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3386 
3387   in.numberofpoints = vEnd - vStart;
3388   if (in.numberofpoints > 0) {
3389     PetscSection coordSection;
3390     Vec          coordinates;
3391     PetscScalar *array;
3392 
3393     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3394     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3395     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3396     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3397     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3398     for (v = vStart; v < vEnd; ++v) {
3399       const PetscInt idx = v - vStart;
3400       PetscInt       off, d;
3401 
3402       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3403       for (d = 0; d < dim; ++d) {
3404         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3405       }
3406       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3407     }
3408     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3409   }
3410   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3411 
3412   in.numberofcorners   = 3;
3413   in.numberoftriangles = cEnd - cStart;
3414 
3415   in.trianglearealist  = (double*) maxVolumes;
3416   if (in.numberoftriangles > 0) {
3417     ierr = PetscMalloc1(in.numberoftriangles*in.numberofcorners, &in.trianglelist);CHKERRQ(ierr);
3418     for (c = cStart; c < cEnd; ++c) {
3419       const PetscInt idx      = c - cStart;
3420       PetscInt      *closure = NULL;
3421       PetscInt       closureSize;
3422 
3423       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3424       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
3425       for (v = 0; v < 3; ++v) {
3426         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
3427       }
3428       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3429     }
3430   }
3431   /* TODO: Segment markers are missing on input */
3432 #if 0 /* Do not currently support holes */
3433   PetscReal *holeCoords;
3434   PetscInt   h, d;
3435 
3436   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3437   if (in.numberofholes > 0) {
3438     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3439     for (h = 0; h < in.numberofholes; ++h) {
3440       for (d = 0; d < dim; ++d) {
3441         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3442       }
3443     }
3444   }
3445 #endif
3446   if (!rank) {
3447     char args[32];
3448 
3449     /* Take away 'Q' for verbose output */
3450     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
3451     triangulate(args, &in, &out, NULL);
3452   }
3453   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3454   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3455   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3456   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3457   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
3458 
3459   {
3460     const PetscInt numCorners  = 3;
3461     const PetscInt numCells    = out.numberoftriangles;
3462     const PetscInt numVertices = out.numberofpoints;
3463     const int     *cells      = out.trianglelist;
3464     const double  *meshCoords = out.pointlist;
3465     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3466 
3467     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3468     /* Set labels */
3469     for (v = 0; v < numVertices; ++v) {
3470       if (out.pointmarkerlist[v]) {
3471         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3472       }
3473     }
3474     if (interpolate) {
3475       PetscInt e;
3476 
3477       for (e = 0; e < out.numberofedges; e++) {
3478         if (out.edgemarkerlist[e]) {
3479           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3480           const PetscInt *edges;
3481           PetscInt        numEdges;
3482 
3483           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3484           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3485           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3486           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3487         }
3488       }
3489     }
3490     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3491   }
3492 #if 0 /* Do not currently support holes */
3493   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3494 #endif
3495   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3496   PetscFunctionReturn(0);
3497 }
3498 #endif
3499 
3500 #if defined(PETSC_HAVE_TETGEN)
3501 #include <tetgen.h>
3502 #undef __FUNCT__
3503 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3504 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3505 {
3506   MPI_Comm       comm;
3507   const PetscInt dim  = 3;
3508   ::tetgenio     in;
3509   ::tetgenio     out;
3510   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3511   PetscMPIInt    rank;
3512   PetscErrorCode ierr;
3513 
3514   PetscFunctionBegin;
3515   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3516   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3517   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3518   in.numberofpoints = vEnd - vStart;
3519   if (in.numberofpoints > 0) {
3520     PetscSection coordSection;
3521     Vec          coordinates;
3522     PetscScalar *array;
3523 
3524     in.pointlist       = new double[in.numberofpoints*dim];
3525     in.pointmarkerlist = new int[in.numberofpoints];
3526 
3527     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3528     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3529     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3530     for (v = vStart; v < vEnd; ++v) {
3531       const PetscInt idx = v - vStart;
3532       PetscInt       off, d;
3533 
3534       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3535       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3536       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3537     }
3538     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3539   }
3540   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3541 
3542   in.numberoffacets = fEnd - fStart;
3543   if (in.numberoffacets > 0) {
3544     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3545     in.facetmarkerlist = new int[in.numberoffacets];
3546     for (f = fStart; f < fEnd; ++f) {
3547       const PetscInt idx     = f - fStart;
3548       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3549 
3550       in.facetlist[idx].numberofpolygons = 1;
3551       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3552       in.facetlist[idx].numberofholes    = 0;
3553       in.facetlist[idx].holelist         = NULL;
3554 
3555       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3556       for (p = 0; p < numPoints*2; p += 2) {
3557         const PetscInt point = points[p];
3558         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3559       }
3560 
3561       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3562       poly->numberofvertices = numVertices;
3563       poly->vertexlist       = new int[poly->numberofvertices];
3564       for (v = 0; v < numVertices; ++v) {
3565         const PetscInt vIdx = points[v] - vStart;
3566         poly->vertexlist[v] = vIdx;
3567       }
3568       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3569       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3570     }
3571   }
3572   if (!rank) {
3573     char args[32];
3574 
3575     /* Take away 'Q' for verbose output */
3576     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3577     ::tetrahedralize(args, &in, &out);
3578   }
3579   {
3580     const PetscInt numCorners  = 4;
3581     const PetscInt numCells    = out.numberoftetrahedra;
3582     const PetscInt numVertices = out.numberofpoints;
3583     const double   *meshCoords = out.pointlist;
3584     int            *cells      = out.tetrahedronlist;
3585 
3586     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3587     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3588     /* Set labels */
3589     for (v = 0; v < numVertices; ++v) {
3590       if (out.pointmarkerlist[v]) {
3591         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3592       }
3593     }
3594     if (interpolate) {
3595       PetscInt e;
3596 
3597       for (e = 0; e < out.numberofedges; e++) {
3598         if (out.edgemarkerlist[e]) {
3599           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3600           const PetscInt *edges;
3601           PetscInt        numEdges;
3602 
3603           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3604           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3605           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3606           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3607         }
3608       }
3609       for (f = 0; f < out.numberoftrifaces; f++) {
3610         if (out.trifacemarkerlist[f]) {
3611           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3612           const PetscInt *faces;
3613           PetscInt        numFaces;
3614 
3615           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3616           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3617           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3618           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3619         }
3620       }
3621     }
3622     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3623   }
3624   PetscFunctionReturn(0);
3625 }
3626 
3627 #undef __FUNCT__
3628 #define __FUNCT__ "DMPlexRefine_Tetgen"
3629 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3630 {
3631   MPI_Comm       comm;
3632   const PetscInt dim  = 3;
3633   ::tetgenio     in;
3634   ::tetgenio     out;
3635   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3636   PetscMPIInt    rank;
3637   PetscErrorCode ierr;
3638 
3639   PetscFunctionBegin;
3640   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3641   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3642   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3643   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3644   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3645 
3646   in.numberofpoints = vEnd - vStart;
3647   if (in.numberofpoints > 0) {
3648     PetscSection coordSection;
3649     Vec          coordinates;
3650     PetscScalar *array;
3651 
3652     in.pointlist       = new double[in.numberofpoints*dim];
3653     in.pointmarkerlist = new int[in.numberofpoints];
3654 
3655     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3656     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3657     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3658     for (v = vStart; v < vEnd; ++v) {
3659       const PetscInt idx = v - vStart;
3660       PetscInt       off, d;
3661 
3662       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3663       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3664       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3665     }
3666     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3667   }
3668   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3669 
3670   in.numberofcorners       = 4;
3671   in.numberoftetrahedra    = cEnd - cStart;
3672   in.tetrahedronvolumelist = (double*) maxVolumes;
3673   if (in.numberoftetrahedra > 0) {
3674     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3675     for (c = cStart; c < cEnd; ++c) {
3676       const PetscInt idx      = c - cStart;
3677       PetscInt      *closure = NULL;
3678       PetscInt       closureSize;
3679 
3680       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3681       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3682       for (v = 0; v < 4; ++v) {
3683         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3684       }
3685       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3686     }
3687   }
3688   /* TODO: Put in boundary faces with markers */
3689   if (!rank) {
3690     char args[32];
3691 
3692     /* Take away 'Q' for verbose output */
3693     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3694     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3695     ::tetrahedralize(args, &in, &out);
3696   }
3697   in.tetrahedronvolumelist = NULL;
3698 
3699   {
3700     const PetscInt numCorners  = 4;
3701     const PetscInt numCells    = out.numberoftetrahedra;
3702     const PetscInt numVertices = out.numberofpoints;
3703     const double   *meshCoords = out.pointlist;
3704     int            *cells      = out.tetrahedronlist;
3705 
3706     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3707 
3708     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3709     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3710     /* Set labels */
3711     for (v = 0; v < numVertices; ++v) {
3712       if (out.pointmarkerlist[v]) {
3713         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3714       }
3715     }
3716     if (interpolate) {
3717       PetscInt e, f;
3718 
3719       for (e = 0; e < out.numberofedges; e++) {
3720         if (out.edgemarkerlist[e]) {
3721           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3722           const PetscInt *edges;
3723           PetscInt        numEdges;
3724 
3725           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3726           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3727           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3728           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3729         }
3730       }
3731       for (f = 0; f < out.numberoftrifaces; f++) {
3732         if (out.trifacemarkerlist[f]) {
3733           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3734           const PetscInt *faces;
3735           PetscInt        numFaces;
3736 
3737           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3738           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3739           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3740           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3741         }
3742       }
3743     }
3744     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3745   }
3746   PetscFunctionReturn(0);
3747 }
3748 #endif
3749 
3750 #if defined(PETSC_HAVE_CTETGEN)
3751 #include <ctetgen.h>
3752 
3753 #undef __FUNCT__
3754 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3755 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3756 {
3757   MPI_Comm       comm;
3758   const PetscInt dim  = 3;
3759   PLC           *in, *out;
3760   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3761   PetscMPIInt    rank;
3762   PetscErrorCode ierr;
3763 
3764   PetscFunctionBegin;
3765   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3766   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3767   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3768   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3769   ierr = PLCCreate(&in);CHKERRQ(ierr);
3770   ierr = PLCCreate(&out);CHKERRQ(ierr);
3771 
3772   in->numberofpoints = vEnd - vStart;
3773   if (in->numberofpoints > 0) {
3774     PetscSection coordSection;
3775     Vec          coordinates;
3776     PetscScalar *array;
3777 
3778     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3779     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3780     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3781     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3782     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3783     for (v = vStart; v < vEnd; ++v) {
3784       const PetscInt idx = v - vStart;
3785       PetscInt       off, d, m;
3786 
3787       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3788       for (d = 0; d < dim; ++d) {
3789         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3790       }
3791       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3792 
3793       in->pointmarkerlist[idx] = (int) m;
3794     }
3795     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3796   }
3797   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3798 
3799   in->numberoffacets = fEnd - fStart;
3800   if (in->numberoffacets > 0) {
3801     ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
3802     ierr = PetscMalloc1(in->numberoffacets,   &in->facetmarkerlist);CHKERRQ(ierr);
3803     for (f = fStart; f < fEnd; ++f) {
3804       const PetscInt idx     = f - fStart;
3805       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3806       polygon       *poly;
3807 
3808       in->facetlist[idx].numberofpolygons = 1;
3809 
3810       ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3811 
3812       in->facetlist[idx].numberofholes    = 0;
3813       in->facetlist[idx].holelist         = NULL;
3814 
3815       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3816       for (p = 0; p < numPoints*2; p += 2) {
3817         const PetscInt point = points[p];
3818         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3819       }
3820 
3821       poly                   = in->facetlist[idx].polygonlist;
3822       poly->numberofvertices = numVertices;
3823       ierr                   = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
3824       for (v = 0; v < numVertices; ++v) {
3825         const PetscInt vIdx = points[v] - vStart;
3826         poly->vertexlist[v] = vIdx;
3827       }
3828       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3829       in->facetmarkerlist[idx] = (int) m;
3830       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3831     }
3832   }
3833   if (!rank) {
3834     TetGenOpts t;
3835 
3836     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3837     t.in        = boundary; /* Should go away */
3838     t.plc       = 1;
3839     t.quality   = 1;
3840     t.edgesout  = 1;
3841     t.zeroindex = 1;
3842     t.quiet     = 1;
3843     t.verbose   = verbose;
3844     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3845     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3846   }
3847   {
3848     const PetscInt numCorners  = 4;
3849     const PetscInt numCells    = out->numberoftetrahedra;
3850     const PetscInt numVertices = out->numberofpoints;
3851     const double   *meshCoords = out->pointlist;
3852     int            *cells      = out->tetrahedronlist;
3853 
3854     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3855     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3856     /* Set labels */
3857     for (v = 0; v < numVertices; ++v) {
3858       if (out->pointmarkerlist[v]) {
3859         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3860       }
3861     }
3862     if (interpolate) {
3863       PetscInt e;
3864 
3865       for (e = 0; e < out->numberofedges; e++) {
3866         if (out->edgemarkerlist[e]) {
3867           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3868           const PetscInt *edges;
3869           PetscInt        numEdges;
3870 
3871           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3872           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3873           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3874           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3875         }
3876       }
3877       for (f = 0; f < out->numberoftrifaces; f++) {
3878         if (out->trifacemarkerlist[f]) {
3879           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3880           const PetscInt *faces;
3881           PetscInt        numFaces;
3882 
3883           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3884           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3885           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3886           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3887         }
3888       }
3889     }
3890     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3891   }
3892 
3893   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3894   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3895   PetscFunctionReturn(0);
3896 }
3897 
3898 #undef __FUNCT__
3899 #define __FUNCT__ "DMPlexRefine_CTetgen"
3900 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3901 {
3902   MPI_Comm       comm;
3903   const PetscInt dim  = 3;
3904   PLC           *in, *out;
3905   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3906   PetscMPIInt    rank;
3907   PetscErrorCode ierr;
3908 
3909   PetscFunctionBegin;
3910   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3911   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3912   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3913   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3914   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3915   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3916   ierr = PLCCreate(&in);CHKERRQ(ierr);
3917   ierr = PLCCreate(&out);CHKERRQ(ierr);
3918 
3919   in->numberofpoints = vEnd - vStart;
3920   if (in->numberofpoints > 0) {
3921     PetscSection coordSection;
3922     Vec          coordinates;
3923     PetscScalar *array;
3924 
3925     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3926     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3927     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3928     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3929     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3930     for (v = vStart; v < vEnd; ++v) {
3931       const PetscInt idx = v - vStart;
3932       PetscInt       off, d, m;
3933 
3934       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3935       for (d = 0; d < dim; ++d) {
3936         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3937       }
3938       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3939 
3940       in->pointmarkerlist[idx] = (int) m;
3941     }
3942     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3943   }
3944   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3945 
3946   in->numberofcorners       = 4;
3947   in->numberoftetrahedra    = cEnd - cStart;
3948   in->tetrahedronvolumelist = maxVolumes;
3949   if (in->numberoftetrahedra > 0) {
3950     ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
3951     for (c = cStart; c < cEnd; ++c) {
3952       const PetscInt idx      = c - cStart;
3953       PetscInt      *closure = NULL;
3954       PetscInt       closureSize;
3955 
3956       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3957       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3958       for (v = 0; v < 4; ++v) {
3959         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3960       }
3961       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3962     }
3963   }
3964   if (!rank) {
3965     TetGenOpts t;
3966 
3967     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3968 
3969     t.in        = dm; /* Should go away */
3970     t.refine    = 1;
3971     t.varvolume = 1;
3972     t.quality   = 1;
3973     t.edgesout  = 1;
3974     t.zeroindex = 1;
3975     t.quiet     = 1;
3976     t.verbose   = verbose; /* Change this */
3977 
3978     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3979     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3980   }
3981   {
3982     const PetscInt numCorners  = 4;
3983     const PetscInt numCells    = out->numberoftetrahedra;
3984     const PetscInt numVertices = out->numberofpoints;
3985     const double   *meshCoords = out->pointlist;
3986     int            *cells      = out->tetrahedronlist;
3987     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3988 
3989     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3990     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3991     /* Set labels */
3992     for (v = 0; v < numVertices; ++v) {
3993       if (out->pointmarkerlist[v]) {
3994         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3995       }
3996     }
3997     if (interpolate) {
3998       PetscInt e, f;
3999 
4000       for (e = 0; e < out->numberofedges; e++) {
4001         if (out->edgemarkerlist[e]) {
4002           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
4003           const PetscInt *edges;
4004           PetscInt        numEdges;
4005 
4006           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
4007           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
4008           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
4009           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
4010         }
4011       }
4012       for (f = 0; f < out->numberoftrifaces; f++) {
4013         if (out->trifacemarkerlist[f]) {
4014           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
4015           const PetscInt *faces;
4016           PetscInt        numFaces;
4017 
4018           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
4019           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
4020           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
4021           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
4022         }
4023       }
4024     }
4025     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
4026   }
4027   ierr = PLCDestroy(&in);CHKERRQ(ierr);
4028   ierr = PLCDestroy(&out);CHKERRQ(ierr);
4029   PetscFunctionReturn(0);
4030 }
4031 #endif
4032 
4033 #undef __FUNCT__
4034 #define __FUNCT__ "DMPlexGenerate"
4035 /*@C
4036   DMPlexGenerate - Generates a mesh.
4037 
4038   Not Collective
4039 
4040   Input Parameters:
4041 + boundary - The DMPlex boundary object
4042 . name - The mesh generation package name
4043 - interpolate - Flag to create intermediate mesh elements
4044 
4045   Output Parameter:
4046 . mesh - The DMPlex object
4047 
4048   Level: intermediate
4049 
4050 .keywords: mesh, elements
4051 .seealso: DMPlexCreate(), DMRefine()
4052 @*/
4053 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
4054 {
4055   PetscInt       dim;
4056   char           genname[1024];
4057   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4058   PetscErrorCode ierr;
4059 
4060   PetscFunctionBegin;
4061   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
4062   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
4063   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
4064   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4065   if (flg) name = genname;
4066   if (name) {
4067     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4068     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4069     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4070   }
4071   switch (dim) {
4072   case 1:
4073     if (!name || isTriangle) {
4074 #if defined(PETSC_HAVE_TRIANGLE)
4075       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
4076 #else
4077       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
4078 #endif
4079     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4080     break;
4081   case 2:
4082     if (!name || isCTetgen) {
4083 #if defined(PETSC_HAVE_CTETGEN)
4084       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4085 #else
4086       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4087 #endif
4088     } else if (isTetgen) {
4089 #if defined(PETSC_HAVE_TETGEN)
4090       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4091 #else
4092       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4093 #endif
4094     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4095     break;
4096   default:
4097     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
4098   }
4099   PetscFunctionReturn(0);
4100 }
4101 
4102 #undef __FUNCT__
4103 #define __FUNCT__ "DMRefine_Plex"
4104 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
4105 {
4106   PetscReal      refinementLimit;
4107   PetscInt       dim, cStart, cEnd;
4108   char           genname[1024], *name = NULL;
4109   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4110   PetscErrorCode ierr;
4111 
4112   PetscFunctionBegin;
4113   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4114   if (isUniform) {
4115     CellRefiner cellRefiner;
4116 
4117     ierr = DMPlexGetCellRefiner_Internal(dm, &cellRefiner);CHKERRQ(ierr);
4118     ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
4119     PetscFunctionReturn(0);
4120   }
4121   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
4122   if (refinementLimit == 0.0) PetscFunctionReturn(0);
4123   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
4124   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4125   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4126   if (flg) name = genname;
4127   if (name) {
4128     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4129     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4130     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4131   }
4132   switch (dim) {
4133   case 2:
4134     if (!name || isTriangle) {
4135 #if defined(PETSC_HAVE_TRIANGLE)
4136       double  *maxVolumes;
4137       PetscInt c;
4138 
4139       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4140       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4141       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4142       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
4143 #else
4144       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
4145 #endif
4146     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4147     break;
4148   case 3:
4149     if (!name || isCTetgen) {
4150 #if defined(PETSC_HAVE_CTETGEN)
4151       PetscReal *maxVolumes;
4152       PetscInt   c;
4153 
4154       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4155       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4156       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4157 #else
4158       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4159 #endif
4160     } else if (isTetgen) {
4161 #if defined(PETSC_HAVE_TETGEN)
4162       double  *maxVolumes;
4163       PetscInt c;
4164 
4165       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4166       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4167       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4168 #else
4169       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4170 #endif
4171     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4172     break;
4173   default:
4174     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
4175   }
4176   PetscFunctionReturn(0);
4177 }
4178 
4179 #undef __FUNCT__
4180 #define __FUNCT__ "DMRefineHierarchy_Plex"
4181 PetscErrorCode DMRefineHierarchy_Plex(DM dm, PetscInt nlevels, DM dmRefined[])
4182 {
4183   DM             cdm = dm;
4184   PetscInt       r;
4185   PetscBool      isUniform;
4186   PetscErrorCode ierr;
4187 
4188   PetscFunctionBegin;
4189   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4190   if (!isUniform) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Non-uniform refinement is incompatible with the hierarchy");
4191   for (r = 0; r < nlevels; ++r) {
4192     CellRefiner cellRefiner;
4193 
4194     ierr = DMPlexGetCellRefiner_Internal(cdm, &cellRefiner);CHKERRQ(ierr);
4195     ierr = DMPlexRefineUniform_Internal(cdm, cellRefiner, &dmRefined[r]);CHKERRQ(ierr);
4196     ierr = DMPlexSetCoarseDM(dmRefined[r], cdm);CHKERRQ(ierr);
4197     cdm  = dmRefined[r];
4198   }
4199   PetscFunctionReturn(0);
4200 }
4201 
4202 #undef __FUNCT__
4203 #define __FUNCT__ "DMCoarsen_Plex"
4204 PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
4205 {
4206   DM_Plex       *mesh = (DM_Plex*) dm->data;
4207   PetscErrorCode ierr;
4208 
4209   PetscFunctionBegin;
4210   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
4211   *dmCoarsened = mesh->coarseMesh;
4212   PetscFunctionReturn(0);
4213 }
4214 
4215 #undef __FUNCT__
4216 #define __FUNCT__ "DMPlexGetDepthLabel"
4217 /*@
4218   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4219 
4220   Not Collective
4221 
4222   Input Parameter:
4223 . dm    - The DMPlex object
4224 
4225   Output Parameter:
4226 . depthLabel - The DMLabel recording point depth
4227 
4228   Level: developer
4229 
4230 .keywords: mesh, points
4231 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4232 @*/
4233 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4234 {
4235   DM_Plex       *mesh = (DM_Plex*) dm->data;
4236   PetscErrorCode ierr;
4237 
4238   PetscFunctionBegin;
4239   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4240   PetscValidPointer(depthLabel, 2);
4241   if (!mesh->depthLabel) {ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);}
4242   *depthLabel = mesh->depthLabel;
4243   PetscFunctionReturn(0);
4244 }
4245 
4246 #undef __FUNCT__
4247 #define __FUNCT__ "DMPlexGetDepth"
4248 /*@
4249   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4250 
4251   Not Collective
4252 
4253   Input Parameter:
4254 . dm    - The DMPlex object
4255 
4256   Output Parameter:
4257 . depth - The number of strata (breadth first levels) in the DAG
4258 
4259   Level: developer
4260 
4261 .keywords: mesh, points
4262 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4263 @*/
4264 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4265 {
4266   DMLabel        label;
4267   PetscInt       d = 0;
4268   PetscErrorCode ierr;
4269 
4270   PetscFunctionBegin;
4271   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4272   PetscValidPointer(depth, 2);
4273   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4274   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4275   *depth = d-1;
4276   PetscFunctionReturn(0);
4277 }
4278 
4279 #undef __FUNCT__
4280 #define __FUNCT__ "DMPlexGetDepthStratum"
4281 /*@
4282   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4283 
4284   Not Collective
4285 
4286   Input Parameters:
4287 + dm           - The DMPlex object
4288 - stratumValue - The requested depth
4289 
4290   Output Parameters:
4291 + start - The first point at this depth
4292 - end   - One beyond the last point at this depth
4293 
4294   Level: developer
4295 
4296 .keywords: mesh, points
4297 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
4298 @*/
4299 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4300 {
4301   DMLabel        label;
4302   PetscInt       pStart, pEnd;
4303   PetscErrorCode ierr;
4304 
4305   PetscFunctionBegin;
4306   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4307   if (start) {PetscValidPointer(start, 3); *start = 0;}
4308   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4309   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4310   if (pStart == pEnd) PetscFunctionReturn(0);
4311   if (stratumValue < 0) {
4312     if (start) *start = pStart;
4313     if (end)   *end   = pEnd;
4314     PetscFunctionReturn(0);
4315   }
4316   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4317   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4318   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4319   PetscFunctionReturn(0);
4320 }
4321 
4322 #undef __FUNCT__
4323 #define __FUNCT__ "DMPlexGetHeightStratum"
4324 /*@
4325   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4326 
4327   Not Collective
4328 
4329   Input Parameters:
4330 + dm           - The DMPlex object
4331 - stratumValue - The requested height
4332 
4333   Output Parameters:
4334 + start - The first point at this height
4335 - end   - One beyond the last point at this height
4336 
4337   Level: developer
4338 
4339 .keywords: mesh, points
4340 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
4341 @*/
4342 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4343 {
4344   DMLabel        label;
4345   PetscInt       depth, pStart, pEnd;
4346   PetscErrorCode ierr;
4347 
4348   PetscFunctionBegin;
4349   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4350   if (start) {PetscValidPointer(start, 3); *start = 0;}
4351   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4352   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4353   if (pStart == pEnd) PetscFunctionReturn(0);
4354   if (stratumValue < 0) {
4355     if (start) *start = pStart;
4356     if (end)   *end   = pEnd;
4357     PetscFunctionReturn(0);
4358   }
4359   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4360   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
4361   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4362   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4363   PetscFunctionReturn(0);
4364 }
4365 
4366 #undef __FUNCT__
4367 #define __FUNCT__ "DMPlexCreateSectionInitial"
4368 /* Set the number of dof on each point and separate by fields */
4369 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
4370 {
4371   PetscInt      *numDofTot;
4372   PetscInt       depth, pStart = 0, pEnd = 0;
4373   PetscInt       p, d, dep, f;
4374   PetscErrorCode ierr;
4375 
4376   PetscFunctionBegin;
4377   ierr = PetscMalloc1((dim+1), &numDofTot);CHKERRQ(ierr);
4378   for (d = 0; d <= dim; ++d) {
4379     numDofTot[d] = 0;
4380     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
4381   }
4382   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
4383   if (numFields > 0) {
4384     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
4385     if (numComp) {
4386       for (f = 0; f < numFields; ++f) {
4387         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
4388       }
4389     }
4390   }
4391   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4392   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
4393   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4394   for (dep = 0; dep <= depth; ++dep) {
4395     d    = dim == depth ? dep : (!dep ? 0 : dim);
4396     ierr = DMPlexGetDepthStratum(dm, dep, &pStart, &pEnd);CHKERRQ(ierr);
4397     for (p = pStart; p < pEnd; ++p) {
4398       for (f = 0; f < numFields; ++f) {
4399         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
4400       }
4401       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
4402     }
4403   }
4404   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
4405   PetscFunctionReturn(0);
4406 }
4407 
4408 #undef __FUNCT__
4409 #define __FUNCT__ "DMPlexCreateSectionBCDof"
4410 /* Set the number of dof on each point and separate by fields
4411    If constDof is PETSC_DETERMINE, constrain every dof on the point
4412 */
4413 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
4414 {
4415   PetscInt       numFields;
4416   PetscInt       bc;
4417   PetscErrorCode ierr;
4418 
4419   PetscFunctionBegin;
4420   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4421   for (bc = 0; bc < numBC; ++bc) {
4422     PetscInt        field = 0;
4423     const PetscInt *idx;
4424     PetscInt        n, i;
4425 
4426     if (numFields) field = bcField[bc];
4427     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
4428     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4429     for (i = 0; i < n; ++i) {
4430       const PetscInt p        = idx[i];
4431       PetscInt       numConst = constDof;
4432 
4433       /* Constrain every dof on the point */
4434       if (numConst < 0) {
4435         if (numFields) {
4436           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
4437         } else {
4438           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
4439         }
4440       }
4441       if (numFields) {
4442         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
4443       }
4444       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
4445     }
4446     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4447   }
4448   PetscFunctionReturn(0);
4449 }
4450 
4451 #undef __FUNCT__
4452 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
4453 /* Set the constrained indices on each point and separate by fields */
4454 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
4455 {
4456   PetscInt      *maxConstraints;
4457   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
4458   PetscErrorCode ierr;
4459 
4460   PetscFunctionBegin;
4461   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4462   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4463   ierr = PetscMalloc1((numFields+1), &maxConstraints);CHKERRQ(ierr);
4464   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
4465   for (p = pStart; p < pEnd; ++p) {
4466     PetscInt cdof;
4467 
4468     if (numFields) {
4469       for (f = 0; f < numFields; ++f) {
4470         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
4471         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
4472       }
4473     } else {
4474       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4475       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
4476     }
4477   }
4478   for (f = 0; f < numFields; ++f) {
4479     maxConstraints[numFields] += maxConstraints[f];
4480   }
4481   if (maxConstraints[numFields]) {
4482     PetscInt *indices;
4483 
4484     ierr = PetscMalloc1(maxConstraints[numFields], &indices);CHKERRQ(ierr);
4485     for (p = pStart; p < pEnd; ++p) {
4486       PetscInt cdof, d;
4487 
4488       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4489       if (cdof) {
4490         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
4491         if (numFields) {
4492           PetscInt numConst = 0, foff = 0;
4493 
4494           for (f = 0; f < numFields; ++f) {
4495             PetscInt cfdof, fdof;
4496 
4497             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4498             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
4499             /* Change constraint numbering from absolute local dof number to field relative local dof number */
4500             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
4501             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
4502             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
4503             numConst += cfdof;
4504             foff     += fdof;
4505           }
4506           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4507         } else {
4508           for (d = 0; d < cdof; ++d) indices[d] = d;
4509         }
4510         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4511       }
4512     }
4513     ierr = PetscFree(indices);CHKERRQ(ierr);
4514   }
4515   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
4516   PetscFunctionReturn(0);
4517 }
4518 
4519 #undef __FUNCT__
4520 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
4521 /* Set the constrained field indices on each point */
4522 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
4523 {
4524   const PetscInt *points, *indices;
4525   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
4526   PetscErrorCode  ierr;
4527 
4528   PetscFunctionBegin;
4529   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4530   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
4531 
4532   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
4533   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
4534   if (!constraintIndices) {
4535     PetscInt *idx, i;
4536 
4537     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4538     ierr = PetscMalloc1(maxDof, &idx);CHKERRQ(ierr);
4539     for (i = 0; i < maxDof; ++i) idx[i] = i;
4540     for (p = 0; p < numPoints; ++p) {
4541       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
4542     }
4543     ierr = PetscFree(idx);CHKERRQ(ierr);
4544   } else {
4545     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
4546     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
4547     for (p = 0; p < numPoints; ++p) {
4548       PetscInt fcdof;
4549 
4550       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
4551       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
4552       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
4553     }
4554     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
4555   }
4556   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
4557   PetscFunctionReturn(0);
4558 }
4559 
4560 #undef __FUNCT__
4561 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
4562 /* Set the constrained indices on each point and separate by fields */
4563 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
4564 {
4565   PetscInt      *indices;
4566   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
4567   PetscErrorCode ierr;
4568 
4569   PetscFunctionBegin;
4570   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4571   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
4572   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4573   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
4574   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4575   for (p = pStart; p < pEnd; ++p) {
4576     PetscInt cdof, d;
4577 
4578     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4579     if (cdof) {
4580       PetscInt numConst = 0, foff = 0;
4581 
4582       for (f = 0; f < numFields; ++f) {
4583         const PetscInt *fcind;
4584         PetscInt        fdof, fcdof;
4585 
4586         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4587         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
4588         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
4589         /* Change constraint numbering from field relative local dof number to absolute local dof number */
4590         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
4591         foff     += fdof;
4592         numConst += fcdof;
4593       }
4594       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4595       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4596     }
4597   }
4598   ierr = PetscFree(indices);CHKERRQ(ierr);
4599   PetscFunctionReturn(0);
4600 }
4601 
4602 #undef __FUNCT__
4603 #define __FUNCT__ "DMPlexCreateSection"
4604 /*@C
4605   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
4606 
4607   Not Collective
4608 
4609   Input Parameters:
4610 + dm        - The DMPlex object
4611 . dim       - The spatial dimension of the problem
4612 . numFields - The number of fields in the problem
4613 . numComp   - An array of size numFields that holds the number of components for each field
4614 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
4615 . numBC     - The number of boundary conditions
4616 . bcField   - An array of size numBC giving the field number for each boundry condition
4617 . bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
4618 - perm      - Optional permutation of the chart, or NULL
4619 
4620   Output Parameter:
4621 . section - The PetscSection object
4622 
4623   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
4624   number of dof for field 0 on each edge.
4625 
4626   The chart permutation is the same one set using PetscSectionSetPermutation()
4627 
4628   Level: developer
4629 
4630   Fortran Notes:
4631   A Fortran 90 version is available as DMPlexCreateSectionF90()
4632 
4633 .keywords: mesh, elements
4634 .seealso: DMPlexCreate(), PetscSectionCreate(), PetscSectionSetPermutation()
4635 @*/
4636 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], IS perm, PetscSection *section)
4637 {
4638   PetscErrorCode ierr;
4639 
4640   PetscFunctionBegin;
4641   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
4642   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
4643   if (perm) {ierr = PetscSectionSetPermutation(*section, perm);CHKERRQ(ierr);}
4644   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
4645   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
4646   ierr = PetscSectionViewFromOptions(*section,NULL,"-section_view");CHKERRQ(ierr);
4647   PetscFunctionReturn(0);
4648 }
4649 
4650 #undef __FUNCT__
4651 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
4652 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4653 {
4654   PetscSection   section;
4655   PetscErrorCode ierr;
4656 
4657   PetscFunctionBegin;
4658   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4659   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4660   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
4661   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4662   PetscFunctionReturn(0);
4663 }
4664 
4665 #undef __FUNCT__
4666 #define __FUNCT__ "DMPlexGetConeSection"
4667 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4668 {
4669   DM_Plex *mesh = (DM_Plex*) dm->data;
4670 
4671   PetscFunctionBegin;
4672   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4673   if (section) *section = mesh->coneSection;
4674   PetscFunctionReturn(0);
4675 }
4676 
4677 #undef __FUNCT__
4678 #define __FUNCT__ "DMPlexGetSupportSection"
4679 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4680 {
4681   DM_Plex *mesh = (DM_Plex*) dm->data;
4682 
4683   PetscFunctionBegin;
4684   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4685   if (section) *section = mesh->supportSection;
4686   PetscFunctionReturn(0);
4687 }
4688 
4689 #undef __FUNCT__
4690 #define __FUNCT__ "DMPlexGetCones"
4691 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4692 {
4693   DM_Plex *mesh = (DM_Plex*) dm->data;
4694 
4695   PetscFunctionBegin;
4696   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4697   if (cones) *cones = mesh->cones;
4698   PetscFunctionReturn(0);
4699 }
4700 
4701 #undef __FUNCT__
4702 #define __FUNCT__ "DMPlexGetConeOrientations"
4703 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4704 {
4705   DM_Plex *mesh = (DM_Plex*) dm->data;
4706 
4707   PetscFunctionBegin;
4708   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4709   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4710   PetscFunctionReturn(0);
4711 }
4712 
4713 /******************************** FEM Support **********************************/
4714 
4715 #undef __FUNCT__
4716 #define __FUNCT__ "DMPlexVecGetClosure_Depth1_Static"
4717 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4718 {
4719   PetscScalar    *array, *vArray;
4720   const PetscInt *cone, *coneO;
4721   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4722   PetscErrorCode  ierr;
4723 
4724   PetscFunctionBeginHot;
4725   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4726   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4727   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4728   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4729   if (!values || !*values) {
4730     if ((point >= pStart) && (point < pEnd)) {
4731       PetscInt dof;
4732 
4733       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4734       size += dof;
4735     }
4736     for (p = 0; p < numPoints; ++p) {
4737       const PetscInt cp = cone[p];
4738       PetscInt       dof;
4739 
4740       if ((cp < pStart) || (cp >= pEnd)) continue;
4741       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4742       size += dof;
4743     }
4744     if (!values) {
4745       if (csize) *csize = size;
4746       PetscFunctionReturn(0);
4747     }
4748     ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
4749   } else {
4750     array = *values;
4751   }
4752   size = 0;
4753   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4754   if ((point >= pStart) && (point < pEnd)) {
4755     PetscInt     dof, off, d;
4756     PetscScalar *varr;
4757 
4758     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4759     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4760     varr = &vArray[off];
4761     for (d = 0; d < dof; ++d, ++offset) {
4762       array[offset] = varr[d];
4763     }
4764     size += dof;
4765   }
4766   for (p = 0; p < numPoints; ++p) {
4767     const PetscInt cp = cone[p];
4768     PetscInt       o  = coneO[p];
4769     PetscInt       dof, off, d;
4770     PetscScalar   *varr;
4771 
4772     if ((cp < pStart) || (cp >= pEnd)) continue;
4773     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4774     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4775     varr = &vArray[off];
4776     if (o >= 0) {
4777       for (d = 0; d < dof; ++d, ++offset) {
4778         array[offset] = varr[d];
4779       }
4780     } else {
4781       for (d = dof-1; d >= 0; --d, ++offset) {
4782         array[offset] = varr[d];
4783       }
4784     }
4785     size += dof;
4786   }
4787   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4788   if (!*values) {
4789     if (csize) *csize = size;
4790     *values = array;
4791   } else {
4792     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4793     *csize = size;
4794   }
4795   PetscFunctionReturn(0);
4796 }
4797 
4798 #undef __FUNCT__
4799 #define __FUNCT__ "DMPlexVecGetClosure_Static"
4800 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4801 {
4802   PetscInt       offset = 0, p;
4803   PetscErrorCode ierr;
4804 
4805   PetscFunctionBeginHot;
4806   *size = 0;
4807   for (p = 0; p < numPoints*2; p += 2) {
4808     const PetscInt point = points[p];
4809     const PetscInt o     = points[p+1];
4810     PetscInt       dof, off, d;
4811     const PetscScalar *varr;
4812 
4813     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4814     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4815     varr = &vArray[off];
4816     if (o >= 0) {
4817       for (d = 0; d < dof; ++d, ++offset)    array[offset] = varr[d];
4818     } else {
4819       for (d = dof-1; d >= 0; --d, ++offset) array[offset] = varr[d];
4820     }
4821   }
4822   *size = offset;
4823   PetscFunctionReturn(0);
4824 }
4825 
4826 #undef __FUNCT__
4827 #define __FUNCT__ "DMPlexVecGetClosure_Fields_Static"
4828 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4829 {
4830   PetscInt       offset = 0, f;
4831   PetscErrorCode ierr;
4832 
4833   PetscFunctionBeginHot;
4834   *size = 0;
4835   for (f = 0; f < numFields; ++f) {
4836     PetscInt fcomp, p;
4837 
4838     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4839     for (p = 0; p < numPoints*2; p += 2) {
4840       const PetscInt point = points[p];
4841       const PetscInt o     = points[p+1];
4842       PetscInt       fdof, foff, d, c;
4843       const PetscScalar *varr;
4844 
4845       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4846       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4847       varr = &vArray[foff];
4848       if (o >= 0) {
4849         for (d = 0; d < fdof; ++d, ++offset) array[offset] = varr[d];
4850       } else {
4851         for (d = fdof/fcomp-1; d >= 0; --d) {
4852           for (c = 0; c < fcomp; ++c, ++offset) {
4853             array[offset] = varr[d*fcomp+c];
4854           }
4855         }
4856       }
4857     }
4858   }
4859   *size = offset;
4860   PetscFunctionReturn(0);
4861 }
4862 
4863 #undef __FUNCT__
4864 #define __FUNCT__ "DMPlexVecGetClosure"
4865 /*@C
4866   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4867 
4868   Not collective
4869 
4870   Input Parameters:
4871 + dm - The DM
4872 . section - The section describing the layout in v, or NULL to use the default section
4873 . v - The local vector
4874 - point - The sieve point in the DM
4875 
4876   Output Parameters:
4877 + csize - The number of values in the closure, or NULL
4878 - values - The array of values, which is a borrowed array and should not be freed
4879 
4880   Fortran Notes:
4881   Since it returns an array, this routine is only available in Fortran 90, and you must
4882   include petsc.h90 in your code.
4883 
4884   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4885 
4886   Level: intermediate
4887 
4888 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4889 @*/
4890 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4891 {
4892   PetscSection    clSection;
4893   IS              clPoints;
4894   PetscScalar    *array, *vArray;
4895   PetscInt       *points = NULL;
4896   const PetscInt *clp;
4897   PetscInt        depth, numFields, numPoints, size;
4898   PetscErrorCode  ierr;
4899 
4900   PetscFunctionBeginHot;
4901   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4902   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4903   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4904   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4905   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4906   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4907   if (depth == 1 && numFields < 2) {
4908     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4909     PetscFunctionReturn(0);
4910   }
4911   /* Get points */
4912   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
4913   if (!clPoints) {
4914     PetscInt pStart, pEnd, p, q;
4915 
4916     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4917     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
4918     /* Compress out points not in the section */
4919     for (p = 0, q = 0; p < numPoints*2; p += 2) {
4920       if ((points[p] >= pStart) && (points[p] < pEnd)) {
4921         points[q*2]   = points[p];
4922         points[q*2+1] = points[p+1];
4923         ++q;
4924       }
4925     }
4926     numPoints = q;
4927   } else {
4928     PetscInt dof, off;
4929 
4930     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
4931     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
4932     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
4933     numPoints = dof/2;
4934     points    = (PetscInt *) &clp[off];
4935   }
4936   /* Get array */
4937   if (!values || !*values) {
4938     PetscInt asize = 0, dof, p;
4939 
4940     for (p = 0; p < numPoints*2; p += 2) {
4941       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4942       asize += dof;
4943     }
4944     if (!values) {
4945       if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4946       else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4947       if (csize) *csize = asize;
4948       PetscFunctionReturn(0);
4949     }
4950     ierr = DMGetWorkArray(dm, asize, PETSC_SCALAR, &array);CHKERRQ(ierr);
4951   } else {
4952     array = *values;
4953   }
4954   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4955   /* Get values */
4956   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(section, numPoints, points, numFields, vArray, &size, array);CHKERRQ(ierr);}
4957   else               {ierr = DMPlexVecGetClosure_Static(section, numPoints, points, vArray, &size, array);CHKERRQ(ierr);}
4958   /* Cleanup points */
4959   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4960   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4961   /* Cleanup array */
4962   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4963   if (!*values) {
4964     if (csize) *csize = size;
4965     *values = array;
4966   } else {
4967     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4968     *csize = size;
4969   }
4970   PetscFunctionReturn(0);
4971 }
4972 
4973 #undef __FUNCT__
4974 #define __FUNCT__ "DMPlexVecRestoreClosure"
4975 /*@C
4976   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4977 
4978   Not collective
4979 
4980   Input Parameters:
4981 + dm - The DM
4982 . section - The section describing the layout in v, or NULL to use the default section
4983 . v - The local vector
4984 . point - The sieve point in the DM
4985 . csize - The number of values in the closure, or NULL
4986 - values - The array of values, which is a borrowed array and should not be freed
4987 
4988   Fortran Notes:
4989   Since it returns an array, this routine is only available in Fortran 90, and you must
4990   include petsc.h90 in your code.
4991 
4992   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4993 
4994   Level: intermediate
4995 
4996 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4997 @*/
4998 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4999 {
5000   PetscInt       size = 0;
5001   PetscErrorCode ierr;
5002 
5003   PetscFunctionBegin;
5004   /* Should work without recalculating size */
5005   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
5006   PetscFunctionReturn(0);
5007 }
5008 
5009 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5010 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5011 
5012 #undef __FUNCT__
5013 #define __FUNCT__ "updatePoint_private"
5014 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
5015 {
5016   PetscInt        cdof;   /* The number of constraints on this point */
5017   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5018   PetscScalar    *a;
5019   PetscInt        off, cind = 0, k;
5020   PetscErrorCode  ierr;
5021 
5022   PetscFunctionBegin;
5023   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5024   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5025   a    = &array[off];
5026   if (!cdof || setBC) {
5027     if (orientation >= 0) {
5028       for (k = 0; k < dof; ++k) {
5029         fuse(&a[k], values[k]);
5030       }
5031     } else {
5032       for (k = 0; k < dof; ++k) {
5033         fuse(&a[k], values[dof-k-1]);
5034       }
5035     }
5036   } else {
5037     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5038     if (orientation >= 0) {
5039       for (k = 0; k < dof; ++k) {
5040         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5041         fuse(&a[k], values[k]);
5042       }
5043     } else {
5044       for (k = 0; k < dof; ++k) {
5045         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5046         fuse(&a[k], values[dof-k-1]);
5047       }
5048     }
5049   }
5050   PetscFunctionReturn(0);
5051 }
5052 
5053 #undef __FUNCT__
5054 #define __FUNCT__ "updatePointBC_private"
5055 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscInt orientation, const PetscScalar values[], PetscScalar array[])
5056 {
5057   PetscInt        cdof;   /* The number of constraints on this point */
5058   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5059   PetscScalar    *a;
5060   PetscInt        off, cind = 0, k;
5061   PetscErrorCode  ierr;
5062 
5063   PetscFunctionBegin;
5064   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5065   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5066   a    = &array[off];
5067   if (cdof) {
5068     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5069     if (orientation >= 0) {
5070       for (k = 0; k < dof; ++k) {
5071         if ((cind < cdof) && (k == cdofs[cind])) {
5072           fuse(&a[k], values[k]);
5073           ++cind;
5074         }
5075       }
5076     } else {
5077       for (k = 0; k < dof; ++k) {
5078         if ((cind < cdof) && (k == cdofs[cind])) {
5079           fuse(&a[k], values[dof-k-1]);
5080           ++cind;
5081         }
5082       }
5083     }
5084   }
5085   PetscFunctionReturn(0);
5086 }
5087 
5088 #undef __FUNCT__
5089 #define __FUNCT__ "updatePointFields_private"
5090 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5091 {
5092   PetscScalar    *a;
5093   PetscInt        fdof, foff, fcdof, foffset = *offset;
5094   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5095   PetscInt        cind = 0, k, c;
5096   PetscErrorCode  ierr;
5097 
5098   PetscFunctionBegin;
5099   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5100   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5101   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5102   a    = &array[foff];
5103   if (!fcdof || setBC) {
5104     if (o >= 0) {
5105       for (k = 0; k < fdof; ++k) fuse(&a[k], values[foffset+k]);
5106     } else {
5107       for (k = fdof/fcomp-1; k >= 0; --k) {
5108         for (c = 0; c < fcomp; ++c) {
5109           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5110         }
5111       }
5112     }
5113   } else {
5114     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5115     if (o >= 0) {
5116       for (k = 0; k < fdof; ++k) {
5117         if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
5118         fuse(&a[k], values[foffset+k]);
5119       }
5120     } else {
5121       for (k = fdof/fcomp-1; k >= 0; --k) {
5122         for (c = 0; c < fcomp; ++c) {
5123           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
5124           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5125         }
5126       }
5127     }
5128   }
5129   *offset += fdof;
5130   PetscFunctionReturn(0);
5131 }
5132 
5133 #undef __FUNCT__
5134 #define __FUNCT__ "updatePointFieldsBC_private"
5135 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5136 {
5137   PetscScalar    *a;
5138   PetscInt        fdof, foff, fcdof, foffset = *offset;
5139   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5140   PetscInt        cind = 0, k, c;
5141   PetscErrorCode  ierr;
5142 
5143   PetscFunctionBegin;
5144   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5145   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5146   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5147   a    = &array[foff];
5148   if (fcdof) {
5149     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5150     if (o >= 0) {
5151       for (k = 0; k < fdof; ++k) {
5152         if ((cind < fcdof) && (k == fcdofs[cind])) {
5153           fuse(&a[k], values[foffset+k]);
5154           ++cind;
5155         }
5156       }
5157     } else {
5158       for (k = fdof/fcomp-1; k >= 0; --k) {
5159         for (c = 0; c < fcomp; ++c) {
5160           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {
5161             fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5162             ++cind;
5163           }
5164         }
5165       }
5166     }
5167   }
5168   *offset += fdof;
5169   PetscFunctionReturn(0);
5170 }
5171 
5172 #undef __FUNCT__
5173 #define __FUNCT__ "DMPlexVecSetClosure_Static"
5174 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5175 {
5176   PetscScalar    *array;
5177   const PetscInt *cone, *coneO;
5178   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5179   PetscErrorCode  ierr;
5180 
5181   PetscFunctionBeginHot;
5182   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5183   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5184   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5185   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5186   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5187   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5188     const PetscInt cp = !p ? point : cone[p-1];
5189     const PetscInt o  = !p ? 0     : coneO[p-1];
5190 
5191     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5192     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5193     /* ADD_VALUES */
5194     {
5195       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5196       PetscScalar    *a;
5197       PetscInt        cdof, coff, cind = 0, k;
5198 
5199       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5200       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5201       a    = &array[coff];
5202       if (!cdof) {
5203         if (o >= 0) {
5204           for (k = 0; k < dof; ++k) {
5205             a[k] += values[off+k];
5206           }
5207         } else {
5208           for (k = 0; k < dof; ++k) {
5209             a[k] += values[off+dof-k-1];
5210           }
5211         }
5212       } else {
5213         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
5214         if (o >= 0) {
5215           for (k = 0; k < dof; ++k) {
5216             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5217             a[k] += values[off+k];
5218           }
5219         } else {
5220           for (k = 0; k < dof; ++k) {
5221             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5222             a[k] += values[off+dof-k-1];
5223           }
5224         }
5225       }
5226     }
5227   }
5228   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5229   PetscFunctionReturn(0);
5230 }
5231 
5232 #undef __FUNCT__
5233 #define __FUNCT__ "DMPlexVecSetClosure"
5234 /*@C
5235   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
5236 
5237   Not collective
5238 
5239   Input Parameters:
5240 + dm - The DM
5241 . section - The section describing the layout in v, or NULL to use the default section
5242 . v - The local vector
5243 . point - The sieve point in the DM
5244 . values - The array of values
5245 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5246 
5247   Fortran Notes:
5248   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5249 
5250   Level: intermediate
5251 
5252 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
5253 @*/
5254 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5255 {
5256   PetscSection    clSection;
5257   IS              clPoints;
5258   PetscScalar    *array;
5259   PetscInt       *points = NULL;
5260   const PetscInt *clp;
5261   PetscInt        depth, numFields, numPoints, p;
5262   PetscErrorCode  ierr;
5263 
5264   PetscFunctionBeginHot;
5265   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5266   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5267   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5268   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5269   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5270   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5271   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
5272     ierr = DMPlexVecSetClosure_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
5273     PetscFunctionReturn(0);
5274   }
5275   /* Get points */
5276   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5277   if (!clPoints) {
5278     PetscInt pStart, pEnd, q;
5279 
5280     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5281     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5282     /* Compress out points not in the section */
5283     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5284       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5285         points[q*2]   = points[p];
5286         points[q*2+1] = points[p+1];
5287         ++q;
5288       }
5289     }
5290     numPoints = q;
5291   } else {
5292     PetscInt dof, off;
5293 
5294     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5295     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5296     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5297     numPoints = dof/2;
5298     points    = (PetscInt *) &clp[off];
5299   }
5300   /* Get array */
5301   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5302   /* Get values */
5303   if (numFields > 0) {
5304     PetscInt offset = 0, fcomp, f;
5305     for (f = 0; f < numFields; ++f) {
5306       ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5307       switch (mode) {
5308       case INSERT_VALUES:
5309         for (p = 0; p < numPoints*2; p += 2) {
5310           const PetscInt point = points[p];
5311           const PetscInt o     = points[p+1];
5312           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
5313         } break;
5314       case INSERT_ALL_VALUES:
5315         for (p = 0; p < numPoints*2; p += 2) {
5316           const PetscInt point = points[p];
5317           const PetscInt o     = points[p+1];
5318           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
5319         } break;
5320       case INSERT_BC_VALUES:
5321         for (p = 0; p < numPoints*2; p += 2) {
5322           const PetscInt point = points[p];
5323           const PetscInt o     = points[p+1];
5324           updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
5325         } break;
5326       case ADD_VALUES:
5327         for (p = 0; p < numPoints*2; p += 2) {
5328           const PetscInt point = points[p];
5329           const PetscInt o     = points[p+1];
5330           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
5331         } break;
5332       case ADD_ALL_VALUES:
5333         for (p = 0; p < numPoints*2; p += 2) {
5334           const PetscInt point = points[p];
5335           const PetscInt o     = points[p+1];
5336           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
5337         } break;
5338       default:
5339         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5340       }
5341     }
5342   } else {
5343     PetscInt dof, off;
5344 
5345     switch (mode) {
5346     case INSERT_VALUES:
5347       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5348         PetscInt o = points[p+1];
5349         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5350         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
5351       } break;
5352     case INSERT_ALL_VALUES:
5353       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5354         PetscInt o = points[p+1];
5355         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5356         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
5357       } break;
5358     case INSERT_BC_VALUES:
5359       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5360         PetscInt o = points[p+1];
5361         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5362         updatePointBC_private(section, points[p], dof, insert,  o, &values[off], array);
5363       } break;
5364     case ADD_VALUES:
5365       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5366         PetscInt o = points[p+1];
5367         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5368         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
5369       } break;
5370     case ADD_ALL_VALUES:
5371       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5372         PetscInt o = points[p+1];
5373         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5374         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
5375       } break;
5376     default:
5377       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5378     }
5379   }
5380   /* Cleanup points */
5381   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5382   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5383   /* Cleanup array */
5384   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5385   PetscFunctionReturn(0);
5386 }
5387 
5388 #undef __FUNCT__
5389 #define __FUNCT__ "DMPlexPrintMatSetValues"
5390 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5391 {
5392   PetscMPIInt    rank;
5393   PetscInt       i, j;
5394   PetscErrorCode ierr;
5395 
5396   PetscFunctionBegin;
5397   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5398   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
5399   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5400   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5401   numCIndices = numCIndices ? numCIndices : numRIndices;
5402   for (i = 0; i < numRIndices; i++) {
5403     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
5404     for (j = 0; j < numCIndices; j++) {
5405 #if defined(PETSC_USE_COMPLEX)
5406       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5407 #else
5408       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5409 #endif
5410     }
5411     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5412   }
5413   PetscFunctionReturn(0);
5414 }
5415 
5416 #undef __FUNCT__
5417 #define __FUNCT__ "indicesPoint_private"
5418 /* . off - The global offset of this point */
5419 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
5420 {
5421   PetscInt        dof;    /* The number of unknowns on this point */
5422   PetscInt        cdof;   /* The number of constraints on this point */
5423   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5424   PetscInt        cind = 0, k;
5425   PetscErrorCode  ierr;
5426 
5427   PetscFunctionBegin;
5428   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5429   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5430   if (!cdof || setBC) {
5431     if (orientation >= 0) {
5432       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
5433     } else {
5434       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
5435     }
5436   } else {
5437     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5438     if (orientation >= 0) {
5439       for (k = 0; k < dof; ++k) {
5440         if ((cind < cdof) && (k == cdofs[cind])) {
5441           /* Insert check for returning constrained indices */
5442           indices[*loff+k] = -(off+k+1);
5443           ++cind;
5444         } else {
5445           indices[*loff+k] = off+k-cind;
5446         }
5447       }
5448     } else {
5449       for (k = 0; k < dof; ++k) {
5450         if ((cind < cdof) && (k == cdofs[cind])) {
5451           /* Insert check for returning constrained indices */
5452           indices[*loff+dof-k-1] = -(off+k+1);
5453           ++cind;
5454         } else {
5455           indices[*loff+dof-k-1] = off+k-cind;
5456         }
5457       }
5458     }
5459   }
5460   *loff += dof;
5461   PetscFunctionReturn(0);
5462 }
5463 
5464 #undef __FUNCT__
5465 #define __FUNCT__ "indicesPointFields_private"
5466 /* . off - The global offset of this point */
5467 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
5468 {
5469   PetscInt       numFields, foff, f;
5470   PetscErrorCode ierr;
5471 
5472   PetscFunctionBegin;
5473   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5474   for (f = 0, foff = 0; f < numFields; ++f) {
5475     PetscInt        fdof, fcomp, cfdof;
5476     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5477     PetscInt        cind = 0, k, c;
5478 
5479     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5480     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5481     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5482     if (!cfdof || setBC) {
5483       if (orientation >= 0) {
5484         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
5485       } else {
5486         for (k = fdof/fcomp-1; k >= 0; --k) {
5487           for (c = 0; c < fcomp; ++c) {
5488             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
5489           }
5490         }
5491       }
5492     } else {
5493       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5494       if (orientation >= 0) {
5495         for (k = 0; k < fdof; ++k) {
5496           if ((cind < cfdof) && (k == fcdofs[cind])) {
5497             indices[foffs[f]+k] = -(off+foff+k+1);
5498             ++cind;
5499           } else {
5500             indices[foffs[f]+k] = off+foff+k-cind;
5501           }
5502         }
5503       } else {
5504         for (k = fdof/fcomp-1; k >= 0; --k) {
5505           for (c = 0; c < fcomp; ++c) {
5506             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
5507               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
5508               ++cind;
5509             } else {
5510               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
5511             }
5512           }
5513         }
5514       }
5515     }
5516     foff     += fdof - cfdof;
5517     foffs[f] += fdof;
5518   }
5519   PetscFunctionReturn(0);
5520 }
5521 
5522 #undef __FUNCT__
5523 #define __FUNCT__ "DMPlexMatSetClosure"
5524 /*@C
5525   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5526 
5527   Not collective
5528 
5529   Input Parameters:
5530 + dm - The DM
5531 . section - The section describing the layout in v, or NULL to use the default section
5532 . globalSection - The section describing the layout in v, or NULL to use the default global section
5533 . A - The matrix
5534 . point - The sieve point in the DM
5535 . values - The array of values
5536 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5537 
5538   Fortran Notes:
5539   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5540 
5541   Level: intermediate
5542 
5543 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5544 @*/
5545 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5546 {
5547   DM_Plex        *mesh   = (DM_Plex*) dm->data;
5548   PetscSection    clSection;
5549   IS              clPoints;
5550   PetscInt       *points = NULL;
5551   const PetscInt *clp;
5552   PetscInt       *indices;
5553   PetscInt        offsets[32];
5554   PetscInt        numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
5555   PetscErrorCode  ierr;
5556 
5557   PetscFunctionBegin;
5558   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5559   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5560   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5561   if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5562   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5563   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5564   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5565   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5566   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5567   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5568   if (!clPoints) {
5569     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5570     /* Compress out points not in the section */
5571     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5572     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5573       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5574         points[q*2]   = points[p];
5575         points[q*2+1] = points[p+1];
5576         ++q;
5577       }
5578     }
5579     numPoints = q;
5580   } else {
5581     PetscInt dof, off;
5582 
5583     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5584     numPoints = dof/2;
5585     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5586     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5587     points = (PetscInt *) &clp[off];
5588   }
5589   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5590     PetscInt fdof;
5591 
5592     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5593     for (f = 0; f < numFields; ++f) {
5594       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5595       offsets[f+1] += fdof;
5596     }
5597     numIndices += dof;
5598   }
5599   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5600 
5601   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
5602   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5603   if (numFields) {
5604     for (p = 0; p < numPoints*2; p += 2) {
5605       PetscInt o = points[p+1];
5606       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5607       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
5608     }
5609   } else {
5610     for (p = 0, off = 0; p < numPoints*2; p += 2) {
5611       PetscInt o = points[p+1];
5612       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5613       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
5614     }
5615   }
5616   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5617   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5618   if (ierr) {
5619     PetscMPIInt    rank;
5620     PetscErrorCode ierr2;
5621 
5622     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5623     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5624     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5625     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
5626     CHKERRQ(ierr);
5627   }
5628   if (!clPoints) {
5629     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5630   } else {
5631     ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
5632   }
5633   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5634   PetscFunctionReturn(0);
5635 }
5636 
5637 #undef __FUNCT__
5638 #define __FUNCT__ "DMPlexMatSetClosureRefined"
5639 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5640 {
5641   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5642   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5643   PetscInt       *cpoints = NULL;
5644   PetscInt       *findices, *cindices;
5645   PetscInt        foffsets[32], coffsets[32];
5646   CellRefiner     cellRefiner;
5647   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5648   PetscErrorCode  ierr;
5649 
5650   PetscFunctionBegin;
5651   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5652   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5653   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5654   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5655   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5656   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5657   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5658   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5659   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5660   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5661   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5662   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5663   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5664   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5665   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5666   /* Column indices */
5667   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5668   maxFPoints = numCPoints;
5669   /* Compress out points not in the section */
5670   /*   TODO: Squeeze out points with 0 dof as well */
5671   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5672   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5673     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5674       cpoints[q*2]   = cpoints[p];
5675       cpoints[q*2+1] = cpoints[p+1];
5676       ++q;
5677     }
5678   }
5679   numCPoints = q;
5680   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5681     PetscInt fdof;
5682 
5683     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5684     if (!dof) continue;
5685     for (f = 0; f < numFields; ++f) {
5686       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5687       coffsets[f+1] += fdof;
5688     }
5689     numCIndices += dof;
5690   }
5691   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5692   /* Row indices */
5693   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5694   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5695   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5696   for (r = 0, q = 0; r < numSubcells; ++r) {
5697     /* TODO Map from coarse to fine cells */
5698     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5699     /* Compress out points not in the section */
5700     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5701     for (p = 0; p < numFPoints*2; p += 2) {
5702       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5703         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5704         if (!dof) continue;
5705         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5706         if (s < q) continue;
5707         ftotpoints[q*2]   = fpoints[p];
5708         ftotpoints[q*2+1] = fpoints[p+1];
5709         ++q;
5710       }
5711     }
5712     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5713   }
5714   numFPoints = q;
5715   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5716     PetscInt fdof;
5717 
5718     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5719     if (!dof) continue;
5720     for (f = 0; f < numFields; ++f) {
5721       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5722       foffsets[f+1] += fdof;
5723     }
5724     numFIndices += dof;
5725   }
5726   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5727 
5728   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", foffsets[numFields], numFIndices);
5729   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", coffsets[numFields], numCIndices);
5730   ierr = DMGetWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5731   ierr = DMGetWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5732   if (numFields) {
5733     for (p = 0; p < numFPoints*2; p += 2) {
5734       PetscInt o = ftotpoints[p+1];
5735       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5736       indicesPointFields_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, o, findices);
5737     }
5738     for (p = 0; p < numCPoints*2; p += 2) {
5739       PetscInt o = cpoints[p+1];
5740       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5741       indicesPointFields_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, o, cindices);
5742     }
5743   } else {
5744     for (p = 0, off = 0; p < numFPoints*2; p += 2) {
5745       PetscInt o = ftotpoints[p+1];
5746       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5747       indicesPoint_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, findices);
5748     }
5749     for (p = 0, off = 0; p < numCPoints*2; p += 2) {
5750       PetscInt o = cpoints[p+1];
5751       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5752       indicesPoint_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, cindices);
5753     }
5754   }
5755   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
5756   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
5757   if (ierr) {
5758     PetscMPIInt    rank;
5759     PetscErrorCode ierr2;
5760 
5761     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5762     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5763     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
5764     ierr2 = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr2);
5765     ierr2 = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr2);
5766     CHKERRQ(ierr);
5767   }
5768   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5769   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5770   ierr = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5771   ierr = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5772   PetscFunctionReturn(0);
5773 }
5774 
5775 #undef __FUNCT__
5776 #define __FUNCT__ "DMPlexGetHybridBounds"
5777 /*@
5778   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
5779 
5780   Input Parameter:
5781 . dm - The DMPlex object
5782 
5783   Output Parameters:
5784 + cMax - The first hybrid cell
5785 . cMax - The first hybrid face
5786 . cMax - The first hybrid edge
5787 - cMax - The first hybrid vertex
5788 
5789   Level: developer
5790 
5791 .seealso DMPlexCreateHybridMesh()
5792 @*/
5793 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
5794 {
5795   DM_Plex       *mesh = (DM_Plex*) dm->data;
5796   PetscInt       dim;
5797   PetscErrorCode ierr;
5798 
5799   PetscFunctionBegin;
5800   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5801   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5802   if (cMax) *cMax = mesh->hybridPointMax[dim];
5803   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
5804   if (eMax) *eMax = mesh->hybridPointMax[1];
5805   if (vMax) *vMax = mesh->hybridPointMax[0];
5806   PetscFunctionReturn(0);
5807 }
5808 
5809 #undef __FUNCT__
5810 #define __FUNCT__ "DMPlexSetHybridBounds"
5811 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
5812 {
5813   DM_Plex       *mesh = (DM_Plex*) dm->data;
5814   PetscInt       dim;
5815   PetscErrorCode ierr;
5816 
5817   PetscFunctionBegin;
5818   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5819   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5820   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
5821   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
5822   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
5823   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
5824   PetscFunctionReturn(0);
5825 }
5826 
5827 #undef __FUNCT__
5828 #define __FUNCT__ "DMPlexGetVTKCellHeight"
5829 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
5830 {
5831   DM_Plex *mesh = (DM_Plex*) dm->data;
5832 
5833   PetscFunctionBegin;
5834   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5835   PetscValidPointer(cellHeight, 2);
5836   *cellHeight = mesh->vtkCellHeight;
5837   PetscFunctionReturn(0);
5838 }
5839 
5840 #undef __FUNCT__
5841 #define __FUNCT__ "DMPlexSetVTKCellHeight"
5842 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
5843 {
5844   DM_Plex *mesh = (DM_Plex*) dm->data;
5845 
5846   PetscFunctionBegin;
5847   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5848   mesh->vtkCellHeight = cellHeight;
5849   PetscFunctionReturn(0);
5850 }
5851 
5852 #undef __FUNCT__
5853 #define __FUNCT__ "DMPlexCreateNumbering_Private"
5854 /* We can easily have a form that takes an IS instead */
5855 PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscSF sf, IS *numbering)
5856 {
5857   PetscSection   section, globalSection;
5858   PetscInt      *numbers, p;
5859   PetscErrorCode ierr;
5860 
5861   PetscFunctionBegin;
5862   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5863   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
5864   for (p = pStart; p < pEnd; ++p) {
5865     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
5866   }
5867   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
5868   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
5869   ierr = PetscMalloc1((pEnd - pStart), &numbers);CHKERRQ(ierr);
5870   for (p = pStart; p < pEnd; ++p) {
5871     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
5872   }
5873   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
5874   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5875   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
5876   PetscFunctionReturn(0);
5877 }
5878 
5879 #undef __FUNCT__
5880 #define __FUNCT__ "DMPlexGetCellNumbering"
5881 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
5882 {
5883   DM_Plex       *mesh = (DM_Plex*) dm->data;
5884   PetscInt       cellHeight, cStart, cEnd, cMax;
5885   PetscErrorCode ierr;
5886 
5887   PetscFunctionBegin;
5888   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5889   if (!mesh->globalCellNumbers) {
5890     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
5891     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5892     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5893     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
5894     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
5895   }
5896   *globalCellNumbers = mesh->globalCellNumbers;
5897   PetscFunctionReturn(0);
5898 }
5899 
5900 #undef __FUNCT__
5901 #define __FUNCT__ "DMPlexGetVertexNumbering"
5902 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
5903 {
5904   DM_Plex       *mesh = (DM_Plex*) dm->data;
5905   PetscInt       vStart, vEnd, vMax;
5906   PetscErrorCode ierr;
5907 
5908   PetscFunctionBegin;
5909   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5910   if (!mesh->globalVertexNumbers) {
5911     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5912     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
5913     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
5914     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
5915   }
5916   *globalVertexNumbers = mesh->globalVertexNumbers;
5917   PetscFunctionReturn(0);
5918 }
5919 
5920 
5921 #undef __FUNCT__
5922 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
5923 /*@C
5924   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
5925   the local section and an SF describing the section point overlap.
5926 
5927   Input Parameters:
5928   + s - The PetscSection for the local field layout
5929   . sf - The SF describing parallel layout of the section points
5930   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
5931   . label - The label specifying the points
5932   - labelValue - The label stratum specifying the points
5933 
5934   Output Parameter:
5935   . gsection - The PetscSection for the global field layout
5936 
5937   Note: This gives negative sizes and offsets to points not owned by this process
5938 
5939   Level: developer
5940 
5941 .seealso: PetscSectionCreate()
5942 @*/
5943 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
5944 {
5945   PetscInt      *neg = NULL, *tmpOff = NULL;
5946   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
5947   PetscErrorCode ierr;
5948 
5949   PetscFunctionBegin;
5950   ierr = PetscSectionCreate(PetscObjectComm((PetscObject) s), gsection);CHKERRQ(ierr);
5951   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
5952   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
5953   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
5954   if (nroots >= 0) {
5955     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
5956     ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
5957     if (nroots > pEnd-pStart) {
5958       ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
5959     } else {
5960       tmpOff = &(*gsection)->atlasDof[-pStart];
5961     }
5962   }
5963   /* Mark ghost points with negative dof */
5964   for (p = pStart; p < pEnd; ++p) {
5965     PetscInt value;
5966 
5967     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
5968     if (value != labelValue) continue;
5969     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
5970     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
5971     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
5972     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
5973     if (neg) neg[p] = -(dof+1);
5974   }
5975   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
5976   if (nroots >= 0) {
5977     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5978     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5979     if (nroots > pEnd-pStart) {
5980       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
5981     }
5982   }
5983   /* Calculate new sizes, get proccess offset, and calculate point offsets */
5984   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5985     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
5986     (*gsection)->atlasOff[p] = off;
5987     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
5988   }
5989   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) s));CHKERRQ(ierr);
5990   globalOff -= off;
5991   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5992     (*gsection)->atlasOff[p] += globalOff;
5993     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
5994   }
5995   /* Put in negative offsets for ghost points */
5996   if (nroots >= 0) {
5997     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5998     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5999     if (nroots > pEnd-pStart) {
6000       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
6001     }
6002   }
6003   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
6004   ierr = PetscFree(neg);CHKERRQ(ierr);
6005   PetscFunctionReturn(0);
6006 }
6007 
6008 #undef __FUNCT__
6009 #define __FUNCT__ "DMPlexCheckSymmetry"
6010 /*@
6011   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6012 
6013   Input Parameters:
6014   + dm - The DMPlex object
6015 
6016   Note: This is a useful diagnostic when creating meshes programmatically.
6017 
6018   Level: developer
6019 
6020 .seealso: DMCreate(), DMCheckSkeleton(), DMCheckFaces()
6021 @*/
6022 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6023 {
6024   PetscSection    coneSection, supportSection;
6025   const PetscInt *cone, *support;
6026   PetscInt        coneSize, c, supportSize, s;
6027   PetscInt        pStart, pEnd, p, csize, ssize;
6028   PetscErrorCode  ierr;
6029 
6030   PetscFunctionBegin;
6031   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6032   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6033   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6034   /* Check that point p is found in the support of its cone points, and vice versa */
6035   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6036   for (p = pStart; p < pEnd; ++p) {
6037     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6038     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6039     for (c = 0; c < coneSize; ++c) {
6040       PetscBool dup = PETSC_FALSE;
6041       PetscInt  d;
6042       for (d = c-1; d >= 0; --d) {
6043         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6044       }
6045       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6046       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6047       for (s = 0; s < supportSize; ++s) {
6048         if (support[s] == p) break;
6049       }
6050       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6051         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", p);
6052         for (s = 0; s < coneSize; ++s) {
6053           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[s]);
6054         }
6055         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6056         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", cone[c]);
6057         for (s = 0; s < supportSize; ++s) {
6058           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[s]);
6059         }
6060         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6061         if (dup) {
6062           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not repeatedly found in support of repeated cone point %d", p, cone[c]);
6063         } else {
6064           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in support of cone point %d", p, cone[c]);
6065         }
6066       }
6067     }
6068     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6069     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6070     for (s = 0; s < supportSize; ++s) {
6071       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6072       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6073       for (c = 0; c < coneSize; ++c) {
6074         if (cone[c] == p) break;
6075       }
6076       if (c >= coneSize) {
6077         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", p);
6078         for (c = 0; c < supportSize; ++c) {
6079           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[c]);
6080         }
6081         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6082         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", support[s]);
6083         for (c = 0; c < coneSize; ++c) {
6084           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[c]);
6085         }
6086         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6087         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in cone of support point %d", p, support[s]);
6088       }
6089     }
6090   }
6091   ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6092   ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6093   if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %d != Total support size %d", csize, ssize);
6094   PetscFunctionReturn(0);
6095 }
6096 
6097 #undef __FUNCT__
6098 #define __FUNCT__ "DMPlexCheckSkeleton"
6099 /*@
6100   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6101 
6102   Input Parameters:
6103 + dm - The DMPlex object
6104 . isSimplex - Are the cells simplices or tensor products
6105 - cellHeight - Normally 0
6106 
6107   Note: This is a useful diagnostic when creating meshes programmatically.
6108 
6109   Level: developer
6110 
6111 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckFaces()
6112 @*/
6113 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6114 {
6115   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6116   PetscErrorCode ierr;
6117 
6118   PetscFunctionBegin;
6119   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6120   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6121   switch (dim) {
6122   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6123   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6124   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6125   default:
6126     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %d", dim);
6127   }
6128   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6129   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6130   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6131   cMax = cMax >= 0 ? cMax : cEnd;
6132   for (c = cStart; c < cMax; ++c) {
6133     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6134 
6135     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6136     for (cl = 0; cl < closureSize*2; cl += 2) {
6137       const PetscInt p = closure[cl];
6138       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6139     }
6140     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6141     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has  %d vertices != %d", c, coneSize, numCorners);
6142   }
6143   for (c = cMax; c < cEnd; ++c) {
6144     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6145 
6146     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6147     for (cl = 0; cl < closureSize*2; cl += 2) {
6148       const PetscInt p = closure[cl];
6149       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6150     }
6151     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6152     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %d has  %d vertices > %d", c, coneSize, numHybridCorners);
6153   }
6154   PetscFunctionReturn(0);
6155 }
6156 
6157 #undef __FUNCT__
6158 #define __FUNCT__ "DMPlexCheckFaces"
6159 /*@
6160   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6161 
6162   Input Parameters:
6163 + dm - The DMPlex object
6164 . isSimplex - Are the cells simplices or tensor products
6165 - cellHeight - Normally 0
6166 
6167   Note: This is a useful diagnostic when creating meshes programmatically.
6168 
6169   Level: developer
6170 
6171 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton()
6172 @*/
6173 PetscErrorCode DMPlexCheckFaces(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6174 {
6175   PetscInt       pMax[4];
6176   PetscInt       dim, vStart, vEnd, cStart, cEnd, c, h;
6177   PetscErrorCode ierr;
6178 
6179   PetscFunctionBegin;
6180   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6181   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6182   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6183   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6184   for (h = cellHeight; h < dim; ++h) {
6185     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6186     for (c = cStart; c < cEnd; ++c) {
6187       const PetscInt *cone, *ornt, *faces;
6188       PetscInt        numFaces, faceSize, coneSize,f;
6189       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6190 
6191       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6192       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6193       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6194       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6195       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6196       for (cl = 0; cl < closureSize*2; cl += 2) {
6197         const PetscInt p = closure[cl];
6198         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6199       }
6200       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6201       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has %d faces but should have %d", c, coneSize, numFaces);
6202       for (f = 0; f < numFaces; ++f) {
6203         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6204 
6205         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6206         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6207           const PetscInt p = fclosure[cl];
6208           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6209         }
6210         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d has %d vertices but should have %d", cone[f], f, c, fnumCorners, faceSize);
6211         for (v = 0; v < fnumCorners; ++v) {
6212           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d vertex %d, %d != %d", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6213         }
6214         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6215       }
6216       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6217       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6218     }
6219   }
6220   PetscFunctionReturn(0);
6221 }
6222 
6223 #undef __FUNCT__
6224 #define __FUNCT__ "DMCreateInterpolation_Plex"
6225 /* Pointwise interpolation
6226      Just code FEM for now
6227      u^f = I u^c
6228      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
6229      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
6230      I_{ij} = psi^f_i phi^c_j
6231 */
6232 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
6233 {
6234   PetscSection   gsc, gsf;
6235   PetscInt       m, n;
6236   void          *ctx;
6237   PetscErrorCode ierr;
6238 
6239   PetscFunctionBegin;
6240   /*
6241   Loop over coarse cells
6242     Loop over coarse basis functions
6243       Loop over fine cells in coarse cell
6244         Loop over fine dual basis functions
6245           Evaluate coarse basis on fine dual basis quad points
6246           Sum
6247           Update local element matrix
6248     Accumulate to interpolation matrix
6249 
6250    Can extend PetscFEIntegrateJacobian_Basic() to do a specialized cell loop
6251   */
6252   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6253   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6254   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6255   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6256   /* We need to preallocate properly */
6257   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
6258   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6259   ierr = MatSetType(*interpolation, dmCoarse->mattype);CHKERRQ(ierr);
6260   ierr = MatSetUp(*interpolation);CHKERRQ(ierr);
6261   ierr = MatSetFromOptions(*interpolation);CHKERRQ(ierr);
6262   ierr = MatSetOption(*interpolation, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);CHKERRQ(ierr);
6263   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6264   ierr = DMPlexComputeInterpolatorFEM(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);
6265   /* Use naive scaling */
6266   ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
6267   PetscFunctionReturn(0);
6268 }
6269 
6270 #undef __FUNCT__
6271 #define __FUNCT__ "DMCreateInjection_Plex"
6272 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, VecScatter *ctx)
6273 {
6274   Vec             cv,  fv;
6275   IS              cis, fis, fpointIS;
6276   PetscSection    sc, gsc, gsf;
6277   const PetscInt *fpoints;
6278   PetscInt       *cindices, *findices;
6279   PetscInt        cpStart, cpEnd, m, off, cp;
6280   PetscErrorCode  ierr;
6281 
6282   PetscFunctionBegin;
6283   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6284   ierr = DMGetGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6285   ierr = DMGetDefaultSection(dmCoarse, &sc);CHKERRQ(ierr);
6286   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6287   ierr = DMGetGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6288   ierr = DMPlexCreateCoarsePointIS(dmCoarse, &fpointIS);CHKERRQ(ierr);
6289   ierr = PetscSectionGetConstrainedStorageSize(gsc, &m);CHKERRQ(ierr);
6290   ierr = PetscMalloc2(m,&cindices,m,&findices);CHKERRQ(ierr);
6291   ierr = PetscSectionGetChart(gsc, &cpStart, &cpEnd);CHKERRQ(ierr);
6292   ierr = ISGetIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6293   for (cp = cpStart, off = 0; cp < cpEnd; ++cp) {
6294     const PetscInt *cdofsC = NULL;
6295     PetscInt        fp     = fpoints[cp-cpStart], dofC, cdofC, dofF, offC, offF, d, e;
6296 
6297     ierr = PetscSectionGetDof(gsc, cp, &dofC);CHKERRQ(ierr);
6298     if (dofC <= 0) continue;
6299     ierr = PetscSectionGetConstraintDof(sc, cp, &cdofC);CHKERRQ(ierr);
6300     ierr = PetscSectionGetDof(gsf, fp, &dofF);CHKERRQ(ierr);
6301     ierr = PetscSectionGetOffset(gsc, cp, &offC);CHKERRQ(ierr);
6302     ierr = PetscSectionGetOffset(gsf, fp, &offF);CHKERRQ(ierr);
6303     if (cdofC) {ierr = PetscSectionGetConstraintIndices(sc, cp, &cdofsC);CHKERRQ(ierr);}
6304     if (dofC != dofF) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has %d coarse dofs != %d fine dofs", cp, fp, dofC, dofF);
6305     if (offC < 0 || offF < 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Coarse point %d has invalid offset %d (%d)", cp, offC, offF);
6306     for (d = 0, e = 0; d < dofC; ++d) {
6307       if (cdofsC && cdofsC[e] == d) {++e; continue;}
6308       cindices[off+d-e] = offC+d; findices[off+d-e] = offF+d;
6309     }
6310     if (e != cdofC) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has invalid number of constraints %d != %d", cp, fp, e, cdofC);
6311     off += dofC-cdofC;
6312   }
6313   ierr = ISRestoreIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6314   if (off != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coarse dofs %d != %d", off, m);
6315   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);CHKERRQ(ierr);
6316   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);CHKERRQ(ierr);
6317   ierr = VecScatterCreate(cv, cis, fv, fis, ctx);CHKERRQ(ierr);
6318   ierr = ISDestroy(&cis);CHKERRQ(ierr);
6319   ierr = ISDestroy(&fis);CHKERRQ(ierr);
6320   ierr = DMRestoreGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6321   ierr = DMRestoreGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6322   ierr = ISDestroy(&fpointIS);CHKERRQ(ierr);
6323   PetscFunctionReturn(0);
6324 }
6325 
6326 #undef __FUNCT__
6327 #define __FUNCT__ "DMCreateDefaultSection_Plex"
6328 /* Pointwise interpolation
6329      Just code FEM for now
6330      u^f = I u^c
6331      sum_k u^f_k phi^f_k = I sum_l u^c_l phi^c_l
6332      u^f_i = sum_l int psi^f_i I phi^c_l u^c_l
6333      I_{ij} = int psi^f_i phi^c_j
6334 */
6335 PetscErrorCode DMCreateDefaultSection_Plex(DM dm)
6336 {
6337   PetscSection   section;
6338   IS            *bcPoints;
6339   PetscInt      *bcFields, *numComp, *numDof;
6340   PetscInt       depth, dim, numBd, numBC = 0, numFields, bd, bc, f;
6341   PetscErrorCode ierr;
6342 
6343   PetscFunctionBegin;
6344   /* Handle boundary conditions */
6345   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6346   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6347   ierr = DMPlexGetNumBoundary(dm, &numBd);CHKERRQ(ierr);
6348   for (bd = 0; bd < numBd; ++bd) {
6349     PetscBool isEssential;
6350     ierr = DMPlexGetBoundary(dm, bd, &isEssential, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6351     if (isEssential) ++numBC;
6352   }
6353   ierr = PetscMalloc2(numBC,&bcFields,numBC,&bcPoints);CHKERRQ(ierr);
6354   for (bd = 0, bc = 0; bd < numBd; ++bd) {
6355     const char     *bdLabel;
6356     DMLabel         label;
6357     const PetscInt *values;
6358     PetscInt        field, numValues;
6359     PetscBool       isEssential, has;
6360 
6361     ierr = DMPlexGetBoundary(dm, bd, &isEssential, &bdLabel, &field, NULL, &numValues, &values, NULL);CHKERRQ(ierr);
6362     if (numValues != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Bug me and I will fix this");
6363     ierr = DMPlexHasLabel(dm, bdLabel, &has);CHKERRQ(ierr);
6364     if (!has) {
6365       ierr = DMPlexCreateLabel(dm, bdLabel);CHKERRQ(ierr);
6366       ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6367       ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
6368     }
6369     ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6370     /* Only want to do this for FEM */
6371     ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
6372     ierr = DMPlexLabelAddCells(dm, label);CHKERRQ(ierr);
6373     /* Filter out cells, if you actually want to constraint cells you need to do things by hand right now */
6374     if (isEssential) {
6375       IS              tmp;
6376       PetscInt       *newidx;
6377       const PetscInt *idx;
6378       PetscInt        cStart, cEnd, n, p, newn = 0;
6379 
6380       bcFields[bc] = field;
6381       ierr = DMPlexGetStratumIS(dm, bdLabel, values[0], &tmp);CHKERRQ(ierr);
6382       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6383       ierr = ISGetLocalSize(tmp, &n);CHKERRQ(ierr);
6384       ierr = ISGetIndices(tmp, &idx);CHKERRQ(ierr);
6385       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) ++newn;
6386       ierr = PetscMalloc1(newn,&newidx);CHKERRQ(ierr);
6387       newn = 0;
6388       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) newidx[newn++] = idx[p];
6389       ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), newn, newidx, PETSC_OWN_POINTER, &bcPoints[bc++]);CHKERRQ(ierr);
6390       ierr = ISRestoreIndices(tmp, &idx);CHKERRQ(ierr);
6391       ierr = ISDestroy(&tmp);CHKERRQ(ierr);
6392     }
6393   }
6394   /* Handle discretization */
6395   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
6396   ierr = PetscMalloc2(numFields,&numComp,numFields*(dim+1),&numDof);CHKERRQ(ierr);
6397   for (f = 0; f < numFields; ++f) {
6398     PetscFE         fe;
6399     const PetscInt *numFieldDof;
6400     PetscInt        d;
6401 
6402     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6403     ierr = PetscFEGetNumComponents(fe, &numComp[f]);CHKERRQ(ierr);
6404     ierr = PetscFEGetNumDof(fe, &numFieldDof);CHKERRQ(ierr);
6405     for (d = 0; d < dim+1; ++d) numDof[f*(dim+1)+d] = numFieldDof[d];
6406   }
6407   for (f = 0; f < numFields; ++f) {
6408     PetscInt d;
6409     for (d = 1; d < dim; ++d) {
6410       if ((numDof[f*(dim+1)+d] > 0) && (depth < dim)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mesh must be interpolated when unknowns are specified on edges or faces.");
6411     }
6412   }
6413   ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcPoints, NULL, &section);CHKERRQ(ierr);
6414   for (f = 0; f < numFields; ++f) {
6415     PetscFE     fe;
6416     const char *name;
6417 
6418     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6419     ierr = PetscObjectGetName((PetscObject) fe, &name);CHKERRQ(ierr);
6420     ierr = PetscSectionSetFieldName(section, f, name);CHKERRQ(ierr);
6421   }
6422   ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
6423   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6424   for (bc = 0; bc < numBC; ++bc) {ierr = ISDestroy(&bcPoints[bc]);CHKERRQ(ierr);}
6425   ierr = PetscFree2(bcFields,bcPoints);CHKERRQ(ierr);
6426   ierr = PetscFree2(numComp,numDof);CHKERRQ(ierr);
6427   PetscFunctionReturn(0);
6428 }
6429 
6430 #undef __FUNCT__
6431 #define __FUNCT__ "DMPlexGetCoarseDM"
6432 /*@
6433   DMPlexGetCoarseDM - Get the coarse mesh from which this was obtained by refinement
6434 
6435   Input Parameter:
6436 . dm - The DMPlex object
6437 
6438   Output Parameter:
6439 . cdm - The coarse DM
6440 
6441   Level: intermediate
6442 
6443 .seealso: DMPlexSetCoarseDM()
6444 @*/
6445 PetscErrorCode DMPlexGetCoarseDM(DM dm, DM *cdm)
6446 {
6447   PetscFunctionBegin;
6448   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6449   PetscValidPointer(cdm, 2);
6450   *cdm = ((DM_Plex *) dm->data)->coarseMesh;
6451   PetscFunctionReturn(0);
6452 }
6453 
6454 #undef __FUNCT__
6455 #define __FUNCT__ "DMPlexSetCoarseDM"
6456 /*@
6457   DMPlexSetCoarseDM - Set the coarse mesh from which this was obtained by refinement
6458 
6459   Input Parameters:
6460 + dm - The DMPlex object
6461 - cdm - The coarse DM
6462 
6463   Level: intermediate
6464 
6465 .seealso: DMPlexGetCoarseDM()
6466 @*/
6467 PetscErrorCode DMPlexSetCoarseDM(DM dm, DM cdm)
6468 {
6469   DM_Plex       *mesh;
6470   PetscErrorCode ierr;
6471 
6472   PetscFunctionBegin;
6473   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6474   if (cdm) PetscValidHeaderSpecific(cdm, DM_CLASSID, 2);
6475   mesh = (DM_Plex *) dm->data;
6476   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
6477   mesh->coarseMesh = cdm;
6478   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
6479   PetscFunctionReturn(0);
6480 }
6481