xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 166d9d0cd0beca779b4a31eb5deb23ec76fb66cf)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petscsf.h>
3 
4 #undef __FUNCT__
5 #define __FUNCT__ "DMPlexMarkBoundaryFaces"
6 /*@
7   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
8 
9   Not Collective
10 
11   Input Parameter:
12 . dm - The original DM
13 
14   Output Parameter:
15 . label - The DMLabel marking boundary faces with value 1
16 
17   Level: developer
18 
19 .seealso: DMLabelCreate(), DMPlexCreateLabel()
20 @*/
21 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, DMLabel label)
22 {
23   PetscInt       fStart, fEnd, f;
24   PetscErrorCode ierr;
25 
26   PetscFunctionBegin;
27   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
28   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
29   for (f = fStart; f < fEnd; ++f) {
30     PetscInt supportSize;
31 
32     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
33     if (supportSize == 1) {
34       ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);
35     }
36   }
37   PetscFunctionReturn(0);
38 }
39 
40 #undef __FUNCT__
41 #define __FUNCT__ "DMPlexLabelComplete"
42 /*@
43   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
44 
45   Input Parameters:
46 + dm - The DM
47 - label - A DMLabel marking the surface points
48 
49   Output Parameter:
50 . label - A DMLabel marking all surface points in the transitive closure
51 
52   Level: developer
53 
54 .seealso: DMPlexLabelCohesiveComplete()
55 @*/
56 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
57 {
58   IS              valueIS;
59   const PetscInt *values;
60   PetscInt        numValues, v;
61   PetscErrorCode  ierr;
62 
63   PetscFunctionBegin;
64   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
65   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
66   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
67   for (v = 0; v < numValues; ++v) {
68     IS              pointIS;
69     const PetscInt *points;
70     PetscInt        numPoints, p;
71 
72     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
73     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
74     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
75     for (p = 0; p < numPoints; ++p) {
76       PetscInt *closure = NULL;
77       PetscInt  closureSize, c;
78 
79       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
80       for (c = 0; c < closureSize*2; c += 2) {
81         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
82       }
83       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
84     }
85     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
86     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
87   }
88   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
89   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
90   PetscFunctionReturn(0);
91 }
92 
93 #undef __FUNCT__
94 #define __FUNCT__ "DMPlexShiftPoint_Internal"
95 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthEnd[], PetscInt depthShift[])
96 {
97   if (depth < 0) return p;
98   /* Cells    */ if (p < depthEnd[depth])   return p;
99   /* Vertices */ if (p < depthEnd[0])       return p + depthShift[depth];
100   /* Faces    */ if (p < depthEnd[depth-1]) return p + depthShift[depth] + depthShift[0];
101   /* Edges    */                            return p + depthShift[depth] + depthShift[0] + depthShift[depth-1];
102 }
103 
104 #undef __FUNCT__
105 #define __FUNCT__ "DMPlexShiftSizes_Internal"
106 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
107 {
108   PetscInt      *depthEnd;
109   PetscInt       depth = 0, d, pStart, pEnd, p;
110   PetscErrorCode ierr;
111 
112   PetscFunctionBegin;
113   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
114   if (depth < 0) PetscFunctionReturn(0);
115   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
116   /* Step 1: Expand chart */
117   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
118   for (d = 0; d <= depth; ++d) {
119     pEnd += depthShift[d];
120     ierr  = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
121   }
122   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
123   /* Step 2: Set cone and support sizes */
124   for (d = 0; d <= depth; ++d) {
125     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
126     for (p = pStart; p < pEnd; ++p) {
127       PetscInt newp = DMPlexShiftPoint_Internal(p, depth, depthEnd, depthShift);
128       PetscInt size;
129 
130       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
131       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
132       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
133       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
134     }
135   }
136   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
137   PetscFunctionReturn(0);
138 }
139 
140 #undef __FUNCT__
141 #define __FUNCT__ "DMPlexShiftPoints_Internal"
142 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
143 {
144   PetscInt      *depthEnd, *newpoints;
145   PetscInt       depth = 0, d, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
146   PetscErrorCode ierr;
147 
148   PetscFunctionBegin;
149   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
150   if (depth < 0) PetscFunctionReturn(0);
151   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
152   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
153   ierr = PetscMalloc2(depth+1,PetscInt,&depthEnd,PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),PetscInt,&newpoints);CHKERRQ(ierr);
154   for (d = 0; d <= depth; ++d) {
155     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
156   }
157   /* Step 5: Set cones and supports */
158   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
159   for (p = pStart; p < pEnd; ++p) {
160     const PetscInt *points = NULL, *orientations = NULL;
161     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthEnd, depthShift);
162 
163     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
164     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
165     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
166     for (i = 0; i < size; ++i) {
167       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthEnd, depthShift);
168     }
169     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
170     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
171     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
172     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
173     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
174     for (i = 0; i < size; ++i) {
175       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthEnd, depthShift);
176     }
177     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
178     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
179   }
180   ierr = PetscFree2(depthEnd,newpoints);CHKERRQ(ierr);
181   PetscFunctionReturn(0);
182 }
183 
184 #undef __FUNCT__
185 #define __FUNCT__ "DMPlexShiftCoordinates_Internal"
186 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
187 {
188   PetscSection   coordSection, newCoordSection;
189   Vec            coordinates, newCoordinates;
190   PetscScalar   *coords, *newCoords;
191   PetscInt      *depthEnd, coordSize;
192   PetscInt       dim, depth = 0, d, vStart, vEnd, vStartNew, vEndNew, v;
193   PetscErrorCode ierr;
194 
195   PetscFunctionBegin;
196   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
197   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
198   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
199   for (d = 0; d <= depth; ++d) {
200     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
201   }
202   /* Step 8: Convert coordinates */
203   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
204   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
205   ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
206   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
207   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
208   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
209   ierr = PetscSectionSetChart(newCoordSection, vStartNew, vEndNew);CHKERRQ(ierr);
210   for (v = vStartNew; v < vEndNew; ++v) {
211     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
212     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
213   }
214   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
215   ierr = DMPlexSetCoordinateSection(dmNew, newCoordSection);CHKERRQ(ierr);
216   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
217   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &newCoordinates);CHKERRQ(ierr);
218   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
219   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
220   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
221   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
222   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
223   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
224   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
225   for (v = vStart; v < vEnd; ++v) {
226     PetscInt dof, off, noff, d;
227 
228     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
229     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
230     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthEnd, depthShift), &noff);CHKERRQ(ierr);
231     for (d = 0; d < dof; ++d) {
232       newCoords[noff+d] = coords[off+d];
233     }
234   }
235   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
236   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
237   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
238   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
239   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
240   PetscFunctionReturn(0);
241 }
242 
243 #undef __FUNCT__
244 #define __FUNCT__ "DMPlexShiftSF_Internal"
245 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
246 {
247   PetscInt          *depthEnd;
248   PetscInt           depth = 0, d;
249   PetscSF            sfPoint, sfPointNew;
250   const PetscSFNode *remotePoints;
251   PetscSFNode       *gremotePoints;
252   const PetscInt    *localPoints;
253   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
254   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
255   PetscErrorCode     ierr;
256 
257   PetscFunctionBegin;
258   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
259   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
260   for (d = 0; d <= depth; ++d) {
261     totShift += depthShift[d];
262     ierr      = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
263   }
264   /* Step 9: Convert pointSF */
265   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
266   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
267   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
268   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
269   if (numRoots >= 0) {
270     ierr = PetscMalloc2(numRoots,PetscInt,&newLocation,pEnd-pStart,PetscInt,&newRemoteLocation);CHKERRQ(ierr);
271     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthEnd, depthShift);
272     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
273     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
274     ierr = PetscMalloc(numLeaves * sizeof(PetscInt),    &glocalPoints);CHKERRQ(ierr);
275     ierr = PetscMalloc(numLeaves * sizeof(PetscSFNode), &gremotePoints);CHKERRQ(ierr);
276     for (l = 0; l < numLeaves; ++l) {
277       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthEnd, depthShift);
278       gremotePoints[l].rank  = remotePoints[l].rank;
279       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
280     }
281     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
282     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
283   }
284   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
285   PetscFunctionReturn(0);
286 }
287 
288 #undef __FUNCT__
289 #define __FUNCT__ "DMPlexShiftLabels_Internal"
290 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
291 {
292   PetscSF            sfPoint;
293   DMLabel            vtkLabel, ghostLabel;
294   PetscInt          *depthEnd;
295   const PetscSFNode *leafRemote;
296   const PetscInt    *leafLocal;
297   PetscInt           depth = 0, d, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
298   PetscMPIInt        rank;
299   PetscErrorCode     ierr;
300 
301   PetscFunctionBegin;
302   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
303   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
304   for (d = 0; d <= depth; ++d) {
305     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
306   }
307   /* Step 10: Convert labels */
308   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
309   for (l = 0; l < numLabels; ++l) {
310     DMLabel         label, newlabel;
311     const char     *lname;
312     PetscBool       isDepth;
313     IS              valueIS;
314     const PetscInt *values;
315     PetscInt        numValues, val;
316 
317     ierr = DMPlexGetLabelName(dm, l, &lname);CHKERRQ(ierr);
318     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
319     if (isDepth) continue;
320     ierr = DMPlexCreateLabel(dmNew, lname);CHKERRQ(ierr);
321     ierr = DMPlexGetLabel(dm, lname, &label);CHKERRQ(ierr);
322     ierr = DMPlexGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
323     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
324     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
325     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
326     for (val = 0; val < numValues; ++val) {
327       IS              pointIS;
328       const PetscInt *points;
329       PetscInt        numPoints, p;
330 
331       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
332       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
333       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
334       for (p = 0; p < numPoints; ++p) {
335         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthEnd, depthShift);
336 
337         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
338       }
339       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
340       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
341     }
342     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
343     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
344   }
345   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
346   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
347   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
348   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
349   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
350   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
351   ierr = DMPlexCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
352   ierr = DMPlexCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
353   ierr = DMPlexGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
354   ierr = DMPlexGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
355   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
356     for (; c < leafLocal[l] && c < cEnd; ++c) {
357       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
358     }
359     if (leafLocal[l] >= cEnd) break;
360     if (leafRemote[l].rank == rank) {
361       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
362     } else {
363       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
364     }
365   }
366   for (; c < cEnd; ++c) {
367     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
368   }
369   if (0) {
370     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
371     ierr = DMLabelView(vtkLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
372     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
373   }
374   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
375   for (f = fStart; f < fEnd; ++f) {
376     PetscInt numCells;
377 
378     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
379     if (numCells < 2) {
380       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
381     } else {
382       const PetscInt *cells = NULL;
383       PetscInt        vA, vB;
384 
385       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
386       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
387       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
388       if (!vA && !vB) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
389     }
390   }
391   if (0) {
392     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
393     ierr = DMLabelView(ghostLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
394     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
395   }
396   PetscFunctionReturn(0);
397 }
398 
399 #undef __FUNCT__
400 #define __FUNCT__ "DMPlexConstructGhostCells_Internal"
401 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
402 {
403   IS              valueIS;
404   const PetscInt *values;
405   PetscInt       *depthShift;
406   PetscInt        depth = 0, numFS, fs, ghostCell, cEnd, c;
407   PetscErrorCode  ierr;
408 
409   PetscFunctionBegin;
410   /* Count ghost cells */
411   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
412   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
413   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
414 
415   *numGhostCells = 0;
416   for (fs = 0; fs < numFS; ++fs) {
417     PetscInt numBdFaces;
418 
419     ierr = DMLabelGetStratumSize(label, values[fs], &numBdFaces);CHKERRQ(ierr);
420 
421     *numGhostCells += numBdFaces;
422   }
423   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
424   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthShift);CHKERRQ(ierr);
425   ierr = PetscMemzero(depthShift, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
426   if (depth >= 0) depthShift[depth] = *numGhostCells;
427   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
428   /* Step 3: Set cone/support sizes for new points */
429   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
430   for (c = cEnd; c < cEnd + *numGhostCells; ++c) {
431     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
432   }
433   for (fs = 0; fs < numFS; ++fs) {
434     IS              faceIS;
435     const PetscInt *faces;
436     PetscInt        numFaces, f;
437 
438     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
439     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
440     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
441     for (f = 0; f < numFaces; ++f) {
442       PetscInt size;
443 
444       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
445       if (size != 1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
446       ierr = DMPlexSetSupportSize(gdm, faces[f] + *numGhostCells, 2);CHKERRQ(ierr);
447     }
448     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
449     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
450   }
451   /* Step 4: Setup ghosted DM */
452   ierr = DMSetUp(gdm);CHKERRQ(ierr);
453   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
454   /* Step 6: Set cones and supports for new points */
455   ghostCell = cEnd;
456   for (fs = 0; fs < numFS; ++fs) {
457     IS              faceIS;
458     const PetscInt *faces;
459     PetscInt        numFaces, f;
460 
461     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
462     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
463     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
464     for (f = 0; f < numFaces; ++f, ++ghostCell) {
465       PetscInt newFace = faces[f] + *numGhostCells;
466 
467       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
468       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
469     }
470     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
471     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
472   }
473   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
474   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
475   /* Step 7: Stratify */
476   ierr = DMPlexStratify(gdm);CHKERRQ(ierr);
477   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
478   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
479   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
480   ierr = PetscFree(depthShift);CHKERRQ(ierr);
481   PetscFunctionReturn(0);
482 }
483 
484 #undef __FUNCT__
485 #define __FUNCT__ "DMPlexConstructGhostCells"
486 /*@C
487   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
488 
489   Collective on dm
490 
491   Input Parameters:
492 + dm - The original DM
493 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
494 
495   Output Parameters:
496 + numGhostCells - The number of ghost cells added to the DM
497 - dmGhosted - The new DM
498 
499   Note: If no label exists of that name, one will be created marking all boundary faces
500 
501   Level: developer
502 
503 .seealso: DMCreate()
504 */
505 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
506 {
507   DM             gdm;
508   DMLabel        label;
509   const char    *name = labelName ? labelName : "Face Sets";
510   PetscInt       dim;
511   PetscErrorCode ierr;
512 
513   PetscFunctionBegin;
514   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
515   PetscValidPointer(numGhostCells, 3);
516   PetscValidPointer(dmGhosted, 4);
517   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
518   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
519   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
520   ierr = DMPlexSetDimension(gdm, dim);CHKERRQ(ierr);
521   ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
522   if (!label) {
523     /* Get label for boundary faces */
524     ierr = DMPlexCreateLabel(dm, name);CHKERRQ(ierr);
525     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
526     ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
527   }
528   ierr = DMPlexConstructGhostCells_Internal(dm, label, numGhostCells, gdm);CHKERRQ(ierr);
529   ierr = DMSetFromOptions(gdm);CHKERRQ(ierr);
530   *dmGhosted = gdm;
531   PetscFunctionReturn(0);
532 }
533 
534 #undef __FUNCT__
535 #define __FUNCT__ "DMPlexConstructCohesiveCells_Internal"
536 /*
537   We are adding two kinds of points here:
538     Replicated: Copies of points which exist in the mesh, such as vertices identified across a fault
539     Hybrid:     Entirely new points, such as cohesive cells
540 */
541 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DM sdm)
542 {
543   MPI_Comm         comm;
544   IS               valueIS;
545   PetscInt         numSP = 0;       /* The number of depths for which we have replicated points */
546   const PetscInt  *values;          /* List of depths for which we have replicated points */
547   IS              *pointIS;
548   PetscInt        *numSplitPoints;  /* The number of replicated points at each depth */
549   PetscInt        *numHybridPoints; /* The number of hybrid points at each depth */
550   const PetscInt **splitPoints;     /* Replicated points for each depth */
551   PetscSection     coordSection;
552   Vec              coordinates;
553   PetscScalar     *coords;
554   PetscInt         depths[4];       /* Depths in the order that plex points are numbered */
555   PetscInt        *depthShift;      /* Number of replicated+hybrid points at each depth */
556   PetscInt        *depthOffset;     /* Prefix sums of depthShift */
557   PetscInt        *pMaxNew;         /* The first replicated point at each depth in the new mesh, hybrids come after this */
558   PetscInt        *coneNew, *coneONew, *supportNew;
559   PetscInt         shift = 100, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
560   PetscErrorCode   ierr;
561 
562   PetscFunctionBegin;
563   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
564   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
565   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
566   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
567   depths[0] = depth;
568   depths[1] = 0;
569   depths[2] = depth-1;
570   depths[3] = 1;
571   /* Count split points and add cohesive cells */
572   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
573   ierr = PetscMalloc3(depth+1,PetscInt,&depthShift,depth+1,PetscInt,&depthOffset,depth+1,PetscInt,&pMaxNew);CHKERRQ(ierr);
574   ierr = PetscMalloc4(depth+1,IS,&pointIS,depth+1,PetscInt,&numSplitPoints,depth+1,PetscInt,&numHybridPoints,depth+1,const PetscInt*,&splitPoints);CHKERRQ(ierr);
575   ierr = PetscMemzero(depthShift,  (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
576   ierr = PetscMemzero(depthOffset, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
577   for (d = 0; d <= depth; ++d) {
578     ierr               = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
579     numSplitPoints[d]  = 0;
580     numHybridPoints[d] = 0;
581     splitPoints[d]     = NULL;
582     pointIS[d]         = NULL;
583   }
584   if (label) {
585     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
586     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
587     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
588   }
589   for (sp = 0; sp < numSP; ++sp) {
590     const PetscInt dep = values[sp];
591 
592     if ((dep < 0) || (dep > depth)) continue;
593     ierr = DMLabelGetStratumIS(label, dep, &pointIS[dep]);CHKERRQ(ierr);
594     if (pointIS[dep]) {
595       ierr = ISGetLocalSize(pointIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
596       ierr = ISGetIndices(pointIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
597     }
598   }
599   /* Calculate number of hybrid points */
600   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
601   for (d = 0; d <= depth; ++d) depthShift[d]          = numSplitPoints[d] + numHybridPoints[d];
602   for (d = 1; d <= depth; ++d) depthOffset[depths[d]] = depthOffset[depths[d-1]] + depthShift[depths[d-1]];
603   for (d = 0; d <= depth; ++d) pMaxNew[d]            += depthOffset[d];
604   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
605   /* Step 3: Set cone/support sizes for new points */
606   for (dep = 0; dep <= depth; ++dep) {
607     for (p = 0; p < numSplitPoints[dep]; ++p) {
608       const PetscInt  oldp   = splitPoints[dep][p];
609       const PetscInt  newp   = oldp + depthOffset[dep];
610       const PetscInt  splitp = p    + pMaxNew[dep];
611       const PetscInt *support;
612       PetscInt        coneSize, supportSize, qf, qn, qp, e;
613 
614       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
615       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
616       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
617       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
618       if (dep == depth-1) {
619         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
620 
621         /* Add cohesive cells, they are prisms */
622         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
623       } else if (dep == 0) {
624         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
625 
626         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
627         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
628           PetscInt val;
629 
630           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
631           if (val == 1) ++qf;
632           if ((val == 1) || (val ==  (shift + 1))) ++qn;
633           if ((val == 1) || (val == -(shift + 1))) ++qp;
634         }
635         /* Split old vertex: Edges into original vertex and new cohesive edge */
636         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
637         /* Split new vertex: Edges into split vertex and new cohesive edge */
638         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
639         /* Add hybrid edge */
640         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
641         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
642       } else if (dep == dim-2) {
643         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
644 
645         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
646         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
647           PetscInt val;
648 
649           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
650           if (val == dim-1) ++qf;
651           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
652           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
653         }
654         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
655         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
656         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
657         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
658         /* Add hybrid face */
659         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
660         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
661       }
662     }
663   }
664   /* Step 4: Setup split DM */
665   ierr = DMSetUp(sdm);CHKERRQ(ierr);
666   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
667   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
668   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,PetscInt,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,PetscInt,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),PetscInt,&supportNew);CHKERRQ(ierr);
669   /* Step 6: Set cones and supports for new points */
670   for (dep = 0; dep <= depth; ++dep) {
671     for (p = 0; p < numSplitPoints[dep]; ++p) {
672       const PetscInt  oldp   = splitPoints[dep][p];
673       const PetscInt  newp   = oldp + depthOffset[dep];
674       const PetscInt  splitp = p    + pMaxNew[dep];
675       const PetscInt *cone, *support, *ornt;
676       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
677 
678       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
679       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
680       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
681       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
682       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
683       if (dep == depth-1) {
684         const PetscInt  hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
685         const PetscInt *supportF;
686 
687         /* Split face:       copy in old face to new face to start */
688         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
689         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
690         /* Split old face:   old vertices/edges in cone so no change */
691         /* Split new face:   new vertices/edges in cone */
692         for (q = 0; q < coneSize; ++q) {
693           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
694 
695           coneNew[2+q] = v + pMaxNew[dep-1];
696         }
697         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
698         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
699         /* Face support */
700         for (s = 0; s < supportSize; ++s) {
701           PetscInt val;
702 
703           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
704           if (val < 0) {
705             /* Split old face:   Replace negative side cell with cohesive cell */
706              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
707           } else {
708             /* Split new face:   Replace positive side cell with cohesive cell */
709             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
710             /* Get orientation for cohesive face */
711             {
712               const PetscInt *ncone, *nconeO;
713               PetscInt        nconeSize, nc;
714 
715               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
716               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
717               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
718               for (nc = 0; nc < nconeSize; ++nc) {
719                 if (ncone[nc] == oldp) {
720                   coneONew[0] = nconeO[nc];
721                   break;
722                 }
723               }
724               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
725             }
726           }
727         }
728         /* Cohesive cell:    Old and new split face, then new cohesive faces */
729         coneNew[0]  = newp;   /* Extracted negative side orientation above */
730         coneNew[1]  = splitp;
731         coneONew[1] = coneONew[0];
732         for (q = 0; q < coneSize; ++q) {
733           coneNew[2+q] += (pMaxNew[dep] - pMaxNew[dep-1]) + numSplitPoints[dep];
734           coneONew[2+q] = 0;
735         }
736         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
737         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
738       } else if (dep == 0) {
739         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
740 
741         /* Split old vertex: Edges in old split faces and new cohesive edge */
742         for (e = 0, qn = 0; e < supportSize; ++e) {
743           PetscInt val;
744 
745           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
746           if ((val == 1) || (val == (shift + 1))) {
747             supportNew[qn++] = support[e] + depthOffset[dep+1];
748           }
749         }
750         supportNew[qn] = hybedge;
751         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
752         /* Split new vertex: Edges in new split faces and new cohesive edge */
753         for (e = 0, qp = 0; e < supportSize; ++e) {
754           PetscInt val, edge;
755 
756           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
757           if (val == 1) {
758             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
759             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
760             supportNew[qp++] = edge + pMaxNew[dep+1];
761           } else if (val == -(shift + 1)) {
762             supportNew[qp++] = support[e] + depthOffset[dep+1];
763           }
764         }
765         supportNew[qp] = hybedge;
766         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
767         /* Hybrid edge:    Old and new split vertex */
768         coneNew[0] = newp;
769         coneNew[1] = splitp;
770         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
771         for (e = 0, qf = 0; e < supportSize; ++e) {
772           PetscInt val, edge;
773 
774           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
775           if (val == 1) {
776             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
777             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
778             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
779           }
780         }
781         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
782       } else if (dep == dim-2) {
783         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
784 
785         /* Split old edge:   old vertices in cone so no change */
786         /* Split new edge:   new vertices in cone */
787         for (q = 0; q < coneSize; ++q) {
788           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
789 
790           coneNew[q] = v + pMaxNew[dep-1];
791         }
792         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
793         /* Split old edge: Faces in positive side cells and old split faces */
794         for (e = 0, q = 0; e < supportSize; ++e) {
795           PetscInt val;
796 
797           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
798           if (val == dim-1) {
799             supportNew[q++] = support[e] + depthOffset[dep+1];
800           } else if (val == (shift + dim-1)) {
801             supportNew[q++] = support[e] + depthOffset[dep+1];
802           }
803         }
804         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
805         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
806         /* Split new edge: Faces in negative side cells and new split faces */
807         for (e = 0, q = 0; e < supportSize; ++e) {
808           PetscInt val, face;
809 
810           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
811           if (val == dim-1) {
812             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
813             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
814             supportNew[q++] = face + pMaxNew[dep+1];
815           } else if (val == -(shift + dim-1)) {
816             supportNew[q++] = support[e] + depthOffset[dep+1];
817           }
818         }
819         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
820         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
821         /* Hybrid face */
822         coneNew[0] = newp;
823         coneNew[1] = splitp;
824         for (v = 0; v < coneSize; ++v) {
825           PetscInt vertex;
826           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
827           if (vertex < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not a split vertex", support[e]);
828           coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
829         }
830         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
831         for (e = 0, qf = 0; e < supportSize; ++e) {
832           PetscInt val, face;
833 
834           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
835           if (val == dim-1) {
836             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
837             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
838             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
839           }
840         }
841         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
842       }
843     }
844   }
845   /* Step 6b: Replace split points in negative side cones */
846   for (sp = 0; sp < numSP; ++sp) {
847     PetscInt        dep = values[sp];
848     IS              pIS;
849     PetscInt        numPoints;
850     const PetscInt *points;
851 
852     if (dep >= 0) continue;
853     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
854     if (!pIS) continue;
855     dep  = -dep - shift;
856     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
857     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
858     for (p = 0; p < numPoints; ++p) {
859       const PetscInt  oldp = points[p];
860       const PetscInt  newp = depthOffset[dep] + oldp;
861       const PetscInt *cone;
862       PetscInt        coneSize, c;
863       /* PetscBool       replaced = PETSC_FALSE; */
864 
865       /* Negative edge: replace split vertex */
866       /* Negative cell: replace split face */
867       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
868       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
869       for (c = 0; c < coneSize; ++c) {
870         const PetscInt coldp = cone[c] - depthOffset[dep-1];
871         PetscInt       csplitp, cp, val;
872 
873         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
874         if (val == dep-1) {
875           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
876           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
877           csplitp  = pMaxNew[dep-1] + cp;
878           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
879           /* replaced = PETSC_TRUE; */
880         }
881       }
882       /* Cells with only a vertex or edge on the submesh have no replacement */
883       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
884     }
885     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
886     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
887   }
888   /* Step 7: Stratify */
889   ierr = DMPlexStratify(sdm);CHKERRQ(ierr);
890   /* Step 8: Coordinates */
891   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
892   ierr = DMPlexGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
893   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
894   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
895   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
896     const PetscInt newp   = depthOffset[0] + splitPoints[0][v];
897     const PetscInt splitp = pMaxNew[0] + v;
898     PetscInt       dof, off, soff, d;
899 
900     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
901     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
902     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
903     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
904   }
905   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
906   /* Step 9: SF, if I can figure this out we can split the mesh in parallel */
907   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
908   /* Step 10: Labels */
909   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
910   ierr = DMPlexGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
911   for (dep = 0; dep <= depth; ++dep) {
912     for (p = 0; p < numSplitPoints[dep]; ++p) {
913       const PetscInt newp   = depthOffset[dep] + splitPoints[dep][p];
914       const PetscInt splitp = pMaxNew[dep] + p;
915       PetscInt       l;
916 
917       for (l = 0; l < numLabels; ++l) {
918         DMLabel     mlabel;
919         const char *lname;
920         PetscInt    val;
921         PetscBool   isDepth;
922 
923         ierr = DMPlexGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
924         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
925         if (isDepth) continue;
926         ierr = DMPlexGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
927         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
928         if (val >= 0) {
929           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
930 #if 0
931           /* Do not put cohesive edges into the label */
932           if (dep == 0) {
933             const PetscInt cedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
934             ierr = DMLabelSetValue(mlabel, cedge, val);CHKERRQ(ierr);
935           } else if (dep == dim-2) {
936             const PetscInt cface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
937             ierr = DMLabelSetValue(mlabel, cface, val);CHKERRQ(ierr);
938           }
939           /* Do not put cohesive faces into the label */
940 #endif
941         }
942       }
943     }
944   }
945   for (sp = 0; sp < numSP; ++sp) {
946     const PetscInt dep = values[sp];
947 
948     if ((dep < 0) || (dep > depth)) continue;
949     if (pointIS[dep]) {ierr = ISRestoreIndices(pointIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
950     ierr = ISDestroy(&pointIS[dep]);CHKERRQ(ierr);
951   }
952   if (label) {
953     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
954     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
955   }
956   for (d = 0; d <= depth; ++d) {
957     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
958     pMaxNew[d] = pEnd - numHybridPoints[d];
959   }
960   ierr = DMPlexSetHybridBounds(sdm, depth >= 0 ? pMaxNew[depth] : PETSC_DETERMINE, depth>1 ? pMaxNew[depth-1] : PETSC_DETERMINE, depth>2 ? pMaxNew[1] : PETSC_DETERMINE, depth >= 0 ? pMaxNew[0] : PETSC_DETERMINE);CHKERRQ(ierr);
961   ierr = PetscFree3(depthShift, depthOffset, pMaxNew);CHKERRQ(ierr);
962   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
963   ierr = PetscFree4(pointIS, numSplitPoints, numHybridPoints, splitPoints);CHKERRQ(ierr);
964   PetscFunctionReturn(0);
965 }
966 
967 #undef __FUNCT__
968 #define __FUNCT__ "DMPlexConstructCohesiveCells"
969 /*@C
970   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
971 
972   Collective on dm
973 
974   Input Parameters:
975 + dm - The original DM
976 - label - The label specifying the boundary faces (this could be auto-generated)
977 
978   Output Parameters:
979 - dmSplit - The new DM
980 
981   Level: developer
982 
983 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
984 @*/
985 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DM *dmSplit)
986 {
987   DM             sdm;
988   PetscInt       dim;
989   PetscErrorCode ierr;
990 
991   PetscFunctionBegin;
992   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
993   PetscValidPointer(dmSplit, 3);
994   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
995   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
996   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
997   ierr = DMPlexSetDimension(sdm, dim);CHKERRQ(ierr);
998   switch (dim) {
999   case 2:
1000   case 3:
1001     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, sdm);CHKERRQ(ierr);
1002     break;
1003   default:
1004     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1005   }
1006   *dmSplit = sdm;
1007   PetscFunctionReturn(0);
1008 }
1009 
1010 #undef __FUNCT__
1011 #define __FUNCT__ "DMPlexLabelCohesiveComplete"
1012 /*@
1013   DMPlexLabelCohesiveComplete - Starting with a label marking vertices on an internal surface, we add all other mesh pieces
1014   to complete the surface
1015 
1016   Input Parameters:
1017 + dm - The DM
1018 . label - A DMLabel marking the surface vertices
1019 . flip  - Flag to flip the submesh normal and replace points on the other side
1020 - subdm - The subDM associated with the label, or NULL
1021 
1022   Output Parameter:
1023 . label - A DMLabel marking all surface points
1024 
1025   Level: developer
1026 
1027 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1028 @*/
1029 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, PetscBool flip, DM subdm)
1030 {
1031   DMLabel         depthLabel;
1032   IS              dimIS, subpointIS;
1033   const PetscInt *points, *subpoints;
1034   const PetscInt  rev   = flip ? -1 : 1;
1035   PetscInt        shift = 100, dim, dep, cStart, cEnd, numPoints, numSubpoints, p, val;
1036   PetscErrorCode  ierr;
1037 
1038   PetscFunctionBegin;
1039   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1040   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1041   if (subdm) {
1042     ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1043     if (subpointIS) {
1044       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1045       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1046     }
1047   }
1048   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1049   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1050   if (!dimIS) PetscFunctionReturn(0);
1051   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1052   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1053   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1054     const PetscInt *support;
1055     PetscInt        supportSize, s;
1056 
1057     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1058     if (supportSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports", points[p], supportSize);
1059     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1060     for (s = 0; s < supportSize; ++s) {
1061       const PetscInt *cone, *ornt;
1062       PetscInt        coneSize, c;
1063       PetscBool       pos = PETSC_TRUE;
1064 
1065       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1066       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1067       ierr = DMPlexGetConeOrientation(dm, support[s], &ornt);CHKERRQ(ierr);
1068       for (c = 0; c < coneSize; ++c) {
1069         if (cone[c] == points[p]) {
1070           PetscInt o = ornt[c];
1071 
1072           if (subdm) {
1073             const PetscInt *subcone, *subornt;
1074             PetscInt        subpoint, subface, subconeSize, sc;
1075 
1076             ierr = PetscFindInt(support[s], numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1077             ierr = PetscFindInt(points[p],  numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1078             ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1079             ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1080             ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1081             for (sc = 0; sc < subconeSize; ++sc) {
1082               if (subcone[sc] == subface) {
1083                 o = subornt[0];
1084                 break;
1085               }
1086             }
1087             if (sc >= subconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find point %d in cone for subpoint %d", points[p], subpoint);
1088           }
1089           if (o >= 0) {
1090             ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);
1091             pos  = rev > 0 ? PETSC_TRUE : PETSC_FALSE;
1092           } else {
1093             ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);
1094             pos  = rev > 0 ? PETSC_FALSE : PETSC_TRUE;
1095           }
1096           break;
1097         }
1098       }
1099       if (c == coneSize) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell split face %d support does not have it in the cone", points[p]);
1100       /* Put faces touching the fault in the label */
1101       for (c = 0; c < coneSize; ++c) {
1102         const PetscInt point = cone[c];
1103 
1104         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1105         if (val == -1) {
1106           PetscInt *closure = NULL;
1107           PetscInt  closureSize, cl;
1108 
1109           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1110           for (cl = 0; cl < closureSize*2; cl += 2) {
1111             const PetscInt clp = closure[cl];
1112 
1113             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1114             if ((val >= 0) && (val < dim-1)) {
1115               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1116               break;
1117             }
1118           }
1119           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1120         }
1121       }
1122     }
1123   }
1124   if (subdm) {
1125     if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1126     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1127   }
1128   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1129   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1130   /* Search for other cells/faces/edges connected to the fault by a vertex */
1131   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1132   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1133   if (!dimIS) PetscFunctionReturn(0);
1134   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1135   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1136   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1137     PetscInt *star = NULL;
1138     PetscInt  starSize, s;
1139     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1140 
1141     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1142     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1143     while (again) {
1144       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1145       again = 0;
1146       for (s = 0; s < starSize*2; s += 2) {
1147         const PetscInt  point = star[s];
1148         const PetscInt *cone;
1149         PetscInt        coneSize, c;
1150 
1151         if ((point < cStart) || (point >= cEnd)) continue;
1152         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1153         if (val != -1) continue;
1154         again = again == 1 ? 1 : 2;
1155         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1156         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1157         for (c = 0; c < coneSize; ++c) {
1158           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1159           if (val != -1) {
1160             const PetscInt *ccone;
1161             PetscInt        cconeSize, cc, side;
1162 
1163             if (abs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1164             if (val > 0) side =  1;
1165             else         side = -1;
1166             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1167             /* Mark cell faces which touch the fault */
1168             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1169             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1170             for (cc = 0; cc < cconeSize; ++cc) {
1171               PetscInt *closure = NULL;
1172               PetscInt  closureSize, cl;
1173 
1174               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1175               if (val != -1) continue;
1176               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1177               for (cl = 0; cl < closureSize*2; cl += 2) {
1178                 const PetscInt clp = closure[cl];
1179 
1180                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1181                 if (val == -1) continue;
1182                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1183                 break;
1184               }
1185               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1186             }
1187             again = 1;
1188             break;
1189           }
1190         }
1191       }
1192     }
1193     /* Classify the rest by cell membership */
1194     for (s = 0; s < starSize*2; s += 2) {
1195       const PetscInt point = star[s];
1196 
1197       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1198       if (val == -1) {
1199         PetscInt  *sstar = NULL;
1200         PetscInt   sstarSize, ss;
1201         PetscBool  marked = PETSC_FALSE;
1202 
1203         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1204         for (ss = 0; ss < sstarSize*2; ss += 2) {
1205           const PetscInt spoint = sstar[ss];
1206 
1207           if ((spoint < cStart) || (spoint >= cEnd)) continue;
1208           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1209           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1210           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1211           if (val > 0) {
1212             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1213           } else {
1214             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1215           }
1216           marked = PETSC_TRUE;
1217           break;
1218         }
1219         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1220         if (!marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1221       }
1222     }
1223     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1224   }
1225   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1226   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1227   PetscFunctionReturn(0);
1228 }
1229 
1230 #undef __FUNCT__
1231 #define __FUNCT__ "DMPlexCreateHybridMesh"
1232 /*@C
1233   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
1234 
1235   Collective on dm
1236 
1237   Input Parameters:
1238 + dm - The original DM
1239 - labelName - The label specifying the interface vertices
1240 
1241   Output Parameters:
1242 + hybridLabel - The label fully marking the interface
1243 - dmHybrid - The new DM
1244 
1245   Level: developer
1246 
1247 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMCreate()
1248 @*/
1249 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel *hybridLabel, DM *dmHybrid)
1250 {
1251   DM             idm;
1252   DMLabel        subpointMap, hlabel;
1253   PetscInt       dim;
1254   PetscErrorCode ierr;
1255 
1256   PetscFunctionBegin;
1257   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1258   if (hybridLabel) PetscValidPointer(hybridLabel, 3);
1259   PetscValidPointer(dmHybrid, 4);
1260   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1261   ierr = DMPlexCreateSubmesh(dm, label, 1, &idm);CHKERRQ(ierr);
1262   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
1263   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
1264   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
1265   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
1266   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, PETSC_FALSE, idm);CHKERRQ(ierr);
1267   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1268   ierr = DMPlexConstructCohesiveCells(dm, hlabel, dmHybrid);CHKERRQ(ierr);
1269   if (hybridLabel) *hybridLabel = hlabel;
1270   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
1271   PetscFunctionReturn(0);
1272 }
1273 
1274 #undef __FUNCT__
1275 #define __FUNCT__ "DMPlexMarkSubmesh_Uninterpolated"
1276 /* Here we need the explicit assumption that:
1277 
1278      For any marked cell, the marked vertices constitute a single face
1279 */
1280 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
1281 {
1282   IS               subvertexIS = NULL;
1283   const PetscInt  *subvertices;
1284   PetscInt        *pStart, *pEnd, *pMax, pSize;
1285   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
1286   PetscErrorCode   ierr;
1287 
1288   PetscFunctionBegin;
1289   *numFaces = 0;
1290   *nFV      = 0;
1291   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1292   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1293   pSize = PetscMax(depth, dim) + 1;
1294   ierr = PetscMalloc3(pSize,PetscInt,&pStart,pSize,PetscInt,&pEnd,pSize,PetscInt,&pMax);CHKERRQ(ierr);
1295   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1296   for (d = 0; d <= depth; ++d) {
1297     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1298     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1299   }
1300   /* Loop over initial vertices and mark all faces in the collective star() */
1301   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
1302   if (subvertexIS) {
1303     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1304     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1305   }
1306   for (v = 0; v < numSubVerticesInitial; ++v) {
1307     const PetscInt vertex = subvertices[v];
1308     PetscInt      *star   = NULL;
1309     PetscInt       starSize, s, numCells = 0, c;
1310 
1311     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1312     for (s = 0; s < starSize*2; s += 2) {
1313       const PetscInt point = star[s];
1314       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
1315     }
1316     for (c = 0; c < numCells; ++c) {
1317       const PetscInt cell    = star[c];
1318       PetscInt      *closure = NULL;
1319       PetscInt       closureSize, cl;
1320       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
1321 
1322       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
1323       if (cellLoc == 2) continue;
1324       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
1325       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1326       for (cl = 0; cl < closureSize*2; cl += 2) {
1327         const PetscInt point = closure[cl];
1328         PetscInt       vertexLoc;
1329 
1330         if ((point >= pStart[0]) && (point < pEnd[0])) {
1331           ++numCorners;
1332           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1333           if (vertexLoc == value) closure[faceSize++] = point;
1334         }
1335       }
1336       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
1337       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
1338       if (faceSize == *nFV) {
1339         const PetscInt *cells = NULL;
1340         PetscInt        numCells, nc;
1341 
1342         ++(*numFaces);
1343         for (cl = 0; cl < faceSize; ++cl) {
1344           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
1345         }
1346         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1347         for (nc = 0; nc < numCells; ++nc) {
1348           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
1349         }
1350         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1351       }
1352       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1353     }
1354     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1355   }
1356   if (subvertexIS) {
1357     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1358   }
1359   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1360   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1361   PetscFunctionReturn(0);
1362 }
1363 
1364 #undef __FUNCT__
1365 #define __FUNCT__ "DMPlexMarkSubmesh_Interpolated"
1366 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, DM subdm)
1367 {
1368   IS               subvertexIS;
1369   const PetscInt  *subvertices;
1370   PetscInt        *pStart, *pEnd, *pMax;
1371   PetscInt         dim, d, numSubVerticesInitial = 0, v;
1372   PetscErrorCode   ierr;
1373 
1374   PetscFunctionBegin;
1375   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1376   ierr = PetscMalloc3(dim+1,PetscInt,&pStart,dim+1,PetscInt,&pEnd,dim+1,PetscInt,&pMax);CHKERRQ(ierr);
1377   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], dim>1 ? &pMax[dim-1] : NULL, dim > 2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1378   for (d = 0; d <= dim; ++d) {
1379     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1380     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1381   }
1382   /* Loop over initial vertices and mark all faces in the collective star() */
1383   ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
1384   if (subvertexIS) {
1385     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1386     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1387   }
1388   for (v = 0; v < numSubVerticesInitial; ++v) {
1389     const PetscInt vertex = subvertices[v];
1390     PetscInt      *star   = NULL;
1391     PetscInt       starSize, s, numFaces = 0, f;
1392 
1393     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1394     for (s = 0; s < starSize*2; s += 2) {
1395       const PetscInt point = star[s];
1396       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) star[numFaces++] = point;
1397     }
1398     for (f = 0; f < numFaces; ++f) {
1399       const PetscInt face    = star[f];
1400       PetscInt      *closure = NULL;
1401       PetscInt       closureSize, c;
1402       PetscInt       faceLoc;
1403 
1404       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
1405       if (faceLoc == dim-1) continue;
1406       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
1407       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1408       for (c = 0; c < closureSize*2; c += 2) {
1409         const PetscInt point = closure[c];
1410         PetscInt       vertexLoc;
1411 
1412         if ((point >= pStart[0]) && (point < pEnd[0])) {
1413           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1414           if (vertexLoc != value) break;
1415         }
1416       }
1417       if (c == closureSize*2) {
1418         const PetscInt *support;
1419         PetscInt        supportSize, s;
1420 
1421         for (c = 0; c < closureSize*2; c += 2) {
1422           const PetscInt point = closure[c];
1423 
1424           for (d = 0; d < dim; ++d) {
1425             if ((point >= pStart[d]) && (point < pEnd[d])) {
1426               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
1427               break;
1428             }
1429           }
1430         }
1431         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
1432         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
1433         for (s = 0; s < supportSize; ++s) {
1434           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
1435         }
1436       }
1437       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1438     }
1439     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1440   }
1441   if (subvertexIS) {
1442     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1443   }
1444   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1445   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1446   PetscFunctionReturn(0);
1447 }
1448 
1449 #undef __FUNCT__
1450 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Uninterpolated"
1451 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
1452 {
1453   DMLabel         label = NULL;
1454   const PetscInt *cone;
1455   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize;
1456   PetscErrorCode  ierr;
1457 
1458   PetscFunctionBegin;
1459   *numFaces = 0;
1460   *nFV = 0;
1461   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1462   *subCells = NULL;
1463   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1464   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1465   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1466   if (cMax < 0) PetscFunctionReturn(0);
1467   if (label) {
1468     for (c = cMax; c < cEnd; ++c) {
1469       PetscInt val;
1470 
1471       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1472       if (val == value) {
1473         ++(*numFaces);
1474         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1475       }
1476     }
1477   } else {
1478     *numFaces = cEnd - cMax;
1479     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
1480   }
1481   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
1482   ierr = PetscMalloc(*numFaces *2 * sizeof(PetscInt), subCells);CHKERRQ(ierr);
1483   for (c = cMax; c < cEnd; ++c) {
1484     const PetscInt *cells;
1485     PetscInt        numCells;
1486 
1487     if (label) {
1488       PetscInt val;
1489 
1490       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1491       if (val != value) continue;
1492     }
1493     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1494     for (p = 0; p < *nFV; ++p) {
1495       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
1496     }
1497     /* Negative face */
1498     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1499     /* Not true in parallel
1500     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
1501     for (p = 0; p < numCells; ++p) {
1502       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
1503       (*subCells)[subc++] = cells[p];
1504     }
1505     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1506     /* Positive face is not included */
1507   }
1508   PetscFunctionReturn(0);
1509 }
1510 
1511 #undef __FUNCT__
1512 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Interpolated"
1513 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, DM subdm)
1514 {
1515   DMLabel        label = NULL;
1516   PetscInt      *pStart, *pEnd;
1517   PetscInt       dim, cMax, cEnd, c, d;
1518   PetscErrorCode ierr;
1519 
1520   PetscFunctionBegin;
1521   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1522   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1523   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1524   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1525   if (cMax < 0) PetscFunctionReturn(0);
1526   ierr = PetscMalloc2(dim+1,PetscInt,&pStart,dim+1,PetscInt,&pEnd);CHKERRQ(ierr);
1527   for (d = 0; d <= dim; ++d) {
1528     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1529   }
1530   for (c = cMax; c < cEnd; ++c) {
1531     const PetscInt *cone;
1532     PetscInt       *closure = NULL;
1533     PetscInt        coneSize, closureSize, cl;
1534 
1535     if (label) {
1536       PetscInt val;
1537 
1538       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1539       if (val != value) continue;
1540     }
1541     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1542     if (hasLagrange) {
1543       if (coneSize != 3) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
1544     } else {
1545       if (coneSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
1546     }
1547     /* Negative face */
1548     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1549     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1550     for (cl = 0; cl < closureSize*2; cl += 2) {
1551       const PetscInt point = closure[cl];
1552 
1553       for (d = 0; d <= dim; ++d) {
1554         if ((point >= pStart[d]) && (point < pEnd[d])) {
1555           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
1556           break;
1557         }
1558       }
1559     }
1560     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1561     /* Cells -- positive face is not included */
1562     for (cl = 0; cl < 1; ++cl) {
1563       const PetscInt *support;
1564       PetscInt        supportSize, s;
1565 
1566       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
1567       if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells");
1568       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
1569       for (s = 0; s < supportSize; ++s) {
1570         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
1571       }
1572     }
1573   }
1574   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
1575   PetscFunctionReturn(0);
1576 }
1577 
1578 #undef __FUNCT__
1579 #define __FUNCT__ "DMPlexGetFaceOrientation"
1580 PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
1581 {
1582   MPI_Comm       comm;
1583   PetscBool      posOrient = PETSC_FALSE;
1584   const PetscInt debug     = 0;
1585   PetscInt       cellDim, faceSize, f;
1586   PetscErrorCode ierr;
1587 
1588   PetscFunctionBegin;
1589   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1590   ierr = DMPlexGetDimension(dm, &cellDim);CHKERRQ(ierr);
1591   if (debug) {PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
1592 
1593   if (cellDim == 1 && numCorners == 2) {
1594     /* Triangle */
1595     faceSize  = numCorners-1;
1596     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1597   } else if (cellDim == 2 && numCorners == 3) {
1598     /* Triangle */
1599     faceSize  = numCorners-1;
1600     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1601   } else if (cellDim == 3 && numCorners == 4) {
1602     /* Tetrahedron */
1603     faceSize  = numCorners-1;
1604     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1605   } else if (cellDim == 1 && numCorners == 3) {
1606     /* Quadratic line */
1607     faceSize  = 1;
1608     posOrient = PETSC_TRUE;
1609   } else if (cellDim == 2 && numCorners == 4) {
1610     /* Quads */
1611     faceSize = 2;
1612     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
1613       posOrient = PETSC_TRUE;
1614     } else if ((indices[0] == 3) && (indices[1] == 0)) {
1615       posOrient = PETSC_TRUE;
1616     } else {
1617       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
1618         posOrient = PETSC_FALSE;
1619       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
1620     }
1621   } else if (cellDim == 2 && numCorners == 6) {
1622     /* Quadratic triangle (I hate this) */
1623     /* Edges are determined by the first 2 vertices (corners of edges) */
1624     const PetscInt faceSizeTri = 3;
1625     PetscInt       sortedIndices[3], i, iFace;
1626     PetscBool      found                    = PETSC_FALSE;
1627     PetscInt       faceVerticesTriSorted[9] = {
1628       0, 3,  4, /* bottom */
1629       1, 4,  5, /* right */
1630       2, 3,  5, /* left */
1631     };
1632     PetscInt       faceVerticesTri[9] = {
1633       0, 3,  4, /* bottom */
1634       1, 4,  5, /* right */
1635       2, 5,  3, /* left */
1636     };
1637 
1638     faceSize = faceSizeTri;
1639     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
1640     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
1641     for (iFace = 0; iFace < 3; ++iFace) {
1642       const PetscInt ii = iFace*faceSizeTri;
1643       PetscInt       fVertex, cVertex;
1644 
1645       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
1646           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
1647         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
1648           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
1649             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
1650               faceVertices[fVertex] = origVertices[cVertex];
1651               break;
1652             }
1653           }
1654         }
1655         found = PETSC_TRUE;
1656         break;
1657       }
1658     }
1659     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
1660     if (posOriented) *posOriented = PETSC_TRUE;
1661     PetscFunctionReturn(0);
1662   } else if (cellDim == 2 && numCorners == 9) {
1663     /* Quadratic quad (I hate this) */
1664     /* Edges are determined by the first 2 vertices (corners of edges) */
1665     const PetscInt faceSizeQuad = 3;
1666     PetscInt       sortedIndices[3], i, iFace;
1667     PetscBool      found                      = PETSC_FALSE;
1668     PetscInt       faceVerticesQuadSorted[12] = {
1669       0, 1,  4, /* bottom */
1670       1, 2,  5, /* right */
1671       2, 3,  6, /* top */
1672       0, 3,  7, /* left */
1673     };
1674     PetscInt       faceVerticesQuad[12] = {
1675       0, 1,  4, /* bottom */
1676       1, 2,  5, /* right */
1677       2, 3,  6, /* top */
1678       3, 0,  7, /* left */
1679     };
1680 
1681     faceSize = faceSizeQuad;
1682     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
1683     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
1684     for (iFace = 0; iFace < 4; ++iFace) {
1685       const PetscInt ii = iFace*faceSizeQuad;
1686       PetscInt       fVertex, cVertex;
1687 
1688       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
1689           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
1690         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
1691           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
1692             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
1693               faceVertices[fVertex] = origVertices[cVertex];
1694               break;
1695             }
1696           }
1697         }
1698         found = PETSC_TRUE;
1699         break;
1700       }
1701     }
1702     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
1703     if (posOriented) *posOriented = PETSC_TRUE;
1704     PetscFunctionReturn(0);
1705   } else if (cellDim == 3 && numCorners == 8) {
1706     /* Hexes
1707        A hex is two oriented quads with the normal of the first
1708        pointing up at the second.
1709 
1710           7---6
1711          /|  /|
1712         4---5 |
1713         | 1-|-2
1714         |/  |/
1715         0---3
1716 
1717         Faces are determined by the first 4 vertices (corners of faces) */
1718     const PetscInt faceSizeHex = 4;
1719     PetscInt       sortedIndices[4], i, iFace;
1720     PetscBool      found                     = PETSC_FALSE;
1721     PetscInt       faceVerticesHexSorted[24] = {
1722       0, 1, 2, 3,  /* bottom */
1723       4, 5, 6, 7,  /* top */
1724       0, 3, 4, 5,  /* front */
1725       2, 3, 5, 6,  /* right */
1726       1, 2, 6, 7,  /* back */
1727       0, 1, 4, 7,  /* left */
1728     };
1729     PetscInt       faceVerticesHex[24] = {
1730       1, 2, 3, 0,  /* bottom */
1731       4, 5, 6, 7,  /* top */
1732       0, 3, 5, 4,  /* front */
1733       3, 2, 6, 5,  /* right */
1734       2, 1, 7, 6,  /* back */
1735       1, 0, 4, 7,  /* left */
1736     };
1737 
1738     faceSize = faceSizeHex;
1739     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
1740     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
1741     for (iFace = 0; iFace < 6; ++iFace) {
1742       const PetscInt ii = iFace*faceSizeHex;
1743       PetscInt       fVertex, cVertex;
1744 
1745       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
1746           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
1747           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
1748           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
1749         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
1750           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
1751             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
1752               faceVertices[fVertex] = origVertices[cVertex];
1753               break;
1754             }
1755           }
1756         }
1757         found = PETSC_TRUE;
1758         break;
1759       }
1760     }
1761     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
1762     if (posOriented) *posOriented = PETSC_TRUE;
1763     PetscFunctionReturn(0);
1764   } else if (cellDim == 3 && numCorners == 10) {
1765     /* Quadratic tet */
1766     /* Faces are determined by the first 3 vertices (corners of faces) */
1767     const PetscInt faceSizeTet = 6;
1768     PetscInt       sortedIndices[6], i, iFace;
1769     PetscBool      found                     = PETSC_FALSE;
1770     PetscInt       faceVerticesTetSorted[24] = {
1771       0, 1, 2,  6, 7, 8, /* bottom */
1772       0, 3, 4,  6, 7, 9,  /* front */
1773       1, 4, 5,  7, 8, 9,  /* right */
1774       2, 3, 5,  6, 8, 9,  /* left */
1775     };
1776     PetscInt       faceVerticesTet[24] = {
1777       0, 1, 2,  6, 7, 8, /* bottom */
1778       0, 4, 3,  6, 7, 9,  /* front */
1779       1, 5, 4,  7, 8, 9,  /* right */
1780       2, 3, 5,  8, 6, 9,  /* left */
1781     };
1782 
1783     faceSize = faceSizeTet;
1784     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
1785     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
1786     for (iFace=0; iFace < 4; ++iFace) {
1787       const PetscInt ii = iFace*faceSizeTet;
1788       PetscInt       fVertex, cVertex;
1789 
1790       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
1791           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
1792           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
1793           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
1794         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
1795           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
1796             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
1797               faceVertices[fVertex] = origVertices[cVertex];
1798               break;
1799             }
1800           }
1801         }
1802         found = PETSC_TRUE;
1803         break;
1804       }
1805     }
1806     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
1807     if (posOriented) *posOriented = PETSC_TRUE;
1808     PetscFunctionReturn(0);
1809   } else if (cellDim == 3 && numCorners == 27) {
1810     /* Quadratic hexes (I hate this)
1811        A hex is two oriented quads with the normal of the first
1812        pointing up at the second.
1813 
1814          7---6
1815         /|  /|
1816        4---5 |
1817        | 3-|-2
1818        |/  |/
1819        0---1
1820 
1821        Faces are determined by the first 4 vertices (corners of faces) */
1822     const PetscInt faceSizeQuadHex = 9;
1823     PetscInt       sortedIndices[9], i, iFace;
1824     PetscBool      found                         = PETSC_FALSE;
1825     PetscInt       faceVerticesQuadHexSorted[54] = {
1826       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
1827       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
1828       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
1829       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
1830       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
1831       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
1832     };
1833     PetscInt       faceVerticesQuadHex[54] = {
1834       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
1835       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
1836       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
1837       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
1838       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
1839       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
1840     };
1841 
1842     faceSize = faceSizeQuadHex;
1843     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
1844     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
1845     for (iFace = 0; iFace < 6; ++iFace) {
1846       const PetscInt ii = iFace*faceSizeQuadHex;
1847       PetscInt       fVertex, cVertex;
1848 
1849       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
1850           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
1851           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
1852           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
1853         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
1854           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
1855             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
1856               faceVertices[fVertex] = origVertices[cVertex];
1857               break;
1858             }
1859           }
1860         }
1861         found = PETSC_TRUE;
1862         break;
1863       }
1864     }
1865     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
1866     if (posOriented) *posOriented = PETSC_TRUE;
1867     PetscFunctionReturn(0);
1868   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
1869   if (!posOrient) {
1870     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
1871     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
1872   } else {
1873     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
1874     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
1875   }
1876   if (posOriented) *posOriented = posOrient;
1877   PetscFunctionReturn(0);
1878 }
1879 
1880 #undef __FUNCT__
1881 #define __FUNCT__ "DMPlexGetOrientedFace"
1882 /*
1883     Given a cell and a face, as a set of vertices,
1884       return the oriented face, as a set of vertices, in faceVertices
1885     The orientation is such that the face normal points out of the cell
1886 */
1887 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
1888 {
1889   const PetscInt *cone = NULL;
1890   PetscInt        coneSize, v, f, v2;
1891   PetscInt        oppositeVertex = -1;
1892   PetscErrorCode  ierr;
1893 
1894   PetscFunctionBegin;
1895   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
1896   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
1897   for (v = 0, v2 = 0; v < coneSize; ++v) {
1898     PetscBool found = PETSC_FALSE;
1899 
1900     for (f = 0; f < faceSize; ++f) {
1901       if (face[f] == cone[v]) {
1902         found = PETSC_TRUE; break;
1903       }
1904     }
1905     if (found) {
1906       indices[v2]      = v;
1907       origVertices[v2] = cone[v];
1908       ++v2;
1909     } else {
1910       oppositeVertex = v;
1911     }
1912   }
1913   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
1914   PetscFunctionReturn(0);
1915 }
1916 
1917 #undef __FUNCT__
1918 #define __FUNCT__ "DMPlexInsertFace_Internal"
1919 /*
1920   DMPlexInsertFace_Internal - Puts a face into the mesh
1921 
1922   Not collective
1923 
1924   Input Parameters:
1925   + dm              - The DMPlex
1926   . numFaceVertex   - The number of vertices in the face
1927   . faceVertices    - The vertices in the face for dm
1928   . subfaceVertices - The vertices in the face for subdm
1929   . numCorners      - The number of vertices in the cell
1930   . cell            - A cell in dm containing the face
1931   . subcell         - A cell in subdm containing the face
1932   . firstFace       - First face in the mesh
1933   - newFacePoint    - Next face in the mesh
1934 
1935   Output Parameters:
1936   . newFacePoint - Contains next face point number on input, updated on output
1937 
1938   Level: developer
1939 */
1940 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
1941 {
1942   MPI_Comm        comm;
1943   DM_Plex        *submesh = (DM_Plex*) subdm->data;
1944   const PetscInt *faces;
1945   PetscInt        numFaces, coneSize;
1946   PetscErrorCode  ierr;
1947 
1948   PetscFunctionBegin;
1949   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1950   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
1951   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
1952 #if 0
1953   /* Cannot use this because support() has not been constructed yet */
1954   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
1955 #else
1956   {
1957     PetscInt f;
1958 
1959     numFaces = 0;
1960     ierr     = DMGetWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
1961     for (f = firstFace; f < *newFacePoint; ++f) {
1962       PetscInt dof, off, d;
1963 
1964       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
1965       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
1966       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
1967       for (d = 0; d < dof; ++d) {
1968         const PetscInt p = submesh->cones[off+d];
1969         PetscInt       v;
1970 
1971         for (v = 0; v < numFaceVertices; ++v) {
1972           if (subfaceVertices[v] == p) break;
1973         }
1974         if (v == numFaceVertices) break;
1975       }
1976       if (d == dof) {
1977         numFaces               = 1;
1978         ((PetscInt*) faces)[0] = f;
1979       }
1980     }
1981   }
1982 #endif
1983   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
1984   else if (numFaces == 1) {
1985     /* Add the other cell neighbor for this face */
1986     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
1987   } else {
1988     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
1989     PetscBool posOriented;
1990 
1991     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
1992     origVertices        = &orientedVertices[numFaceVertices];
1993     indices             = &orientedVertices[numFaceVertices*2];
1994     orientedSubVertices = &orientedVertices[numFaceVertices*3];
1995     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
1996     /* TODO: I know that routine should return a permutation, not the indices */
1997     for (v = 0; v < numFaceVertices; ++v) {
1998       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
1999       for (ov = 0; ov < numFaceVertices; ++ov) {
2000         if (orientedVertices[ov] == vertex) {
2001           orientedSubVertices[ov] = subvertex;
2002           break;
2003         }
2004       }
2005       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2006     }
2007     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2008     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2009     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2010     ++(*newFacePoint);
2011   }
2012 #if 0
2013   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2014 #else
2015   ierr = DMRestoreWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2016 #endif
2017   PetscFunctionReturn(0);
2018 }
2019 
2020 #undef __FUNCT__
2021 #define __FUNCT__ "DMPlexCreateSubmesh_Uninterpolated"
2022 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2023 {
2024   MPI_Comm        comm;
2025   DMLabel         subpointMap;
2026   IS              subvertexIS,  subcellIS;
2027   const PetscInt *subVertices, *subCells;
2028   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2029   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2030   PetscInt        vStart, vEnd, c, f;
2031   PetscErrorCode  ierr;
2032 
2033   PetscFunctionBegin;
2034   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2035   /* Create subpointMap which marks the submesh */
2036   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2037   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2038   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2039   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2040   /* Setup chart */
2041   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2042   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2043   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2044   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2045   /* Set cone sizes */
2046   firstSubVertex = numSubCells;
2047   firstSubFace   = numSubCells+numSubVertices;
2048   newFacePoint   = firstSubFace;
2049   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2050   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2051   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2052   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2053   for (c = 0; c < numSubCells; ++c) {
2054     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2055   }
2056   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2057     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2058   }
2059   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2060   /* Create face cones */
2061   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2062   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2063   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2064   for (c = 0; c < numSubCells; ++c) {
2065     const PetscInt cell    = subCells[c];
2066     const PetscInt subcell = c;
2067     PetscInt      *closure = NULL;
2068     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2069 
2070     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2071     for (cl = 0; cl < closureSize*2; cl += 2) {
2072       const PetscInt point = closure[cl];
2073       PetscInt       subVertex;
2074 
2075       if ((point >= vStart) && (point < vEnd)) {
2076         ++numCorners;
2077         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2078         if (subVertex >= 0) {
2079           closure[faceSize] = point;
2080           subface[faceSize] = firstSubVertex+subVertex;
2081           ++faceSize;
2082         }
2083       }
2084     }
2085     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2086     if (faceSize == nFV) {
2087       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2088     }
2089     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2090   }
2091   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2092   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2093   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2094   /* Build coordinates */
2095   {
2096     PetscSection coordSection, subCoordSection;
2097     Vec          coordinates, subCoordinates;
2098     PetscScalar *coords, *subCoords;
2099     PetscInt     numComp, coordSize, v;
2100 
2101     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2102     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2103     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2104     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2105     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2106     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2107     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2108     for (v = 0; v < numSubVertices; ++v) {
2109       const PetscInt vertex    = subVertices[v];
2110       const PetscInt subvertex = firstSubVertex+v;
2111       PetscInt       dof;
2112 
2113       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2114       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2115       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2116     }
2117     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2118     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2119     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2120     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2121     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2122     if (coordSize) {
2123       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2124       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2125       for (v = 0; v < numSubVertices; ++v) {
2126         const PetscInt vertex    = subVertices[v];
2127         const PetscInt subvertex = firstSubVertex+v;
2128         PetscInt       dof, off, sdof, soff, d;
2129 
2130         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2131         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2132         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2133         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2134         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2135         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2136       }
2137       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2138       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2139     }
2140     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2141     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2142   }
2143   /* Cleanup */
2144   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2145   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2146   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2147   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
2148   PetscFunctionReturn(0);
2149 }
2150 
2151 #undef __FUNCT__
2152 #define __FUNCT__ "DMPlexCreateSubmesh_Interpolated"
2153 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2154 {
2155   MPI_Comm         comm;
2156   DMLabel          subpointMap;
2157   IS              *subpointIS;
2158   const PetscInt **subpoints;
2159   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *coneONew;
2160   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2161   PetscErrorCode   ierr;
2162 
2163   PetscFunctionBegin;
2164   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2165   /* Create subpointMap which marks the submesh */
2166   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2167   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2168   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2169   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Interpolated(dm, vertexLabel, value, subpointMap, subdm);CHKERRQ(ierr);}
2170   /* Setup chart */
2171   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2172   ierr = PetscMalloc4(dim+1,PetscInt,&numSubPoints,dim+1,PetscInt,&firstSubPoint,dim+1,IS,&subpointIS,dim+1,const PetscInt *,&subpoints);CHKERRQ(ierr);
2173   for (d = 0; d <= dim; ++d) {
2174     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2175     totSubPoints += numSubPoints[d];
2176   }
2177   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2178   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2179   /* Set cone sizes */
2180   firstSubPoint[dim] = 0;
2181   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2182   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2183   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2184   for (d = 0; d <= dim; ++d) {
2185     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2186     ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2187   }
2188   for (d = 0; d <= dim; ++d) {
2189     for (p = 0; p < numSubPoints[d]; ++p) {
2190       const PetscInt  point    = subpoints[d][p];
2191       const PetscInt  subpoint = firstSubPoint[d] + p;
2192       const PetscInt *cone;
2193       PetscInt        coneSize, coneSizeNew, c, val;
2194 
2195       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2196       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2197       if (d == dim) {
2198         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2199         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2200           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2201           if (val >= 0) coneSizeNew++;
2202         }
2203         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2204       }
2205     }
2206   }
2207   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2208   /* Set cones */
2209   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2210   ierr = PetscMalloc2(maxConeSize,PetscInt,&coneNew,maxConeSize,PetscInt,&coneONew);CHKERRQ(ierr);
2211   for (d = 0; d <= dim; ++d) {
2212     for (p = 0; p < numSubPoints[d]; ++p) {
2213       const PetscInt  point    = subpoints[d][p];
2214       const PetscInt  subpoint = firstSubPoint[d] + p;
2215       const PetscInt *cone, *ornt;
2216       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2217 
2218       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2219       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2220       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2221       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
2222       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2223         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2224         if (subc >= 0) {
2225           coneNew[coneSizeNew]  = firstSubPoint[d-1] + subc;
2226           coneONew[coneSizeNew] = ornt[c];
2227           ++coneSizeNew;
2228         }
2229       }
2230       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2231       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2232       ierr = DMPlexSetConeOrientation(subdm, subpoint, coneONew);CHKERRQ(ierr);
2233     }
2234   }
2235   ierr = PetscFree2(coneNew,coneONew);CHKERRQ(ierr);
2236   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2237   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2238   /* Build coordinates */
2239   {
2240     PetscSection coordSection, subCoordSection;
2241     Vec          coordinates, subCoordinates;
2242     PetscScalar *coords, *subCoords;
2243     PetscInt     numComp, coordSize;
2244 
2245     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2246     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2247     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2248     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2249     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2250     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2251     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2252     for (v = 0; v < numSubPoints[0]; ++v) {
2253       const PetscInt vertex    = subpoints[0][v];
2254       const PetscInt subvertex = firstSubPoint[0]+v;
2255       PetscInt       dof;
2256 
2257       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2258       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2259       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2260     }
2261     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2262     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2263     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2264     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2265     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2266     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2267     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2268     for (v = 0; v < numSubPoints[0]; ++v) {
2269       const PetscInt vertex    = subpoints[0][v];
2270       const PetscInt subvertex = firstSubPoint[0]+v;
2271       PetscInt dof, off, sdof, soff, d;
2272 
2273       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2274       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2275       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2276       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2277       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2278       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2279     }
2280     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2281     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2282     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2283     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2284   }
2285   /* Cleanup */
2286   for (d = 0; d <= dim; ++d) {
2287     ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2288     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
2289   }
2290   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
2291   PetscFunctionReturn(0);
2292 }
2293 
2294 #undef __FUNCT__
2295 #define __FUNCT__ "DMPlexCreateSubmesh"
2296 /*@
2297   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
2298 
2299   Input Parameters:
2300 + dm           - The original mesh
2301 . vertexLabel  - The DMLabel marking vertices contained in the surface
2302 - value        - The label value to use
2303 
2304   Output Parameter:
2305 . subdm - The surface mesh
2306 
2307   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
2308 
2309   Level: developer
2310 
2311 .seealso: DMPlexGetSubpointMap(), DMPlexGetLabel(), DMLabelSetValue()
2312 @*/
2313 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, DM *subdm)
2314 {
2315   PetscInt       dim, depth;
2316   PetscErrorCode ierr;
2317 
2318   PetscFunctionBegin;
2319   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2320   PetscValidPointer(subdm, 3);
2321   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2322   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2323   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
2324   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
2325   ierr = DMPlexSetDimension(*subdm, dim-1);CHKERRQ(ierr);
2326   if (depth == dim) {
2327     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2328   } else {
2329     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2330   }
2331   PetscFunctionReturn(0);
2332 }
2333 
2334 #undef __FUNCT__
2335 #define __FUNCT__ "DMPlexFilterPoint_Internal"
2336 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
2337 {
2338   PetscInt       subPoint;
2339   PetscErrorCode ierr;
2340 
2341   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
2342   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
2343 }
2344 
2345 #undef __FUNCT__
2346 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Uninterpolated"
2347 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
2348 {
2349   MPI_Comm        comm;
2350   DMLabel         subpointMap;
2351   IS              subvertexIS;
2352   const PetscInt *subVertices;
2353   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
2354   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
2355   PetscInt        cMax, c, f;
2356   PetscErrorCode  ierr;
2357 
2358   PetscFunctionBegin;
2359   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
2360   /* Create subpointMap which marks the submesh */
2361   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2362   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2363   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2364   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
2365   /* Setup chart */
2366   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2367   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2368   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2369   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2370   /* Set cone sizes */
2371   firstSubVertex = numSubCells;
2372   firstSubFace   = numSubCells+numSubVertices;
2373   newFacePoint   = firstSubFace;
2374   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2375   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2376   for (c = 0; c < numSubCells; ++c) {
2377     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2378   }
2379   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2380     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2381   }
2382   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2383   /* Create face cones */
2384   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2385   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2386   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2387   for (c = 0; c < numSubCells; ++c) {
2388     const PetscInt  cell    = subCells[c];
2389     const PetscInt  subcell = c;
2390     const PetscInt *cone, *cells;
2391     PetscInt        numCells, subVertex, p, v;
2392 
2393     if (cell < cMax) continue;
2394     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2395     for (v = 0; v < nFV; ++v) {
2396       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2397       subface[v] = firstSubVertex+subVertex;
2398     }
2399     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
2400     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
2401     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2402     /* Not true in parallel
2403     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2404     for (p = 0; p < numCells; ++p) {
2405       PetscInt negsubcell;
2406 
2407       if (cells[p] >= cMax) continue;
2408       /* I know this is a crap search */
2409       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
2410         if (subCells[negsubcell] == cells[p]) break;
2411       }
2412       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
2413       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
2414     }
2415     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2416     ++newFacePoint;
2417   }
2418   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2419   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2420   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2421   /* Build coordinates */
2422   {
2423     PetscSection coordSection, subCoordSection;
2424     Vec          coordinates, subCoordinates;
2425     PetscScalar *coords, *subCoords;
2426     PetscInt     numComp, coordSize, v;
2427 
2428     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2429     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2430     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2431     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2432     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2433     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2434     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2435     for (v = 0; v < numSubVertices; ++v) {
2436       const PetscInt vertex    = subVertices[v];
2437       const PetscInt subvertex = firstSubVertex+v;
2438       PetscInt       dof;
2439 
2440       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2441       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2442       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2443     }
2444     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2445     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2446     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2447     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2448     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2449     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2450     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2451     for (v = 0; v < numSubVertices; ++v) {
2452       const PetscInt vertex    = subVertices[v];
2453       const PetscInt subvertex = firstSubVertex+v;
2454       PetscInt       dof, off, sdof, soff, d;
2455 
2456       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2457       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2458       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2459       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2460       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2461       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2462     }
2463     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2464     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2465     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2466     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2467   }
2468   /* Build SF */
2469   CHKMEMQ;
2470   {
2471     PetscSF            sfPoint, sfPointSub;
2472     const PetscSFNode *remotePoints;
2473     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
2474     const PetscInt    *localPoints;
2475     PetscInt          *slocalPoints;
2476     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
2477     PetscMPIInt        rank;
2478 
2479     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2480     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2481     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
2482     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2483     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2484     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
2485     if (numRoots >= 0) {
2486       /* Only vertices should be shared */
2487       ierr = PetscMalloc2(pEnd-pStart,PetscSFNode,&newLocalPoints,numRoots,PetscSFNode,&newOwners);CHKERRQ(ierr);
2488       for (p = 0; p < pEnd-pStart; ++p) {
2489         newLocalPoints[p].rank  = -2;
2490         newLocalPoints[p].index = -2;
2491       }
2492       /* Set subleaves */
2493       for (l = 0; l < numLeaves; ++l) {
2494         const PetscInt point    = localPoints[l];
2495         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2496 
2497         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
2498         if (subPoint < 0) continue;
2499         newLocalPoints[point-pStart].rank  = rank;
2500         newLocalPoints[point-pStart].index = subPoint;
2501         ++numSubLeaves;
2502       }
2503       /* Must put in owned subpoints */
2504       for (p = pStart; p < pEnd; ++p) {
2505         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
2506 
2507         if (subPoint < 0) {
2508           newOwners[p-pStart].rank  = -3;
2509           newOwners[p-pStart].index = -3;
2510         } else {
2511           newOwners[p-pStart].rank  = rank;
2512           newOwners[p-pStart].index = subPoint;
2513         }
2514       }
2515       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2516       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2517       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2518       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2519       ierr = PetscMalloc(numSubLeaves * sizeof(PetscInt),    &slocalPoints);CHKERRQ(ierr);
2520       ierr = PetscMalloc(numSubLeaves * sizeof(PetscSFNode), &sremotePoints);CHKERRQ(ierr);
2521       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
2522         const PetscInt point    = localPoints[l];
2523         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2524 
2525         if (subPoint < 0) continue;
2526         if (newLocalPoints[point].rank == rank) {++ll; continue;}
2527         slocalPoints[sl]        = subPoint;
2528         sremotePoints[sl].rank  = newLocalPoints[point].rank;
2529         sremotePoints[sl].index = newLocalPoints[point].index;
2530         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
2531         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
2532         ++sl;
2533       }
2534       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
2535       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
2536       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
2537     }
2538   }
2539   CHKMEMQ;
2540   /* Cleanup */
2541   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2542   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2543   ierr = PetscFree(subCells);CHKERRQ(ierr);
2544   PetscFunctionReturn(0);
2545 }
2546 
2547 #undef __FUNCT__
2548 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Interpolated"
2549 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
2550 {
2551   MPI_Comm         comm;
2552   DMLabel          subpointMap;
2553   IS              *subpointIS;
2554   const PetscInt **subpoints;
2555   PetscInt        *numSubPoints, *firstSubPoint, *coneNew;
2556   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2557   PetscErrorCode   ierr;
2558 
2559   PetscFunctionBegin;
2560   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2561   /* Create subpointMap which marks the submesh */
2562   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2563   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2564   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2565   ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, hasLagrange, label, value, subpointMap, subdm);CHKERRQ(ierr);
2566   /* Setup chart */
2567   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2568   ierr = PetscMalloc4(dim+1,PetscInt,&numSubPoints,dim+1,PetscInt,&firstSubPoint,dim+1,IS,&subpointIS,dim+1,const PetscInt *,&subpoints);CHKERRQ(ierr);
2569   for (d = 0; d <= dim; ++d) {
2570     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2571     totSubPoints += numSubPoints[d];
2572   }
2573   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2574   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2575   /* Set cone sizes */
2576   firstSubPoint[dim] = 0;
2577   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2578   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2579   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2580   for (d = 0; d <= dim; ++d) {
2581     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2582     ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2583   }
2584   for (d = 0; d <= dim; ++d) {
2585     for (p = 0; p < numSubPoints[d]; ++p) {
2586       const PetscInt  point    = subpoints[d][p];
2587       const PetscInt  subpoint = firstSubPoint[d] + p;
2588       const PetscInt *cone;
2589       PetscInt        coneSize, coneSizeNew, c, val;
2590 
2591       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2592       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2593       if (d == dim) {
2594         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2595         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2596           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2597           if (val >= 0) coneSizeNew++;
2598         }
2599         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2600       }
2601     }
2602   }
2603   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2604   /* Set cones */
2605   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2606   ierr = PetscMalloc(maxConeSize * sizeof(PetscInt), &coneNew);CHKERRQ(ierr);
2607   for (d = 0; d <= dim; ++d) {
2608     for (p = 0; p < numSubPoints[d]; ++p) {
2609       const PetscInt  point    = subpoints[d][p];
2610       const PetscInt  subpoint = firstSubPoint[d] + p;
2611       const PetscInt *cone;
2612       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2613 
2614       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2615       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2616       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2617       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2618         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2619         if (subc >= 0) coneNew[coneSizeNew++] = firstSubPoint[d-1] + subc;
2620       }
2621       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2622       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2623     }
2624   }
2625   ierr = PetscFree(coneNew);CHKERRQ(ierr);
2626   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2627   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2628   /* Build coordinates */
2629   {
2630     PetscSection coordSection, subCoordSection;
2631     Vec          coordinates, subCoordinates;
2632     PetscScalar *coords, *subCoords;
2633     PetscInt     numComp, coordSize;
2634 
2635     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2636     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2637     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2638     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2639     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2640     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2641     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2642     for (v = 0; v < numSubPoints[0]; ++v) {
2643       const PetscInt vertex    = subpoints[0][v];
2644       const PetscInt subvertex = firstSubPoint[0]+v;
2645       PetscInt       dof;
2646 
2647       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2648       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2649       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2650     }
2651     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2652     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2653     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2654     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2655     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2656     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2657     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2658     for (v = 0; v < numSubPoints[0]; ++v) {
2659       const PetscInt vertex    = subpoints[0][v];
2660       const PetscInt subvertex = firstSubPoint[0]+v;
2661       PetscInt dof, off, sdof, soff, d;
2662 
2663       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2664       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2665       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2666       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2667       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2668       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2669     }
2670     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2671     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2672     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2673     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2674   }
2675   /* Cleanup */
2676   for (d = 0; d <= dim; ++d) {
2677     ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2678     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
2679   }
2680   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
2681   PetscFunctionReturn(0);
2682 }
2683 
2684 #undef __FUNCT__
2685 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh"
2686 /*
2687   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
2688 
2689   Input Parameters:
2690 + dm          - The original mesh
2691 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
2692 . label       - A label name, or PETSC_NULL
2693 - value  - A label value
2694 
2695   Output Parameter:
2696 . subdm - The surface mesh
2697 
2698   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
2699 
2700   Level: developer
2701 
2702 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
2703 */
2704 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
2705 {
2706   PetscInt       dim, depth;
2707   PetscErrorCode ierr;
2708 
2709   PetscFunctionBegin;
2710   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2711   PetscValidPointer(subdm, 5);
2712   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2713   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2714   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
2715   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
2716   ierr = DMPlexSetDimension(*subdm, dim-1);CHKERRQ(ierr);
2717   if (depth == dim) {
2718     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
2719   } else {
2720     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
2721   }
2722   PetscFunctionReturn(0);
2723 }
2724 
2725 #undef __FUNCT__
2726 #define __FUNCT__ "DMPlexGetSubpointMap"
2727 /*@
2728   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
2729 
2730   Input Parameter:
2731 . dm - The submesh DM
2732 
2733   Output Parameter:
2734 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
2735 
2736   Level: developer
2737 
2738 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
2739 @*/
2740 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
2741 {
2742   DM_Plex *mesh = (DM_Plex*) dm->data;
2743 
2744   PetscFunctionBegin;
2745   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2746   PetscValidPointer(subpointMap, 2);
2747   *subpointMap = mesh->subpointMap;
2748   PetscFunctionReturn(0);
2749 }
2750 
2751 #undef __FUNCT__
2752 #define __FUNCT__ "DMPlexSetSubpointMap"
2753 /* Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh() */
2754 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
2755 {
2756   DM_Plex       *mesh = (DM_Plex *) dm->data;
2757   DMLabel        tmp;
2758   PetscErrorCode ierr;
2759 
2760   PetscFunctionBegin;
2761   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2762   tmp  = mesh->subpointMap;
2763   mesh->subpointMap = subpointMap;
2764   ++mesh->subpointMap->refct;
2765   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
2766   PetscFunctionReturn(0);
2767 }
2768 
2769 #undef __FUNCT__
2770 #define __FUNCT__ "DMPlexCreateSubpointIS"
2771 /*@
2772   DMPlexCreateSubpointIS - Creates an IS covering the entire subdm chart with the original points as data
2773 
2774   Input Parameter:
2775 . dm - The submesh DM
2776 
2777   Output Parameter:
2778 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
2779 
2780   Note: This is IS is guaranteed to be sorted by the construction of the submesh
2781 
2782   Level: developer
2783 
2784 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
2785 @*/
2786 PetscErrorCode DMPlexCreateSubpointIS(DM dm, IS *subpointIS)
2787 {
2788   MPI_Comm        comm;
2789   DMLabel         subpointMap;
2790   IS              is;
2791   const PetscInt *opoints;
2792   PetscInt       *points, *depths;
2793   PetscInt        depth, depStart, depEnd, d, pStart, pEnd, p, n, off;
2794   PetscErrorCode  ierr;
2795 
2796   PetscFunctionBegin;
2797   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2798   PetscValidPointer(subpointIS, 2);
2799   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2800   *subpointIS = NULL;
2801   ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
2802   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2803   if (subpointMap && depth >= 0) {
2804     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2805     if (pStart) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
2806     ierr = DMGetWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
2807     depths[0] = depth;
2808     depths[1] = 0;
2809     for(d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
2810     ierr = PetscMalloc(pEnd * sizeof(PetscInt), &points);CHKERRQ(ierr);
2811     for(d = 0, off = 0; d <= depth; ++d) {
2812       const PetscInt dep = depths[d];
2813 
2814       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
2815       ierr = DMLabelGetStratumSize(subpointMap, dep, &n);CHKERRQ(ierr);
2816       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
2817         if (n != depEnd-depStart) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
2818       } else {
2819         if (!n) {
2820           if (d == 0) {
2821             /* Missing cells */
2822             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
2823           } else {
2824             /* Missing faces */
2825             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
2826           }
2827         }
2828       }
2829       if (n) {
2830         ierr = DMLabelGetStratumIS(subpointMap, dep, &is);CHKERRQ(ierr);
2831         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
2832         for(p = 0; p < n; ++p, ++off) points[off] = opoints[p];
2833         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
2834         ierr = ISDestroy(&is);CHKERRQ(ierr);
2835       }
2836     }
2837     ierr = DMRestoreWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
2838     if (off != pEnd) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
2839     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
2840   }
2841   PetscFunctionReturn(0);
2842 }
2843