xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 36dbac8238fc7b848ba9e501c70f7583e8f6918f)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petscsf.h>
3 
4 #undef __FUNCT__
5 #define __FUNCT__ "DMPlexMarkBoundaryFaces"
6 /*@
7   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
8 
9   Not Collective
10 
11   Input Parameter:
12 . dm - The original DM
13 
14   Output Parameter:
15 . label - The DMLabel marking boundary faces with value 1
16 
17   Level: developer
18 
19 .seealso: DMLabelCreate(), DMPlexCreateLabel()
20 @*/
21 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, DMLabel label)
22 {
23   PetscInt       fStart, fEnd, f;
24   PetscErrorCode ierr;
25 
26   PetscFunctionBegin;
27   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
28   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
29   for (f = fStart; f < fEnd; ++f) {
30     PetscInt supportSize;
31 
32     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
33     if (supportSize == 1) {
34       ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);
35     }
36   }
37   PetscFunctionReturn(0);
38 }
39 
40 #undef __FUNCT__
41 #define __FUNCT__ "DMPlexLabelComplete"
42 /*@
43   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
44 
45   Input Parameters:
46 + dm - The DM
47 - label - A DMLabel marking the surface points
48 
49   Output Parameter:
50 . label - A DMLabel marking all surface points in the transitive closure
51 
52   Level: developer
53 
54 .seealso: DMPlexLabelCohesiveComplete()
55 @*/
56 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
57 {
58   IS              valueIS;
59   const PetscInt *values;
60   PetscInt        numValues, v;
61   PetscErrorCode  ierr;
62 
63   PetscFunctionBegin;
64   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
65   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
66   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
67   for (v = 0; v < numValues; ++v) {
68     IS              pointIS;
69     const PetscInt *points;
70     PetscInt        numPoints, p;
71 
72     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
73     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
74     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
75     for (p = 0; p < numPoints; ++p) {
76       PetscInt *closure = NULL;
77       PetscInt  closureSize, c;
78 
79       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
80       for (c = 0; c < closureSize*2; c += 2) {
81         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
82       }
83       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
84     }
85     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
86     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
87   }
88   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
89   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
90   PetscFunctionReturn(0);
91 }
92 
93 #undef __FUNCT__
94 #define __FUNCT__ "DMPlexShiftPoint_Internal"
95 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthEnd[], PetscInt depthShift[])
96 {
97   if (depth < 0) return p;
98   /* Cells    */ if (p < depthEnd[depth])   return p;
99   /* Vertices */ if (p < depthEnd[0])       return p + depthShift[depth];
100   /* Faces    */ if (p < depthEnd[depth-1]) return p + depthShift[depth] + depthShift[0];
101   /* Edges    */                            return p + depthShift[depth] + depthShift[0] + depthShift[depth-1];
102 }
103 
104 #undef __FUNCT__
105 #define __FUNCT__ "DMPlexShiftSizes_Internal"
106 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
107 {
108   PetscInt      *depthEnd;
109   PetscInt       depth = 0, d, pStart, pEnd, p;
110   PetscErrorCode ierr;
111 
112   PetscFunctionBegin;
113   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
114   if (depth < 0) PetscFunctionReturn(0);
115   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
116   /* Step 1: Expand chart */
117   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
118   for (d = 0; d <= depth; ++d) {
119     pEnd += depthShift[d];
120     ierr  = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
121   }
122   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
123   /* Step 2: Set cone and support sizes */
124   for (d = 0; d <= depth; ++d) {
125     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
126     for (p = pStart; p < pEnd; ++p) {
127       PetscInt newp = DMPlexShiftPoint_Internal(p, depth, depthEnd, depthShift);
128       PetscInt size;
129 
130       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
131       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
132       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
133       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
134     }
135   }
136   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
137   PetscFunctionReturn(0);
138 }
139 
140 #undef __FUNCT__
141 #define __FUNCT__ "DMPlexShiftPoints_Internal"
142 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
143 {
144   PetscInt      *depthEnd, *newpoints;
145   PetscInt       depth = 0, d, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
146   PetscErrorCode ierr;
147 
148   PetscFunctionBegin;
149   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
150   if (depth < 0) PetscFunctionReturn(0);
151   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
152   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
153   ierr = PetscMalloc2(depth+1,PetscInt,&depthEnd,PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),PetscInt,&newpoints);CHKERRQ(ierr);
154   for (d = 0; d <= depth; ++d) {
155     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
156   }
157   /* Step 5: Set cones and supports */
158   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
159   for (p = pStart; p < pEnd; ++p) {
160     const PetscInt *points = NULL, *orientations = NULL;
161     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthEnd, depthShift);
162 
163     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
164     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
165     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
166     for (i = 0; i < size; ++i) {
167       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthEnd, depthShift);
168     }
169     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
170     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
171     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
172     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
173     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
174     for (i = 0; i < size; ++i) {
175       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthEnd, depthShift);
176     }
177     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
178     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
179   }
180   ierr = PetscFree2(depthEnd,newpoints);CHKERRQ(ierr);
181   PetscFunctionReturn(0);
182 }
183 
184 #undef __FUNCT__
185 #define __FUNCT__ "DMPlexShiftCoordinates_Internal"
186 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
187 {
188   PetscSection   coordSection, newCoordSection;
189   Vec            coordinates, newCoordinates;
190   PetscScalar   *coords, *newCoords;
191   PetscInt      *depthEnd, coordSize;
192   PetscInt       dim, depth = 0, d, vStart, vEnd, vStartNew, vEndNew, v;
193   PetscErrorCode ierr;
194 
195   PetscFunctionBegin;
196   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
197   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
198   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
199   for (d = 0; d <= depth; ++d) {
200     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
201   }
202   /* Step 8: Convert coordinates */
203   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
204   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
205   ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
206   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
207   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
208   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
209   ierr = PetscSectionSetChart(newCoordSection, vStartNew, vEndNew);CHKERRQ(ierr);
210   for (v = vStartNew; v < vEndNew; ++v) {
211     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
212     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
213   }
214   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
215   ierr = DMPlexSetCoordinateSection(dmNew, newCoordSection);CHKERRQ(ierr);
216   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
217   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &newCoordinates);CHKERRQ(ierr);
218   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
219   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
220   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
221   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
222   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
223   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
224   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
225   for (v = vStart; v < vEnd; ++v) {
226     PetscInt dof, off, noff, d;
227 
228     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
229     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
230     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthEnd, depthShift), &noff);CHKERRQ(ierr);
231     for (d = 0; d < dof; ++d) {
232       newCoords[noff+d] = coords[off+d];
233     }
234   }
235   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
236   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
237   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
238   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
239   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
240   PetscFunctionReturn(0);
241 }
242 
243 #undef __FUNCT__
244 #define __FUNCT__ "DMPlexShiftSF_Internal"
245 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
246 {
247   PetscInt          *depthEnd;
248   PetscInt           depth = 0, d;
249   PetscSF            sfPoint, sfPointNew;
250   const PetscSFNode *remotePoints;
251   PetscSFNode       *gremotePoints;
252   const PetscInt    *localPoints;
253   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
254   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
255   PetscErrorCode     ierr;
256 
257   PetscFunctionBegin;
258   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
259   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
260   for (d = 0; d <= depth; ++d) {
261     totShift += depthShift[d];
262     ierr      = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
263   }
264   /* Step 9: Convert pointSF */
265   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
266   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
267   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
268   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
269   if (numRoots >= 0) {
270     ierr = PetscMalloc2(numRoots,PetscInt,&newLocation,pEnd-pStart,PetscInt,&newRemoteLocation);CHKERRQ(ierr);
271     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthEnd, depthShift);
272     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
273     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
274     ierr = PetscMalloc(numLeaves * sizeof(PetscInt),    &glocalPoints);CHKERRQ(ierr);
275     ierr = PetscMalloc(numLeaves * sizeof(PetscSFNode), &gremotePoints);CHKERRQ(ierr);
276     for (l = 0; l < numLeaves; ++l) {
277       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthEnd, depthShift);
278       gremotePoints[l].rank  = remotePoints[l].rank;
279       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
280     }
281     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
282     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
283   }
284   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
285   PetscFunctionReturn(0);
286 }
287 
288 #undef __FUNCT__
289 #define __FUNCT__ "DMPlexShiftLabels_Internal"
290 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
291 {
292   PetscSF            sfPoint;
293   DMLabel            vtkLabel, ghostLabel;
294   PetscInt          *depthEnd;
295   const PetscSFNode *leafRemote;
296   const PetscInt    *leafLocal;
297   PetscInt           depth = 0, d, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
298   PetscMPIInt        rank;
299   PetscErrorCode     ierr;
300 
301   PetscFunctionBegin;
302   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
303   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
304   for (d = 0; d <= depth; ++d) {
305     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
306   }
307   /* Step 10: Convert labels */
308   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
309   for (l = 0; l < numLabels; ++l) {
310     DMLabel         label, newlabel;
311     const char     *lname;
312     PetscBool       isDepth;
313     IS              valueIS;
314     const PetscInt *values;
315     PetscInt        numValues, val;
316 
317     ierr = DMPlexGetLabelName(dm, l, &lname);CHKERRQ(ierr);
318     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
319     if (isDepth) continue;
320     ierr = DMPlexCreateLabel(dmNew, lname);CHKERRQ(ierr);
321     ierr = DMPlexGetLabel(dm, lname, &label);CHKERRQ(ierr);
322     ierr = DMPlexGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
323     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
324     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
325     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
326     for (val = 0; val < numValues; ++val) {
327       IS              pointIS;
328       const PetscInt *points;
329       PetscInt        numPoints, p;
330 
331       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
332       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
333       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
334       for (p = 0; p < numPoints; ++p) {
335         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthEnd, depthShift);
336 
337         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
338       }
339       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
340       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
341     }
342     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
343     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
344   }
345   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
346   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
347   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
348   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
349   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
350   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
351   ierr = DMPlexCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
352   ierr = DMPlexCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
353   ierr = DMPlexGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
354   ierr = DMPlexGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
355   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
356     for (; c < leafLocal[l] && c < cEnd; ++c) {
357       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
358     }
359     if (leafLocal[l] >= cEnd) break;
360     if (leafRemote[l].rank == rank) {
361       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
362     } else {
363       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
364     }
365   }
366   for (; c < cEnd; ++c) {
367     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
368   }
369   if (0) {
370     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
371     ierr = DMLabelView(vtkLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
372     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
373   }
374   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
375   for (f = fStart; f < fEnd; ++f) {
376     PetscInt numCells;
377 
378     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
379     if (numCells < 2) {
380       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
381     } else {
382       const PetscInt *cells = NULL;
383       PetscInt        vA, vB;
384 
385       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
386       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
387       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
388       if (!vA && !vB) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
389     }
390   }
391   if (0) {
392     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
393     ierr = DMLabelView(ghostLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
394     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
395   }
396   PetscFunctionReturn(0);
397 }
398 
399 #undef __FUNCT__
400 #define __FUNCT__ "DMPlexConstructGhostCells_Internal"
401 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
402 {
403   IS              valueIS;
404   const PetscInt *values;
405   PetscInt       *depthShift;
406   PetscInt        depth = 0, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
407   PetscErrorCode  ierr;
408 
409   PetscFunctionBegin;
410   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
411   /* Count ghost cells */
412   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
413   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
414   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
415   *numGhostCells = 0;
416   for (fs = 0; fs < numFS; ++fs) {
417     IS              faceIS;
418     const PetscInt *faces;
419     PetscInt        numFaces, f, numBdFaces = 0;
420 
421     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
422     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
423     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
424     for (f = 0; f < numFaces; ++f) {
425       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
426     }
427     *numGhostCells += numBdFaces;
428     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
429   }
430   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
431   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthShift);CHKERRQ(ierr);
432   ierr = PetscMemzero(depthShift, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
433   if (depth >= 0) depthShift[depth] = *numGhostCells;
434   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
435   /* Step 3: Set cone/support sizes for new points */
436   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
437   for (c = cEnd; c < cEnd + *numGhostCells; ++c) {
438     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
439   }
440   for (fs = 0; fs < numFS; ++fs) {
441     IS              faceIS;
442     const PetscInt *faces;
443     PetscInt        numFaces, f;
444 
445     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
446     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
447     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
448     for (f = 0; f < numFaces; ++f) {
449       PetscInt size;
450 
451       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
452       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
453       if (size != 1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
454       ierr = DMPlexSetSupportSize(gdm, faces[f] + *numGhostCells, 2);CHKERRQ(ierr);
455     }
456     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
457     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
458   }
459   /* Step 4: Setup ghosted DM */
460   ierr = DMSetUp(gdm);CHKERRQ(ierr);
461   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
462   /* Step 6: Set cones and supports for new points */
463   ghostCell = cEnd;
464   for (fs = 0; fs < numFS; ++fs) {
465     IS              faceIS;
466     const PetscInt *faces;
467     PetscInt        numFaces, f;
468 
469     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
470     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
471     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
472     for (f = 0; f < numFaces; ++f) {
473       PetscInt newFace = faces[f] + *numGhostCells;
474 
475       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
476       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
477       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
478       ++ghostCell;
479     }
480     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
481     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
482   }
483   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
484   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
485   /* Step 7: Stratify */
486   ierr = DMPlexStratify(gdm);CHKERRQ(ierr);
487   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
488   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
489   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
490   ierr = PetscFree(depthShift);CHKERRQ(ierr);
491   PetscFunctionReturn(0);
492 }
493 
494 #undef __FUNCT__
495 #define __FUNCT__ "DMPlexConstructGhostCells"
496 /*@C
497   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
498 
499   Collective on dm
500 
501   Input Parameters:
502 + dm - The original DM
503 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
504 
505   Output Parameters:
506 + numGhostCells - The number of ghost cells added to the DM
507 - dmGhosted - The new DM
508 
509   Note: If no label exists of that name, one will be created marking all boundary faces
510 
511   Level: developer
512 
513 .seealso: DMCreate()
514 @*/
515 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
516 {
517   DM             gdm;
518   DMLabel        label;
519   const char    *name = labelName ? labelName : "Face Sets";
520   PetscInt       dim;
521   PetscErrorCode ierr;
522 
523   PetscFunctionBegin;
524   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
525   PetscValidPointer(numGhostCells, 3);
526   PetscValidPointer(dmGhosted, 4);
527   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
528   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
529   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
530   ierr = DMPlexSetDimension(gdm, dim);CHKERRQ(ierr);
531   ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
532   if (!label) {
533     /* Get label for boundary faces */
534     ierr = DMPlexCreateLabel(dm, name);CHKERRQ(ierr);
535     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
536     ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
537   }
538   ierr = DMPlexConstructGhostCells_Internal(dm, label, numGhostCells, gdm);CHKERRQ(ierr);
539   ierr = DMSetFromOptions(gdm);CHKERRQ(ierr);
540   *dmGhosted = gdm;
541   PetscFunctionReturn(0);
542 }
543 
544 #undef __FUNCT__
545 #define __FUNCT__ "DMPlexConstructCohesiveCells_Internal"
546 /*
547   We are adding two kinds of points here:
548     Replicated: Copies of points which exist in the mesh, such as vertices identified across a fault
549     Hybrid:     Entirely new points, such as cohesive cells
550 */
551 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DM sdm)
552 {
553   MPI_Comm         comm;
554   IS               valueIS;
555   PetscInt         numSP = 0;       /* The number of depths for which we have replicated points */
556   const PetscInt  *values;          /* List of depths for which we have replicated points */
557   IS              *splitIS;
558   IS              *unsplitIS;
559   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
560   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
561   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
562   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
563   const PetscInt **splitPoints;        /* Replicated points for each depth */
564   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
565   PetscSection     coordSection;
566   Vec              coordinates;
567   PetscScalar     *coords;
568   PetscInt         depths[4];          /* Depths in the order that plex points are numbered */
569   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
570   PetscInt        *depthOffset;        /* Prefix sums of depthShift */
571   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
572   PetscInt        *coneNew, *coneONew, *supportNew;
573   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
574   PetscErrorCode   ierr;
575 
576   PetscFunctionBegin;
577   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
578   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
579   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
580   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
581   depths[0] = depth;
582   depths[1] = 0;
583   depths[2] = depth-1;
584   depths[3] = 1;
585   /* Count split points and add cohesive cells */
586   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
587   ierr = PetscMalloc4(depth+1,PetscInt,&depthShift,depth+1,PetscInt,&depthOffset,depth+1,PetscInt,&pMaxNew,depth+1,PetscInt,&numHybridPointsOld);CHKERRQ(ierr);
588   ierr = PetscMalloc7(depth+1,IS,&splitIS,depth+1,IS,&unsplitIS,depth+1,PetscInt,&numSplitPoints,depth+1,PetscInt,&numUnsplitPoints,depth+1,PetscInt,&numHybridPoints,depth+1,const PetscInt*,&splitPoints,depth+1,const PetscInt*,&unsplitPoints);CHKERRQ(ierr);
589   ierr = PetscMemzero(depthShift,  (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
590   ierr = PetscMemzero(depthOffset, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
591   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &numHybridPointsOld[depth] : NULL, depth>1 ? &numHybridPointsOld[depth-1] : NULL, depth>2 ? &numHybridPointsOld[1] : NULL, depth >= 0 ? &numHybridPointsOld[0] : NULL);CHKERRQ(ierr);
592   for (d = 0; d <= depth; ++d) {
593     ierr = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
594     numSplitPoints[d]     = 0;
595     numUnsplitPoints[d]   = 0;
596     numHybridPoints[d]    = 0;
597     numHybridPointsOld[d] = PetscMax(0, numHybridPointsOld[d]);
598     splitPoints[d]        = NULL;
599     unsplitPoints[d]      = NULL;
600     splitIS[d]            = NULL;
601     unsplitIS[d]          = NULL;
602   }
603   if (label) {
604     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
605     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
606     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
607   }
608   for (sp = 0; sp < numSP; ++sp) {
609     const PetscInt dep = values[sp];
610 
611     if ((dep < 0) || (dep > depth)) continue;
612     ierr = DMLabelGetStratumIS(label, dep, &splitIS[dep]);CHKERRQ(ierr);
613     if (splitIS[dep]) {
614       ierr = ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
615       ierr = ISGetIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
616     }
617     ierr = DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]);CHKERRQ(ierr);
618     if (unsplitIS[dep]) {
619       ierr = ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]);CHKERRQ(ierr);
620       ierr = ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);
621     }
622   }
623   /* Calculate number of hybrid points */
624   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
625   for (d = 0; d <= depth; ++d) depthShift[d]          = numSplitPoints[d] + numHybridPoints[d];
626   for (d = 1; d <= depth; ++d) depthOffset[depths[d]] = depthOffset[depths[d-1]] + depthShift[depths[d-1]];
627   for (d = 0; d <= depth; ++d) pMaxNew[d]            += depthOffset[d];
628   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
629   /* Step 3: Set cone/support sizes for new points */
630   for (dep = 0; dep <= depth; ++dep) {
631     for (p = 0; p < numSplitPoints[dep]; ++p) {
632       const PetscInt  oldp   = splitPoints[dep][p];
633       const PetscInt  newp   = oldp + depthOffset[dep];
634       const PetscInt  splitp = p    + pMaxNew[dep];
635       const PetscInt *support;
636       PetscInt        coneSize, supportSize, qf, qn, qp, e;
637 
638       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
639       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
640       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
641       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
642       if (dep == depth-1) {
643         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
644 
645         /* Add cohesive cells, they are prisms */
646         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
647       } else if (dep == 0) {
648         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
649 
650         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
651         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
652           PetscInt val;
653 
654           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
655           if (val == 1) ++qf;
656           if ((val == 1) || (val ==  (shift + 1))) ++qn;
657           if ((val == 1) || (val == -(shift + 1))) ++qp;
658         }
659         /* Split old vertex: Edges into original vertex and new cohesive edge */
660         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
661         /* Split new vertex: Edges into split vertex and new cohesive edge */
662         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
663         /* Add hybrid edge */
664         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
665         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
666       } else if (dep == dim-2) {
667         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
668 
669         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
670         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
671           PetscInt val;
672 
673           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
674           if (val == dim-1) ++qf;
675           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
676           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
677         }
678         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
679         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
680         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
681         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
682         /* Add hybrid face */
683         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
684         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
685       }
686     }
687   }
688   for (dep = 0; dep <= depth; ++dep) {
689     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
690       const PetscInt  oldp   = unsplitPoints[dep][p];
691       const PetscInt  newp   = oldp + depthOffset[dep];
692       const PetscInt *support;
693       PetscInt        coneSize, supportSize, qf, qn, qp, e;
694 
695       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
696       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
697       if (dep == 0) {
698         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
699 
700         /* Unsplit vertex: Edges into original vertex, split edge, and new cohesive edge twice */
701         ierr = DMPlexSetSupportSize(sdm, newp, supportSize+3);CHKERRQ(ierr);
702         /* Add hybrid edge */
703         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
704         ierr = DMPlexSetSupportSize(sdm, hybedge, 1);CHKERRQ(ierr);
705       } else if (dep == dim-2) {
706         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
707 
708         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
709         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
710           PetscInt val;
711 
712           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
713           if (val == dim-1) ++qf;
714           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
715           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
716         }
717         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
718         ierr = DMPlexSetSupportSize(sdm, newp, supportSize+3);CHKERRQ(ierr);
719         /* Add hybrid face */
720         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
721         ierr = DMPlexSetSupportSize(sdm, hybface, 1);CHKERRQ(ierr);
722       }
723     }
724   }
725   /* Step 4: Setup split DM */
726   ierr = DMSetUp(sdm);CHKERRQ(ierr);
727   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
728   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
729   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,PetscInt,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,PetscInt,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),PetscInt,&supportNew);CHKERRQ(ierr);
730   /* Step 6: Set cones and supports for new points */
731   for (dep = 0; dep <= depth; ++dep) {
732     for (p = 0; p < numSplitPoints[dep]; ++p) {
733       const PetscInt  oldp   = splitPoints[dep][p];
734       const PetscInt  newp   = oldp + depthOffset[dep];
735       const PetscInt  splitp = p    + pMaxNew[dep];
736       const PetscInt *cone, *support, *ornt;
737       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
738 
739       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
740       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
741       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
742       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
743       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
744       if (dep == depth-1) {
745         const PetscInt  hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
746         const PetscInt *supportF;
747 
748         /* Split face:       copy in old face to new face to start */
749         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
750         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
751         /* Split old face:   old vertices/edges in cone so no change */
752         /* Split new face:   new vertices/edges in cone */
753         for (q = 0; q < coneSize; ++q) {
754           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
755           if (v < 0) {
756             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
757             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
758             coneNew[2+q] = cone[q] + depthOffset[dep-1];
759           } else {
760             coneNew[2+q] = v + pMaxNew[dep-1];
761           }
762         }
763         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
764         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
765         /* Face support */
766         for (s = 0; s < supportSize; ++s) {
767           PetscInt val;
768 
769           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
770           if (val < 0) {
771             /* Split old face:   Replace negative side cell with cohesive cell */
772              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
773           } else {
774             /* Split new face:   Replace positive side cell with cohesive cell */
775             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
776             /* Get orientation for cohesive face */
777             {
778               const PetscInt *ncone, *nconeO;
779               PetscInt        nconeSize, nc;
780 
781               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
782               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
783               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
784               for (nc = 0; nc < nconeSize; ++nc) {
785                 if (ncone[nc] == oldp) {
786                   coneONew[0] = nconeO[nc];
787                   break;
788                 }
789               }
790               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
791             }
792           }
793         }
794         /* Cohesive cell:    Old and new split face, then new cohesive faces */
795         coneNew[0]  = newp;   /* Extracted negative side orientation above */
796         coneNew[1]  = splitp;
797         coneONew[1] = coneONew[0];
798         for (q = 0; q < coneSize; ++q) {
799           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
800           if (v < 0) {
801             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
802             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
803             coneONew[2+q] = 0;
804           } else {
805             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
806           }
807           coneONew[2+q] = 0;
808         }
809         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
810         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
811       } else if (dep == 0) {
812         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
813 
814         /* Split old vertex: Edges in old split faces and new cohesive edge */
815         for (e = 0, qn = 0; e < supportSize; ++e) {
816           PetscInt val;
817 
818           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
819           if ((val == 1) || (val == (shift + 1))) {
820             supportNew[qn++] = support[e] + depthOffset[dep+1];
821           }
822         }
823         supportNew[qn] = hybedge;
824         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
825         /* Split new vertex: Edges in new split faces and new cohesive edge */
826         for (e = 0, qp = 0; e < supportSize; ++e) {
827           PetscInt val, edge;
828 
829           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
830           if (val == 1) {
831             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
832             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
833             supportNew[qp++] = edge + pMaxNew[dep+1];
834           } else if (val == -(shift + 1)) {
835             supportNew[qp++] = support[e] + depthOffset[dep+1];
836           }
837         }
838         supportNew[qp] = hybedge;
839         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
840         /* Hybrid edge:    Old and new split vertex */
841         coneNew[0] = newp;
842         coneNew[1] = splitp;
843         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
844         for (e = 0, qf = 0; e < supportSize; ++e) {
845           PetscInt val, edge;
846 
847           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
848           if (val == 1) {
849             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
850             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
851             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
852           }
853         }
854         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
855       } else if (dep == dim-2) {
856         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
857 
858         /* Split old edge:   old vertices in cone so no change */
859         /* Split new edge:   new vertices in cone */
860         for (q = 0; q < coneSize; ++q) {
861           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
862 
863           coneNew[q] = v + pMaxNew[dep-1];
864         }
865         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
866         /* Split old edge: Faces in positive side cells and old split faces */
867         for (e = 0, q = 0; e < supportSize; ++e) {
868           PetscInt val;
869 
870           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
871           if (val == dim-1) {
872             supportNew[q++] = support[e] + depthOffset[dep+1];
873           } else if (val == (shift + dim-1)) {
874             supportNew[q++] = support[e] + depthOffset[dep+1];
875           }
876         }
877         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
878         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
879         /* Split new edge: Faces in negative side cells and new split faces */
880         for (e = 0, q = 0; e < supportSize; ++e) {
881           PetscInt val, face;
882 
883           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
884           if (val == dim-1) {
885             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
886             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
887             supportNew[q++] = face + pMaxNew[dep+1];
888           } else if (val == -(shift + dim-1)) {
889             supportNew[q++] = support[e] + depthOffset[dep+1];
890           }
891         }
892         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
893         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
894         /* Hybrid face */
895         coneNew[0] = newp;
896         coneNew[1] = splitp;
897         for (v = 0; v < coneSize; ++v) {
898           PetscInt vertex;
899           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
900           if (vertex < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not a split vertex", support[e]);
901           coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
902         }
903         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
904         for (e = 0, qf = 0; e < supportSize; ++e) {
905           PetscInt val, face;
906 
907           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
908           if (val == dim-1) {
909             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
910             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
911             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
912           }
913         }
914         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
915       }
916     }
917   }
918   for (dep = 0; dep <= depth; ++dep) {
919     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
920       const PetscInt  oldp   = unsplitPoints[dep][p];
921       const PetscInt  newp   = oldp + depthOffset[dep];
922       const PetscInt *cone, *support, *ornt;
923       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, s;
924 
925       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
926       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
927       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
928       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
929       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
930       if (dep == 0) {
931         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
932 
933         /* Unsplit vertex */
934         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
935         for (s = 0, q = 0; s < supportSize; ++s) {
936           supportNew[q++] = support[s] + depthOffset[dep+1];
937           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
938           if (e >= 0) {
939             supportNew[q++] = e + pMaxNew[dep+1];
940           }
941         }
942         supportNew[q++] = hybedge;
943         supportNew[q++] = hybedge;
944         if (q != supportSizeNew) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Support size %d != %d for vertex %d", q, supportSizeNew, newp);
945         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
946         /* Hybrid edge */
947         coneNew[0] = newp;
948         coneNew[1] = newp;
949         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
950         for (e = 0, qf = 0; e < supportSize; ++e) {
951           PetscInt val, edge;
952 
953           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
954           if (val == 1) {
955             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
956             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
957             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
958           }
959         }
960         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
961       }
962     }
963   }
964   /* Step 6b: Replace split points in negative side cones */
965   for (sp = 0; sp < numSP; ++sp) {
966     PetscInt        dep = values[sp];
967     IS              pIS;
968     PetscInt        numPoints;
969     const PetscInt *points;
970 
971     if (dep >= 0) continue;
972     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
973     if (!pIS) continue;
974     dep  = -dep - shift;
975     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
976     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
977     for (p = 0; p < numPoints; ++p) {
978       const PetscInt  oldp = points[p];
979       const PetscInt  newp = depthOffset[dep] + oldp;
980       const PetscInt *cone;
981       PetscInt        coneSize, c;
982       /* PetscBool       replaced = PETSC_FALSE; */
983 
984       /* Negative edge: replace split vertex */
985       /* Negative cell: replace split face */
986       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
987       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
988       for (c = 0; c < coneSize; ++c) {
989         const PetscInt coldp = cone[c] - depthOffset[dep-1];
990         PetscInt       csplitp, cp, val;
991 
992         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
993         if (val == dep-1) {
994           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
995           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
996           csplitp  = pMaxNew[dep-1] + cp;
997           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
998           /* replaced = PETSC_TRUE; */
999         }
1000       }
1001       /* Cells with only a vertex or edge on the submesh have no replacement */
1002       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1003     }
1004     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
1005     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
1006   }
1007   /* Step 7: Stratify */
1008   ierr = DMPlexStratify(sdm);CHKERRQ(ierr);
1009   /* Step 8: Coordinates */
1010   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1011   ierr = DMPlexGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
1012   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
1013   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1014   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1015     const PetscInt newp   = depthOffset[0] + splitPoints[0][v];
1016     const PetscInt splitp = pMaxNew[0] + v;
1017     PetscInt       dof, off, soff, d;
1018 
1019     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
1020     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
1021     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
1022     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1023   }
1024   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1025   /* Step 9: SF, if I can figure this out we can split the mesh in parallel */
1026   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1027   /* Step 10: Labels */
1028   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1029   ierr = DMPlexGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
1030   for (dep = 0; dep <= depth; ++dep) {
1031     for (p = 0; p < numSplitPoints[dep]; ++p) {
1032       const PetscInt newp   = depthOffset[dep] + splitPoints[dep][p];
1033       const PetscInt splitp = pMaxNew[dep] + p;
1034       PetscInt       l;
1035 
1036       for (l = 0; l < numLabels; ++l) {
1037         DMLabel     mlabel;
1038         const char *lname;
1039         PetscInt    val;
1040         PetscBool   isDepth;
1041 
1042         ierr = DMPlexGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
1043         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
1044         if (isDepth) continue;
1045         ierr = DMPlexGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
1046         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
1047         if (val >= 0) {
1048           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
1049 #if 0
1050           /* Do not put cohesive edges into the label */
1051           if (dep == 0) {
1052             const PetscInt cedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1053             ierr = DMLabelSetValue(mlabel, cedge, val);CHKERRQ(ierr);
1054           } else if (dep == dim-2) {
1055             const PetscInt cface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1056             ierr = DMLabelSetValue(mlabel, cface, val);CHKERRQ(ierr);
1057           }
1058           /* Do not put cohesive faces into the label */
1059 #endif
1060         }
1061       }
1062     }
1063   }
1064   for (sp = 0; sp < numSP; ++sp) {
1065     const PetscInt dep = values[sp];
1066 
1067     if ((dep < 0) || (dep > depth)) continue;
1068     if (splitIS[dep]) {ierr = ISRestoreIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
1069     ierr = ISDestroy(&splitIS[dep]);CHKERRQ(ierr);
1070     if (unsplitIS[dep]) {ierr = ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);}
1071     ierr = ISDestroy(&unsplitIS[dep]);CHKERRQ(ierr);
1072   }
1073   if (label) {
1074     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1075     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1076   }
1077   for (d = 0; d <= depth; ++d) {
1078     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
1079     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1080   }
1081   ierr = DMPlexSetHybridBounds(sdm, depth >= 0 ? pMaxNew[depth] : PETSC_DETERMINE, depth>1 ? pMaxNew[depth-1] : PETSC_DETERMINE, depth>2 ? pMaxNew[1] : PETSC_DETERMINE, depth >= 0 ? pMaxNew[0] : PETSC_DETERMINE);CHKERRQ(ierr);
1082   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
1083   ierr = PetscFree4(depthShift, depthOffset, pMaxNew, numHybridPointsOld);CHKERRQ(ierr);
1084   ierr = PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints);CHKERRQ(ierr);
1085   PetscFunctionReturn(0);
1086 }
1087 
1088 #undef __FUNCT__
1089 #define __FUNCT__ "DMPlexConstructCohesiveCells"
1090 /*@C
1091   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1092 
1093   Collective on dm
1094 
1095   Input Parameters:
1096 + dm - The original DM
1097 - label - The label specifying the boundary faces (this could be auto-generated)
1098 
1099   Output Parameters:
1100 - dmSplit - The new DM
1101 
1102   Level: developer
1103 
1104 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
1105 @*/
1106 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DM *dmSplit)
1107 {
1108   DM             sdm;
1109   PetscInt       dim;
1110   PetscErrorCode ierr;
1111 
1112   PetscFunctionBegin;
1113   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1114   PetscValidPointer(dmSplit, 3);
1115   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
1116   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
1117   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1118   ierr = DMPlexSetDimension(sdm, dim);CHKERRQ(ierr);
1119   switch (dim) {
1120   case 2:
1121   case 3:
1122     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, sdm);CHKERRQ(ierr);
1123     break;
1124   default:
1125     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1126   }
1127   *dmSplit = sdm;
1128   PetscFunctionReturn(0);
1129 }
1130 
1131 #undef __FUNCT__
1132 #define __FUNCT__ "DMPlexLabelCohesiveComplete"
1133 /*@
1134   DMPlexLabelCohesiveComplete - Starting with a label marking vertices on an internal surface, we add all other mesh pieces
1135   to complete the surface
1136 
1137   Input Parameters:
1138 + dm - The DM
1139 . label - A DMLabel marking the surface vertices
1140 . flip  - Flag to flip the submesh normal and replace points on the other side
1141 - subdm - The subDM associated with the label, or NULL
1142 
1143   Output Parameter:
1144 . label - A DMLabel marking all surface points
1145 
1146   Level: developer
1147 
1148 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1149 @*/
1150 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, PetscBool flip, DM subdm)
1151 {
1152   DMLabel         depthLabel;
1153   IS              dimIS, subpointIS, facePosIS, faceNegIS;
1154   const PetscInt *points, *subpoints;
1155   const PetscInt  rev   = flip ? -1 : 1;
1156   PetscInt        shift = 100, shift2 = 200, dim, dep, cStart, cEnd, numPoints, numSubpoints, p, val;
1157   PetscErrorCode  ierr;
1158 
1159   PetscFunctionBegin;
1160   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1161   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1162   if (subdm) {
1163     ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1164     if (subpointIS) {
1165       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1166       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1167     }
1168   }
1169   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1170   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1171   if (!dimIS) PetscFunctionReturn(0);
1172   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1173   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1174   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1175     const PetscInt *support;
1176     PetscInt        supportSize, s;
1177 
1178     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1179     if (supportSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports", points[p], supportSize);
1180     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1181     for (s = 0; s < supportSize; ++s) {
1182       const PetscInt *cone, *ornt;
1183       PetscInt        coneSize, c;
1184       PetscBool       pos = PETSC_TRUE;
1185 
1186       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1187       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1188       ierr = DMPlexGetConeOrientation(dm, support[s], &ornt);CHKERRQ(ierr);
1189       for (c = 0; c < coneSize; ++c) {
1190         if (cone[c] == points[p]) {
1191           PetscInt o = ornt[c];
1192 
1193           if (subdm) {
1194             const PetscInt *subcone, *subornt;
1195             PetscInt        subpoint, subface, subconeSize, sc;
1196 
1197             ierr = PetscFindInt(support[s], numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1198             ierr = PetscFindInt(points[p],  numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1199             ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1200             ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1201             ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1202             for (sc = 0; sc < subconeSize; ++sc) {
1203               if (subcone[sc] == subface) {
1204                 o = subornt[0];
1205                 break;
1206               }
1207             }
1208             if (sc >= subconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find point %d in cone for subpoint %d", points[p], subpoint);
1209           }
1210           if (o >= 0) {
1211             ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);
1212             pos  = rev > 0 ? PETSC_TRUE : PETSC_FALSE;
1213           } else {
1214             ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);
1215             pos  = rev > 0 ? PETSC_FALSE : PETSC_TRUE;
1216           }
1217           break;
1218         }
1219       }
1220       if (c == coneSize) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell split face %d support does not have it in the cone", points[p]);
1221       /* Put faces touching the fault in the label */
1222       for (c = 0; c < coneSize; ++c) {
1223         const PetscInt point = cone[c];
1224 
1225         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1226         if (val == -1) {
1227           PetscInt *closure = NULL;
1228           PetscInt  closureSize, cl;
1229 
1230           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1231           for (cl = 0; cl < closureSize*2; cl += 2) {
1232             const PetscInt clp = closure[cl];
1233 
1234             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1235             if ((val >= 0) && (val < dim-1)) {
1236               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1237               break;
1238             }
1239           }
1240           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1241         }
1242       }
1243     }
1244   }
1245   if (subdm) {
1246     if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1247     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1248   }
1249   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1250   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1251   /* Search for other cells/faces/edges connected to the fault by a vertex */
1252   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1253   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1254   if (!dimIS) PetscFunctionReturn(0);
1255   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1256   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1257   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1258     PetscInt *star = NULL;
1259     PetscInt  starSize, s;
1260     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1261 
1262     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1263     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1264     while (again) {
1265       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1266       again = 0;
1267       for (s = 0; s < starSize*2; s += 2) {
1268         const PetscInt  point = star[s];
1269         const PetscInt *cone;
1270         PetscInt        coneSize, c;
1271 
1272         if ((point < cStart) || (point >= cEnd)) continue;
1273         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1274         if (val != -1) continue;
1275         again = again == 1 ? 1 : 2;
1276         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1277         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1278         for (c = 0; c < coneSize; ++c) {
1279           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1280           if (val != -1) {
1281             const PetscInt *ccone;
1282             PetscInt        cconeSize, cc, side;
1283 
1284             if (abs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1285             if (val > 0) side =  1;
1286             else         side = -1;
1287             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1288             /* Mark cell faces which touch the fault */
1289             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1290             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1291             for (cc = 0; cc < cconeSize; ++cc) {
1292               PetscInt *closure = NULL;
1293               PetscInt  closureSize, cl;
1294 
1295               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1296               if (val != -1) continue;
1297               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1298               for (cl = 0; cl < closureSize*2; cl += 2) {
1299                 const PetscInt clp = closure[cl];
1300 
1301                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1302                 if (val == -1) continue;
1303                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1304                 break;
1305               }
1306               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1307             }
1308             again = 1;
1309             break;
1310           }
1311         }
1312       }
1313     }
1314     /* Classify the rest by cell membership */
1315     for (s = 0; s < starSize*2; s += 2) {
1316       const PetscInt point = star[s];
1317 
1318       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1319       if (val == -1) {
1320         PetscInt  *sstar = NULL;
1321         PetscInt   sstarSize, ss;
1322         PetscBool  marked = PETSC_FALSE;
1323 
1324         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1325         for (ss = 0; ss < sstarSize*2; ss += 2) {
1326           const PetscInt spoint = sstar[ss];
1327 
1328           if ((spoint < cStart) || (spoint >= cEnd)) continue;
1329           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1330           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1331           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1332           if (val > 0) {
1333             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1334           } else {
1335             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1336           }
1337           marked = PETSC_TRUE;
1338           break;
1339         }
1340         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1341         if (!marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1342       }
1343     }
1344     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1345   }
1346   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1347   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1348   /* If any faces touching the fault divide cells on either side, split them */
1349   ierr = DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS);CHKERRQ(ierr);
1350   ierr = DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS);CHKERRQ(ierr);
1351   ierr = ISExpand(facePosIS, faceNegIS, &dimIS);CHKERRQ(ierr);
1352   ierr = ISDestroy(&facePosIS);CHKERRQ(ierr);
1353   ierr = ISDestroy(&faceNegIS);CHKERRQ(ierr);
1354   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1355   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1356   for (p = 0; p < numPoints; ++p) {
1357     const PetscInt  point = points[p];
1358     const PetscInt *support;
1359     PetscInt        supportSize, valA, valB;
1360 
1361     ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1362     if (supportSize != 2) continue;
1363     ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1364     ierr = DMLabelGetValue(label, support[0], &valA);CHKERRQ(ierr);
1365     ierr = DMLabelGetValue(label, support[1], &valB);CHKERRQ(ierr);
1366     if ((valA == -1) || (valB == -1)) continue;
1367     if (valA*valB > 0) continue;
1368     /* Split the face */
1369     ierr = DMLabelGetValue(label, point, &valA);CHKERRQ(ierr);
1370     ierr = DMLabelClearValue(label, point, valA);CHKERRQ(ierr);
1371     ierr = DMLabelSetValue(label, point, dim-1);CHKERRQ(ierr);
1372     /* Label its closure as unsplit */
1373     {
1374       PetscInt *closure = NULL;
1375       PetscInt  closureSize, cl;
1376 
1377       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1378       for (cl = 0; cl < closureSize*2; cl += 2) {
1379         ierr = DMLabelGetValue(label, closure[cl], &valA);CHKERRQ(ierr);
1380         if (valA != -1) continue;
1381         ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1382         ierr = DMLabelSetValue(label, closure[cl], shift2+dep);CHKERRQ(ierr);
1383       }
1384       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1385     }
1386   }
1387   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1388   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1389   PetscFunctionReturn(0);
1390 }
1391 
1392 #undef __FUNCT__
1393 #define __FUNCT__ "DMPlexCreateHybridMesh"
1394 /*@C
1395   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
1396 
1397   Collective on dm
1398 
1399   Input Parameters:
1400 + dm - The original DM
1401 - labelName - The label specifying the interface vertices
1402 
1403   Output Parameters:
1404 + hybridLabel - The label fully marking the interface
1405 - dmHybrid - The new DM
1406 
1407   Level: developer
1408 
1409 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMCreate()
1410 @*/
1411 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel *hybridLabel, DM *dmHybrid)
1412 {
1413   DM             idm;
1414   DMLabel        subpointMap, hlabel;
1415   PetscInt       dim;
1416   PetscErrorCode ierr;
1417 
1418   PetscFunctionBegin;
1419   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1420   if (hybridLabel) PetscValidPointer(hybridLabel, 3);
1421   PetscValidPointer(dmHybrid, 4);
1422   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1423   ierr = DMPlexCreateSubmesh(dm, label, 1, &idm);CHKERRQ(ierr);
1424   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
1425   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
1426   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
1427   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
1428   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, PETSC_FALSE, idm);CHKERRQ(ierr);
1429   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1430   ierr = DMPlexConstructCohesiveCells(dm, hlabel, dmHybrid);CHKERRQ(ierr);
1431   if (hybridLabel) *hybridLabel = hlabel;
1432   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
1433   PetscFunctionReturn(0);
1434 }
1435 
1436 #undef __FUNCT__
1437 #define __FUNCT__ "DMPlexMarkSubmesh_Uninterpolated"
1438 /* Here we need the explicit assumption that:
1439 
1440      For any marked cell, the marked vertices constitute a single face
1441 */
1442 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
1443 {
1444   IS               subvertexIS = NULL;
1445   const PetscInt  *subvertices;
1446   PetscInt        *pStart, *pEnd, *pMax, pSize;
1447   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
1448   PetscErrorCode   ierr;
1449 
1450   PetscFunctionBegin;
1451   *numFaces = 0;
1452   *nFV      = 0;
1453   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1454   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1455   pSize = PetscMax(depth, dim) + 1;
1456   ierr = PetscMalloc3(pSize,PetscInt,&pStart,pSize,PetscInt,&pEnd,pSize,PetscInt,&pMax);CHKERRQ(ierr);
1457   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1458   for (d = 0; d <= depth; ++d) {
1459     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1460     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1461   }
1462   /* Loop over initial vertices and mark all faces in the collective star() */
1463   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
1464   if (subvertexIS) {
1465     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1466     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1467   }
1468   for (v = 0; v < numSubVerticesInitial; ++v) {
1469     const PetscInt vertex = subvertices[v];
1470     PetscInt      *star   = NULL;
1471     PetscInt       starSize, s, numCells = 0, c;
1472 
1473     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1474     for (s = 0; s < starSize*2; s += 2) {
1475       const PetscInt point = star[s];
1476       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
1477     }
1478     for (c = 0; c < numCells; ++c) {
1479       const PetscInt cell    = star[c];
1480       PetscInt      *closure = NULL;
1481       PetscInt       closureSize, cl;
1482       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
1483 
1484       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
1485       if (cellLoc == 2) continue;
1486       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
1487       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1488       for (cl = 0; cl < closureSize*2; cl += 2) {
1489         const PetscInt point = closure[cl];
1490         PetscInt       vertexLoc;
1491 
1492         if ((point >= pStart[0]) && (point < pEnd[0])) {
1493           ++numCorners;
1494           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1495           if (vertexLoc == value) closure[faceSize++] = point;
1496         }
1497       }
1498       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
1499       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
1500       if (faceSize == *nFV) {
1501         const PetscInt *cells = NULL;
1502         PetscInt        numCells, nc;
1503 
1504         ++(*numFaces);
1505         for (cl = 0; cl < faceSize; ++cl) {
1506           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
1507         }
1508         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1509         for (nc = 0; nc < numCells; ++nc) {
1510           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
1511         }
1512         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1513       }
1514       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1515     }
1516     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1517   }
1518   if (subvertexIS) {
1519     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1520   }
1521   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1522   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1523   PetscFunctionReturn(0);
1524 }
1525 
1526 #undef __FUNCT__
1527 #define __FUNCT__ "DMPlexMarkSubmesh_Interpolated"
1528 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, DM subdm)
1529 {
1530   IS               subvertexIS;
1531   const PetscInt  *subvertices;
1532   PetscInt        *pStart, *pEnd, *pMax;
1533   PetscInt         dim, d, numSubVerticesInitial = 0, v;
1534   PetscErrorCode   ierr;
1535 
1536   PetscFunctionBegin;
1537   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1538   ierr = PetscMalloc3(dim+1,PetscInt,&pStart,dim+1,PetscInt,&pEnd,dim+1,PetscInt,&pMax);CHKERRQ(ierr);
1539   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], dim>1 ? &pMax[dim-1] : NULL, dim > 2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1540   for (d = 0; d <= dim; ++d) {
1541     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1542     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1543   }
1544   /* Loop over initial vertices and mark all faces in the collective star() */
1545   ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
1546   if (subvertexIS) {
1547     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1548     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1549   }
1550   for (v = 0; v < numSubVerticesInitial; ++v) {
1551     const PetscInt vertex = subvertices[v];
1552     PetscInt      *star   = NULL;
1553     PetscInt       starSize, s, numFaces = 0, f;
1554 
1555     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1556     for (s = 0; s < starSize*2; s += 2) {
1557       const PetscInt point = star[s];
1558       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) star[numFaces++] = point;
1559     }
1560     for (f = 0; f < numFaces; ++f) {
1561       const PetscInt face    = star[f];
1562       PetscInt      *closure = NULL;
1563       PetscInt       closureSize, c;
1564       PetscInt       faceLoc;
1565 
1566       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
1567       if (faceLoc == dim-1) continue;
1568       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
1569       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1570       for (c = 0; c < closureSize*2; c += 2) {
1571         const PetscInt point = closure[c];
1572         PetscInt       vertexLoc;
1573 
1574         if ((point >= pStart[0]) && (point < pEnd[0])) {
1575           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1576           if (vertexLoc != value) break;
1577         }
1578       }
1579       if (c == closureSize*2) {
1580         const PetscInt *support;
1581         PetscInt        supportSize, s;
1582 
1583         for (c = 0; c < closureSize*2; c += 2) {
1584           const PetscInt point = closure[c];
1585 
1586           for (d = 0; d < dim; ++d) {
1587             if ((point >= pStart[d]) && (point < pEnd[d])) {
1588               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
1589               break;
1590             }
1591           }
1592         }
1593         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
1594         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
1595         for (s = 0; s < supportSize; ++s) {
1596           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
1597         }
1598       }
1599       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1600     }
1601     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1602   }
1603   if (subvertexIS) {
1604     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1605   }
1606   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1607   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1608   PetscFunctionReturn(0);
1609 }
1610 
1611 #undef __FUNCT__
1612 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Uninterpolated"
1613 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
1614 {
1615   DMLabel         label = NULL;
1616   const PetscInt *cone;
1617   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize;
1618   PetscErrorCode  ierr;
1619 
1620   PetscFunctionBegin;
1621   *numFaces = 0;
1622   *nFV = 0;
1623   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1624   *subCells = NULL;
1625   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1626   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1627   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1628   if (cMax < 0) PetscFunctionReturn(0);
1629   if (label) {
1630     for (c = cMax; c < cEnd; ++c) {
1631       PetscInt val;
1632 
1633       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1634       if (val == value) {
1635         ++(*numFaces);
1636         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1637       }
1638     }
1639   } else {
1640     *numFaces = cEnd - cMax;
1641     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
1642   }
1643   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
1644   ierr = PetscMalloc(*numFaces *2 * sizeof(PetscInt), subCells);CHKERRQ(ierr);
1645   for (c = cMax; c < cEnd; ++c) {
1646     const PetscInt *cells;
1647     PetscInt        numCells;
1648 
1649     if (label) {
1650       PetscInt val;
1651 
1652       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1653       if (val != value) continue;
1654     }
1655     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1656     for (p = 0; p < *nFV; ++p) {
1657       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
1658     }
1659     /* Negative face */
1660     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1661     /* Not true in parallel
1662     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
1663     for (p = 0; p < numCells; ++p) {
1664       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
1665       (*subCells)[subc++] = cells[p];
1666     }
1667     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1668     /* Positive face is not included */
1669   }
1670   PetscFunctionReturn(0);
1671 }
1672 
1673 #undef __FUNCT__
1674 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Interpolated"
1675 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DMLabel subpointMap, DM subdm)
1676 {
1677   DMLabel        label = NULL;
1678   PetscInt      *pStart, *pEnd;
1679   PetscInt       dim, cMax, cEnd, c, d;
1680   PetscErrorCode ierr;
1681 
1682   PetscFunctionBegin;
1683   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1684   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1685   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1686   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1687   if (cMax < 0) PetscFunctionReturn(0);
1688   ierr = PetscMalloc2(dim+1,PetscInt,&pStart,dim+1,PetscInt,&pEnd);CHKERRQ(ierr);
1689   for (d = 0; d <= dim; ++d) {ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);}
1690   for (c = cMax; c < cEnd; ++c) {
1691     const PetscInt *cone;
1692     PetscInt       *closure = NULL;
1693     PetscInt        fconeSize, coneSize, closureSize, cl, val;
1694 
1695     if (label) {
1696       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1697       if (val != value) continue;
1698     }
1699     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1700     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1701     ierr = DMPlexGetConeSize(dm, cone[0], &fconeSize);CHKERRQ(ierr);
1702     if (coneSize != (fconeSize ? fconeSize : 1) + 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
1703     /* Negative face */
1704     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1705     for (cl = 0; cl < closureSize*2; cl += 2) {
1706       const PetscInt point = closure[cl];
1707 
1708       for (d = 0; d <= dim; ++d) {
1709         if ((point >= pStart[d]) && (point < pEnd[d])) {
1710           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
1711           break;
1712         }
1713       }
1714     }
1715     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1716     /* Cells -- positive face is not included */
1717     for (cl = 0; cl < 1; ++cl) {
1718       const PetscInt *support;
1719       PetscInt        supportSize, s;
1720 
1721       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
1722       if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells");
1723       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
1724       for (s = 0; s < supportSize; ++s) {
1725         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
1726       }
1727     }
1728   }
1729   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
1730   PetscFunctionReturn(0);
1731 }
1732 
1733 #undef __FUNCT__
1734 #define __FUNCT__ "DMPlexGetFaceOrientation"
1735 PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
1736 {
1737   MPI_Comm       comm;
1738   PetscBool      posOrient = PETSC_FALSE;
1739   const PetscInt debug     = 0;
1740   PetscInt       cellDim, faceSize, f;
1741   PetscErrorCode ierr;
1742 
1743   PetscFunctionBegin;
1744   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1745   ierr = DMPlexGetDimension(dm, &cellDim);CHKERRQ(ierr);
1746   if (debug) {PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
1747 
1748   if (cellDim == 1 && numCorners == 2) {
1749     /* Triangle */
1750     faceSize  = numCorners-1;
1751     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1752   } else if (cellDim == 2 && numCorners == 3) {
1753     /* Triangle */
1754     faceSize  = numCorners-1;
1755     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1756   } else if (cellDim == 3 && numCorners == 4) {
1757     /* Tetrahedron */
1758     faceSize  = numCorners-1;
1759     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1760   } else if (cellDim == 1 && numCorners == 3) {
1761     /* Quadratic line */
1762     faceSize  = 1;
1763     posOrient = PETSC_TRUE;
1764   } else if (cellDim == 2 && numCorners == 4) {
1765     /* Quads */
1766     faceSize = 2;
1767     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
1768       posOrient = PETSC_TRUE;
1769     } else if ((indices[0] == 3) && (indices[1] == 0)) {
1770       posOrient = PETSC_TRUE;
1771     } else {
1772       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
1773         posOrient = PETSC_FALSE;
1774       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
1775     }
1776   } else if (cellDim == 2 && numCorners == 6) {
1777     /* Quadratic triangle (I hate this) */
1778     /* Edges are determined by the first 2 vertices (corners of edges) */
1779     const PetscInt faceSizeTri = 3;
1780     PetscInt       sortedIndices[3], i, iFace;
1781     PetscBool      found                    = PETSC_FALSE;
1782     PetscInt       faceVerticesTriSorted[9] = {
1783       0, 3,  4, /* bottom */
1784       1, 4,  5, /* right */
1785       2, 3,  5, /* left */
1786     };
1787     PetscInt       faceVerticesTri[9] = {
1788       0, 3,  4, /* bottom */
1789       1, 4,  5, /* right */
1790       2, 5,  3, /* left */
1791     };
1792 
1793     faceSize = faceSizeTri;
1794     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
1795     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
1796     for (iFace = 0; iFace < 3; ++iFace) {
1797       const PetscInt ii = iFace*faceSizeTri;
1798       PetscInt       fVertex, cVertex;
1799 
1800       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
1801           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
1802         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
1803           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
1804             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
1805               faceVertices[fVertex] = origVertices[cVertex];
1806               break;
1807             }
1808           }
1809         }
1810         found = PETSC_TRUE;
1811         break;
1812       }
1813     }
1814     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
1815     if (posOriented) *posOriented = PETSC_TRUE;
1816     PetscFunctionReturn(0);
1817   } else if (cellDim == 2 && numCorners == 9) {
1818     /* Quadratic quad (I hate this) */
1819     /* Edges are determined by the first 2 vertices (corners of edges) */
1820     const PetscInt faceSizeQuad = 3;
1821     PetscInt       sortedIndices[3], i, iFace;
1822     PetscBool      found                      = PETSC_FALSE;
1823     PetscInt       faceVerticesQuadSorted[12] = {
1824       0, 1,  4, /* bottom */
1825       1, 2,  5, /* right */
1826       2, 3,  6, /* top */
1827       0, 3,  7, /* left */
1828     };
1829     PetscInt       faceVerticesQuad[12] = {
1830       0, 1,  4, /* bottom */
1831       1, 2,  5, /* right */
1832       2, 3,  6, /* top */
1833       3, 0,  7, /* left */
1834     };
1835 
1836     faceSize = faceSizeQuad;
1837     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
1838     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
1839     for (iFace = 0; iFace < 4; ++iFace) {
1840       const PetscInt ii = iFace*faceSizeQuad;
1841       PetscInt       fVertex, cVertex;
1842 
1843       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
1844           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
1845         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
1846           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
1847             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
1848               faceVertices[fVertex] = origVertices[cVertex];
1849               break;
1850             }
1851           }
1852         }
1853         found = PETSC_TRUE;
1854         break;
1855       }
1856     }
1857     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
1858     if (posOriented) *posOriented = PETSC_TRUE;
1859     PetscFunctionReturn(0);
1860   } else if (cellDim == 3 && numCorners == 8) {
1861     /* Hexes
1862        A hex is two oriented quads with the normal of the first
1863        pointing up at the second.
1864 
1865           7---6
1866          /|  /|
1867         4---5 |
1868         | 1-|-2
1869         |/  |/
1870         0---3
1871 
1872         Faces are determined by the first 4 vertices (corners of faces) */
1873     const PetscInt faceSizeHex = 4;
1874     PetscInt       sortedIndices[4], i, iFace;
1875     PetscBool      found                     = PETSC_FALSE;
1876     PetscInt       faceVerticesHexSorted[24] = {
1877       0, 1, 2, 3,  /* bottom */
1878       4, 5, 6, 7,  /* top */
1879       0, 3, 4, 5,  /* front */
1880       2, 3, 5, 6,  /* right */
1881       1, 2, 6, 7,  /* back */
1882       0, 1, 4, 7,  /* left */
1883     };
1884     PetscInt       faceVerticesHex[24] = {
1885       1, 2, 3, 0,  /* bottom */
1886       4, 5, 6, 7,  /* top */
1887       0, 3, 5, 4,  /* front */
1888       3, 2, 6, 5,  /* right */
1889       2, 1, 7, 6,  /* back */
1890       1, 0, 4, 7,  /* left */
1891     };
1892 
1893     faceSize = faceSizeHex;
1894     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
1895     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
1896     for (iFace = 0; iFace < 6; ++iFace) {
1897       const PetscInt ii = iFace*faceSizeHex;
1898       PetscInt       fVertex, cVertex;
1899 
1900       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
1901           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
1902           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
1903           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
1904         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
1905           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
1906             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
1907               faceVertices[fVertex] = origVertices[cVertex];
1908               break;
1909             }
1910           }
1911         }
1912         found = PETSC_TRUE;
1913         break;
1914       }
1915     }
1916     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
1917     if (posOriented) *posOriented = PETSC_TRUE;
1918     PetscFunctionReturn(0);
1919   } else if (cellDim == 3 && numCorners == 10) {
1920     /* Quadratic tet */
1921     /* Faces are determined by the first 3 vertices (corners of faces) */
1922     const PetscInt faceSizeTet = 6;
1923     PetscInt       sortedIndices[6], i, iFace;
1924     PetscBool      found                     = PETSC_FALSE;
1925     PetscInt       faceVerticesTetSorted[24] = {
1926       0, 1, 2,  6, 7, 8, /* bottom */
1927       0, 3, 4,  6, 7, 9,  /* front */
1928       1, 4, 5,  7, 8, 9,  /* right */
1929       2, 3, 5,  6, 8, 9,  /* left */
1930     };
1931     PetscInt       faceVerticesTet[24] = {
1932       0, 1, 2,  6, 7, 8, /* bottom */
1933       0, 4, 3,  6, 7, 9,  /* front */
1934       1, 5, 4,  7, 8, 9,  /* right */
1935       2, 3, 5,  8, 6, 9,  /* left */
1936     };
1937 
1938     faceSize = faceSizeTet;
1939     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
1940     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
1941     for (iFace=0; iFace < 4; ++iFace) {
1942       const PetscInt ii = iFace*faceSizeTet;
1943       PetscInt       fVertex, cVertex;
1944 
1945       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
1946           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
1947           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
1948           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
1949         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
1950           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
1951             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
1952               faceVertices[fVertex] = origVertices[cVertex];
1953               break;
1954             }
1955           }
1956         }
1957         found = PETSC_TRUE;
1958         break;
1959       }
1960     }
1961     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
1962     if (posOriented) *posOriented = PETSC_TRUE;
1963     PetscFunctionReturn(0);
1964   } else if (cellDim == 3 && numCorners == 27) {
1965     /* Quadratic hexes (I hate this)
1966        A hex is two oriented quads with the normal of the first
1967        pointing up at the second.
1968 
1969          7---6
1970         /|  /|
1971        4---5 |
1972        | 3-|-2
1973        |/  |/
1974        0---1
1975 
1976        Faces are determined by the first 4 vertices (corners of faces) */
1977     const PetscInt faceSizeQuadHex = 9;
1978     PetscInt       sortedIndices[9], i, iFace;
1979     PetscBool      found                         = PETSC_FALSE;
1980     PetscInt       faceVerticesQuadHexSorted[54] = {
1981       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
1982       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
1983       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
1984       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
1985       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
1986       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
1987     };
1988     PetscInt       faceVerticesQuadHex[54] = {
1989       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
1990       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
1991       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
1992       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
1993       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
1994       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
1995     };
1996 
1997     faceSize = faceSizeQuadHex;
1998     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
1999     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
2000     for (iFace = 0; iFace < 6; ++iFace) {
2001       const PetscInt ii = iFace*faceSizeQuadHex;
2002       PetscInt       fVertex, cVertex;
2003 
2004       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2005           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2006           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2007           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2008         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2009           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2010             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2011               faceVertices[fVertex] = origVertices[cVertex];
2012               break;
2013             }
2014           }
2015         }
2016         found = PETSC_TRUE;
2017         break;
2018       }
2019     }
2020     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2021     if (posOriented) *posOriented = PETSC_TRUE;
2022     PetscFunctionReturn(0);
2023   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2024   if (!posOrient) {
2025     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
2026     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2027   } else {
2028     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
2029     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2030   }
2031   if (posOriented) *posOriented = posOrient;
2032   PetscFunctionReturn(0);
2033 }
2034 
2035 #undef __FUNCT__
2036 #define __FUNCT__ "DMPlexGetOrientedFace"
2037 /*
2038     Given a cell and a face, as a set of vertices,
2039       return the oriented face, as a set of vertices, in faceVertices
2040     The orientation is such that the face normal points out of the cell
2041 */
2042 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2043 {
2044   const PetscInt *cone = NULL;
2045   PetscInt        coneSize, v, f, v2;
2046   PetscInt        oppositeVertex = -1;
2047   PetscErrorCode  ierr;
2048 
2049   PetscFunctionBegin;
2050   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
2051   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2052   for (v = 0, v2 = 0; v < coneSize; ++v) {
2053     PetscBool found = PETSC_FALSE;
2054 
2055     for (f = 0; f < faceSize; ++f) {
2056       if (face[f] == cone[v]) {
2057         found = PETSC_TRUE; break;
2058       }
2059     }
2060     if (found) {
2061       indices[v2]      = v;
2062       origVertices[v2] = cone[v];
2063       ++v2;
2064     } else {
2065       oppositeVertex = v;
2066     }
2067   }
2068   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
2069   PetscFunctionReturn(0);
2070 }
2071 
2072 #undef __FUNCT__
2073 #define __FUNCT__ "DMPlexInsertFace_Internal"
2074 /*
2075   DMPlexInsertFace_Internal - Puts a face into the mesh
2076 
2077   Not collective
2078 
2079   Input Parameters:
2080   + dm              - The DMPlex
2081   . numFaceVertex   - The number of vertices in the face
2082   . faceVertices    - The vertices in the face for dm
2083   . subfaceVertices - The vertices in the face for subdm
2084   . numCorners      - The number of vertices in the cell
2085   . cell            - A cell in dm containing the face
2086   . subcell         - A cell in subdm containing the face
2087   . firstFace       - First face in the mesh
2088   - newFacePoint    - Next face in the mesh
2089 
2090   Output Parameters:
2091   . newFacePoint - Contains next face point number on input, updated on output
2092 
2093   Level: developer
2094 */
2095 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2096 {
2097   MPI_Comm        comm;
2098   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2099   const PetscInt *faces;
2100   PetscInt        numFaces, coneSize;
2101   PetscErrorCode  ierr;
2102 
2103   PetscFunctionBegin;
2104   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2105   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
2106   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
2107 #if 0
2108   /* Cannot use this because support() has not been constructed yet */
2109   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2110 #else
2111   {
2112     PetscInt f;
2113 
2114     numFaces = 0;
2115     ierr     = DMGetWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2116     for (f = firstFace; f < *newFacePoint; ++f) {
2117       PetscInt dof, off, d;
2118 
2119       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
2120       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
2121       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2122       for (d = 0; d < dof; ++d) {
2123         const PetscInt p = submesh->cones[off+d];
2124         PetscInt       v;
2125 
2126         for (v = 0; v < numFaceVertices; ++v) {
2127           if (subfaceVertices[v] == p) break;
2128         }
2129         if (v == numFaceVertices) break;
2130       }
2131       if (d == dof) {
2132         numFaces               = 1;
2133         ((PetscInt*) faces)[0] = f;
2134       }
2135     }
2136   }
2137 #endif
2138   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
2139   else if (numFaces == 1) {
2140     /* Add the other cell neighbor for this face */
2141     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
2142   } else {
2143     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2144     PetscBool posOriented;
2145 
2146     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2147     origVertices        = &orientedVertices[numFaceVertices];
2148     indices             = &orientedVertices[numFaceVertices*2];
2149     orientedSubVertices = &orientedVertices[numFaceVertices*3];
2150     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
2151     /* TODO: I know that routine should return a permutation, not the indices */
2152     for (v = 0; v < numFaceVertices; ++v) {
2153       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2154       for (ov = 0; ov < numFaceVertices; ++ov) {
2155         if (orientedVertices[ov] == vertex) {
2156           orientedSubVertices[ov] = subvertex;
2157           break;
2158         }
2159       }
2160       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2161     }
2162     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2163     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2164     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2165     ++(*newFacePoint);
2166   }
2167 #if 0
2168   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2169 #else
2170   ierr = DMRestoreWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2171 #endif
2172   PetscFunctionReturn(0);
2173 }
2174 
2175 #undef __FUNCT__
2176 #define __FUNCT__ "DMPlexCreateSubmesh_Uninterpolated"
2177 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2178 {
2179   MPI_Comm        comm;
2180   DMLabel         subpointMap;
2181   IS              subvertexIS,  subcellIS;
2182   const PetscInt *subVertices, *subCells;
2183   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2184   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2185   PetscInt        vStart, vEnd, c, f;
2186   PetscErrorCode  ierr;
2187 
2188   PetscFunctionBegin;
2189   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2190   /* Create subpointMap which marks the submesh */
2191   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2192   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2193   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2194   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2195   /* Setup chart */
2196   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2197   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2198   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2199   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2200   /* Set cone sizes */
2201   firstSubVertex = numSubCells;
2202   firstSubFace   = numSubCells+numSubVertices;
2203   newFacePoint   = firstSubFace;
2204   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2205   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2206   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2207   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2208   for (c = 0; c < numSubCells; ++c) {
2209     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2210   }
2211   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2212     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2213   }
2214   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2215   /* Create face cones */
2216   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2217   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2218   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2219   for (c = 0; c < numSubCells; ++c) {
2220     const PetscInt cell    = subCells[c];
2221     const PetscInt subcell = c;
2222     PetscInt      *closure = NULL;
2223     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2224 
2225     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2226     for (cl = 0; cl < closureSize*2; cl += 2) {
2227       const PetscInt point = closure[cl];
2228       PetscInt       subVertex;
2229 
2230       if ((point >= vStart) && (point < vEnd)) {
2231         ++numCorners;
2232         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2233         if (subVertex >= 0) {
2234           closure[faceSize] = point;
2235           subface[faceSize] = firstSubVertex+subVertex;
2236           ++faceSize;
2237         }
2238       }
2239     }
2240     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2241     if (faceSize == nFV) {
2242       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2243     }
2244     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2245   }
2246   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2247   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2248   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2249   /* Build coordinates */
2250   {
2251     PetscSection coordSection, subCoordSection;
2252     Vec          coordinates, subCoordinates;
2253     PetscScalar *coords, *subCoords;
2254     PetscInt     numComp, coordSize, v;
2255 
2256     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2257     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2258     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2259     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2260     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2261     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2262     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2263     for (v = 0; v < numSubVertices; ++v) {
2264       const PetscInt vertex    = subVertices[v];
2265       const PetscInt subvertex = firstSubVertex+v;
2266       PetscInt       dof;
2267 
2268       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2269       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2270       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2271     }
2272     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2273     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2274     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2275     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2276     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2277     if (coordSize) {
2278       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2279       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2280       for (v = 0; v < numSubVertices; ++v) {
2281         const PetscInt vertex    = subVertices[v];
2282         const PetscInt subvertex = firstSubVertex+v;
2283         PetscInt       dof, off, sdof, soff, d;
2284 
2285         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2286         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2287         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2288         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2289         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2290         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2291       }
2292       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2293       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2294     }
2295     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2296     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2297   }
2298   /* Cleanup */
2299   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2300   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2301   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2302   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
2303   PetscFunctionReturn(0);
2304 }
2305 
2306 #undef __FUNCT__
2307 #define __FUNCT__ "DMPlexCreateSubmesh_Interpolated"
2308 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2309 {
2310   MPI_Comm         comm;
2311   DMLabel          subpointMap;
2312   IS              *subpointIS;
2313   const PetscInt **subpoints;
2314   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *coneONew;
2315   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2316   PetscErrorCode   ierr;
2317 
2318   PetscFunctionBegin;
2319   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2320   /* Create subpointMap which marks the submesh */
2321   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2322   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2323   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2324   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Interpolated(dm, vertexLabel, value, subpointMap, subdm);CHKERRQ(ierr);}
2325   /* Setup chart */
2326   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2327   ierr = PetscMalloc4(dim+1,PetscInt,&numSubPoints,dim+1,PetscInt,&firstSubPoint,dim+1,IS,&subpointIS,dim+1,const PetscInt *,&subpoints);CHKERRQ(ierr);
2328   for (d = 0; d <= dim; ++d) {
2329     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2330     totSubPoints += numSubPoints[d];
2331   }
2332   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2333   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2334   /* Set cone sizes */
2335   firstSubPoint[dim] = 0;
2336   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2337   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2338   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2339   for (d = 0; d <= dim; ++d) {
2340     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2341     ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2342   }
2343   for (d = 0; d <= dim; ++d) {
2344     for (p = 0; p < numSubPoints[d]; ++p) {
2345       const PetscInt  point    = subpoints[d][p];
2346       const PetscInt  subpoint = firstSubPoint[d] + p;
2347       const PetscInt *cone;
2348       PetscInt        coneSize, coneSizeNew, c, val;
2349 
2350       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2351       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2352       if (d == dim) {
2353         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2354         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2355           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2356           if (val >= 0) coneSizeNew++;
2357         }
2358         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2359       }
2360     }
2361   }
2362   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2363   /* Set cones */
2364   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2365   ierr = PetscMalloc2(maxConeSize,PetscInt,&coneNew,maxConeSize,PetscInt,&coneONew);CHKERRQ(ierr);
2366   for (d = 0; d <= dim; ++d) {
2367     for (p = 0; p < numSubPoints[d]; ++p) {
2368       const PetscInt  point    = subpoints[d][p];
2369       const PetscInt  subpoint = firstSubPoint[d] + p;
2370       const PetscInt *cone, *ornt;
2371       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2372 
2373       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2374       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2375       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2376       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
2377       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2378         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2379         if (subc >= 0) {
2380           coneNew[coneSizeNew]  = firstSubPoint[d-1] + subc;
2381           coneONew[coneSizeNew] = ornt[c];
2382           ++coneSizeNew;
2383         }
2384       }
2385       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2386       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2387       ierr = DMPlexSetConeOrientation(subdm, subpoint, coneONew);CHKERRQ(ierr);
2388     }
2389   }
2390   ierr = PetscFree2(coneNew,coneONew);CHKERRQ(ierr);
2391   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2392   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2393   /* Build coordinates */
2394   {
2395     PetscSection coordSection, subCoordSection;
2396     Vec          coordinates, subCoordinates;
2397     PetscScalar *coords, *subCoords;
2398     PetscInt     numComp, coordSize;
2399 
2400     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2401     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2402     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2403     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2404     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2405     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2406     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2407     for (v = 0; v < numSubPoints[0]; ++v) {
2408       const PetscInt vertex    = subpoints[0][v];
2409       const PetscInt subvertex = firstSubPoint[0]+v;
2410       PetscInt       dof;
2411 
2412       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2413       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2414       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2415     }
2416     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2417     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2418     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2419     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2420     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2421     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2422     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2423     for (v = 0; v < numSubPoints[0]; ++v) {
2424       const PetscInt vertex    = subpoints[0][v];
2425       const PetscInt subvertex = firstSubPoint[0]+v;
2426       PetscInt dof, off, sdof, soff, d;
2427 
2428       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2429       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2430       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2431       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2432       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2433       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2434     }
2435     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2436     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2437     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2438     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2439   }
2440   /* Cleanup */
2441   for (d = 0; d <= dim; ++d) {
2442     ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2443     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
2444   }
2445   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
2446   PetscFunctionReturn(0);
2447 }
2448 
2449 #undef __FUNCT__
2450 #define __FUNCT__ "DMPlexCreateSubmesh"
2451 /*@
2452   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
2453 
2454   Input Parameters:
2455 + dm           - The original mesh
2456 . vertexLabel  - The DMLabel marking vertices contained in the surface
2457 - value        - The label value to use
2458 
2459   Output Parameter:
2460 . subdm - The surface mesh
2461 
2462   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
2463 
2464   Level: developer
2465 
2466 .seealso: DMPlexGetSubpointMap(), DMPlexGetLabel(), DMLabelSetValue()
2467 @*/
2468 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, DM *subdm)
2469 {
2470   PetscInt       dim, depth;
2471   PetscErrorCode ierr;
2472 
2473   PetscFunctionBegin;
2474   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2475   PetscValidPointer(subdm, 3);
2476   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2477   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2478   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
2479   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
2480   ierr = DMPlexSetDimension(*subdm, dim-1);CHKERRQ(ierr);
2481   if (depth == dim) {
2482     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2483   } else {
2484     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2485   }
2486   PetscFunctionReturn(0);
2487 }
2488 
2489 #undef __FUNCT__
2490 #define __FUNCT__ "DMPlexFilterPoint_Internal"
2491 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
2492 {
2493   PetscInt       subPoint;
2494   PetscErrorCode ierr;
2495 
2496   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
2497   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
2498 }
2499 
2500 #undef __FUNCT__
2501 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Uninterpolated"
2502 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
2503 {
2504   MPI_Comm        comm;
2505   DMLabel         subpointMap;
2506   IS              subvertexIS;
2507   const PetscInt *subVertices;
2508   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
2509   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
2510   PetscInt        cMax, c, f;
2511   PetscErrorCode  ierr;
2512 
2513   PetscFunctionBegin;
2514   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
2515   /* Create subpointMap which marks the submesh */
2516   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2517   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2518   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2519   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
2520   /* Setup chart */
2521   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2522   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2523   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2524   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2525   /* Set cone sizes */
2526   firstSubVertex = numSubCells;
2527   firstSubFace   = numSubCells+numSubVertices;
2528   newFacePoint   = firstSubFace;
2529   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2530   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2531   for (c = 0; c < numSubCells; ++c) {
2532     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2533   }
2534   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2535     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2536   }
2537   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2538   /* Create face cones */
2539   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2540   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2541   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2542   for (c = 0; c < numSubCells; ++c) {
2543     const PetscInt  cell    = subCells[c];
2544     const PetscInt  subcell = c;
2545     const PetscInt *cone, *cells;
2546     PetscInt        numCells, subVertex, p, v;
2547 
2548     if (cell < cMax) continue;
2549     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2550     for (v = 0; v < nFV; ++v) {
2551       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2552       subface[v] = firstSubVertex+subVertex;
2553     }
2554     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
2555     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
2556     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2557     /* Not true in parallel
2558     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2559     for (p = 0; p < numCells; ++p) {
2560       PetscInt negsubcell;
2561 
2562       if (cells[p] >= cMax) continue;
2563       /* I know this is a crap search */
2564       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
2565         if (subCells[negsubcell] == cells[p]) break;
2566       }
2567       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
2568       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
2569     }
2570     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2571     ++newFacePoint;
2572   }
2573   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2574   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2575   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2576   /* Build coordinates */
2577   {
2578     PetscSection coordSection, subCoordSection;
2579     Vec          coordinates, subCoordinates;
2580     PetscScalar *coords, *subCoords;
2581     PetscInt     numComp, coordSize, v;
2582 
2583     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2584     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2585     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2586     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2587     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2588     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2589     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2590     for (v = 0; v < numSubVertices; ++v) {
2591       const PetscInt vertex    = subVertices[v];
2592       const PetscInt subvertex = firstSubVertex+v;
2593       PetscInt       dof;
2594 
2595       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2596       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2597       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2598     }
2599     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2600     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2601     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2602     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2603     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2604     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2605     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2606     for (v = 0; v < numSubVertices; ++v) {
2607       const PetscInt vertex    = subVertices[v];
2608       const PetscInt subvertex = firstSubVertex+v;
2609       PetscInt       dof, off, sdof, soff, d;
2610 
2611       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2612       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2613       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2614       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2615       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2616       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2617     }
2618     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2619     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2620     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2621     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2622   }
2623   /* Build SF */
2624   CHKMEMQ;
2625   {
2626     PetscSF            sfPoint, sfPointSub;
2627     const PetscSFNode *remotePoints;
2628     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
2629     const PetscInt    *localPoints;
2630     PetscInt          *slocalPoints;
2631     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
2632     PetscMPIInt        rank;
2633 
2634     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2635     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2636     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
2637     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2638     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2639     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
2640     if (numRoots >= 0) {
2641       /* Only vertices should be shared */
2642       ierr = PetscMalloc2(pEnd-pStart,PetscSFNode,&newLocalPoints,numRoots,PetscSFNode,&newOwners);CHKERRQ(ierr);
2643       for (p = 0; p < pEnd-pStart; ++p) {
2644         newLocalPoints[p].rank  = -2;
2645         newLocalPoints[p].index = -2;
2646       }
2647       /* Set subleaves */
2648       for (l = 0; l < numLeaves; ++l) {
2649         const PetscInt point    = localPoints[l];
2650         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2651 
2652         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
2653         if (subPoint < 0) continue;
2654         newLocalPoints[point-pStart].rank  = rank;
2655         newLocalPoints[point-pStart].index = subPoint;
2656         ++numSubLeaves;
2657       }
2658       /* Must put in owned subpoints */
2659       for (p = pStart; p < pEnd; ++p) {
2660         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
2661 
2662         if (subPoint < 0) {
2663           newOwners[p-pStart].rank  = -3;
2664           newOwners[p-pStart].index = -3;
2665         } else {
2666           newOwners[p-pStart].rank  = rank;
2667           newOwners[p-pStart].index = subPoint;
2668         }
2669       }
2670       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2671       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2672       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2673       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2674       ierr = PetscMalloc(numSubLeaves * sizeof(PetscInt),    &slocalPoints);CHKERRQ(ierr);
2675       ierr = PetscMalloc(numSubLeaves * sizeof(PetscSFNode), &sremotePoints);CHKERRQ(ierr);
2676       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
2677         const PetscInt point    = localPoints[l];
2678         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2679 
2680         if (subPoint < 0) continue;
2681         if (newLocalPoints[point].rank == rank) {++ll; continue;}
2682         slocalPoints[sl]        = subPoint;
2683         sremotePoints[sl].rank  = newLocalPoints[point].rank;
2684         sremotePoints[sl].index = newLocalPoints[point].index;
2685         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
2686         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
2687         ++sl;
2688       }
2689       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
2690       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
2691       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
2692     }
2693   }
2694   CHKMEMQ;
2695   /* Cleanup */
2696   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2697   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2698   ierr = PetscFree(subCells);CHKERRQ(ierr);
2699   PetscFunctionReturn(0);
2700 }
2701 
2702 #undef __FUNCT__
2703 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Interpolated"
2704 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char label[], PetscInt value, DM subdm)
2705 {
2706   MPI_Comm         comm;
2707   DMLabel          subpointMap;
2708   IS              *subpointIS;
2709   const PetscInt **subpoints;
2710   PetscInt        *numSubPoints, *firstSubPoint, *coneNew;
2711   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2712   PetscErrorCode   ierr;
2713 
2714   PetscFunctionBegin;
2715   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2716   /* Create subpointMap which marks the submesh */
2717   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2718   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2719   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2720   ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);
2721   /* Setup chart */
2722   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2723   ierr = PetscMalloc4(dim+1,PetscInt,&numSubPoints,dim+1,PetscInt,&firstSubPoint,dim+1,IS,&subpointIS,dim+1,const PetscInt *,&subpoints);CHKERRQ(ierr);
2724   for (d = 0; d <= dim; ++d) {
2725     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2726     totSubPoints += numSubPoints[d];
2727   }
2728   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2729   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2730   /* Set cone sizes */
2731   firstSubPoint[dim] = 0;
2732   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2733   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2734   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2735   for (d = 0; d <= dim; ++d) {
2736     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2737     ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2738   }
2739   for (d = 0; d <= dim; ++d) {
2740     for (p = 0; p < numSubPoints[d]; ++p) {
2741       const PetscInt  point    = subpoints[d][p];
2742       const PetscInt  subpoint = firstSubPoint[d] + p;
2743       const PetscInt *cone;
2744       PetscInt        coneSize, coneSizeNew, c, val;
2745 
2746       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2747       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2748       if (d == dim) {
2749         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2750         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2751           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2752           if (val >= 0) coneSizeNew++;
2753         }
2754         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2755       }
2756     }
2757   }
2758   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2759   /* Set cones */
2760   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2761   ierr = PetscMalloc(maxConeSize * sizeof(PetscInt), &coneNew);CHKERRQ(ierr);
2762   for (d = 0; d <= dim; ++d) {
2763     for (p = 0; p < numSubPoints[d]; ++p) {
2764       const PetscInt  point    = subpoints[d][p];
2765       const PetscInt  subpoint = firstSubPoint[d] + p;
2766       const PetscInt *cone;
2767       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2768 
2769       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2770       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2771       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2772       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2773         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2774         if (subc >= 0) coneNew[coneSizeNew++] = firstSubPoint[d-1] + subc;
2775       }
2776       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2777       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2778     }
2779   }
2780   ierr = PetscFree(coneNew);CHKERRQ(ierr);
2781   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2782   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2783   /* Build coordinates */
2784   {
2785     PetscSection coordSection, subCoordSection;
2786     Vec          coordinates, subCoordinates;
2787     PetscScalar *coords, *subCoords;
2788     PetscInt     numComp, coordSize;
2789 
2790     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2791     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2792     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2793     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2794     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2795     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2796     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2797     for (v = 0; v < numSubPoints[0]; ++v) {
2798       const PetscInt vertex    = subpoints[0][v];
2799       const PetscInt subvertex = firstSubPoint[0]+v;
2800       PetscInt       dof;
2801 
2802       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2803       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2804       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2805     }
2806     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2807     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2808     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2809     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2810     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2811     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2812     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2813     for (v = 0; v < numSubPoints[0]; ++v) {
2814       const PetscInt vertex    = subpoints[0][v];
2815       const PetscInt subvertex = firstSubPoint[0]+v;
2816       PetscInt dof, off, sdof, soff, d;
2817 
2818       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2819       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2820       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2821       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2822       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2823       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2824     }
2825     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2826     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2827     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2828     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2829   }
2830   /* Cleanup */
2831   for (d = 0; d <= dim; ++d) {
2832     ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2833     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
2834   }
2835   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
2836   PetscFunctionReturn(0);
2837 }
2838 
2839 #undef __FUNCT__
2840 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh"
2841 /*
2842   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
2843 
2844   Input Parameters:
2845 + dm          - The original mesh
2846 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
2847 . label       - A label name, or PETSC_NULL
2848 - value  - A label value
2849 
2850   Output Parameter:
2851 . subdm - The surface mesh
2852 
2853   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
2854 
2855   Level: developer
2856 
2857 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
2858 */
2859 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
2860 {
2861   PetscInt       dim, depth;
2862   PetscErrorCode ierr;
2863 
2864   PetscFunctionBegin;
2865   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2866   PetscValidPointer(subdm, 5);
2867   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2868   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2869   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
2870   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
2871   ierr = DMPlexSetDimension(*subdm, dim-1);CHKERRQ(ierr);
2872   if (depth == dim) {
2873     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm);CHKERRQ(ierr);
2874   } else {
2875     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
2876   }
2877   PetscFunctionReturn(0);
2878 }
2879 
2880 #undef __FUNCT__
2881 #define __FUNCT__ "DMPlexGetSubpointMap"
2882 /*@
2883   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
2884 
2885   Input Parameter:
2886 . dm - The submesh DM
2887 
2888   Output Parameter:
2889 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
2890 
2891   Level: developer
2892 
2893 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
2894 @*/
2895 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
2896 {
2897   DM_Plex *mesh = (DM_Plex*) dm->data;
2898 
2899   PetscFunctionBegin;
2900   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2901   PetscValidPointer(subpointMap, 2);
2902   *subpointMap = mesh->subpointMap;
2903   PetscFunctionReturn(0);
2904 }
2905 
2906 #undef __FUNCT__
2907 #define __FUNCT__ "DMPlexSetSubpointMap"
2908 /* Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh() */
2909 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
2910 {
2911   DM_Plex       *mesh = (DM_Plex *) dm->data;
2912   DMLabel        tmp;
2913   PetscErrorCode ierr;
2914 
2915   PetscFunctionBegin;
2916   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2917   tmp  = mesh->subpointMap;
2918   mesh->subpointMap = subpointMap;
2919   ++mesh->subpointMap->refct;
2920   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
2921   PetscFunctionReturn(0);
2922 }
2923 
2924 #undef __FUNCT__
2925 #define __FUNCT__ "DMPlexCreateSubpointIS"
2926 /*@
2927   DMPlexCreateSubpointIS - Creates an IS covering the entire subdm chart with the original points as data
2928 
2929   Input Parameter:
2930 . dm - The submesh DM
2931 
2932   Output Parameter:
2933 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
2934 
2935   Note: This is IS is guaranteed to be sorted by the construction of the submesh
2936 
2937   Level: developer
2938 
2939 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
2940 @*/
2941 PetscErrorCode DMPlexCreateSubpointIS(DM dm, IS *subpointIS)
2942 {
2943   MPI_Comm        comm;
2944   DMLabel         subpointMap;
2945   IS              is;
2946   const PetscInt *opoints;
2947   PetscInt       *points, *depths;
2948   PetscInt        depth, depStart, depEnd, d, pStart, pEnd, p, n, off;
2949   PetscErrorCode  ierr;
2950 
2951   PetscFunctionBegin;
2952   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2953   PetscValidPointer(subpointIS, 2);
2954   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2955   *subpointIS = NULL;
2956   ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
2957   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2958   if (subpointMap && depth >= 0) {
2959     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2960     if (pStart) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
2961     ierr = DMGetWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
2962     depths[0] = depth;
2963     depths[1] = 0;
2964     for(d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
2965     ierr = PetscMalloc(pEnd * sizeof(PetscInt), &points);CHKERRQ(ierr);
2966     for(d = 0, off = 0; d <= depth; ++d) {
2967       const PetscInt dep = depths[d];
2968 
2969       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
2970       ierr = DMLabelGetStratumSize(subpointMap, dep, &n);CHKERRQ(ierr);
2971       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
2972         if (n != depEnd-depStart) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
2973       } else {
2974         if (!n) {
2975           if (d == 0) {
2976             /* Missing cells */
2977             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
2978           } else {
2979             /* Missing faces */
2980             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
2981           }
2982         }
2983       }
2984       if (n) {
2985         ierr = DMLabelGetStratumIS(subpointMap, dep, &is);CHKERRQ(ierr);
2986         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
2987         for(p = 0; p < n; ++p, ++off) points[off] = opoints[p];
2988         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
2989         ierr = ISDestroy(&is);CHKERRQ(ierr);
2990       }
2991     }
2992     ierr = DMRestoreWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
2993     if (off != pEnd) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
2994     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
2995   }
2996   PetscFunctionReturn(0);
2997 }
2998