xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision e175754804bcdac87c4b1f89cd2c3e12de21c4dd)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petscsf.h>
3 
4 #undef __FUNCT__
5 #define __FUNCT__ "DMPlexMarkBoundaryFaces"
6 /*@
7   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
8 
9   Not Collective
10 
11   Input Parameter:
12 . dm - The original DM
13 
14   Output Parameter:
15 . label - The DMLabel marking boundary faces with value 1
16 
17   Level: developer
18 
19 .seealso: DMLabelCreate(), DMPlexCreateLabel()
20 @*/
21 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, DMLabel label)
22 {
23   PetscInt       fStart, fEnd, f;
24   PetscErrorCode ierr;
25 
26   PetscFunctionBegin;
27   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
28   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
29   for (f = fStart; f < fEnd; ++f) {
30     PetscInt supportSize;
31 
32     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
33     if (supportSize == 1) {
34       ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);
35     }
36   }
37   PetscFunctionReturn(0);
38 }
39 
40 #undef __FUNCT__
41 #define __FUNCT__ "DMPlexLabelComplete"
42 /*@
43   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
44 
45   Input Parameters:
46 + dm - The DM
47 - label - A DMLabel marking the surface points
48 
49   Output Parameter:
50 . label - A DMLabel marking all surface points in the transitive closure
51 
52   Level: developer
53 
54 .seealso: DMPlexLabelCohesiveComplete()
55 @*/
56 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
57 {
58   IS              valueIS;
59   const PetscInt *values;
60   PetscInt        numValues, v;
61   PetscErrorCode  ierr;
62 
63   PetscFunctionBegin;
64   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
65   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
66   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
67   for (v = 0; v < numValues; ++v) {
68     IS              pointIS;
69     const PetscInt *points;
70     PetscInt        numPoints, p;
71 
72     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
73     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
74     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
75     for (p = 0; p < numPoints; ++p) {
76       PetscInt *closure = NULL;
77       PetscInt  closureSize, c;
78 
79       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
80       for (c = 0; c < closureSize*2; c += 2) {
81         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
82       }
83       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
84     }
85     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
86     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
87   }
88   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
89   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
90   PetscFunctionReturn(0);
91 }
92 
93 #undef __FUNCT__
94 #define __FUNCT__ "DMPlexShiftPoint_Internal"
95 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthEnd[], PetscInt depthShift[])
96 {
97   if (depth < 0) return p;
98   /* Cells    */ if (p < depthEnd[depth])   return p;
99   /* Vertices */ if (p < depthEnd[0])       return p + depthShift[depth];
100   /* Faces    */ if (p < depthEnd[depth-1]) return p + depthShift[depth] + depthShift[0];
101   /* Edges    */                            return p + depthShift[depth] + depthShift[0] + depthShift[depth-1];
102 }
103 
104 #undef __FUNCT__
105 #define __FUNCT__ "DMPlexShiftSizes_Internal"
106 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
107 {
108   PetscInt      *depthEnd;
109   PetscInt       depth = 0, d, pStart, pEnd, p;
110   PetscErrorCode ierr;
111 
112   PetscFunctionBegin;
113   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
114   if (depth < 0) PetscFunctionReturn(0);
115   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
116   /* Step 1: Expand chart */
117   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
118   for (d = 0; d <= depth; ++d) {
119     pEnd += depthShift[d];
120     ierr  = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
121   }
122   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
123   /* Step 2: Set cone and support sizes */
124   for (d = 0; d <= depth; ++d) {
125     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
126     for (p = pStart; p < pEnd; ++p) {
127       PetscInt newp = DMPlexShiftPoint_Internal(p, depth, depthEnd, depthShift);
128       PetscInt size;
129 
130       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
131       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
132       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
133       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
134     }
135   }
136   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
137   PetscFunctionReturn(0);
138 }
139 
140 #undef __FUNCT__
141 #define __FUNCT__ "DMPlexShiftPoints_Internal"
142 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
143 {
144   PetscInt      *depthEnd, *newpoints;
145   PetscInt       depth = 0, d, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
146   PetscErrorCode ierr;
147 
148   PetscFunctionBegin;
149   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
150   if (depth < 0) PetscFunctionReturn(0);
151   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
152   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
153   ierr = PetscMalloc2(depth+1,PetscInt,&depthEnd,PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),PetscInt,&newpoints);CHKERRQ(ierr);
154   for (d = 0; d <= depth; ++d) {
155     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
156   }
157   /* Step 5: Set cones and supports */
158   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
159   for (p = pStart; p < pEnd; ++p) {
160     const PetscInt *points = NULL, *orientations = NULL;
161     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthEnd, depthShift);
162 
163     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
164     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
165     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
166     for (i = 0; i < size; ++i) {
167       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthEnd, depthShift);
168     }
169     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
170     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
171     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
172     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
173     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
174     for (i = 0; i < size; ++i) {
175       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthEnd, depthShift);
176     }
177     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
178     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
179   }
180   ierr = PetscFree2(depthEnd,newpoints);CHKERRQ(ierr);
181   PetscFunctionReturn(0);
182 }
183 
184 #undef __FUNCT__
185 #define __FUNCT__ "DMPlexShiftCoordinates_Internal"
186 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
187 {
188   PetscSection   coordSection, newCoordSection;
189   Vec            coordinates, newCoordinates;
190   PetscScalar   *coords, *newCoords;
191   PetscInt      *depthEnd, coordSize;
192   PetscInt       dim, depth = 0, d, vStart, vEnd, vStartNew, vEndNew, v;
193   PetscErrorCode ierr;
194 
195   PetscFunctionBegin;
196   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
197   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
198   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
199   for (d = 0; d <= depth; ++d) {
200     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
201   }
202   /* Step 8: Convert coordinates */
203   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
204   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
205   ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
206   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
207   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
208   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
209   ierr = PetscSectionSetChart(newCoordSection, vStartNew, vEndNew);CHKERRQ(ierr);
210   for (v = vStartNew; v < vEndNew; ++v) {
211     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
212     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
213   }
214   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
215   ierr = DMPlexSetCoordinateSection(dmNew, newCoordSection);CHKERRQ(ierr);
216   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
217   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &newCoordinates);CHKERRQ(ierr);
218   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
219   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
220   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
221   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
222   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
223   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
224   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
225   for (v = vStart; v < vEnd; ++v) {
226     PetscInt dof, off, noff, d;
227 
228     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
229     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
230     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthEnd, depthShift), &noff);CHKERRQ(ierr);
231     for (d = 0; d < dof; ++d) {
232       newCoords[noff+d] = coords[off+d];
233     }
234   }
235   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
236   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
237   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
238   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
239   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
240   PetscFunctionReturn(0);
241 }
242 
243 #undef __FUNCT__
244 #define __FUNCT__ "DMPlexShiftSF_Internal"
245 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
246 {
247   PetscInt          *depthEnd;
248   PetscInt           depth = 0, d;
249   PetscSF            sfPoint, sfPointNew;
250   const PetscSFNode *remotePoints;
251   PetscSFNode       *gremotePoints;
252   const PetscInt    *localPoints;
253   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
254   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
255   PetscErrorCode     ierr;
256 
257   PetscFunctionBegin;
258   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
259   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
260   for (d = 0; d <= depth; ++d) {
261     totShift += depthShift[d];
262     ierr      = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
263   }
264   /* Step 9: Convert pointSF */
265   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
266   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
267   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
268   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
269   if (numRoots >= 0) {
270     ierr = PetscMalloc2(numRoots,PetscInt,&newLocation,pEnd-pStart,PetscInt,&newRemoteLocation);CHKERRQ(ierr);
271     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthEnd, depthShift);
272     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
273     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
274     ierr = PetscMalloc(numLeaves * sizeof(PetscInt),    &glocalPoints);CHKERRQ(ierr);
275     ierr = PetscMalloc(numLeaves * sizeof(PetscSFNode), &gremotePoints);CHKERRQ(ierr);
276     for (l = 0; l < numLeaves; ++l) {
277       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthEnd, depthShift);
278       gremotePoints[l].rank  = remotePoints[l].rank;
279       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
280     }
281     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
282     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
283   }
284   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
285   PetscFunctionReturn(0);
286 }
287 
288 #undef __FUNCT__
289 #define __FUNCT__ "DMPlexShiftLabels_Internal"
290 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
291 {
292   PetscSF            sfPoint;
293   DMLabel            vtkLabel, ghostLabel;
294   PetscInt          *depthEnd;
295   const PetscSFNode *leafRemote;
296   const PetscInt    *leafLocal;
297   PetscInt           depth = 0, d, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
298   PetscMPIInt        rank;
299   PetscErrorCode     ierr;
300 
301   PetscFunctionBegin;
302   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
303   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthEnd);CHKERRQ(ierr);
304   for (d = 0; d <= depth; ++d) {
305     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
306   }
307   /* Step 10: Convert labels */
308   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
309   for (l = 0; l < numLabels; ++l) {
310     DMLabel         label, newlabel;
311     const char     *lname;
312     PetscBool       isDepth;
313     IS              valueIS;
314     const PetscInt *values;
315     PetscInt        numValues, val;
316 
317     ierr = DMPlexGetLabelName(dm, l, &lname);CHKERRQ(ierr);
318     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
319     if (isDepth) continue;
320     ierr = DMPlexCreateLabel(dmNew, lname);CHKERRQ(ierr);
321     ierr = DMPlexGetLabel(dm, lname, &label);CHKERRQ(ierr);
322     ierr = DMPlexGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
323     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
324     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
325     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
326     for (val = 0; val < numValues; ++val) {
327       IS              pointIS;
328       const PetscInt *points;
329       PetscInt        numPoints, p;
330 
331       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
332       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
333       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
334       for (p = 0; p < numPoints; ++p) {
335         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthEnd, depthShift);
336 
337         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
338       }
339       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
340       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
341     }
342     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
343     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
344   }
345   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
346   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
347   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
348   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
349   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
350   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
351   ierr = DMPlexCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
352   ierr = DMPlexCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
353   ierr = DMPlexGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
354   ierr = DMPlexGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
355   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
356     for (; c < leafLocal[l] && c < cEnd; ++c) {
357       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
358     }
359     if (leafLocal[l] >= cEnd) break;
360     if (leafRemote[l].rank == rank) {
361       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
362     } else {
363       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
364     }
365   }
366   for (; c < cEnd; ++c) {
367     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
368   }
369   if (0) {
370     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
371     ierr = DMLabelView(vtkLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
372     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
373   }
374   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
375   for (f = fStart; f < fEnd; ++f) {
376     PetscInt numCells;
377 
378     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
379     if (numCells < 2) {
380       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
381     } else {
382       const PetscInt *cells = NULL;
383       PetscInt        vA, vB;
384 
385       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
386       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
387       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
388       if (!vA && !vB) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
389     }
390   }
391   if (0) {
392     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
393     ierr = DMLabelView(ghostLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
394     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
395   }
396   PetscFunctionReturn(0);
397 }
398 
399 #undef __FUNCT__
400 #define __FUNCT__ "DMPlexConstructGhostCells_Internal"
401 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
402 {
403   IS              valueIS;
404   const PetscInt *values;
405   PetscInt       *depthShift;
406   PetscInt        depth = 0, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
407   PetscErrorCode  ierr;
408 
409   PetscFunctionBegin;
410   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
411   /* Count ghost cells */
412   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
413   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
414   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
415   *numGhostCells = 0;
416   for (fs = 0; fs < numFS; ++fs) {
417     IS              faceIS;
418     const PetscInt *faces;
419     PetscInt        numFaces, f, numBdFaces = 0;
420 
421     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
422     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
423     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
424     for (f = 0; f < numFaces; ++f) {
425       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
426     }
427     *numGhostCells += numBdFaces;
428     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
429   }
430   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
431   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthShift);CHKERRQ(ierr);
432   ierr = PetscMemzero(depthShift, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
433   if (depth >= 0) depthShift[depth] = *numGhostCells;
434   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
435   /* Step 3: Set cone/support sizes for new points */
436   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
437   for (c = cEnd; c < cEnd + *numGhostCells; ++c) {
438     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
439   }
440   for (fs = 0; fs < numFS; ++fs) {
441     IS              faceIS;
442     const PetscInt *faces;
443     PetscInt        numFaces, f;
444 
445     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
446     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
447     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
448     for (f = 0; f < numFaces; ++f) {
449       PetscInt size;
450 
451       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
452       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
453       if (size != 1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
454       ierr = DMPlexSetSupportSize(gdm, faces[f] + *numGhostCells, 2);CHKERRQ(ierr);
455     }
456     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
457     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
458   }
459   /* Step 4: Setup ghosted DM */
460   ierr = DMSetUp(gdm);CHKERRQ(ierr);
461   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
462   /* Step 6: Set cones and supports for new points */
463   ghostCell = cEnd;
464   for (fs = 0; fs < numFS; ++fs) {
465     IS              faceIS;
466     const PetscInt *faces;
467     PetscInt        numFaces, f;
468 
469     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
470     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
471     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
472     for (f = 0; f < numFaces; ++f) {
473       PetscInt newFace = faces[f] + *numGhostCells;
474 
475       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
476       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
477       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
478       ++ghostCell;
479     }
480     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
481     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
482   }
483   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
484   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
485   /* Step 7: Stratify */
486   ierr = DMPlexStratify(gdm);CHKERRQ(ierr);
487   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
488   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
489   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
490   ierr = PetscFree(depthShift);CHKERRQ(ierr);
491   PetscFunctionReturn(0);
492 }
493 
494 #undef __FUNCT__
495 #define __FUNCT__ "DMPlexConstructGhostCells"
496 /*@C
497   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
498 
499   Collective on dm
500 
501   Input Parameters:
502 + dm - The original DM
503 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
504 
505   Output Parameters:
506 + numGhostCells - The number of ghost cells added to the DM
507 - dmGhosted - The new DM
508 
509   Note: If no label exists of that name, one will be created marking all boundary faces
510 
511   Level: developer
512 
513 .seealso: DMCreate()
514 @*/
515 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
516 {
517   DM             gdm;
518   DMLabel        label;
519   const char    *name = labelName ? labelName : "Face Sets";
520   PetscInt       dim;
521   PetscErrorCode ierr;
522 
523   PetscFunctionBegin;
524   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
525   PetscValidPointer(numGhostCells, 3);
526   PetscValidPointer(dmGhosted, 4);
527   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
528   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
529   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
530   ierr = DMPlexSetDimension(gdm, dim);CHKERRQ(ierr);
531   ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
532   if (!label) {
533     /* Get label for boundary faces */
534     ierr = DMPlexCreateLabel(dm, name);CHKERRQ(ierr);
535     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
536     ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
537   }
538   ierr = DMPlexConstructGhostCells_Internal(dm, label, numGhostCells, gdm);CHKERRQ(ierr);
539   ierr = DMSetFromOptions(gdm);CHKERRQ(ierr);
540   *dmGhosted = gdm;
541   PetscFunctionReturn(0);
542 }
543 
544 #undef __FUNCT__
545 #define __FUNCT__ "DMPlexConstructCohesiveCells_Internal"
546 /*
547   We are adding two kinds of points here:
548     Replicated: Copies of points which exist in the mesh, such as vertices identified across a fault
549     Hybrid:     Entirely new points, such as cohesive cells
550 */
551 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DM sdm)
552 {
553   MPI_Comm         comm;
554   IS               valueIS;
555   PetscInt         numSP = 0;       /* The number of depths for which we have replicated points */
556   const PetscInt  *values;          /* List of depths for which we have replicated points */
557   IS              *splitIS;
558   IS              *unsplitIS;
559   PetscInt        *numSplitPoints;   /* The number of replicated points at each depth */
560   PetscInt        *numUnsplitPoints; /* The number of non-replicated points at each depth which still give rise to hybrid points */
561   PetscInt        *numHybridPoints;  /* The number of hybrid points at each depth */
562   const PetscInt **splitPoints;      /* Replicated points for each depth */
563   const PetscInt **unsplitPoints;    /* Non-replicated points for each depth */
564   PetscSection     coordSection;
565   Vec              coordinates;
566   PetscScalar     *coords;
567   PetscInt         depths[4];       /* Depths in the order that plex points are numbered */
568   PetscInt        *depthShift;      /* Number of replicated+hybrid points at each depth */
569   PetscInt        *depthOffset;     /* Prefix sums of depthShift */
570   PetscInt        *pMaxNew;         /* The first replicated point at each depth in the new mesh, hybrids come after this */
571   PetscInt        *coneNew, *coneONew, *supportNew;
572   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
573   PetscErrorCode   ierr;
574 
575   PetscFunctionBegin;
576   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
577   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
578   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
579   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
580   depths[0] = depth;
581   depths[1] = 0;
582   depths[2] = depth-1;
583   depths[3] = 1;
584   /* Count split points and add cohesive cells */
585   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
586   ierr = PetscMalloc3(depth+1,PetscInt,&depthShift,depth+1,PetscInt,&depthOffset,depth+1,PetscInt,&pMaxNew);CHKERRQ(ierr);
587   ierr = PetscMalloc7(depth+1,IS,&splitIS,depth+1,IS,&unsplitIS,depth+1,PetscInt,&numSplitPoints,depth+1,PetscInt,&numUnsplitPoints,depth+1,PetscInt,&numHybridPoints,depth+1,const PetscInt*,&splitPoints,depth+1,const PetscInt*,&unsplitPoints);CHKERRQ(ierr);
588   ierr = PetscMemzero(depthShift,  (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
589   ierr = PetscMemzero(depthOffset, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
590   for (d = 0; d <= depth; ++d) {
591     ierr                = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
592     numSplitPoints[d]   = 0;
593     numUnsplitPoints[d] = 0;
594     numHybridPoints[d]  = 0;
595     splitPoints[d]      = NULL;
596     unsplitPoints[d]    = NULL;
597     splitIS[d]          = NULL;
598     unsplitIS[d]        = NULL;
599   }
600   if (label) {
601     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
602     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
603     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
604   }
605   for (sp = 0; sp < numSP; ++sp) {
606     const PetscInt dep = values[sp];
607 
608     if ((dep < 0) || (dep > depth)) continue;
609     ierr = DMLabelGetStratumIS(label, dep, &splitIS[dep]);CHKERRQ(ierr);
610     if (splitIS[dep]) {
611       ierr = ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
612       ierr = ISGetIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
613     }
614     ierr = DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]);CHKERRQ(ierr);
615     if (unsplitIS[dep]) {
616       ierr = ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]);CHKERRQ(ierr);
617       ierr = ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);
618     }
619   }
620   /* Calculate number of hybrid points */
621   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
622   for (d = 0; d <= depth; ++d) depthShift[d]          = numSplitPoints[d] + numHybridPoints[d];
623   for (d = 1; d <= depth; ++d) depthOffset[depths[d]] = depthOffset[depths[d-1]] + depthShift[depths[d-1]];
624   for (d = 0; d <= depth; ++d) pMaxNew[d]            += depthOffset[d];
625   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
626   /* Step 3: Set cone/support sizes for new points */
627   for (dep = 0; dep <= depth; ++dep) {
628     for (p = 0; p < numSplitPoints[dep]; ++p) {
629       const PetscInt  oldp   = splitPoints[dep][p];
630       const PetscInt  newp   = oldp + depthOffset[dep];
631       const PetscInt  splitp = p    + pMaxNew[dep];
632       const PetscInt *support;
633       PetscInt        coneSize, supportSize, qf, qn, qp, e;
634 
635       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
636       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
637       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
638       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
639       if (dep == depth-1) {
640         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
641 
642         /* Add cohesive cells, they are prisms */
643         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
644       } else if (dep == 0) {
645         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
646 
647         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
648         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
649           PetscInt val;
650 
651           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
652           if (val == 1) ++qf;
653           if ((val == 1) || (val ==  (shift + 1))) ++qn;
654           if ((val == 1) || (val == -(shift + 1))) ++qp;
655         }
656         /* Split old vertex: Edges into original vertex and new cohesive edge */
657         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
658         /* Split new vertex: Edges into split vertex and new cohesive edge */
659         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
660         /* Add hybrid edge */
661         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
662         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
663       } else if (dep == dim-2) {
664         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
665 
666         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
667         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
668           PetscInt val;
669 
670           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
671           if (val == dim-1) ++qf;
672           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
673           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
674         }
675         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
676         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
677         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
678         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
679         /* Add hybrid face */
680         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
681         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
682       }
683     }
684   }
685   for (dep = 0; dep <= depth; ++dep) {
686     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
687       const PetscInt  oldp   = unsplitPoints[dep][p];
688       const PetscInt  newp   = oldp + depthOffset[dep];
689       const PetscInt *support;
690       PetscInt        coneSize, supportSize, qf, qn, qp, e;
691 
692       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
693       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
694       if (dep == 0) {
695         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
696 
697         /* Unsplit vertex: Edges into original vertex, split edge, and new cohesive edge twice */
698         ierr = DMPlexSetSupportSize(sdm, newp, supportSize+3);CHKERRQ(ierr);
699         /* Add hybrid edge */
700         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
701         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
702         for (e = 0, qf = 0; e < supportSize; ++e) {
703           PetscInt val;
704 
705           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
706           /* Split and unsplit edges produce hybrid faces */
707           if ((val == 1) || (val ==  (shift2 + 1))) ++qf;
708         }
709         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
710       } else if (dep == dim-2) {
711         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
712 
713         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
714         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
715           PetscInt val;
716 
717           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
718           if (val == dim-1) ++qf;
719           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
720           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
721         }
722         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
723         ierr = DMPlexSetSupportSize(sdm, newp, supportSize+3);CHKERRQ(ierr);
724         /* Add hybrid face */
725         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
726         /* TODO: Here we should check for neighboring faults for which this is unsplit */
727         ierr = DMPlexSetSupportSize(sdm, hybface, 1);CHKERRQ(ierr);
728       }
729     }
730   }
731   /* Step 4: Setup split DM */
732   ierr = DMSetUp(sdm);CHKERRQ(ierr);
733   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
734   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
735   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,PetscInt,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,PetscInt,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),PetscInt,&supportNew);CHKERRQ(ierr);
736   /* Step 6: Set cones and supports for new points */
737   for (dep = 0; dep <= depth; ++dep) {
738     for (p = 0; p < numSplitPoints[dep]; ++p) {
739       const PetscInt  oldp   = splitPoints[dep][p];
740       const PetscInt  newp   = oldp + depthOffset[dep];
741       const PetscInt  splitp = p    + pMaxNew[dep];
742       const PetscInt *cone, *support, *ornt;
743       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
744 
745       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
746       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
747       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
748       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
749       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
750       if (dep == depth-1) {
751         const PetscInt  hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
752         const PetscInt *supportF;
753 
754         /* Split face:       copy in old face to new face to start */
755         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
756         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
757         /* Split old face:   old vertices/edges in cone so no change */
758         /* Split new face:   new vertices/edges in cone */
759         for (q = 0; q < coneSize; ++q) {
760           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
761           if (v < 0) {
762             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
763             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
764             coneNew[2+q] = cone[q] + depthOffset[dep-1];
765           } else {
766             coneNew[2+q] = v + pMaxNew[dep-1];
767           }
768         }
769         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
770         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
771         /* Face support */
772         for (s = 0; s < supportSize; ++s) {
773           PetscInt val;
774 
775           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
776           if (val < 0) {
777             /* Split old face:   Replace negative side cell with cohesive cell */
778              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
779           } else {
780             /* Split new face:   Replace positive side cell with cohesive cell */
781             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
782             /* Get orientation for cohesive face */
783             {
784               const PetscInt *ncone, *nconeO;
785               PetscInt        nconeSize, nc;
786 
787               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
788               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
789               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
790               for (nc = 0; nc < nconeSize; ++nc) {
791                 if (ncone[nc] == oldp) {
792                   coneONew[0] = nconeO[nc];
793                   break;
794                 }
795               }
796               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
797             }
798           }
799         }
800         /* Cohesive cell:    Old and new split face, then new cohesive faces */
801         coneNew[0]  = newp;   /* Extracted negative side orientation above */
802         coneNew[1]  = splitp;
803         coneONew[1] = coneONew[0];
804         for (q = 0; q < coneSize; ++q) {
805           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
806           if (v < 0) {
807             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
808             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
809             coneONew[2+q] = 0;
810           } else {
811             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
812           }
813           coneONew[2+q] = 0;
814         }
815         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
816         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
817       } else if (dep == 0) {
818         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
819 
820         /* Split old vertex: Edges in old split faces and new cohesive edge */
821         for (e = 0, qn = 0; e < supportSize; ++e) {
822           PetscInt val;
823 
824           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
825           if ((val == 1) || (val == (shift + 1))) {
826             supportNew[qn++] = support[e] + depthOffset[dep+1];
827           }
828         }
829         supportNew[qn] = hybedge;
830         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
831         /* Split new vertex: Edges in new split faces and new cohesive edge */
832         for (e = 0, qp = 0; e < supportSize; ++e) {
833           PetscInt val, edge;
834 
835           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
836           if (val == 1) {
837             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
838             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
839             supportNew[qp++] = edge + pMaxNew[dep+1];
840           } else if (val == -(shift + 1)) {
841             supportNew[qp++] = support[e] + depthOffset[dep+1];
842           }
843         }
844         supportNew[qp] = hybedge;
845         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
846         /* Hybrid edge:    Old and new split vertex */
847         coneNew[0] = newp;
848         coneNew[1] = splitp;
849         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
850         for (e = 0, qf = 0; e < supportSize; ++e) {
851           PetscInt val, edge;
852 
853           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
854           if (val == 1) {
855             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
856             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
857             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
858           }
859         }
860         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
861       } else if (dep == dim-2) {
862         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
863 
864         /* Split old edge:   old vertices in cone so no change */
865         /* Split new edge:   new vertices in cone */
866         for (q = 0; q < coneSize; ++q) {
867           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
868           if (v < 0) {
869             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
870             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
871             coneNew[q] = cone[q] + depthOffset[dep-1];
872           } else {
873             coneNew[q] = v + pMaxNew[dep-1];
874           }
875         }
876         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
877         /* Split old edge: Faces in positive side cells and old split faces */
878         for (e = 0, q = 0; e < supportSize; ++e) {
879           PetscInt val;
880 
881           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
882           if (val == dim-1) {
883             supportNew[q++] = support[e] + depthOffset[dep+1];
884           } else if (val == (shift + dim-1)) {
885             supportNew[q++] = support[e] + depthOffset[dep+1];
886           }
887         }
888         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
889         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
890         /* Split new edge: Faces in negative side cells and new split faces */
891         for (e = 0, q = 0; e < supportSize; ++e) {
892           PetscInt val, face;
893 
894           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
895           if (val == dim-1) {
896             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
897             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
898             supportNew[q++] = face + pMaxNew[dep+1];
899           } else if (val == -(shift + dim-1)) {
900             supportNew[q++] = support[e] + depthOffset[dep+1];
901           }
902         }
903         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
904         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
905         /* Hybrid face */
906         coneNew[0] = newp;
907         coneNew[1] = splitp;
908         for (v = 0; v < coneSize; ++v) {
909           PetscInt vertex;
910           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
911           if (vertex < 0) {
912             ierr = PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex);CHKERRQ(ierr);
913             if (vertex < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[v], dep-1);
914             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
915           } else {
916             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
917           }
918         }
919         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
920         for (e = 0, qf = 0; e < supportSize; ++e) {
921           PetscInt val, face;
922 
923           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
924           if (val == dim-1) {
925             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
926             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
927             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
928           }
929         }
930         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
931       }
932     }
933   }
934   for (dep = 0; dep <= depth; ++dep) {
935     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
936       const PetscInt  oldp   = unsplitPoints[dep][p];
937       const PetscInt  newp   = oldp + depthOffset[dep];
938       const PetscInt *cone, *support, *ornt;
939       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
940 
941       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
942       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
943       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
944       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
945       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
946       if (dep == 0) {
947         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
948 
949         /* Unsplit vertex */
950         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
951         for (s = 0, q = 0; s < supportSize; ++s) {
952           supportNew[q++] = support[s] + depthOffset[dep+1];
953           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
954           if (e >= 0) {
955             supportNew[q++] = e + pMaxNew[dep+1];
956           }
957         }
958         supportNew[q++] = hybedge;
959         supportNew[q++] = hybedge;
960         if (q != supportSizeNew) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Support size %d != %d for vertex %d", q, supportSizeNew, newp);
961         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
962         /* Hybrid edge */
963         coneNew[0] = newp;
964         coneNew[1] = newp;
965         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
966         for (e = 0, qf = 0; e < supportSize; ++e) {
967           PetscInt val, edge;
968 
969           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
970           if (val == 1) {
971             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
972             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
973             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
974           } else if  (val ==  (shift2 + 1)) {
975             ierr = PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge);CHKERRQ(ierr);
976             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a unsplit edge", support[e]);
977             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
978           }
979         }
980         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
981       } else if (dep == dim-2) {
982         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
983         PetscInt       hybcell;
984 
985         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
986         for (f = 0, qf = 0; f < supportSize; ++f) {
987           PetscInt val, face;
988 
989           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
990           if (val == dim-1) {
991             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
992             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[f]);
993             supportNew[qf++] = support[f] + depthOffset[dep+1];
994             supportNew[qf++] = face + pMaxNew[dep+1];
995             hybcell          = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
996           } else {
997             supportNew[qf++] = support[f] + depthOffset[dep+1];
998           }
999         }
1000         supportNew[qf++] = hybface;
1001         supportNew[qf++] = hybface;
1002         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1003         /* Add hybrid face */
1004         coneNew[0] = newp;
1005         ierr = PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1006         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[0]);
1007         coneNew[1] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1008         coneNew[2] = newp;
1009         ierr = PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1010         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[1]);
1011         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1012         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1013         supportNew[0] = hybcell;
1014         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1015       }
1016     }
1017   }
1018   /* Step 6b: Replace split points in negative side cones */
1019   for (sp = 0; sp < numSP; ++sp) {
1020     PetscInt        dep = values[sp];
1021     IS              pIS;
1022     PetscInt        numPoints;
1023     const PetscInt *points;
1024 
1025     if (dep >= 0) continue;
1026     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
1027     if (!pIS) continue;
1028     dep  = -dep - shift;
1029     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
1030     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
1031     for (p = 0; p < numPoints; ++p) {
1032       const PetscInt  oldp = points[p];
1033       const PetscInt  newp = depthOffset[dep] + oldp;
1034       const PetscInt *cone;
1035       PetscInt        coneSize, c;
1036       /* PetscBool       replaced = PETSC_FALSE; */
1037 
1038       /* Negative edge: replace split vertex */
1039       /* Negative cell: replace split face */
1040       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
1041       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
1042       for (c = 0; c < coneSize; ++c) {
1043         const PetscInt coldp = cone[c] - depthOffset[dep-1];
1044         PetscInt       csplitp, cp, val;
1045 
1046         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
1047         if (val == dep-1) {
1048           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
1049           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
1050           csplitp  = pMaxNew[dep-1] + cp;
1051           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
1052           /* replaced = PETSC_TRUE; */
1053         }
1054       }
1055       /* Cells with only a vertex or edge on the submesh have no replacement */
1056       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1057     }
1058     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
1059     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
1060   }
1061   /* Step 7: Stratify */
1062   ierr = DMPlexStratify(sdm);CHKERRQ(ierr);
1063   /* Step 8: Coordinates */
1064   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1065   ierr = DMPlexGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
1066   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
1067   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1068   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1069     const PetscInt newp   = depthOffset[0] + splitPoints[0][v];
1070     const PetscInt splitp = pMaxNew[0] + v;
1071     PetscInt       dof, off, soff, d;
1072 
1073     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
1074     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
1075     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
1076     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1077   }
1078   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1079   /* Step 9: SF, if I can figure this out we can split the mesh in parallel */
1080   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1081   /* Step 10: Labels */
1082   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1083   ierr = DMPlexGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
1084   for (dep = 0; dep <= depth; ++dep) {
1085     for (p = 0; p < numSplitPoints[dep]; ++p) {
1086       const PetscInt newp   = depthOffset[dep] + splitPoints[dep][p];
1087       const PetscInt splitp = pMaxNew[dep] + p;
1088       PetscInt       l;
1089 
1090       for (l = 0; l < numLabels; ++l) {
1091         DMLabel     mlabel;
1092         const char *lname;
1093         PetscInt    val;
1094         PetscBool   isDepth;
1095 
1096         ierr = DMPlexGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
1097         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
1098         if (isDepth) continue;
1099         ierr = DMPlexGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
1100         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
1101         if (val >= 0) {
1102           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
1103 #if 0
1104           /* Do not put cohesive edges into the label */
1105           if (dep == 0) {
1106             const PetscInt cedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1107             ierr = DMLabelSetValue(mlabel, cedge, val);CHKERRQ(ierr);
1108           } else if (dep == dim-2) {
1109             const PetscInt cface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1110             ierr = DMLabelSetValue(mlabel, cface, val);CHKERRQ(ierr);
1111           }
1112           /* Do not put cohesive faces into the label */
1113 #endif
1114         }
1115       }
1116     }
1117   }
1118   for (sp = 0; sp < numSP; ++sp) {
1119     const PetscInt dep = values[sp];
1120 
1121     if ((dep < 0) || (dep > depth)) continue;
1122     if (splitIS[dep]) {ierr = ISRestoreIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
1123     ierr = ISDestroy(&splitIS[dep]);CHKERRQ(ierr);
1124     if (unsplitIS[dep]) {ierr = ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);}
1125     ierr = ISDestroy(&unsplitIS[dep]);CHKERRQ(ierr);
1126   }
1127   if (label) {
1128     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1129     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1130   }
1131   for (d = 0; d <= depth; ++d) {
1132     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
1133     pMaxNew[d] = pEnd - numHybridPoints[d];
1134   }
1135   ierr = DMPlexSetHybridBounds(sdm, depth >= 0 ? pMaxNew[depth] : PETSC_DETERMINE, depth>1 ? pMaxNew[depth-1] : PETSC_DETERMINE, depth>2 ? pMaxNew[1] : PETSC_DETERMINE, depth >= 0 ? pMaxNew[0] : PETSC_DETERMINE);CHKERRQ(ierr);
1136   ierr = PetscFree3(depthShift, depthOffset, pMaxNew);CHKERRQ(ierr);
1137   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
1138   ierr = PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints);CHKERRQ(ierr);
1139   PetscFunctionReturn(0);
1140 }
1141 
1142 #undef __FUNCT__
1143 #define __FUNCT__ "DMPlexConstructCohesiveCells"
1144 /*@C
1145   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1146 
1147   Collective on dm
1148 
1149   Input Parameters:
1150 + dm - The original DM
1151 - label - The label specifying the boundary faces (this could be auto-generated)
1152 
1153   Output Parameters:
1154 - dmSplit - The new DM
1155 
1156   Level: developer
1157 
1158 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
1159 @*/
1160 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DM *dmSplit)
1161 {
1162   DM             sdm;
1163   PetscInt       dim;
1164   PetscErrorCode ierr;
1165 
1166   PetscFunctionBegin;
1167   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1168   PetscValidPointer(dmSplit, 3);
1169   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
1170   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
1171   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1172   ierr = DMPlexSetDimension(sdm, dim);CHKERRQ(ierr);
1173   switch (dim) {
1174   case 2:
1175   case 3:
1176     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, sdm);CHKERRQ(ierr);
1177     break;
1178   default:
1179     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1180   }
1181   *dmSplit = sdm;
1182   PetscFunctionReturn(0);
1183 }
1184 
1185 #undef __FUNCT__
1186 #define __FUNCT__ "DMPlexLabelCohesiveComplete"
1187 /*@
1188   DMPlexLabelCohesiveComplete - Starting with a label marking vertices on an internal surface, we add all other mesh pieces
1189   to complete the surface
1190 
1191   Input Parameters:
1192 + dm - The DM
1193 . label - A DMLabel marking the surface vertices
1194 . flip  - Flag to flip the submesh normal and replace points on the other side
1195 - subdm - The subDM associated with the label, or NULL
1196 
1197   Output Parameter:
1198 . label - A DMLabel marking all surface points
1199 
1200   Level: developer
1201 
1202 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1203 @*/
1204 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, PetscBool flip, DM subdm)
1205 {
1206   DMLabel         depthLabel;
1207   IS              dimIS, subpointIS, facePosIS, faceNegIS;
1208   const PetscInt *points, *subpoints;
1209   const PetscInt  rev   = flip ? -1 : 1;
1210   PetscInt        shift = 100, shift2 = 200, dim, dep, cStart, cEnd, numPoints, numSubpoints, p, val;
1211   PetscErrorCode  ierr;
1212 
1213   PetscFunctionBegin;
1214   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1215   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1216   if (subdm) {
1217     ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1218     if (subpointIS) {
1219       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1220       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1221     }
1222   }
1223   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1224   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1225   if (!dimIS) PetscFunctionReturn(0);
1226   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1227   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1228   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1229     const PetscInt *support;
1230     PetscInt        supportSize, s;
1231 
1232     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1233     if (supportSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports", points[p], supportSize);
1234     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1235     for (s = 0; s < supportSize; ++s) {
1236       const PetscInt *cone, *ornt;
1237       PetscInt        coneSize, c;
1238       PetscBool       pos = PETSC_TRUE;
1239 
1240       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1241       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1242       ierr = DMPlexGetConeOrientation(dm, support[s], &ornt);CHKERRQ(ierr);
1243       for (c = 0; c < coneSize; ++c) {
1244         if (cone[c] == points[p]) {
1245           PetscInt o = ornt[c];
1246 
1247           if (subdm) {
1248             const PetscInt *subcone, *subornt;
1249             PetscInt        subpoint, subface, subconeSize, sc;
1250 
1251             ierr = PetscFindInt(support[s], numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1252             ierr = PetscFindInt(points[p],  numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1253             ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1254             ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1255             ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1256             for (sc = 0; sc < subconeSize; ++sc) {
1257               if (subcone[sc] == subface) {
1258                 o = subornt[0];
1259                 break;
1260               }
1261             }
1262             if (sc >= subconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find point %d in cone for subpoint %d", points[p], subpoint);
1263           }
1264           if (o >= 0) {
1265             ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);
1266             pos  = rev > 0 ? PETSC_TRUE : PETSC_FALSE;
1267           } else {
1268             ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);
1269             pos  = rev > 0 ? PETSC_FALSE : PETSC_TRUE;
1270           }
1271           break;
1272         }
1273       }
1274       if (c == coneSize) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell split face %d support does not have it in the cone", points[p]);
1275       /* Put faces touching the fault in the label */
1276       for (c = 0; c < coneSize; ++c) {
1277         const PetscInt point = cone[c];
1278 
1279         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1280         if (val == -1) {
1281           PetscInt *closure = NULL;
1282           PetscInt  closureSize, cl;
1283 
1284           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1285           for (cl = 0; cl < closureSize*2; cl += 2) {
1286             const PetscInt clp = closure[cl];
1287 
1288             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1289             if ((val >= 0) && (val < dim-1)) {
1290               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1291               break;
1292             }
1293           }
1294           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1295         }
1296       }
1297     }
1298   }
1299   if (subdm) {
1300     if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1301     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1302   }
1303   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1304   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1305   /* Search for other cells/faces/edges connected to the fault by a vertex */
1306   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1307   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1308   if (!dimIS) PetscFunctionReturn(0);
1309   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1310   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1311   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1312     PetscInt *star = NULL;
1313     PetscInt  starSize, s;
1314     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1315 
1316     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1317     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1318     while (again) {
1319       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1320       again = 0;
1321       for (s = 0; s < starSize*2; s += 2) {
1322         const PetscInt  point = star[s];
1323         const PetscInt *cone;
1324         PetscInt        coneSize, c;
1325 
1326         if ((point < cStart) || (point >= cEnd)) continue;
1327         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1328         if (val != -1) continue;
1329         again = again == 1 ? 1 : 2;
1330         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1331         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1332         for (c = 0; c < coneSize; ++c) {
1333           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1334           if (val != -1) {
1335             const PetscInt *ccone;
1336             PetscInt        cconeSize, cc, side;
1337 
1338             if (abs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1339             if (val > 0) side =  1;
1340             else         side = -1;
1341             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1342             /* Mark cell faces which touch the fault */
1343             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1344             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1345             for (cc = 0; cc < cconeSize; ++cc) {
1346               PetscInt *closure = NULL;
1347               PetscInt  closureSize, cl;
1348 
1349               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1350               if (val != -1) continue;
1351               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1352               for (cl = 0; cl < closureSize*2; cl += 2) {
1353                 const PetscInt clp = closure[cl];
1354 
1355                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1356                 if (val == -1) continue;
1357                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1358                 break;
1359               }
1360               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1361             }
1362             again = 1;
1363             break;
1364           }
1365         }
1366       }
1367     }
1368     /* Classify the rest by cell membership */
1369     for (s = 0; s < starSize*2; s += 2) {
1370       const PetscInt point = star[s];
1371 
1372       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1373       if (val == -1) {
1374         PetscInt  *sstar = NULL;
1375         PetscInt   sstarSize, ss;
1376         PetscBool  marked = PETSC_FALSE;
1377 
1378         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1379         for (ss = 0; ss < sstarSize*2; ss += 2) {
1380           const PetscInt spoint = sstar[ss];
1381 
1382           if ((spoint < cStart) || (spoint >= cEnd)) continue;
1383           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1384           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1385           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1386           if (val > 0) {
1387             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1388           } else {
1389             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1390           }
1391           marked = PETSC_TRUE;
1392           break;
1393         }
1394         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1395         if (!marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1396       }
1397     }
1398     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1399   }
1400   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1401   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1402   /* If any faces touching the fault divide cells on either side, split them */
1403   ierr = DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS);CHKERRQ(ierr);
1404   ierr = DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS);CHKERRQ(ierr);
1405   ierr = ISExpand(facePosIS, faceNegIS, &dimIS);CHKERRQ(ierr);
1406   ierr = ISDestroy(&facePosIS);CHKERRQ(ierr);
1407   ierr = ISDestroy(&faceNegIS);CHKERRQ(ierr);
1408   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1409   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1410   for (p = 0; p < numPoints; ++p) {
1411     const PetscInt  point = points[p];
1412     const PetscInt *support;
1413     PetscInt        supportSize, valA, valB;
1414 
1415     ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1416     if (supportSize != 2) continue;
1417     ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1418     ierr = DMLabelGetValue(label, support[0], &valA);CHKERRQ(ierr);
1419     ierr = DMLabelGetValue(label, support[1], &valB);CHKERRQ(ierr);
1420     if ((valA == -1) || (valB == -1)) continue;
1421     if (valA*valB > 0) continue;
1422     /* Split the face */
1423     ierr = DMLabelGetValue(label, point, &valA);CHKERRQ(ierr);
1424     ierr = DMLabelClearValue(label, point, valA);CHKERRQ(ierr);
1425     ierr = DMLabelSetValue(label, point, dim-1);CHKERRQ(ierr);
1426     /* Label its closure:
1427       unmarked: label as unsplit
1428       incident: relabel as split
1429       split:    do nothing
1430     */
1431     {
1432       PetscInt *closure = NULL;
1433       PetscInt  closureSize, cl;
1434 
1435       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1436       for (cl = 0; cl < closureSize*2; cl += 2) {
1437         ierr = DMLabelGetValue(label, closure[cl], &valA);CHKERRQ(ierr);
1438         if (valA < 0) { /* Mark as unsplit */
1439           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1440           ierr = DMLabelSetValue(label, closure[cl], shift2+dep);CHKERRQ(ierr);
1441         } else if (valA >= shift) {
1442           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1443           ierr = DMLabelClearValue(label, closure[cl], valA);CHKERRQ(ierr);
1444           ierr = DMLabelSetValue(label, closure[cl], dep);CHKERRQ(ierr);
1445         }
1446       }
1447       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1448     }
1449   }
1450   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1451   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1452   PetscFunctionReturn(0);
1453 }
1454 
1455 #undef __FUNCT__
1456 #define __FUNCT__ "DMPlexCreateHybridMesh"
1457 /*@C
1458   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
1459 
1460   Collective on dm
1461 
1462   Input Parameters:
1463 + dm - The original DM
1464 - labelName - The label specifying the interface vertices
1465 
1466   Output Parameters:
1467 + hybridLabel - The label fully marking the interface
1468 - dmHybrid - The new DM
1469 
1470   Level: developer
1471 
1472 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMCreate()
1473 @*/
1474 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel *hybridLabel, DM *dmHybrid)
1475 {
1476   DM             idm;
1477   DMLabel        subpointMap, hlabel;
1478   PetscInt       dim;
1479   PetscErrorCode ierr;
1480 
1481   PetscFunctionBegin;
1482   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1483   if (hybridLabel) PetscValidPointer(hybridLabel, 3);
1484   PetscValidPointer(dmHybrid, 4);
1485   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1486   ierr = DMPlexCreateSubmesh(dm, label, 1, &idm);CHKERRQ(ierr);
1487   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
1488   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
1489   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
1490   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
1491   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, PETSC_FALSE, idm);CHKERRQ(ierr);
1492   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1493   ierr = DMPlexConstructCohesiveCells(dm, hlabel, dmHybrid);CHKERRQ(ierr);
1494   if (hybridLabel) *hybridLabel = hlabel;
1495   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
1496   PetscFunctionReturn(0);
1497 }
1498 
1499 #undef __FUNCT__
1500 #define __FUNCT__ "DMPlexMarkSubmesh_Uninterpolated"
1501 /* Here we need the explicit assumption that:
1502 
1503      For any marked cell, the marked vertices constitute a single face
1504 */
1505 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
1506 {
1507   IS               subvertexIS = NULL;
1508   const PetscInt  *subvertices;
1509   PetscInt        *pStart, *pEnd, *pMax, pSize;
1510   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
1511   PetscErrorCode   ierr;
1512 
1513   PetscFunctionBegin;
1514   *numFaces = 0;
1515   *nFV      = 0;
1516   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1517   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1518   pSize = PetscMax(depth, dim) + 1;
1519   ierr = PetscMalloc3(pSize,PetscInt,&pStart,pSize,PetscInt,&pEnd,pSize,PetscInt,&pMax);CHKERRQ(ierr);
1520   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1521   for (d = 0; d <= depth; ++d) {
1522     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1523     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1524   }
1525   /* Loop over initial vertices and mark all faces in the collective star() */
1526   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
1527   if (subvertexIS) {
1528     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1529     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1530   }
1531   for (v = 0; v < numSubVerticesInitial; ++v) {
1532     const PetscInt vertex = subvertices[v];
1533     PetscInt      *star   = NULL;
1534     PetscInt       starSize, s, numCells = 0, c;
1535 
1536     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1537     for (s = 0; s < starSize*2; s += 2) {
1538       const PetscInt point = star[s];
1539       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
1540     }
1541     for (c = 0; c < numCells; ++c) {
1542       const PetscInt cell    = star[c];
1543       PetscInt      *closure = NULL;
1544       PetscInt       closureSize, cl;
1545       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
1546 
1547       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
1548       if (cellLoc == 2) continue;
1549       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
1550       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1551       for (cl = 0; cl < closureSize*2; cl += 2) {
1552         const PetscInt point = closure[cl];
1553         PetscInt       vertexLoc;
1554 
1555         if ((point >= pStart[0]) && (point < pEnd[0])) {
1556           ++numCorners;
1557           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1558           if (vertexLoc == value) closure[faceSize++] = point;
1559         }
1560       }
1561       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
1562       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
1563       if (faceSize == *nFV) {
1564         const PetscInt *cells = NULL;
1565         PetscInt        numCells, nc;
1566 
1567         ++(*numFaces);
1568         for (cl = 0; cl < faceSize; ++cl) {
1569           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
1570         }
1571         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1572         for (nc = 0; nc < numCells; ++nc) {
1573           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
1574         }
1575         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1576       }
1577       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1578     }
1579     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1580   }
1581   if (subvertexIS) {
1582     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1583   }
1584   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1585   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1586   PetscFunctionReturn(0);
1587 }
1588 
1589 #undef __FUNCT__
1590 #define __FUNCT__ "DMPlexMarkSubmesh_Interpolated"
1591 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, DM subdm)
1592 {
1593   IS               subvertexIS;
1594   const PetscInt  *subvertices;
1595   PetscInt        *pStart, *pEnd, *pMax;
1596   PetscInt         dim, d, numSubVerticesInitial = 0, v;
1597   PetscErrorCode   ierr;
1598 
1599   PetscFunctionBegin;
1600   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1601   ierr = PetscMalloc3(dim+1,PetscInt,&pStart,dim+1,PetscInt,&pEnd,dim+1,PetscInt,&pMax);CHKERRQ(ierr);
1602   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], dim>1 ? &pMax[dim-1] : NULL, dim > 2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1603   for (d = 0; d <= dim; ++d) {
1604     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1605     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1606   }
1607   /* Loop over initial vertices and mark all faces in the collective star() */
1608   ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
1609   if (subvertexIS) {
1610     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1611     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1612   }
1613   for (v = 0; v < numSubVerticesInitial; ++v) {
1614     const PetscInt vertex = subvertices[v];
1615     PetscInt      *star   = NULL;
1616     PetscInt       starSize, s, numFaces = 0, f;
1617 
1618     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1619     for (s = 0; s < starSize*2; s += 2) {
1620       const PetscInt point = star[s];
1621       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) star[numFaces++] = point;
1622     }
1623     for (f = 0; f < numFaces; ++f) {
1624       const PetscInt face    = star[f];
1625       PetscInt      *closure = NULL;
1626       PetscInt       closureSize, c;
1627       PetscInt       faceLoc;
1628 
1629       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
1630       if (faceLoc == dim-1) continue;
1631       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
1632       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1633       for (c = 0; c < closureSize*2; c += 2) {
1634         const PetscInt point = closure[c];
1635         PetscInt       vertexLoc;
1636 
1637         if ((point >= pStart[0]) && (point < pEnd[0])) {
1638           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1639           if (vertexLoc != value) break;
1640         }
1641       }
1642       if (c == closureSize*2) {
1643         const PetscInt *support;
1644         PetscInt        supportSize, s;
1645 
1646         for (c = 0; c < closureSize*2; c += 2) {
1647           const PetscInt point = closure[c];
1648 
1649           for (d = 0; d < dim; ++d) {
1650             if ((point >= pStart[d]) && (point < pEnd[d])) {
1651               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
1652               break;
1653             }
1654           }
1655         }
1656         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
1657         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
1658         for (s = 0; s < supportSize; ++s) {
1659           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
1660         }
1661       }
1662       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1663     }
1664     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1665   }
1666   if (subvertexIS) {
1667     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1668   }
1669   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1670   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1671   PetscFunctionReturn(0);
1672 }
1673 
1674 #undef __FUNCT__
1675 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Uninterpolated"
1676 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
1677 {
1678   DMLabel         label = NULL;
1679   const PetscInt *cone;
1680   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize;
1681   PetscErrorCode  ierr;
1682 
1683   PetscFunctionBegin;
1684   *numFaces = 0;
1685   *nFV = 0;
1686   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1687   *subCells = NULL;
1688   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1689   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1690   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1691   if (cMax < 0) PetscFunctionReturn(0);
1692   if (label) {
1693     for (c = cMax; c < cEnd; ++c) {
1694       PetscInt val;
1695 
1696       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1697       if (val == value) {
1698         ++(*numFaces);
1699         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1700       }
1701     }
1702   } else {
1703     *numFaces = cEnd - cMax;
1704     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
1705   }
1706   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
1707   ierr = PetscMalloc(*numFaces *2 * sizeof(PetscInt), subCells);CHKERRQ(ierr);
1708   for (c = cMax; c < cEnd; ++c) {
1709     const PetscInt *cells;
1710     PetscInt        numCells;
1711 
1712     if (label) {
1713       PetscInt val;
1714 
1715       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1716       if (val != value) continue;
1717     }
1718     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1719     for (p = 0; p < *nFV; ++p) {
1720       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
1721     }
1722     /* Negative face */
1723     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1724     /* Not true in parallel
1725     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
1726     for (p = 0; p < numCells; ++p) {
1727       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
1728       (*subCells)[subc++] = cells[p];
1729     }
1730     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1731     /* Positive face is not included */
1732   }
1733   PetscFunctionReturn(0);
1734 }
1735 
1736 #undef __FUNCT__
1737 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Interpolated"
1738 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DMLabel subpointMap, DM subdm)
1739 {
1740   DMLabel        label = NULL;
1741   PetscInt      *pStart, *pEnd;
1742   PetscInt       dim, cMax, cEnd, c, d;
1743   PetscErrorCode ierr;
1744 
1745   PetscFunctionBegin;
1746   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1747   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1748   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1749   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1750   if (cMax < 0) PetscFunctionReturn(0);
1751   ierr = PetscMalloc2(dim+1,PetscInt,&pStart,dim+1,PetscInt,&pEnd);CHKERRQ(ierr);
1752   for (d = 0; d <= dim; ++d) {ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);}
1753   for (c = cMax; c < cEnd; ++c) {
1754     const PetscInt *cone;
1755     PetscInt       *closure = NULL;
1756     PetscInt        fconeSize, coneSize, closureSize, cl, val;
1757 
1758     if (label) {
1759       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1760       if (val != value) continue;
1761     }
1762     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1763     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1764     ierr = DMPlexGetConeSize(dm, cone[0], &fconeSize);CHKERRQ(ierr);
1765     if (coneSize != (fconeSize ? fconeSize : 1) + 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
1766     /* Negative face */
1767     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1768     for (cl = 0; cl < closureSize*2; cl += 2) {
1769       const PetscInt point = closure[cl];
1770 
1771       for (d = 0; d <= dim; ++d) {
1772         if ((point >= pStart[d]) && (point < pEnd[d])) {
1773           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
1774           break;
1775         }
1776       }
1777     }
1778     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1779     /* Cells -- positive face is not included */
1780     for (cl = 0; cl < 1; ++cl) {
1781       const PetscInt *support;
1782       PetscInt        supportSize, s;
1783 
1784       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
1785       if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells");
1786       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
1787       for (s = 0; s < supportSize; ++s) {
1788         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
1789       }
1790     }
1791   }
1792   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
1793   PetscFunctionReturn(0);
1794 }
1795 
1796 #undef __FUNCT__
1797 #define __FUNCT__ "DMPlexGetFaceOrientation"
1798 PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
1799 {
1800   MPI_Comm       comm;
1801   PetscBool      posOrient = PETSC_FALSE;
1802   const PetscInt debug     = 0;
1803   PetscInt       cellDim, faceSize, f;
1804   PetscErrorCode ierr;
1805 
1806   PetscFunctionBegin;
1807   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1808   ierr = DMPlexGetDimension(dm, &cellDim);CHKERRQ(ierr);
1809   if (debug) {PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
1810 
1811   if (cellDim == 1 && numCorners == 2) {
1812     /* Triangle */
1813     faceSize  = numCorners-1;
1814     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1815   } else if (cellDim == 2 && numCorners == 3) {
1816     /* Triangle */
1817     faceSize  = numCorners-1;
1818     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1819   } else if (cellDim == 3 && numCorners == 4) {
1820     /* Tetrahedron */
1821     faceSize  = numCorners-1;
1822     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1823   } else if (cellDim == 1 && numCorners == 3) {
1824     /* Quadratic line */
1825     faceSize  = 1;
1826     posOrient = PETSC_TRUE;
1827   } else if (cellDim == 2 && numCorners == 4) {
1828     /* Quads */
1829     faceSize = 2;
1830     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
1831       posOrient = PETSC_TRUE;
1832     } else if ((indices[0] == 3) && (indices[1] == 0)) {
1833       posOrient = PETSC_TRUE;
1834     } else {
1835       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
1836         posOrient = PETSC_FALSE;
1837       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
1838     }
1839   } else if (cellDim == 2 && numCorners == 6) {
1840     /* Quadratic triangle (I hate this) */
1841     /* Edges are determined by the first 2 vertices (corners of edges) */
1842     const PetscInt faceSizeTri = 3;
1843     PetscInt       sortedIndices[3], i, iFace;
1844     PetscBool      found                    = PETSC_FALSE;
1845     PetscInt       faceVerticesTriSorted[9] = {
1846       0, 3,  4, /* bottom */
1847       1, 4,  5, /* right */
1848       2, 3,  5, /* left */
1849     };
1850     PetscInt       faceVerticesTri[9] = {
1851       0, 3,  4, /* bottom */
1852       1, 4,  5, /* right */
1853       2, 5,  3, /* left */
1854     };
1855 
1856     faceSize = faceSizeTri;
1857     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
1858     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
1859     for (iFace = 0; iFace < 3; ++iFace) {
1860       const PetscInt ii = iFace*faceSizeTri;
1861       PetscInt       fVertex, cVertex;
1862 
1863       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
1864           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
1865         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
1866           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
1867             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
1868               faceVertices[fVertex] = origVertices[cVertex];
1869               break;
1870             }
1871           }
1872         }
1873         found = PETSC_TRUE;
1874         break;
1875       }
1876     }
1877     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
1878     if (posOriented) *posOriented = PETSC_TRUE;
1879     PetscFunctionReturn(0);
1880   } else if (cellDim == 2 && numCorners == 9) {
1881     /* Quadratic quad (I hate this) */
1882     /* Edges are determined by the first 2 vertices (corners of edges) */
1883     const PetscInt faceSizeQuad = 3;
1884     PetscInt       sortedIndices[3], i, iFace;
1885     PetscBool      found                      = PETSC_FALSE;
1886     PetscInt       faceVerticesQuadSorted[12] = {
1887       0, 1,  4, /* bottom */
1888       1, 2,  5, /* right */
1889       2, 3,  6, /* top */
1890       0, 3,  7, /* left */
1891     };
1892     PetscInt       faceVerticesQuad[12] = {
1893       0, 1,  4, /* bottom */
1894       1, 2,  5, /* right */
1895       2, 3,  6, /* top */
1896       3, 0,  7, /* left */
1897     };
1898 
1899     faceSize = faceSizeQuad;
1900     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
1901     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
1902     for (iFace = 0; iFace < 4; ++iFace) {
1903       const PetscInt ii = iFace*faceSizeQuad;
1904       PetscInt       fVertex, cVertex;
1905 
1906       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
1907           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
1908         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
1909           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
1910             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
1911               faceVertices[fVertex] = origVertices[cVertex];
1912               break;
1913             }
1914           }
1915         }
1916         found = PETSC_TRUE;
1917         break;
1918       }
1919     }
1920     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
1921     if (posOriented) *posOriented = PETSC_TRUE;
1922     PetscFunctionReturn(0);
1923   } else if (cellDim == 3 && numCorners == 8) {
1924     /* Hexes
1925        A hex is two oriented quads with the normal of the first
1926        pointing up at the second.
1927 
1928           7---6
1929          /|  /|
1930         4---5 |
1931         | 1-|-2
1932         |/  |/
1933         0---3
1934 
1935         Faces are determined by the first 4 vertices (corners of faces) */
1936     const PetscInt faceSizeHex = 4;
1937     PetscInt       sortedIndices[4], i, iFace;
1938     PetscBool      found                     = PETSC_FALSE;
1939     PetscInt       faceVerticesHexSorted[24] = {
1940       0, 1, 2, 3,  /* bottom */
1941       4, 5, 6, 7,  /* top */
1942       0, 3, 4, 5,  /* front */
1943       2, 3, 5, 6,  /* right */
1944       1, 2, 6, 7,  /* back */
1945       0, 1, 4, 7,  /* left */
1946     };
1947     PetscInt       faceVerticesHex[24] = {
1948       1, 2, 3, 0,  /* bottom */
1949       4, 5, 6, 7,  /* top */
1950       0, 3, 5, 4,  /* front */
1951       3, 2, 6, 5,  /* right */
1952       2, 1, 7, 6,  /* back */
1953       1, 0, 4, 7,  /* left */
1954     };
1955 
1956     faceSize = faceSizeHex;
1957     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
1958     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
1959     for (iFace = 0; iFace < 6; ++iFace) {
1960       const PetscInt ii = iFace*faceSizeHex;
1961       PetscInt       fVertex, cVertex;
1962 
1963       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
1964           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
1965           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
1966           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
1967         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
1968           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
1969             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
1970               faceVertices[fVertex] = origVertices[cVertex];
1971               break;
1972             }
1973           }
1974         }
1975         found = PETSC_TRUE;
1976         break;
1977       }
1978     }
1979     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
1980     if (posOriented) *posOriented = PETSC_TRUE;
1981     PetscFunctionReturn(0);
1982   } else if (cellDim == 3 && numCorners == 10) {
1983     /* Quadratic tet */
1984     /* Faces are determined by the first 3 vertices (corners of faces) */
1985     const PetscInt faceSizeTet = 6;
1986     PetscInt       sortedIndices[6], i, iFace;
1987     PetscBool      found                     = PETSC_FALSE;
1988     PetscInt       faceVerticesTetSorted[24] = {
1989       0, 1, 2,  6, 7, 8, /* bottom */
1990       0, 3, 4,  6, 7, 9,  /* front */
1991       1, 4, 5,  7, 8, 9,  /* right */
1992       2, 3, 5,  6, 8, 9,  /* left */
1993     };
1994     PetscInt       faceVerticesTet[24] = {
1995       0, 1, 2,  6, 7, 8, /* bottom */
1996       0, 4, 3,  6, 7, 9,  /* front */
1997       1, 5, 4,  7, 8, 9,  /* right */
1998       2, 3, 5,  8, 6, 9,  /* left */
1999     };
2000 
2001     faceSize = faceSizeTet;
2002     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2003     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
2004     for (iFace=0; iFace < 4; ++iFace) {
2005       const PetscInt ii = iFace*faceSizeTet;
2006       PetscInt       fVertex, cVertex;
2007 
2008       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2009           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2010           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2011           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2012         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2013           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2014             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2015               faceVertices[fVertex] = origVertices[cVertex];
2016               break;
2017             }
2018           }
2019         }
2020         found = PETSC_TRUE;
2021         break;
2022       }
2023     }
2024     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2025     if (posOriented) *posOriented = PETSC_TRUE;
2026     PetscFunctionReturn(0);
2027   } else if (cellDim == 3 && numCorners == 27) {
2028     /* Quadratic hexes (I hate this)
2029        A hex is two oriented quads with the normal of the first
2030        pointing up at the second.
2031 
2032          7---6
2033         /|  /|
2034        4---5 |
2035        | 3-|-2
2036        |/  |/
2037        0---1
2038 
2039        Faces are determined by the first 4 vertices (corners of faces) */
2040     const PetscInt faceSizeQuadHex = 9;
2041     PetscInt       sortedIndices[9], i, iFace;
2042     PetscBool      found                         = PETSC_FALSE;
2043     PetscInt       faceVerticesQuadHexSorted[54] = {
2044       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2045       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2046       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2047       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2048       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2049       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2050     };
2051     PetscInt       faceVerticesQuadHex[54] = {
2052       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2053       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2054       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2055       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2056       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2057       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2058     };
2059 
2060     faceSize = faceSizeQuadHex;
2061     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2062     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
2063     for (iFace = 0; iFace < 6; ++iFace) {
2064       const PetscInt ii = iFace*faceSizeQuadHex;
2065       PetscInt       fVertex, cVertex;
2066 
2067       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2068           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2069           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2070           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2071         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2072           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2073             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2074               faceVertices[fVertex] = origVertices[cVertex];
2075               break;
2076             }
2077           }
2078         }
2079         found = PETSC_TRUE;
2080         break;
2081       }
2082     }
2083     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2084     if (posOriented) *posOriented = PETSC_TRUE;
2085     PetscFunctionReturn(0);
2086   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2087   if (!posOrient) {
2088     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
2089     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2090   } else {
2091     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
2092     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2093   }
2094   if (posOriented) *posOriented = posOrient;
2095   PetscFunctionReturn(0);
2096 }
2097 
2098 #undef __FUNCT__
2099 #define __FUNCT__ "DMPlexGetOrientedFace"
2100 /*
2101     Given a cell and a face, as a set of vertices,
2102       return the oriented face, as a set of vertices, in faceVertices
2103     The orientation is such that the face normal points out of the cell
2104 */
2105 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2106 {
2107   const PetscInt *cone = NULL;
2108   PetscInt        coneSize, v, f, v2;
2109   PetscInt        oppositeVertex = -1;
2110   PetscErrorCode  ierr;
2111 
2112   PetscFunctionBegin;
2113   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
2114   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2115   for (v = 0, v2 = 0; v < coneSize; ++v) {
2116     PetscBool found = PETSC_FALSE;
2117 
2118     for (f = 0; f < faceSize; ++f) {
2119       if (face[f] == cone[v]) {
2120         found = PETSC_TRUE; break;
2121       }
2122     }
2123     if (found) {
2124       indices[v2]      = v;
2125       origVertices[v2] = cone[v];
2126       ++v2;
2127     } else {
2128       oppositeVertex = v;
2129     }
2130   }
2131   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
2132   PetscFunctionReturn(0);
2133 }
2134 
2135 #undef __FUNCT__
2136 #define __FUNCT__ "DMPlexInsertFace_Internal"
2137 /*
2138   DMPlexInsertFace_Internal - Puts a face into the mesh
2139 
2140   Not collective
2141 
2142   Input Parameters:
2143   + dm              - The DMPlex
2144   . numFaceVertex   - The number of vertices in the face
2145   . faceVertices    - The vertices in the face for dm
2146   . subfaceVertices - The vertices in the face for subdm
2147   . numCorners      - The number of vertices in the cell
2148   . cell            - A cell in dm containing the face
2149   . subcell         - A cell in subdm containing the face
2150   . firstFace       - First face in the mesh
2151   - newFacePoint    - Next face in the mesh
2152 
2153   Output Parameters:
2154   . newFacePoint - Contains next face point number on input, updated on output
2155 
2156   Level: developer
2157 */
2158 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2159 {
2160   MPI_Comm        comm;
2161   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2162   const PetscInt *faces;
2163   PetscInt        numFaces, coneSize;
2164   PetscErrorCode  ierr;
2165 
2166   PetscFunctionBegin;
2167   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2168   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
2169   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
2170 #if 0
2171   /* Cannot use this because support() has not been constructed yet */
2172   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2173 #else
2174   {
2175     PetscInt f;
2176 
2177     numFaces = 0;
2178     ierr     = DMGetWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2179     for (f = firstFace; f < *newFacePoint; ++f) {
2180       PetscInt dof, off, d;
2181 
2182       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
2183       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
2184       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2185       for (d = 0; d < dof; ++d) {
2186         const PetscInt p = submesh->cones[off+d];
2187         PetscInt       v;
2188 
2189         for (v = 0; v < numFaceVertices; ++v) {
2190           if (subfaceVertices[v] == p) break;
2191         }
2192         if (v == numFaceVertices) break;
2193       }
2194       if (d == dof) {
2195         numFaces               = 1;
2196         ((PetscInt*) faces)[0] = f;
2197       }
2198     }
2199   }
2200 #endif
2201   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
2202   else if (numFaces == 1) {
2203     /* Add the other cell neighbor for this face */
2204     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
2205   } else {
2206     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2207     PetscBool posOriented;
2208 
2209     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2210     origVertices        = &orientedVertices[numFaceVertices];
2211     indices             = &orientedVertices[numFaceVertices*2];
2212     orientedSubVertices = &orientedVertices[numFaceVertices*3];
2213     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
2214     /* TODO: I know that routine should return a permutation, not the indices */
2215     for (v = 0; v < numFaceVertices; ++v) {
2216       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2217       for (ov = 0; ov < numFaceVertices; ++ov) {
2218         if (orientedVertices[ov] == vertex) {
2219           orientedSubVertices[ov] = subvertex;
2220           break;
2221         }
2222       }
2223       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2224     }
2225     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2226     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2227     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2228     ++(*newFacePoint);
2229   }
2230 #if 0
2231   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2232 #else
2233   ierr = DMRestoreWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2234 #endif
2235   PetscFunctionReturn(0);
2236 }
2237 
2238 #undef __FUNCT__
2239 #define __FUNCT__ "DMPlexCreateSubmesh_Uninterpolated"
2240 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2241 {
2242   MPI_Comm        comm;
2243   DMLabel         subpointMap;
2244   IS              subvertexIS,  subcellIS;
2245   const PetscInt *subVertices, *subCells;
2246   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2247   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2248   PetscInt        vStart, vEnd, c, f;
2249   PetscErrorCode  ierr;
2250 
2251   PetscFunctionBegin;
2252   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2253   /* Create subpointMap which marks the submesh */
2254   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2255   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2256   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2257   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2258   /* Setup chart */
2259   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2260   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2261   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2262   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2263   /* Set cone sizes */
2264   firstSubVertex = numSubCells;
2265   firstSubFace   = numSubCells+numSubVertices;
2266   newFacePoint   = firstSubFace;
2267   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2268   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2269   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2270   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2271   for (c = 0; c < numSubCells; ++c) {
2272     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2273   }
2274   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2275     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2276   }
2277   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2278   /* Create face cones */
2279   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2280   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2281   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2282   for (c = 0; c < numSubCells; ++c) {
2283     const PetscInt cell    = subCells[c];
2284     const PetscInt subcell = c;
2285     PetscInt      *closure = NULL;
2286     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2287 
2288     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2289     for (cl = 0; cl < closureSize*2; cl += 2) {
2290       const PetscInt point = closure[cl];
2291       PetscInt       subVertex;
2292 
2293       if ((point >= vStart) && (point < vEnd)) {
2294         ++numCorners;
2295         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2296         if (subVertex >= 0) {
2297           closure[faceSize] = point;
2298           subface[faceSize] = firstSubVertex+subVertex;
2299           ++faceSize;
2300         }
2301       }
2302     }
2303     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2304     if (faceSize == nFV) {
2305       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2306     }
2307     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2308   }
2309   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2310   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2311   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2312   /* Build coordinates */
2313   {
2314     PetscSection coordSection, subCoordSection;
2315     Vec          coordinates, subCoordinates;
2316     PetscScalar *coords, *subCoords;
2317     PetscInt     numComp, coordSize, v;
2318 
2319     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2320     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2321     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2322     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2323     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2324     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2325     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2326     for (v = 0; v < numSubVertices; ++v) {
2327       const PetscInt vertex    = subVertices[v];
2328       const PetscInt subvertex = firstSubVertex+v;
2329       PetscInt       dof;
2330 
2331       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2332       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2333       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2334     }
2335     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2336     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2337     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2338     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2339     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2340     if (coordSize) {
2341       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2342       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2343       for (v = 0; v < numSubVertices; ++v) {
2344         const PetscInt vertex    = subVertices[v];
2345         const PetscInt subvertex = firstSubVertex+v;
2346         PetscInt       dof, off, sdof, soff, d;
2347 
2348         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2349         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2350         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2351         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2352         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2353         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2354       }
2355       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2356       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2357     }
2358     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2359     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2360   }
2361   /* Cleanup */
2362   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2363   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2364   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2365   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
2366   PetscFunctionReturn(0);
2367 }
2368 
2369 #undef __FUNCT__
2370 #define __FUNCT__ "DMPlexCreateSubmesh_Interpolated"
2371 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2372 {
2373   MPI_Comm         comm;
2374   DMLabel          subpointMap;
2375   IS              *subpointIS;
2376   const PetscInt **subpoints;
2377   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *coneONew;
2378   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2379   PetscErrorCode   ierr;
2380 
2381   PetscFunctionBegin;
2382   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2383   /* Create subpointMap which marks the submesh */
2384   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2385   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2386   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2387   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Interpolated(dm, vertexLabel, value, subpointMap, subdm);CHKERRQ(ierr);}
2388   /* Setup chart */
2389   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2390   ierr = PetscMalloc4(dim+1,PetscInt,&numSubPoints,dim+1,PetscInt,&firstSubPoint,dim+1,IS,&subpointIS,dim+1,const PetscInt *,&subpoints);CHKERRQ(ierr);
2391   for (d = 0; d <= dim; ++d) {
2392     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2393     totSubPoints += numSubPoints[d];
2394   }
2395   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2396   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2397   /* Set cone sizes */
2398   firstSubPoint[dim] = 0;
2399   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2400   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2401   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2402   for (d = 0; d <= dim; ++d) {
2403     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2404     ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2405   }
2406   for (d = 0; d <= dim; ++d) {
2407     for (p = 0; p < numSubPoints[d]; ++p) {
2408       const PetscInt  point    = subpoints[d][p];
2409       const PetscInt  subpoint = firstSubPoint[d] + p;
2410       const PetscInt *cone;
2411       PetscInt        coneSize, coneSizeNew, c, val;
2412 
2413       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2414       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2415       if (d == dim) {
2416         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2417         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2418           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2419           if (val >= 0) coneSizeNew++;
2420         }
2421         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2422       }
2423     }
2424   }
2425   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2426   /* Set cones */
2427   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2428   ierr = PetscMalloc2(maxConeSize,PetscInt,&coneNew,maxConeSize,PetscInt,&coneONew);CHKERRQ(ierr);
2429   for (d = 0; d <= dim; ++d) {
2430     for (p = 0; p < numSubPoints[d]; ++p) {
2431       const PetscInt  point    = subpoints[d][p];
2432       const PetscInt  subpoint = firstSubPoint[d] + p;
2433       const PetscInt *cone, *ornt;
2434       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2435 
2436       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2437       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2438       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2439       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
2440       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2441         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2442         if (subc >= 0) {
2443           coneNew[coneSizeNew]  = firstSubPoint[d-1] + subc;
2444           coneONew[coneSizeNew] = ornt[c];
2445           ++coneSizeNew;
2446         }
2447       }
2448       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2449       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2450       ierr = DMPlexSetConeOrientation(subdm, subpoint, coneONew);CHKERRQ(ierr);
2451     }
2452   }
2453   ierr = PetscFree2(coneNew,coneONew);CHKERRQ(ierr);
2454   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2455   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2456   /* Build coordinates */
2457   {
2458     PetscSection coordSection, subCoordSection;
2459     Vec          coordinates, subCoordinates;
2460     PetscScalar *coords, *subCoords;
2461     PetscInt     numComp, coordSize;
2462 
2463     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2464     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2465     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2466     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2467     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2468     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2469     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2470     for (v = 0; v < numSubPoints[0]; ++v) {
2471       const PetscInt vertex    = subpoints[0][v];
2472       const PetscInt subvertex = firstSubPoint[0]+v;
2473       PetscInt       dof;
2474 
2475       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2476       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2477       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2478     }
2479     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2480     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2481     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2482     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2483     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2484     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2485     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2486     for (v = 0; v < numSubPoints[0]; ++v) {
2487       const PetscInt vertex    = subpoints[0][v];
2488       const PetscInt subvertex = firstSubPoint[0]+v;
2489       PetscInt dof, off, sdof, soff, d;
2490 
2491       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2492       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2493       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2494       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2495       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2496       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2497     }
2498     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2499     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2500     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2501     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2502   }
2503   /* Cleanup */
2504   for (d = 0; d <= dim; ++d) {
2505     ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2506     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
2507   }
2508   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
2509   PetscFunctionReturn(0);
2510 }
2511 
2512 #undef __FUNCT__
2513 #define __FUNCT__ "DMPlexCreateSubmesh"
2514 /*@
2515   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
2516 
2517   Input Parameters:
2518 + dm           - The original mesh
2519 . vertexLabel  - The DMLabel marking vertices contained in the surface
2520 - value        - The label value to use
2521 
2522   Output Parameter:
2523 . subdm - The surface mesh
2524 
2525   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
2526 
2527   Level: developer
2528 
2529 .seealso: DMPlexGetSubpointMap(), DMPlexGetLabel(), DMLabelSetValue()
2530 @*/
2531 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, DM *subdm)
2532 {
2533   PetscInt       dim, depth;
2534   PetscErrorCode ierr;
2535 
2536   PetscFunctionBegin;
2537   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2538   PetscValidPointer(subdm, 3);
2539   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2540   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2541   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
2542   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
2543   ierr = DMPlexSetDimension(*subdm, dim-1);CHKERRQ(ierr);
2544   if (depth == dim) {
2545     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2546   } else {
2547     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2548   }
2549   PetscFunctionReturn(0);
2550 }
2551 
2552 #undef __FUNCT__
2553 #define __FUNCT__ "DMPlexFilterPoint_Internal"
2554 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
2555 {
2556   PetscInt       subPoint;
2557   PetscErrorCode ierr;
2558 
2559   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
2560   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
2561 }
2562 
2563 #undef __FUNCT__
2564 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Uninterpolated"
2565 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
2566 {
2567   MPI_Comm        comm;
2568   DMLabel         subpointMap;
2569   IS              subvertexIS;
2570   const PetscInt *subVertices;
2571   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
2572   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
2573   PetscInt        cMax, c, f;
2574   PetscErrorCode  ierr;
2575 
2576   PetscFunctionBegin;
2577   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
2578   /* Create subpointMap which marks the submesh */
2579   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2580   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2581   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2582   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
2583   /* Setup chart */
2584   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2585   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2586   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2587   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2588   /* Set cone sizes */
2589   firstSubVertex = numSubCells;
2590   firstSubFace   = numSubCells+numSubVertices;
2591   newFacePoint   = firstSubFace;
2592   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2593   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2594   for (c = 0; c < numSubCells; ++c) {
2595     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2596   }
2597   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2598     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2599   }
2600   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2601   /* Create face cones */
2602   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2603   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2604   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2605   for (c = 0; c < numSubCells; ++c) {
2606     const PetscInt  cell    = subCells[c];
2607     const PetscInt  subcell = c;
2608     const PetscInt *cone, *cells;
2609     PetscInt        numCells, subVertex, p, v;
2610 
2611     if (cell < cMax) continue;
2612     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2613     for (v = 0; v < nFV; ++v) {
2614       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2615       subface[v] = firstSubVertex+subVertex;
2616     }
2617     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
2618     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
2619     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2620     /* Not true in parallel
2621     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2622     for (p = 0; p < numCells; ++p) {
2623       PetscInt negsubcell;
2624 
2625       if (cells[p] >= cMax) continue;
2626       /* I know this is a crap search */
2627       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
2628         if (subCells[negsubcell] == cells[p]) break;
2629       }
2630       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
2631       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
2632     }
2633     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2634     ++newFacePoint;
2635   }
2636   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2637   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2638   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2639   /* Build coordinates */
2640   {
2641     PetscSection coordSection, subCoordSection;
2642     Vec          coordinates, subCoordinates;
2643     PetscScalar *coords, *subCoords;
2644     PetscInt     numComp, coordSize, v;
2645 
2646     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2647     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2648     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2649     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2650     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2651     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2652     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2653     for (v = 0; v < numSubVertices; ++v) {
2654       const PetscInt vertex    = subVertices[v];
2655       const PetscInt subvertex = firstSubVertex+v;
2656       PetscInt       dof;
2657 
2658       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2659       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2660       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2661     }
2662     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2663     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2664     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2665     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2666     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2667     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2668     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2669     for (v = 0; v < numSubVertices; ++v) {
2670       const PetscInt vertex    = subVertices[v];
2671       const PetscInt subvertex = firstSubVertex+v;
2672       PetscInt       dof, off, sdof, soff, d;
2673 
2674       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2675       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2676       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2677       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2678       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2679       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2680     }
2681     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2682     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2683     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2684     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2685   }
2686   /* Build SF */
2687   CHKMEMQ;
2688   {
2689     PetscSF            sfPoint, sfPointSub;
2690     const PetscSFNode *remotePoints;
2691     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
2692     const PetscInt    *localPoints;
2693     PetscInt          *slocalPoints;
2694     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
2695     PetscMPIInt        rank;
2696 
2697     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2698     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2699     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
2700     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2701     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2702     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
2703     if (numRoots >= 0) {
2704       /* Only vertices should be shared */
2705       ierr = PetscMalloc2(pEnd-pStart,PetscSFNode,&newLocalPoints,numRoots,PetscSFNode,&newOwners);CHKERRQ(ierr);
2706       for (p = 0; p < pEnd-pStart; ++p) {
2707         newLocalPoints[p].rank  = -2;
2708         newLocalPoints[p].index = -2;
2709       }
2710       /* Set subleaves */
2711       for (l = 0; l < numLeaves; ++l) {
2712         const PetscInt point    = localPoints[l];
2713         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2714 
2715         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
2716         if (subPoint < 0) continue;
2717         newLocalPoints[point-pStart].rank  = rank;
2718         newLocalPoints[point-pStart].index = subPoint;
2719         ++numSubLeaves;
2720       }
2721       /* Must put in owned subpoints */
2722       for (p = pStart; p < pEnd; ++p) {
2723         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
2724 
2725         if (subPoint < 0) {
2726           newOwners[p-pStart].rank  = -3;
2727           newOwners[p-pStart].index = -3;
2728         } else {
2729           newOwners[p-pStart].rank  = rank;
2730           newOwners[p-pStart].index = subPoint;
2731         }
2732       }
2733       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2734       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2735       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2736       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2737       ierr = PetscMalloc(numSubLeaves * sizeof(PetscInt),    &slocalPoints);CHKERRQ(ierr);
2738       ierr = PetscMalloc(numSubLeaves * sizeof(PetscSFNode), &sremotePoints);CHKERRQ(ierr);
2739       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
2740         const PetscInt point    = localPoints[l];
2741         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2742 
2743         if (subPoint < 0) continue;
2744         if (newLocalPoints[point].rank == rank) {++ll; continue;}
2745         slocalPoints[sl]        = subPoint;
2746         sremotePoints[sl].rank  = newLocalPoints[point].rank;
2747         sremotePoints[sl].index = newLocalPoints[point].index;
2748         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
2749         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
2750         ++sl;
2751       }
2752       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
2753       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
2754       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
2755     }
2756   }
2757   CHKMEMQ;
2758   /* Cleanup */
2759   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2760   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2761   ierr = PetscFree(subCells);CHKERRQ(ierr);
2762   PetscFunctionReturn(0);
2763 }
2764 
2765 #undef __FUNCT__
2766 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Interpolated"
2767 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char label[], PetscInt value, DM subdm)
2768 {
2769   MPI_Comm         comm;
2770   DMLabel          subpointMap;
2771   IS              *subpointIS;
2772   const PetscInt **subpoints;
2773   PetscInt        *numSubPoints, *firstSubPoint, *coneNew;
2774   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2775   PetscErrorCode   ierr;
2776 
2777   PetscFunctionBegin;
2778   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2779   /* Create subpointMap which marks the submesh */
2780   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2781   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2782   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2783   ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);
2784   /* Setup chart */
2785   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2786   ierr = PetscMalloc4(dim+1,PetscInt,&numSubPoints,dim+1,PetscInt,&firstSubPoint,dim+1,IS,&subpointIS,dim+1,const PetscInt *,&subpoints);CHKERRQ(ierr);
2787   for (d = 0; d <= dim; ++d) {
2788     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2789     totSubPoints += numSubPoints[d];
2790   }
2791   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2792   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2793   /* Set cone sizes */
2794   firstSubPoint[dim] = 0;
2795   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2796   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2797   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2798   for (d = 0; d <= dim; ++d) {
2799     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2800     ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2801   }
2802   for (d = 0; d <= dim; ++d) {
2803     for (p = 0; p < numSubPoints[d]; ++p) {
2804       const PetscInt  point    = subpoints[d][p];
2805       const PetscInt  subpoint = firstSubPoint[d] + p;
2806       const PetscInt *cone;
2807       PetscInt        coneSize, coneSizeNew, c, val;
2808 
2809       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2810       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2811       if (d == dim) {
2812         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2813         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2814           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2815           if (val >= 0) coneSizeNew++;
2816         }
2817         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2818       }
2819     }
2820   }
2821   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2822   /* Set cones */
2823   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2824   ierr = PetscMalloc(maxConeSize * sizeof(PetscInt), &coneNew);CHKERRQ(ierr);
2825   for (d = 0; d <= dim; ++d) {
2826     for (p = 0; p < numSubPoints[d]; ++p) {
2827       const PetscInt  point    = subpoints[d][p];
2828       const PetscInt  subpoint = firstSubPoint[d] + p;
2829       const PetscInt *cone;
2830       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2831 
2832       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2833       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2834       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2835       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2836         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2837         if (subc >= 0) coneNew[coneSizeNew++] = firstSubPoint[d-1] + subc;
2838       }
2839       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2840       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2841     }
2842   }
2843   ierr = PetscFree(coneNew);CHKERRQ(ierr);
2844   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2845   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2846   /* Build coordinates */
2847   {
2848     PetscSection coordSection, subCoordSection;
2849     Vec          coordinates, subCoordinates;
2850     PetscScalar *coords, *subCoords;
2851     PetscInt     numComp, coordSize;
2852 
2853     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2854     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2855     ierr = DMPlexGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2856     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2857     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2858     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2859     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2860     for (v = 0; v < numSubPoints[0]; ++v) {
2861       const PetscInt vertex    = subpoints[0][v];
2862       const PetscInt subvertex = firstSubPoint[0]+v;
2863       PetscInt       dof;
2864 
2865       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2866       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2867       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2868     }
2869     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2870     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2871     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2872     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2873     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2874     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2875     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2876     for (v = 0; v < numSubPoints[0]; ++v) {
2877       const PetscInt vertex    = subpoints[0][v];
2878       const PetscInt subvertex = firstSubPoint[0]+v;
2879       PetscInt dof, off, sdof, soff, d;
2880 
2881       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2882       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2883       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2884       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2885       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2886       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2887     }
2888     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2889     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2890     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2891     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2892   }
2893   /* Cleanup */
2894   for (d = 0; d <= dim; ++d) {
2895     ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);
2896     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
2897   }
2898   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
2899   PetscFunctionReturn(0);
2900 }
2901 
2902 #undef __FUNCT__
2903 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh"
2904 /*
2905   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
2906 
2907   Input Parameters:
2908 + dm          - The original mesh
2909 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
2910 . label       - A label name, or PETSC_NULL
2911 - value  - A label value
2912 
2913   Output Parameter:
2914 . subdm - The surface mesh
2915 
2916   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
2917 
2918   Level: developer
2919 
2920 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
2921 */
2922 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
2923 {
2924   PetscInt       dim, depth;
2925   PetscErrorCode ierr;
2926 
2927   PetscFunctionBegin;
2928   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2929   PetscValidPointer(subdm, 5);
2930   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2931   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2932   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
2933   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
2934   ierr = DMPlexSetDimension(*subdm, dim-1);CHKERRQ(ierr);
2935   if (depth == dim) {
2936     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm);CHKERRQ(ierr);
2937   } else {
2938     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
2939   }
2940   PetscFunctionReturn(0);
2941 }
2942 
2943 #undef __FUNCT__
2944 #define __FUNCT__ "DMPlexGetSubpointMap"
2945 /*@
2946   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
2947 
2948   Input Parameter:
2949 . dm - The submesh DM
2950 
2951   Output Parameter:
2952 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
2953 
2954   Level: developer
2955 
2956 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
2957 @*/
2958 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
2959 {
2960   DM_Plex *mesh = (DM_Plex*) dm->data;
2961 
2962   PetscFunctionBegin;
2963   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2964   PetscValidPointer(subpointMap, 2);
2965   *subpointMap = mesh->subpointMap;
2966   PetscFunctionReturn(0);
2967 }
2968 
2969 #undef __FUNCT__
2970 #define __FUNCT__ "DMPlexSetSubpointMap"
2971 /* Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh() */
2972 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
2973 {
2974   DM_Plex       *mesh = (DM_Plex *) dm->data;
2975   DMLabel        tmp;
2976   PetscErrorCode ierr;
2977 
2978   PetscFunctionBegin;
2979   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2980   tmp  = mesh->subpointMap;
2981   mesh->subpointMap = subpointMap;
2982   ++mesh->subpointMap->refct;
2983   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
2984   PetscFunctionReturn(0);
2985 }
2986 
2987 #undef __FUNCT__
2988 #define __FUNCT__ "DMPlexCreateSubpointIS"
2989 /*@
2990   DMPlexCreateSubpointIS - Creates an IS covering the entire subdm chart with the original points as data
2991 
2992   Input Parameter:
2993 . dm - The submesh DM
2994 
2995   Output Parameter:
2996 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
2997 
2998   Note: This is IS is guaranteed to be sorted by the construction of the submesh
2999 
3000   Level: developer
3001 
3002 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
3003 @*/
3004 PetscErrorCode DMPlexCreateSubpointIS(DM dm, IS *subpointIS)
3005 {
3006   MPI_Comm        comm;
3007   DMLabel         subpointMap;
3008   IS              is;
3009   const PetscInt *opoints;
3010   PetscInt       *points, *depths;
3011   PetscInt        depth, depStart, depEnd, d, pStart, pEnd, p, n, off;
3012   PetscErrorCode  ierr;
3013 
3014   PetscFunctionBegin;
3015   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3016   PetscValidPointer(subpointIS, 2);
3017   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3018   *subpointIS = NULL;
3019   ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
3020   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3021   if (subpointMap && depth >= 0) {
3022     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3023     if (pStart) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
3024     ierr = DMGetWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3025     depths[0] = depth;
3026     depths[1] = 0;
3027     for(d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3028     ierr = PetscMalloc(pEnd * sizeof(PetscInt), &points);CHKERRQ(ierr);
3029     for(d = 0, off = 0; d <= depth; ++d) {
3030       const PetscInt dep = depths[d];
3031 
3032       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
3033       ierr = DMLabelGetStratumSize(subpointMap, dep, &n);CHKERRQ(ierr);
3034       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3035         if (n != depEnd-depStart) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
3036       } else {
3037         if (!n) {
3038           if (d == 0) {
3039             /* Missing cells */
3040             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3041           } else {
3042             /* Missing faces */
3043             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3044           }
3045         }
3046       }
3047       if (n) {
3048         ierr = DMLabelGetStratumIS(subpointMap, dep, &is);CHKERRQ(ierr);
3049         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
3050         for(p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3051         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
3052         ierr = ISDestroy(&is);CHKERRQ(ierr);
3053       }
3054     }
3055     ierr = DMRestoreWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3056     if (off != pEnd) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
3057     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
3058   }
3059   PetscFunctionReturn(0);
3060 }
3061