xref: /petsc/src/dm/impls/plex/tests/ex18.c (revision 589a23caa660d2a5f330cc8d1ed213e9cfaf51a7) 
1c4762a1bSJed Brown static char help[] = "Tests for parallel mesh loading and parallel topological interpolation\n\n";
2c4762a1bSJed Brown 
3c4762a1bSJed Brown #include <petsc/private/dmpleximpl.h>
4c4762a1bSJed Brown /* List of test meshes
5c4762a1bSJed Brown 
6c4762a1bSJed Brown Network
7c4762a1bSJed Brown -------
8c4762a1bSJed Brown Test 0 (2 ranks):
9c4762a1bSJed Brown 
10c4762a1bSJed Brown network=0:
11c4762a1bSJed Brown ---------
12c4762a1bSJed Brown   cell 0   cell 1   cell 2          nCells-1       (edge)
13c4762a1bSJed Brown 0 ------ 1 ------ 2 ------ 3 -- -- v --  -- nCells (vertex)
14c4762a1bSJed Brown 
15c4762a1bSJed Brown   vertex distribution:
16c4762a1bSJed Brown     rank 0: 0 1
17c4762a1bSJed Brown     rank 1: 2 3 ... nCells
18c4762a1bSJed Brown   cell(edge) distribution:
19c4762a1bSJed Brown     rank 0: 0 1
20c4762a1bSJed Brown     rank 1: 2 ... nCells-1
21c4762a1bSJed Brown 
22c4762a1bSJed Brown network=1:
23c4762a1bSJed Brown ---------
24c4762a1bSJed Brown                v2
25c4762a1bSJed Brown                 ^
26c4762a1bSJed Brown                 |
27c4762a1bSJed Brown                cell 2
28c4762a1bSJed Brown                 |
29c4762a1bSJed Brown  v0 --cell 0--> v3--cell 1--> v1
30c4762a1bSJed Brown 
31c4762a1bSJed Brown   vertex distribution:
32c4762a1bSJed Brown     rank 0: 0 1 3
33c4762a1bSJed Brown     rank 1: 2
34c4762a1bSJed Brown   cell(edge) distribution:
35c4762a1bSJed Brown     rank 0: 0 1
36c4762a1bSJed Brown     rank 1: 2
37c4762a1bSJed Brown 
38c4762a1bSJed Brown   example:
39c4762a1bSJed Brown     mpiexec -n 2 ./ex18 -distribute 1 -dim 1 -orig_dm_view -dist_dm_view -dist_dm_view -petscpartitioner_type parmetis -ncells 50
40c4762a1bSJed Brown 
41c4762a1bSJed Brown Triangle
42c4762a1bSJed Brown --------
43c4762a1bSJed Brown Test 0 (2 ranks):
44c4762a1bSJed Brown Two triangles sharing a face
45c4762a1bSJed Brown 
46c4762a1bSJed Brown         2
47c4762a1bSJed Brown       / | \
48c4762a1bSJed Brown      /  |  \
49c4762a1bSJed Brown     /   |   \
50c4762a1bSJed Brown    0  0 | 1  3
51c4762a1bSJed Brown     \   |   /
52c4762a1bSJed Brown      \  |  /
53c4762a1bSJed Brown       \ | /
54c4762a1bSJed Brown         1
55c4762a1bSJed Brown 
56c4762a1bSJed Brown   vertex distribution:
57c4762a1bSJed Brown     rank 0: 0 1
58c4762a1bSJed Brown     rank 1: 2 3
59c4762a1bSJed Brown   cell distribution:
60c4762a1bSJed Brown     rank 0: 0
61c4762a1bSJed Brown     rank 1: 1
62c4762a1bSJed Brown 
63c4762a1bSJed Brown Test 1 (3 ranks):
64c4762a1bSJed Brown Four triangles partitioned across 3 ranks
65c4762a1bSJed Brown 
66c4762a1bSJed Brown    0 _______ 3
67c4762a1bSJed Brown    | \     / |
68c4762a1bSJed Brown    |  \ 1 /  |
69c4762a1bSJed Brown    |   \ /   |
70c4762a1bSJed Brown    | 0  2  2 |
71c4762a1bSJed Brown    |   / \   |
72c4762a1bSJed Brown    |  / 3 \  |
73c4762a1bSJed Brown    | /     \ |
74c4762a1bSJed Brown    1 ------- 4
75c4762a1bSJed Brown 
76c4762a1bSJed Brown   vertex distribution:
77c4762a1bSJed Brown     rank 0: 0 1
78c4762a1bSJed Brown     rank 1: 2 3
79c4762a1bSJed Brown     rank 2: 4
80c4762a1bSJed Brown   cell distribution:
81c4762a1bSJed Brown     rank 0: 0
82c4762a1bSJed Brown     rank 1: 1
83c4762a1bSJed Brown     rank 2: 2 3
84c4762a1bSJed Brown 
85c4762a1bSJed Brown Test 2 (3 ranks):
86c4762a1bSJed Brown Four triangles partitioned across 3 ranks
87c4762a1bSJed Brown 
88c4762a1bSJed Brown    1 _______ 3
89c4762a1bSJed Brown    | \     / |
90c4762a1bSJed Brown    |  \ 1 /  |
91c4762a1bSJed Brown    |   \ /   |
92c4762a1bSJed Brown    | 0  0  2 |
93c4762a1bSJed Brown    |   / \   |
94c4762a1bSJed Brown    |  / 3 \  |
95c4762a1bSJed Brown    | /     \ |
96c4762a1bSJed Brown    2 ------- 4
97c4762a1bSJed Brown 
98c4762a1bSJed Brown   vertex distribution:
99c4762a1bSJed Brown     rank 0: 0 1
100c4762a1bSJed Brown     rank 1: 2 3
101c4762a1bSJed Brown     rank 2: 4
102c4762a1bSJed Brown   cell distribution:
103c4762a1bSJed Brown     rank 0: 0
104c4762a1bSJed Brown     rank 1: 1
105c4762a1bSJed Brown     rank 2: 2 3
106c4762a1bSJed Brown 
107c4762a1bSJed Brown Tetrahedron
108c4762a1bSJed Brown -----------
109c4762a1bSJed Brown Test 0:
110c4762a1bSJed Brown Two tets sharing a face
111c4762a1bSJed Brown 
112c4762a1bSJed Brown  cell   3 _______    cell
113c4762a1bSJed Brown  0    / | \      \   1
114c4762a1bSJed Brown      /  |  \      \
115c4762a1bSJed Brown     /   |   \      \
116c4762a1bSJed Brown    0----|----4-----2
117c4762a1bSJed Brown     \   |   /      /
118c4762a1bSJed Brown      \  |  /      /
119c4762a1bSJed Brown       \ | /      /
120c4762a1bSJed Brown         1-------
121c4762a1bSJed Brown    y
122c4762a1bSJed Brown    | x
123c4762a1bSJed Brown    |/
124c4762a1bSJed Brown    *----z
125c4762a1bSJed Brown 
126c4762a1bSJed Brown   vertex distribution:
127c4762a1bSJed Brown     rank 0: 0 1
128c4762a1bSJed Brown     rank 1: 2 3 4
129c4762a1bSJed Brown   cell distribution:
130c4762a1bSJed Brown     rank 0: 0
131c4762a1bSJed Brown     rank 1: 1
132c4762a1bSJed Brown 
133c4762a1bSJed Brown Quadrilateral
134c4762a1bSJed Brown -------------
135c4762a1bSJed Brown Test 0 (2 ranks):
136c4762a1bSJed Brown Two quads sharing a face
137c4762a1bSJed Brown 
138c4762a1bSJed Brown    3-------2-------5
139c4762a1bSJed Brown    |       |       |
140c4762a1bSJed Brown    |   0   |   1   |
141c4762a1bSJed Brown    |       |       |
142c4762a1bSJed Brown    0-------1-------4
143c4762a1bSJed Brown 
144c4762a1bSJed Brown   vertex distribution:
145c4762a1bSJed Brown     rank 0: 0 1 2
146c4762a1bSJed Brown     rank 1: 3 4 5
147c4762a1bSJed Brown   cell distribution:
148c4762a1bSJed Brown     rank 0: 0
149c4762a1bSJed Brown     rank 1: 1
150c4762a1bSJed Brown 
151c4762a1bSJed Brown TODO Test 1:
152c4762a1bSJed Brown A quad and a triangle sharing a face
153c4762a1bSJed Brown 
154c4762a1bSJed Brown    5-------4
155c4762a1bSJed Brown    |       | \
156c4762a1bSJed Brown    |   0   |  \
157c4762a1bSJed Brown    |       | 1 \
158c4762a1bSJed Brown    2-------3----6
159c4762a1bSJed Brown 
160c4762a1bSJed Brown Hexahedron
161c4762a1bSJed Brown ----------
162c4762a1bSJed Brown Test 0 (2 ranks):
163c4762a1bSJed Brown Two hexes sharing a face
164c4762a1bSJed Brown 
165c4762a1bSJed Brown cell   7-------------6-------------11 cell
166c4762a1bSJed Brown 0     /|            /|            /|     1
167c4762a1bSJed Brown      / |   F1      / |   F7      / |
168c4762a1bSJed Brown     /  |          /  |          /  |
169c4762a1bSJed Brown    4-------------5-------------10  |
170c4762a1bSJed Brown    |   |     F4  |   |     F10 |   |
171c4762a1bSJed Brown    |   |         |   |         |   |
172c4762a1bSJed Brown    |F5 |         |F3 |         |F9 |
173c4762a1bSJed Brown    |   |  F2     |   |   F8    |   |
174c4762a1bSJed Brown    |   3---------|---2---------|---9
175c4762a1bSJed Brown    |  /          |  /          |  /
176c4762a1bSJed Brown    | /   F0      | /    F6     | /
177c4762a1bSJed Brown    |/            |/            |/
178c4762a1bSJed Brown    0-------------1-------------8
179c4762a1bSJed Brown 
180c4762a1bSJed Brown   vertex distribution:
181c4762a1bSJed Brown     rank 0: 0 1 2 3 4 5
182c4762a1bSJed Brown     rank 1: 6 7 8 9 10 11
183c4762a1bSJed Brown   cell distribution:
184c4762a1bSJed Brown     rank 0: 0
185c4762a1bSJed Brown     rank 1: 1
186c4762a1bSJed Brown 
187c4762a1bSJed Brown */
188c4762a1bSJed Brown 
189c4762a1bSJed Brown typedef enum {NONE, CREATE, AFTER_CREATE, AFTER_DISTRIBUTE} InterpType;
190c4762a1bSJed Brown 
191c4762a1bSJed Brown typedef struct {
192c4762a1bSJed Brown   PetscInt   debug;                        /* The debugging level */
193c4762a1bSJed Brown   PetscInt   testNum;                      /* Indicates the mesh to create */
194c4762a1bSJed Brown   PetscInt   dim;                          /* The topological mesh dimension */
195c4762a1bSJed Brown   PetscBool  cellSimplex;                  /* Use simplices or hexes */
196c4762a1bSJed Brown   PetscBool  distribute;                   /* Distribute the mesh */
197c4762a1bSJed Brown   InterpType interpolate;                  /* Interpolate the mesh before or after DMPlexDistribute() */
198c4762a1bSJed Brown   PetscBool  useGenerator;                 /* Construct mesh with a mesh generator */
199c4762a1bSJed Brown   PetscBool  testOrientIF;                 /* Test for different original interface orientations */
200c4762a1bSJed Brown   PetscBool  testHeavy;                    /* Run the heavy PointSF test */
201c4762a1bSJed Brown   PetscBool  customView;                   /* Show results of DMPlexIsInterpolated() etc. */
202c4762a1bSJed Brown   PetscInt   ornt[2];                      /* Orientation of interface on rank 0 and rank 1 */
203c4762a1bSJed Brown   PetscInt   faces[3];                     /* Number of faces per dimension for generator */
204c4762a1bSJed Brown   PetscReal  coords[128];
205c4762a1bSJed Brown   PetscReal  coordsTol;
206c4762a1bSJed Brown   PetscInt   ncoords;
207c4762a1bSJed Brown   PetscInt   pointsToExpand[128];
208c4762a1bSJed Brown   PetscInt   nPointsToExpand;
209c4762a1bSJed Brown   PetscBool  testExpandPointsEmpty;
210c4762a1bSJed Brown   char       filename[PETSC_MAX_PATH_LEN]; /* Import mesh from file */
211c4762a1bSJed Brown } AppCtx;
212c4762a1bSJed Brown 
213c4762a1bSJed Brown struct _n_PortableBoundary {
214c4762a1bSJed Brown   Vec coordinates;
215c4762a1bSJed Brown   PetscInt depth;
216c4762a1bSJed Brown   PetscSection *sections;
217c4762a1bSJed Brown };
218c4762a1bSJed Brown typedef struct _n_PortableBoundary * PortableBoundary;
219c4762a1bSJed Brown 
220c4762a1bSJed Brown static PetscLogStage  stage[3];
221c4762a1bSJed Brown 
222c4762a1bSJed Brown static PetscErrorCode DMPlexCheckPointSFHeavy(DM, PortableBoundary);
223c4762a1bSJed Brown static PetscErrorCode DMPlexSetOrientInterface_Private(DM,PetscBool);
224c4762a1bSJed Brown static PetscErrorCode DMPlexGetExpandedBoundary_Private(DM, PortableBoundary *);
225c4762a1bSJed Brown static PetscErrorCode DMPlexExpandedConesToFaces_Private(DM, IS, PetscSection, IS *);
226c4762a1bSJed Brown 
227c4762a1bSJed Brown static PetscErrorCode PortableBoundaryDestroy(PortableBoundary *bnd)
228c4762a1bSJed Brown {
229c4762a1bSJed Brown   PetscInt       d;
230c4762a1bSJed Brown   PetscErrorCode ierr;
231c4762a1bSJed Brown 
232c4762a1bSJed Brown   PetscFunctionBegin;
233c4762a1bSJed Brown   if (!*bnd) PetscFunctionReturn(0);
234c4762a1bSJed Brown   ierr = VecDestroy(&(*bnd)->coordinates);CHKERRQ(ierr);
235c4762a1bSJed Brown   for (d=0; d < (*bnd)->depth; d++) {
236c4762a1bSJed Brown     ierr = PetscSectionDestroy(&(*bnd)->sections[d]);CHKERRQ(ierr);
237c4762a1bSJed Brown   }
238c4762a1bSJed Brown   ierr = PetscFree((*bnd)->sections);CHKERRQ(ierr);
239c4762a1bSJed Brown   ierr = PetscFree(*bnd);CHKERRQ(ierr);
240c4762a1bSJed Brown   PetscFunctionReturn(0);
241c4762a1bSJed Brown }
242c4762a1bSJed Brown 
243c4762a1bSJed Brown static PetscErrorCode ProcessOptions(MPI_Comm comm, AppCtx *options)
244c4762a1bSJed Brown {
245c4762a1bSJed Brown   const char    *interpTypes[4]  = {"none", "create", "after_create", "after_distribute"};
246c4762a1bSJed Brown   PetscInt       interp=NONE, dim;
247c4762a1bSJed Brown   PetscBool      flg1, flg2;
248c4762a1bSJed Brown   PetscErrorCode ierr;
249c4762a1bSJed Brown 
250c4762a1bSJed Brown   PetscFunctionBegin;
251c4762a1bSJed Brown   options->debug        = 0;
252c4762a1bSJed Brown   options->testNum      = 0;
253c4762a1bSJed Brown   options->dim          = 2;
254c4762a1bSJed Brown   options->cellSimplex  = PETSC_TRUE;
255c4762a1bSJed Brown   options->distribute   = PETSC_FALSE;
256c4762a1bSJed Brown   options->interpolate  = NONE;
257c4762a1bSJed Brown   options->useGenerator = PETSC_FALSE;
258c4762a1bSJed Brown   options->testOrientIF = PETSC_FALSE;
259c4762a1bSJed Brown   options->testHeavy    = PETSC_TRUE;
260c4762a1bSJed Brown   options->customView   = PETSC_FALSE;
261c4762a1bSJed Brown   options->testExpandPointsEmpty = PETSC_FALSE;
262c4762a1bSJed Brown   options->ornt[0]      = 0;
263c4762a1bSJed Brown   options->ornt[1]      = 0;
264c4762a1bSJed Brown   options->faces[0]     = 2;
265c4762a1bSJed Brown   options->faces[1]     = 2;
266c4762a1bSJed Brown   options->faces[2]     = 2;
267c4762a1bSJed Brown   options->filename[0]  = '\0';
268c4762a1bSJed Brown   options->coordsTol    = PETSC_DEFAULT;
269c4762a1bSJed Brown 
270c4762a1bSJed Brown   ierr = PetscOptionsBegin(comm, "", "Meshing Interpolation Test Options", "DMPLEX");CHKERRQ(ierr);
271c4762a1bSJed Brown   ierr = PetscOptionsBoundedInt("-debug", "The debugging level", "ex18.c", options->debug, &options->debug, NULL,0);CHKERRQ(ierr);
272c4762a1bSJed Brown   ierr = PetscOptionsBoundedInt("-testnum", "The mesh to create", "ex18.c", options->testNum, &options->testNum, NULL,0);CHKERRQ(ierr);
273c4762a1bSJed Brown   ierr = PetscOptionsBool("-cell_simplex", "Generate simplices if true, otherwise hexes", "ex18.c", options->cellSimplex, &options->cellSimplex, NULL);CHKERRQ(ierr);
274c4762a1bSJed Brown   ierr = PetscOptionsBool("-distribute", "Distribute the mesh", "ex18.c", options->distribute, &options->distribute, NULL);CHKERRQ(ierr);
275c4762a1bSJed Brown   ierr = PetscOptionsEList("-interpolate", "Type of mesh interpolation (none, create, after_create, after_distribute)", "ex18.c", interpTypes, 4, interpTypes[options->interpolate], &interp, NULL);CHKERRQ(ierr);
276c4762a1bSJed Brown   options->interpolate = (InterpType) interp;
277c4762a1bSJed Brown   if (!options->distribute && options->interpolate == AFTER_DISTRIBUTE) SETERRQ(comm, PETSC_ERR_SUP, "-interpolate after_distribute  needs  -distribute 1");
278c4762a1bSJed Brown   ierr = PetscOptionsBool("-use_generator", "Use a mesh generator to build the mesh", "ex18.c", options->useGenerator, &options->useGenerator, NULL);CHKERRQ(ierr);
279c4762a1bSJed Brown   options->ncoords = 128;
280c4762a1bSJed Brown   ierr = PetscOptionsRealArray("-view_vertices_from_coords", "Print DAG points corresponding to vertices with given coordinates", "ex18.c", options->coords, &options->ncoords, NULL);CHKERRQ(ierr);
281c4762a1bSJed Brown   ierr = PetscOptionsReal("-view_vertices_from_coords_tol", "Tolerance for -view_vertices_from_coords", "ex18.c", options->coordsTol, &options->coordsTol, NULL);CHKERRQ(ierr);
282c4762a1bSJed Brown   options->nPointsToExpand = 128;
283c4762a1bSJed Brown   ierr = PetscOptionsIntArray("-test_expand_points", "Expand given array of DAG point using DMPlexGetConeRecursive() and print results", "ex18.c", options->pointsToExpand, &options->nPointsToExpand, NULL);CHKERRQ(ierr);
284c4762a1bSJed Brown   if (options->nPointsToExpand) {
285c4762a1bSJed Brown     ierr = PetscOptionsBool("-test_expand_points_empty", "For -test_expand_points, rank 0 will have empty input array", "ex18.c", options->testExpandPointsEmpty, &options->testExpandPointsEmpty, NULL);CHKERRQ(ierr);
286c4762a1bSJed Brown   }
287c4762a1bSJed Brown   ierr = PetscOptionsBool("-test_heavy", "Run the heavy PointSF test", "ex18.c", options->testHeavy, &options->testHeavy, NULL);CHKERRQ(ierr);
288c4762a1bSJed Brown   ierr = PetscOptionsBool("-custom_view", "Custom DMPlex view", "ex18.c", options->customView, &options->customView, NULL);CHKERRQ(ierr);
289c4762a1bSJed Brown   ierr = PetscOptionsRangeInt("-dim", "The topological mesh dimension", "ex18.c", options->dim, &options->dim, &flg1,1,3);CHKERRQ(ierr);
290c4762a1bSJed Brown   dim = 3;
291c4762a1bSJed Brown   ierr = PetscOptionsIntArray("-faces", "Number of faces per dimension", "ex18.c", options->faces, &dim, &flg2);CHKERRQ(ierr);
292c4762a1bSJed Brown   if (flg2) {
293c4762a1bSJed Brown     if (flg1 && dim != options->dim) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "specified -dim %D is not equal to length %D of -faces (note that -dim can be omitted)", options->dim, dim);
294c4762a1bSJed Brown     options->dim = dim;
295c4762a1bSJed Brown   }
296*589a23caSBarry Smith   ierr = PetscOptionsString("-filename", "The mesh file", "ex18.c", options->filename, options->filename, sizeof(options->filename), NULL);CHKERRQ(ierr);
297c4762a1bSJed Brown   ierr = PetscOptionsBoundedInt("-rotate_interface_0", "Rotation (relative orientation) of interface on rank 0; implies -interpolate create -distribute 0", "ex18.c", options->ornt[0], &options->ornt[0], &options->testOrientIF,0);CHKERRQ(ierr);
298c4762a1bSJed Brown   ierr = PetscOptionsBoundedInt("-rotate_interface_1", "Rotation (relative orientation) of interface on rank 1; implies -interpolate create -distribute 0", "ex18.c", options->ornt[1], &options->ornt[1], &flg2,0);CHKERRQ(ierr);
299c4762a1bSJed Brown   if (flg2 != options->testOrientIF) SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "neither or both -rotate_interface_0 -rotate_interface_1 must be set");
300c4762a1bSJed Brown   if (options->testOrientIF) {
301c4762a1bSJed Brown     PetscInt i;
302c4762a1bSJed Brown     for (i=0; i<2; i++) {
303c4762a1bSJed Brown       if (options->ornt[i] >= 10) options->ornt[i] = -(options->ornt[i]-10);  /* 11 12 13 become -1 -2 -3 */
304c4762a1bSJed Brown     }
305c4762a1bSJed Brown     options->filename[0]  = 0;
306c4762a1bSJed Brown     options->useGenerator = PETSC_FALSE;
307c4762a1bSJed Brown     options->dim          = 3;
308c4762a1bSJed Brown     options->cellSimplex  = PETSC_TRUE;
309c4762a1bSJed Brown     options->interpolate  = CREATE;
310c4762a1bSJed Brown     options->distribute   = PETSC_FALSE;
311c4762a1bSJed Brown   }
312c4762a1bSJed Brown   ierr = PetscOptionsEnd();
313c4762a1bSJed Brown   PetscFunctionReturn(0);
314c4762a1bSJed Brown }
315c4762a1bSJed Brown 
316c4762a1bSJed Brown static PetscErrorCode CreateMesh_1D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
317c4762a1bSJed Brown {
318c4762a1bSJed Brown   PetscInt       testNum = user->testNum;
319c4762a1bSJed Brown   PetscMPIInt    rank,size;
320c4762a1bSJed Brown   PetscErrorCode ierr;
321c4762a1bSJed Brown   PetscInt       numCorners=2,i;
322c4762a1bSJed Brown   PetscInt       numCells,numVertices,network;
323c4762a1bSJed Brown   int            *cells;
324c4762a1bSJed Brown   PetscReal      *coords;
325c4762a1bSJed Brown 
326c4762a1bSJed Brown   PetscFunctionBegin;
327c4762a1bSJed Brown   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
328c4762a1bSJed Brown   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
329c4762a1bSJed Brown   if (size > 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for <=2 processes",testNum);
330c4762a1bSJed Brown 
331c4762a1bSJed Brown   numCells = 3;
332c4762a1bSJed Brown   ierr = PetscOptionsGetInt(NULL, NULL, "-ncells", &numCells, NULL);CHKERRQ(ierr);
333c4762a1bSJed Brown   if (numCells < 3) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test ncells must >=3",numCells);
334c4762a1bSJed Brown 
335c4762a1bSJed Brown   if (size == 1) {
336c4762a1bSJed Brown     double *dcoords;
337c4762a1bSJed Brown     numVertices = numCells + 1;
338c4762a1bSJed Brown     ierr = PetscMalloc2(2*numCells,&cells,2*numVertices,&dcoords);CHKERRQ(ierr);
339c4762a1bSJed Brown     for (i=0; i<numCells; i++) {
340c4762a1bSJed Brown       cells[2*i] = i; cells[2*i+1] = i + 1;
341c4762a1bSJed Brown       dcoords[2*i] = i; dcoords[2*i+1] = i + 1;
342c4762a1bSJed Brown     }
343c4762a1bSJed Brown 
344c4762a1bSJed Brown     ierr = DMPlexCreateFromCellList(comm, user->dim, numCells, numVertices, numCorners, PETSC_FALSE, cells, user->dim, dcoords, dm);CHKERRQ(ierr);
345c4762a1bSJed Brown     ierr = PetscFree2(cells,coords);CHKERRQ(ierr);
346c4762a1bSJed Brown     PetscFunctionReturn(0);
347c4762a1bSJed Brown   }
348c4762a1bSJed Brown 
349c4762a1bSJed Brown   network = 0;
350c4762a1bSJed Brown   ierr = PetscOptionsGetInt(NULL, NULL, "-network_case", &network, NULL);CHKERRQ(ierr);
351c4762a1bSJed Brown   if (network == 0) {
352c4762a1bSJed Brown     switch (rank) {
353c4762a1bSJed Brown     case 0:
354c4762a1bSJed Brown     {
355c4762a1bSJed Brown       numCells    = 2;
356c4762a1bSJed Brown       numVertices = numCells;
357c4762a1bSJed Brown       ierr = PetscMalloc2(2*numCells,&cells,2*numCells,&coords);CHKERRQ(ierr);
358c4762a1bSJed Brown       cells[0] = 0; cells[1] = 1;
359c4762a1bSJed Brown       cells[2] = 1; cells[3] = 2;
360c4762a1bSJed Brown       coords[0] = 0.; coords[1] = 1.;
361c4762a1bSJed Brown       coords[2] = 1.; coords[3] = 2.;
362c4762a1bSJed Brown     }
363c4762a1bSJed Brown     break;
364c4762a1bSJed Brown     case 1:
365c4762a1bSJed Brown     {
366c4762a1bSJed Brown       numCells    -= 2;
367c4762a1bSJed Brown       numVertices = numCells + 1;
368c4762a1bSJed Brown       ierr = PetscMalloc2(2*numCells,&cells,2*numCells,&coords);CHKERRQ(ierr);
369c4762a1bSJed Brown       for (i=0; i<numCells; i++) {
370c4762a1bSJed Brown         cells[2*i] = 2+i; cells[2*i+1] = 2 + i + 1;
371c4762a1bSJed Brown         coords[2*i] = 2+i; coords[2*i+1] = 2 + i + 1;
372c4762a1bSJed Brown       }
373c4762a1bSJed Brown     }
374c4762a1bSJed Brown     break;
375c4762a1bSJed Brown     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
376c4762a1bSJed Brown     }
377c4762a1bSJed Brown   } else { /* network_case = 1 */
378c4762a1bSJed Brown     /* ----------------------- */
379c4762a1bSJed Brown     switch (rank) {
380c4762a1bSJed Brown     case 0:
381c4762a1bSJed Brown     {
382c4762a1bSJed Brown       numCells    = 2;
383c4762a1bSJed Brown       numVertices = 3;
384c4762a1bSJed Brown       ierr = PetscMalloc2(2*numCells,&cells,2*numCells,&coords);CHKERRQ(ierr);
385c4762a1bSJed Brown       cells[0] = 0; cells[1] = 3;
386c4762a1bSJed Brown       cells[2] = 3; cells[3] = 1;
387c4762a1bSJed Brown     }
388c4762a1bSJed Brown     break;
389c4762a1bSJed Brown     case 1:
390c4762a1bSJed Brown     {
391c4762a1bSJed Brown       numCells    = 1;
392c4762a1bSJed Brown       numVertices = 1;
393c4762a1bSJed Brown       ierr = PetscMalloc2(2*numCells,&cells,2*numCells,&coords);CHKERRQ(ierr);
394c4762a1bSJed Brown       cells[0] = 3; cells[1] = 2;
395c4762a1bSJed Brown     }
396c4762a1bSJed Brown     break;
397c4762a1bSJed Brown     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
398c4762a1bSJed Brown     }
399c4762a1bSJed Brown   }
400c4762a1bSJed Brown   ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, PETSC_FALSE, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
401c4762a1bSJed Brown   ierr = PetscFree2(cells,coords);CHKERRQ(ierr);
402c4762a1bSJed Brown   PetscFunctionReturn(0);
403c4762a1bSJed Brown }
404c4762a1bSJed Brown 
405c4762a1bSJed Brown static PetscErrorCode CreateSimplex_2D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
406c4762a1bSJed Brown {
407c4762a1bSJed Brown   PetscInt       testNum = user->testNum, p;
408c4762a1bSJed Brown   PetscMPIInt    rank, size;
409c4762a1bSJed Brown   PetscErrorCode ierr;
410c4762a1bSJed Brown 
411c4762a1bSJed Brown   PetscFunctionBegin;
412c4762a1bSJed Brown   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
413c4762a1bSJed Brown   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
414c4762a1bSJed Brown   switch (testNum) {
415c4762a1bSJed Brown   case 0:
416c4762a1bSJed Brown     if (size != 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 2 processes", testNum);
417c4762a1bSJed Brown     switch (rank) {
418c4762a1bSJed Brown       case 0:
419c4762a1bSJed Brown       {
420c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
421c4762a1bSJed Brown         const int      cells[3]  = {0, 1, 2};
422c4762a1bSJed Brown         PetscReal      coords[4] = {-0.5, 0.5, 0.0, 0.0};
423c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
424c4762a1bSJed Brown 
425c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
426c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
427c4762a1bSJed Brown       }
428c4762a1bSJed Brown       break;
429c4762a1bSJed Brown       case 1:
430c4762a1bSJed Brown       {
431c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
432c4762a1bSJed Brown         const int      cells[3]  = {1, 3, 2};
433c4762a1bSJed Brown         PetscReal      coords[4] = {0.0, 1.0, 0.5, 0.5};
434c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
435c4762a1bSJed Brown 
436c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
437c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
438c4762a1bSJed Brown       }
439c4762a1bSJed Brown       break;
440c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
441c4762a1bSJed Brown     }
442c4762a1bSJed Brown     break;
443c4762a1bSJed Brown   case 1:
444c4762a1bSJed Brown     if (size != 3) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 3 processes", testNum);
445c4762a1bSJed Brown     switch (rank) {
446c4762a1bSJed Brown       case 0:
447c4762a1bSJed Brown       {
448c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
449c4762a1bSJed Brown         const int      cells[3]  = {0, 1, 2};
450c4762a1bSJed Brown         PetscReal      coords[4] = {0.0, 1.0, 0.0, 0.0};
451c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
452c4762a1bSJed Brown 
453c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
454c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
455c4762a1bSJed Brown       }
456c4762a1bSJed Brown       break;
457c4762a1bSJed Brown       case 1:
458c4762a1bSJed Brown       {
459c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
460c4762a1bSJed Brown         const int      cells[3]  = {0, 2, 3};
461c4762a1bSJed Brown         PetscReal      coords[4] = {0.5, 0.5, 1.0, 1.0};
462c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
463c4762a1bSJed Brown 
464c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
465c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
466c4762a1bSJed Brown       }
467c4762a1bSJed Brown       break;
468c4762a1bSJed Brown       case 2:
469c4762a1bSJed Brown       {
470c4762a1bSJed Brown         const PetscInt numCells  = 2, numVertices = 1, numCorners = 3;
471c4762a1bSJed Brown         const int      cells[6]  = {2, 4, 3, 2, 1, 4};
472c4762a1bSJed Brown         PetscReal      coords[2] = {1.0, 0.0};
473c4762a1bSJed Brown         PetscInt       markerPoints[10] = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
474c4762a1bSJed Brown 
475c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
476c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
477c4762a1bSJed Brown       }
478c4762a1bSJed Brown       break;
479c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
480c4762a1bSJed Brown     }
481c4762a1bSJed Brown     break;
482c4762a1bSJed Brown   case 2:
483c4762a1bSJed Brown     if (size != 3) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 3 processes", testNum);
484c4762a1bSJed Brown     switch (rank) {
485c4762a1bSJed Brown       case 0:
486c4762a1bSJed Brown       {
487c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
488c4762a1bSJed Brown         const int      cells[3]  = {1, 2, 0};
489c4762a1bSJed Brown         PetscReal      coords[4] = {0.5, 0.5, 0.0, 1.0};
490c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
491c4762a1bSJed Brown 
492c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
493c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
494c4762a1bSJed Brown       }
495c4762a1bSJed Brown       break;
496c4762a1bSJed Brown       case 1:
497c4762a1bSJed Brown       {
498c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
499c4762a1bSJed Brown         const int      cells[3]  = {1, 0, 3};
500c4762a1bSJed Brown         PetscReal      coords[4] = {0.0, 0.0, 1.0, 1.0};
501c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
502c4762a1bSJed Brown 
503c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
504c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
505c4762a1bSJed Brown       }
506c4762a1bSJed Brown       break;
507c4762a1bSJed Brown       case 2:
508c4762a1bSJed Brown       {
509c4762a1bSJed Brown         const PetscInt numCells  = 2, numVertices = 1, numCorners = 3;
510c4762a1bSJed Brown         const int      cells[6]  = {0, 4, 3, 0, 2, 4};
511c4762a1bSJed Brown         PetscReal      coords[2] = {1.0, 0.0};
512c4762a1bSJed Brown         PetscInt       markerPoints[10] = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
513c4762a1bSJed Brown 
514c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
515c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
516c4762a1bSJed Brown       }
517c4762a1bSJed Brown       break;
518c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
519c4762a1bSJed Brown     }
520c4762a1bSJed Brown     break;
521c4762a1bSJed Brown   default: SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh %D", testNum);
522c4762a1bSJed Brown   }
523c4762a1bSJed Brown   PetscFunctionReturn(0);
524c4762a1bSJed Brown }
525c4762a1bSJed Brown 
526c4762a1bSJed Brown static PetscErrorCode CreateSimplex_3D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
527c4762a1bSJed Brown {
528c4762a1bSJed Brown   PetscInt       testNum = user->testNum, p;
529c4762a1bSJed Brown   PetscMPIInt    rank, size;
530c4762a1bSJed Brown   PetscErrorCode ierr;
531c4762a1bSJed Brown 
532c4762a1bSJed Brown   PetscFunctionBegin;
533c4762a1bSJed Brown   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
534c4762a1bSJed Brown   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
535c4762a1bSJed Brown   switch (testNum) {
536c4762a1bSJed Brown   case 0:
537c4762a1bSJed Brown     if (size != 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 2 processes", testNum);
538c4762a1bSJed Brown     switch (rank) {
539c4762a1bSJed Brown       case 0:
540c4762a1bSJed Brown       {
541c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 4;
542c4762a1bSJed Brown         const int      cells[4]  = {0, 2, 1, 3};
543c4762a1bSJed Brown         PetscReal      coords[6] = {0.0, 0.0, -0.5,  0.0, -0.5, 0.0};
544c4762a1bSJed Brown         PetscInt       markerPoints[8] = {1, 1, 2, 1, 3, 1, 4, 1};
545c4762a1bSJed Brown 
546c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
547c4762a1bSJed Brown         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
548c4762a1bSJed Brown       }
549c4762a1bSJed Brown       break;
550c4762a1bSJed Brown       case 1:
551c4762a1bSJed Brown       {
552c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 3, numCorners = 4;
553c4762a1bSJed Brown         const int      cells[4]  = {1, 2, 4, 3};
554c4762a1bSJed Brown         PetscReal      coords[9] = {1.0, 0.0, 0.0,  0.0, 0.5, 0.0,  0.0, 0.0, 0.5};
555c4762a1bSJed Brown         PetscInt       markerPoints[8] = {1, 1, 2, 1, 3, 1, 4, 1};
556c4762a1bSJed Brown 
557c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
558c4762a1bSJed Brown         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
559c4762a1bSJed Brown       }
560c4762a1bSJed Brown       break;
561c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
562c4762a1bSJed Brown     }
563c4762a1bSJed Brown     break;
564c4762a1bSJed Brown   default: SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh %D", testNum);
565c4762a1bSJed Brown   }
566c4762a1bSJed Brown   if (user->testOrientIF) {
567c4762a1bSJed Brown     PetscInt start;
568c4762a1bSJed Brown     PetscBool reverse;
569c4762a1bSJed Brown     PetscInt ifp[] = {8, 6};
570c4762a1bSJed Brown 
571c4762a1bSJed Brown     ierr = PetscObjectSetName((PetscObject) *dm, "Mesh before orientation");CHKERRQ(ierr);
572c4762a1bSJed Brown     ierr = DMViewFromOptions(*dm, NULL, "-before_orientation_dm_view");CHKERRQ(ierr);
573c4762a1bSJed Brown     /* rotate interface face ifp[rank] by given orientation ornt[rank] */
574c4762a1bSJed Brown     ierr = DMPlexFixFaceOrientations_Translate_Private(user->ornt[rank], &start, &reverse);CHKERRQ(ierr);
575c4762a1bSJed Brown     ierr = DMPlexOrientCell_Internal(*dm, ifp[rank], start, reverse);CHKERRQ(ierr);
576c4762a1bSJed Brown     ierr = DMPlexCheckFaces(*dm, 0);CHKERRQ(ierr);
577c4762a1bSJed Brown     ierr = DMPlexOrientInterface_Internal(*dm);CHKERRQ(ierr);
578c4762a1bSJed Brown     ierr = PetscPrintf(comm, "Orientation test PASSED\n");CHKERRQ(ierr);
579c4762a1bSJed Brown   }
580c4762a1bSJed Brown   PetscFunctionReturn(0);
581c4762a1bSJed Brown }
582c4762a1bSJed Brown 
583c4762a1bSJed Brown static PetscErrorCode CreateQuad_2D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
584c4762a1bSJed Brown {
585c4762a1bSJed Brown   PetscInt       testNum = user->testNum, p;
586c4762a1bSJed Brown   PetscMPIInt    rank, size;
587c4762a1bSJed Brown   PetscErrorCode ierr;
588c4762a1bSJed Brown 
589c4762a1bSJed Brown   PetscFunctionBegin;
590c4762a1bSJed Brown   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
591c4762a1bSJed Brown   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
592c4762a1bSJed Brown   switch (testNum) {
593c4762a1bSJed Brown   case 0:
594c4762a1bSJed Brown     if (size != 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 2 processes", testNum);
595c4762a1bSJed Brown     switch (rank) {
596c4762a1bSJed Brown       case 0:
597c4762a1bSJed Brown       {
598c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 3, numCorners = 4;
599c4762a1bSJed Brown         const int      cells[4]  = {0, 1, 2, 3};
600c4762a1bSJed Brown         PetscReal      coords[6] = {-0.5, 0.0, 0.0, 0.0, 0.0, 1.0};
601c4762a1bSJed Brown         PetscInt       markerPoints[4*2] = {1, 1, 2, 1, 3, 1, 4, 1};
602c4762a1bSJed Brown 
603c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
604c4762a1bSJed Brown         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
605c4762a1bSJed Brown       }
606c4762a1bSJed Brown       break;
607c4762a1bSJed Brown       case 1:
608c4762a1bSJed Brown       {
609c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 3, numCorners = 4;
610c4762a1bSJed Brown         const int      cells[4]  = {1, 4, 5, 2};
611c4762a1bSJed Brown         PetscReal      coords[6] = {-0.5, 1.0, 0.5, 0.0, 0.5, 1.0};
612c4762a1bSJed Brown         PetscInt       markerPoints[4*2] = {1, 1, 2, 1, 3, 1, 4, 1};
613c4762a1bSJed Brown 
614c4762a1bSJed Brown         ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
615c4762a1bSJed Brown         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
616c4762a1bSJed Brown       }
617c4762a1bSJed Brown       break;
618c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
619c4762a1bSJed Brown     }
620c4762a1bSJed Brown     break;
621c4762a1bSJed Brown   default: SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh %D", testNum);
622c4762a1bSJed Brown   }
623c4762a1bSJed Brown   PetscFunctionReturn(0);
624c4762a1bSJed Brown }
625c4762a1bSJed Brown 
626c4762a1bSJed Brown static PetscErrorCode CreateHex_3D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
627c4762a1bSJed Brown {
628c4762a1bSJed Brown   PetscInt       testNum = user->testNum, p;
629c4762a1bSJed Brown   PetscMPIInt    rank, size;
630c4762a1bSJed Brown   PetscErrorCode ierr;
631c4762a1bSJed Brown 
632c4762a1bSJed Brown   PetscFunctionBegin;
633c4762a1bSJed Brown   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
634c4762a1bSJed Brown   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
635c4762a1bSJed Brown   switch (testNum) {
636c4762a1bSJed Brown   case 0:
637c4762a1bSJed Brown     if (size != 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 2 processes", testNum);
638c4762a1bSJed Brown     switch (rank) {
639c4762a1bSJed Brown     case 0:
640c4762a1bSJed Brown     {
641c4762a1bSJed Brown       const PetscInt numCells  = 1, numVertices = 6, numCorners = 8;
642c4762a1bSJed Brown       const int      cells[8]  = {0, 3, 2, 1, 4, 5, 6, 7};
643c4762a1bSJed Brown       PetscReal      coords[6*3] = {-0.5,0.0,0.0, 0.0,0.0,0.0, 0.0,1.0,0.0, -0.5,1.0,0.0, -0.5,0.0,1.0, 0.0,0.0,1.0};
644c4762a1bSJed Brown       PetscInt       markerPoints[8*2] = {2,1,3,1,4,1,5,1,6,1,7,1,8,1,9,1};
645c4762a1bSJed Brown 
646c4762a1bSJed Brown       ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
647c4762a1bSJed Brown       for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
648c4762a1bSJed Brown     }
649c4762a1bSJed Brown     break;
650c4762a1bSJed Brown     case 1:
651c4762a1bSJed Brown     {
652c4762a1bSJed Brown       const PetscInt numCells  = 1, numVertices = 6, numCorners = 8;
653c4762a1bSJed Brown       const int      cells[8]  = {1, 2, 9, 8, 5, 10, 11, 6};
654c4762a1bSJed Brown       PetscReal      coords[6*3] = {0.0,1.0,1.0, -0.5,1.0,1.0, 0.5,0.0,0.0, 0.5,1.0,0.0, 0.5,0.0,1.0,  0.5,1.0,1.0};
655c4762a1bSJed Brown       PetscInt       markerPoints[8*2] = {2,1,3,1,4,1,5,1,6,1,7,1,8,1,9,1};
656c4762a1bSJed Brown 
657c4762a1bSJed Brown       ierr = DMPlexCreateFromCellListParallel(comm, user->dim, numCells, numVertices, numCorners, interpolate, cells, user->dim, coords, NULL, dm);CHKERRQ(ierr);
658c4762a1bSJed Brown       for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
659c4762a1bSJed Brown     }
660c4762a1bSJed Brown     break;
661c4762a1bSJed Brown     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
662c4762a1bSJed Brown     }
663c4762a1bSJed Brown   break;
664c4762a1bSJed Brown   default: SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh %D", testNum);
665c4762a1bSJed Brown   }
666c4762a1bSJed Brown   PetscFunctionReturn(0);
667c4762a1bSJed Brown }
668c4762a1bSJed Brown 
669c4762a1bSJed Brown static PetscErrorCode CustomView(DM dm, PetscViewer v)
670c4762a1bSJed Brown {
671c4762a1bSJed Brown   DMPlexInterpolatedFlag interpolated;
672c4762a1bSJed Brown   PetscBool              distributed;
673c4762a1bSJed Brown   PetscErrorCode         ierr;
674c4762a1bSJed Brown 
675c4762a1bSJed Brown   PetscFunctionBegin;
676c4762a1bSJed Brown   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
677c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(dm, &interpolated);CHKERRQ(ierr);
678c4762a1bSJed Brown   ierr = PetscViewerASCIIPrintf(v, "DMPlexIsDistributed: %s\n", PetscBools[distributed]);CHKERRQ(ierr);
679c4762a1bSJed Brown   ierr = PetscViewerASCIIPrintf(v, "DMPlexIsInterpolatedCollective: %s\n", DMPlexInterpolatedFlags[interpolated]);CHKERRQ(ierr);
680c4762a1bSJed Brown   PetscFunctionReturn(0);
681c4762a1bSJed Brown }
682c4762a1bSJed Brown 
683c4762a1bSJed Brown static PetscErrorCode CreateMeshFromFile(MPI_Comm comm, AppCtx *user, DM *dm, DM *serialDM)
684c4762a1bSJed Brown {
685c4762a1bSJed Brown   const char    *filename       = user->filename;
686c4762a1bSJed Brown   PetscBool      testHeavy      = user->testHeavy;
687c4762a1bSJed Brown   PetscBool      interpCreate   = user->interpolate == CREATE ? PETSC_TRUE : PETSC_FALSE;
688c4762a1bSJed Brown   PetscBool      distributed    = PETSC_FALSE;
689c4762a1bSJed Brown   PetscErrorCode ierr;
690c4762a1bSJed Brown 
691c4762a1bSJed Brown   PetscFunctionBegin;
692c4762a1bSJed Brown   *serialDM = NULL;
693c4762a1bSJed Brown   if (testHeavy && interpCreate) {ierr = DMPlexSetOrientInterface_Private(NULL, PETSC_FALSE);CHKERRQ(ierr);}
694c4762a1bSJed Brown   ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr);
695c4762a1bSJed Brown   ierr = DMPlexCreateFromFile(comm, filename, interpCreate, dm);CHKERRQ(ierr); /* with DMPlexOrientInterface_Internal() call skipped so that PointSF issues are left to DMPlexCheckPointSFHeavy() */
696c4762a1bSJed Brown   ierr = PetscLogStagePop();CHKERRQ(ierr);
697c4762a1bSJed Brown   if (testHeavy && interpCreate) {ierr = DMPlexSetOrientInterface_Private(NULL, PETSC_TRUE);CHKERRQ(ierr);}
698c4762a1bSJed Brown   ierr = DMPlexIsDistributed(*dm, &distributed);CHKERRQ(ierr);
699c4762a1bSJed Brown   ierr = PetscPrintf(comm, "DMPlexCreateFromFile produced %s mesh.\n", distributed ? "distributed" : "serial");CHKERRQ(ierr);
700c4762a1bSJed Brown   if (testHeavy && distributed) {
701c4762a1bSJed Brown     ierr = PetscOptionsSetValue(NULL, "-dm_plex_hdf5_force_sequential", NULL);CHKERRQ(ierr);
702c4762a1bSJed Brown     ierr = DMPlexCreateFromFile(comm, filename, interpCreate, serialDM);CHKERRQ(ierr);
703c4762a1bSJed Brown     ierr = DMPlexIsDistributed(*serialDM, &distributed);CHKERRQ(ierr);
704c4762a1bSJed Brown     if (distributed) SETERRQ(comm, PETSC_ERR_PLIB, "unable to create a serial DM from file");
705c4762a1bSJed Brown   }
706c4762a1bSJed Brown   ierr = DMGetDimension(*dm, &user->dim);CHKERRQ(ierr);
707c4762a1bSJed Brown   PetscFunctionReturn(0);
708c4762a1bSJed Brown }
709c4762a1bSJed Brown 
710c4762a1bSJed Brown static PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
711c4762a1bSJed Brown {
712c4762a1bSJed Brown   PetscPartitioner part;
713c4762a1bSJed Brown   PortableBoundary boundary     = NULL;
714c4762a1bSJed Brown   DM             serialDM       = NULL;
715c4762a1bSJed Brown   PetscBool      cellSimplex    = user->cellSimplex;
716c4762a1bSJed Brown   PetscBool      useGenerator   = user->useGenerator;
717c4762a1bSJed Brown   PetscBool      interpCreate   = user->interpolate == CREATE ? PETSC_TRUE : PETSC_FALSE;
718c4762a1bSJed Brown   PetscBool      interpSerial   = user->interpolate == AFTER_CREATE ? PETSC_TRUE : PETSC_FALSE;
719c4762a1bSJed Brown   PetscBool      interpParallel = user->interpolate == AFTER_DISTRIBUTE ? PETSC_TRUE : PETSC_FALSE;
720c4762a1bSJed Brown   PetscBool      testHeavy      = user->testHeavy;
721c4762a1bSJed Brown   PetscMPIInt    rank;
722c4762a1bSJed Brown   PetscErrorCode ierr;
723c4762a1bSJed Brown 
724c4762a1bSJed Brown   PetscFunctionBegin;
725c4762a1bSJed Brown   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
726c4762a1bSJed Brown   if (user->filename[0]) {
727c4762a1bSJed Brown     ierr = CreateMeshFromFile(comm, user, dm, &serialDM);CHKERRQ(ierr);
728c4762a1bSJed Brown   } else if (useGenerator) {
729c4762a1bSJed Brown     ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr);
730c4762a1bSJed Brown     ierr = DMPlexCreateBoxMesh(comm, user->dim, cellSimplex, user->faces, NULL, NULL, NULL, interpCreate, dm);CHKERRQ(ierr);
731c4762a1bSJed Brown     ierr = PetscLogStagePop();CHKERRQ(ierr);
732c4762a1bSJed Brown   } else {
733c4762a1bSJed Brown     ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr);
734c4762a1bSJed Brown     switch (user->dim) {
735c4762a1bSJed Brown     case 1:
736c4762a1bSJed Brown       ierr = CreateMesh_1D(comm, interpCreate, user, dm);CHKERRQ(ierr);
737c4762a1bSJed Brown       break;
738c4762a1bSJed Brown     case 2:
739c4762a1bSJed Brown       if (cellSimplex) {
740c4762a1bSJed Brown         ierr = CreateSimplex_2D(comm, interpCreate, user, dm);CHKERRQ(ierr);
741c4762a1bSJed Brown       } else {
742c4762a1bSJed Brown         ierr = CreateQuad_2D(comm, interpCreate, user, dm);CHKERRQ(ierr);
743c4762a1bSJed Brown       }
744c4762a1bSJed Brown       break;
745c4762a1bSJed Brown     case 3:
746c4762a1bSJed Brown       if (cellSimplex) {
747c4762a1bSJed Brown         ierr = CreateSimplex_3D(comm, interpCreate, user, dm);CHKERRQ(ierr);
748c4762a1bSJed Brown       } else {
749c4762a1bSJed Brown         ierr = CreateHex_3D(comm, interpCreate, user, dm);CHKERRQ(ierr);
750c4762a1bSJed Brown       }
751c4762a1bSJed Brown       break;
752c4762a1bSJed Brown     default:
753c4762a1bSJed Brown       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %D", user->dim);
754c4762a1bSJed Brown     }
755c4762a1bSJed Brown     ierr = PetscLogStagePop();CHKERRQ(ierr);
756c4762a1bSJed Brown   }
757c4762a1bSJed Brown   if (user->ncoords % user->dim) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "length of coordinates array %D must be divisable by spatial dimension %D", user->ncoords, user->dim);
758c4762a1bSJed Brown   ierr = PetscObjectSetName((PetscObject) *dm, "Original Mesh");CHKERRQ(ierr);
759c4762a1bSJed Brown   ierr = DMViewFromOptions(*dm, NULL, "-orig_dm_view");CHKERRQ(ierr);
760c4762a1bSJed Brown 
761c4762a1bSJed Brown   if (interpSerial) {
762c4762a1bSJed Brown     DM idm;
763c4762a1bSJed Brown 
764c4762a1bSJed Brown     if (testHeavy) {ierr = DMPlexSetOrientInterface_Private(*dm, PETSC_FALSE);CHKERRQ(ierr);}
765c4762a1bSJed Brown     ierr = PetscLogStagePush(stage[2]);CHKERRQ(ierr);
766c4762a1bSJed Brown     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr); /* with DMPlexOrientInterface_Internal() call skipped so that PointSF issues are left to DMPlexCheckPointSFHeavy() */
767c4762a1bSJed Brown     ierr = PetscLogStagePop();CHKERRQ(ierr);
768c4762a1bSJed Brown     if (testHeavy) {ierr = DMPlexSetOrientInterface_Private(*dm, PETSC_TRUE);CHKERRQ(ierr);}
769c4762a1bSJed Brown     ierr = DMDestroy(dm);CHKERRQ(ierr);
770c4762a1bSJed Brown     *dm = idm;
771c4762a1bSJed Brown     ierr = PetscObjectSetName((PetscObject) *dm, "Interpolated Mesh");CHKERRQ(ierr);
772c4762a1bSJed Brown     ierr = DMViewFromOptions(*dm, NULL, "-intp_dm_view");CHKERRQ(ierr);
773c4762a1bSJed Brown   }
774c4762a1bSJed Brown 
775c4762a1bSJed Brown   /* Set partitioner options */
776c4762a1bSJed Brown   ierr = DMPlexGetPartitioner(*dm, &part);CHKERRQ(ierr);
777c4762a1bSJed Brown   if (part) {
778c4762a1bSJed Brown     ierr = PetscPartitionerSetType(part, PETSCPARTITIONERSIMPLE);CHKERRQ(ierr);
779c4762a1bSJed Brown     ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
780c4762a1bSJed Brown   }
781c4762a1bSJed Brown 
782c4762a1bSJed Brown   if (user->customView) {ierr = CustomView(*dm, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);}
783c4762a1bSJed Brown   if (testHeavy) {
784c4762a1bSJed Brown     PetscBool distributed;
785c4762a1bSJed Brown 
786c4762a1bSJed Brown     ierr = DMPlexIsDistributed(*dm, &distributed);CHKERRQ(ierr);
787c4762a1bSJed Brown     if (!serialDM && !distributed) {
788c4762a1bSJed Brown       serialDM = *dm;
789c4762a1bSJed Brown       ierr = PetscObjectReference((PetscObject)*dm);CHKERRQ(ierr);
790c4762a1bSJed Brown     }
791c4762a1bSJed Brown     if (serialDM) {
792c4762a1bSJed Brown       ierr = DMPlexGetExpandedBoundary_Private(serialDM, &boundary);CHKERRQ(ierr);
793c4762a1bSJed Brown     }
794c4762a1bSJed Brown     if (boundary) {
795c4762a1bSJed Brown       /* check DM which has been created in parallel and already interpolated */
796c4762a1bSJed Brown       ierr = DMPlexCheckPointSFHeavy(*dm, boundary);CHKERRQ(ierr);
797c4762a1bSJed Brown     }
798c4762a1bSJed Brown     /* Orient interface because it could be deliberately skipped above. It is idempotent. */
799c4762a1bSJed Brown     ierr = DMPlexOrientInterface_Internal(*dm);CHKERRQ(ierr);
800c4762a1bSJed Brown   }
801c4762a1bSJed Brown   if (user->distribute) {
802c4762a1bSJed Brown     DM               pdm = NULL;
803c4762a1bSJed Brown 
804c4762a1bSJed Brown     /* Redistribute mesh over processes using that partitioner */
805c4762a1bSJed Brown     ierr = PetscLogStagePush(stage[1]);CHKERRQ(ierr);
806c4762a1bSJed Brown     ierr = DMPlexDistribute(*dm, 0, NULL, &pdm);CHKERRQ(ierr);
807c4762a1bSJed Brown     ierr = PetscLogStagePop();CHKERRQ(ierr);
808c4762a1bSJed Brown     if (pdm) {
809c4762a1bSJed Brown       ierr = DMDestroy(dm);CHKERRQ(ierr);
810c4762a1bSJed Brown       *dm  = pdm;
811c4762a1bSJed Brown       ierr = PetscObjectSetName((PetscObject) *dm, "Redistributed Mesh");CHKERRQ(ierr);
812c4762a1bSJed Brown       ierr = DMViewFromOptions(*dm, NULL, "-dist_dm_view");CHKERRQ(ierr);
813c4762a1bSJed Brown     }
814c4762a1bSJed Brown 
815c4762a1bSJed Brown     if (interpParallel) {
816c4762a1bSJed Brown       DM idm;
817c4762a1bSJed Brown 
818c4762a1bSJed Brown       if (testHeavy) {ierr = DMPlexSetOrientInterface_Private(*dm, PETSC_FALSE);CHKERRQ(ierr);}
819c4762a1bSJed Brown       ierr = PetscLogStagePush(stage[2]);CHKERRQ(ierr);
820c4762a1bSJed Brown       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr); /* with DMPlexOrientInterface_Internal() call skipped so that PointSF issues are left to DMPlexCheckPointSFHeavy() */
821c4762a1bSJed Brown       ierr = PetscLogStagePop();CHKERRQ(ierr);
822c4762a1bSJed Brown       if (testHeavy) {ierr = DMPlexSetOrientInterface_Private(*dm, PETSC_TRUE);CHKERRQ(ierr);}
823c4762a1bSJed Brown       ierr = DMDestroy(dm);CHKERRQ(ierr);
824c4762a1bSJed Brown       *dm = idm;
825c4762a1bSJed Brown       ierr = PetscObjectSetName((PetscObject) *dm, "Interpolated Redistributed Mesh");CHKERRQ(ierr);
826c4762a1bSJed Brown       ierr = DMViewFromOptions(*dm, NULL, "-intp_dm_view");CHKERRQ(ierr);
827c4762a1bSJed Brown     }
828c4762a1bSJed Brown   }
829c4762a1bSJed Brown   if (testHeavy) {
830c4762a1bSJed Brown     if (boundary) {
831c4762a1bSJed Brown       ierr = DMPlexCheckPointSFHeavy(*dm, boundary);CHKERRQ(ierr);
832c4762a1bSJed Brown     }
833c4762a1bSJed Brown     /* Orient interface because it could be deliberately skipped above. It is idempotent. */
834c4762a1bSJed Brown     ierr = DMPlexOrientInterface_Internal(*dm);CHKERRQ(ierr);
835c4762a1bSJed Brown   }
836c4762a1bSJed Brown 
837c4762a1bSJed Brown   ierr = PetscObjectSetName((PetscObject) *dm, "Parallel Mesh");CHKERRQ(ierr);
838c4762a1bSJed Brown   ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
839c4762a1bSJed Brown   ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
840c4762a1bSJed Brown 
841c4762a1bSJed Brown   if (user->customView) {ierr = CustomView(*dm, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);}
842c4762a1bSJed Brown   ierr = DMDestroy(&serialDM);CHKERRQ(ierr);
843c4762a1bSJed Brown   ierr = PortableBoundaryDestroy(&boundary);CHKERRQ(ierr);
844c4762a1bSJed Brown   PetscFunctionReturn(0);
845c4762a1bSJed Brown }
846c4762a1bSJed Brown 
847c4762a1bSJed Brown PETSC_STATIC_INLINE PetscErrorCode coord2str(char buf[], size_t len, PetscInt dim, PetscReal *coords, PetscReal tol)
848c4762a1bSJed Brown {
849c4762a1bSJed Brown   PetscErrorCode ierr;
850c4762a1bSJed Brown 
851c4762a1bSJed Brown   PetscFunctionBegin;
852c4762a1bSJed Brown   if (dim > 3) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "dim must be less than or equal 3");
853c4762a1bSJed Brown   if (tol >= 1e-3) {
854c4762a1bSJed Brown     switch (dim) {
855c4762a1bSJed Brown       case 1: ierr = PetscSNPrintf(buf,len,"(%12.3f)",(double)coords[0]);CHKERRQ(ierr);
856c4762a1bSJed Brown       case 2: ierr = PetscSNPrintf(buf,len,"(%12.3f, %12.3f)",(double)coords[0],(double)coords[1]);CHKERRQ(ierr);
857c4762a1bSJed Brown       case 3: ierr = PetscSNPrintf(buf,len,"(%12.3f, %12.3f, %12.3f)",(double)coords[0],(double)coords[1],(double)coords[2]);CHKERRQ(ierr);
858c4762a1bSJed Brown     }
859c4762a1bSJed Brown   } else {
860c4762a1bSJed Brown     switch (dim) {
861c4762a1bSJed Brown       case 1: ierr = PetscSNPrintf(buf,len,"(%12.6f)",(double)coords[0]);CHKERRQ(ierr);
862c4762a1bSJed Brown       case 2: ierr = PetscSNPrintf(buf,len,"(%12.6f, %12.6f)",(double)coords[0],(double)coords[1]);CHKERRQ(ierr);
863c4762a1bSJed Brown       case 3: ierr = PetscSNPrintf(buf,len,"(%12.6f, %12.6f, %12.6f)",(double)coords[0],(double)coords[1],(double)coords[2]);CHKERRQ(ierr);
864c4762a1bSJed Brown     }
865c4762a1bSJed Brown   }
866c4762a1bSJed Brown   PetscFunctionReturn(0);
867c4762a1bSJed Brown }
868c4762a1bSJed Brown 
869c4762a1bSJed Brown static PetscErrorCode ViewVerticesFromCoords(DM dm, PetscInt npoints, PetscReal coords[], PetscReal tol, PetscViewer viewer)
870c4762a1bSJed Brown {
871c4762a1bSJed Brown   PetscInt       dim, i;
872c4762a1bSJed Brown   PetscInt       *points;
873c4762a1bSJed Brown   char           coordstr[128];
874c4762a1bSJed Brown   MPI_Comm       comm;
875c4762a1bSJed Brown   PetscMPIInt    rank;
876c4762a1bSJed Brown   PetscErrorCode ierr;
877c4762a1bSJed Brown 
878c4762a1bSJed Brown   PetscFunctionBegin;
879c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
880c4762a1bSJed Brown   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
881c4762a1bSJed Brown   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
882c4762a1bSJed Brown   ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
883c4762a1bSJed Brown   ierr = PetscMalloc1(npoints, &points);CHKERRQ(ierr);
884c4762a1bSJed Brown   ierr = DMPlexFindVertices(dm, npoints, coords, tol, points);CHKERRQ(ierr);
885c4762a1bSJed Brown   for (i=0; i < npoints; i++) {
886c4762a1bSJed Brown     ierr = coord2str(coordstr, sizeof(coordstr), dim, &coords[i*dim], tol);CHKERRQ(ierr);
887c4762a1bSJed Brown     if (!rank && i) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, "-----\n");CHKERRQ(ierr);}
888c4762a1bSJed Brown     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] %s --> points[%D] = %D\n", rank, coordstr, i, points[i]);CHKERRQ(ierr);
889c4762a1bSJed Brown     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
890c4762a1bSJed Brown   }
891c4762a1bSJed Brown   ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
892c4762a1bSJed Brown   ierr = PetscFree(points);CHKERRQ(ierr);
893c4762a1bSJed Brown   PetscFunctionReturn(0);
894c4762a1bSJed Brown }
895c4762a1bSJed Brown 
896c4762a1bSJed Brown static PetscErrorCode TestExpandPoints(DM dm, AppCtx *user)
897c4762a1bSJed Brown {
898c4762a1bSJed Brown   IS                is;
899c4762a1bSJed Brown   PetscSection      *sects;
900c4762a1bSJed Brown   IS                *iss;
901c4762a1bSJed Brown   PetscInt          d,depth;
902c4762a1bSJed Brown   PetscMPIInt       rank;
903c4762a1bSJed Brown   PetscErrorCode    ierr;
904c4762a1bSJed Brown   PetscViewer       viewer=PETSC_VIEWER_STDOUT_WORLD, sviewer;
905c4762a1bSJed Brown 
906c4762a1bSJed Brown   PetscFunctionBegin;
907c4762a1bSJed Brown   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm),&rank);CHKERRQ(ierr);
908c4762a1bSJed Brown   if (user->testExpandPointsEmpty && !rank) {
909c4762a1bSJed Brown     ierr = ISCreateGeneral(PETSC_COMM_SELF, 0, NULL, PETSC_USE_POINTER, &is);CHKERRQ(ierr);
910c4762a1bSJed Brown   } else {
911c4762a1bSJed Brown     ierr = ISCreateGeneral(PETSC_COMM_SELF, user->nPointsToExpand, user->pointsToExpand, PETSC_USE_POINTER, &is);CHKERRQ(ierr);
912c4762a1bSJed Brown   }
913c4762a1bSJed Brown   ierr = DMPlexGetConeRecursive(dm, is, &depth, &iss, &sects);CHKERRQ(ierr);
914c4762a1bSJed Brown   ierr = PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
915c4762a1bSJed Brown   ierr = PetscViewerASCIIPrintf(sviewer, "[%d] ==========================\n",rank);CHKERRQ(ierr);
916c4762a1bSJed Brown   for (d=depth-1; d>=0; d--) {
917c4762a1bSJed Brown     IS          checkIS;
918c4762a1bSJed Brown     PetscBool   flg;
919c4762a1bSJed Brown 
920c4762a1bSJed Brown     ierr = PetscViewerASCIIPrintf(sviewer, "depth %D ---------------\n",d);CHKERRQ(ierr);
921c4762a1bSJed Brown     ierr = PetscSectionView(sects[d], sviewer);CHKERRQ(ierr);
922c4762a1bSJed Brown     ierr = ISView(iss[d], sviewer);CHKERRQ(ierr);
923c4762a1bSJed Brown     /* check reverse operation */
924c4762a1bSJed Brown     if (d < depth-1) {
925c4762a1bSJed Brown       ierr = DMPlexExpandedConesToFaces_Private(dm, iss[d], sects[d], &checkIS);CHKERRQ(ierr);
926c4762a1bSJed Brown       ierr = ISEqualUnsorted(checkIS, iss[d+1], &flg);CHKERRQ(ierr);
927c4762a1bSJed Brown       if (!flg) SETERRQ(PetscObjectComm((PetscObject) checkIS), PETSC_ERR_PLIB, "DMPlexExpandedConesToFaces_Private produced wrong IS");
928c4762a1bSJed Brown       ierr = ISDestroy(&checkIS);CHKERRQ(ierr);
929c4762a1bSJed Brown     }
930c4762a1bSJed Brown   }
931c4762a1bSJed Brown   ierr = PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
932c4762a1bSJed Brown   ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
933c4762a1bSJed Brown   ierr = DMPlexRestoreConeRecursive(dm, is, &depth, &iss, &sects);CHKERRQ(ierr);
934c4762a1bSJed Brown   ierr = ISDestroy(&is);CHKERRQ(ierr);
935c4762a1bSJed Brown   PetscFunctionReturn(0);
936c4762a1bSJed Brown }
937c4762a1bSJed Brown 
938c4762a1bSJed Brown static PetscErrorCode DMPlexExpandedConesToFaces_Private(DM dm, IS is, PetscSection section, IS *newis)
939c4762a1bSJed Brown {
940c4762a1bSJed Brown   PetscInt          n,n1,ncone,numCoveredPoints,o,p,q,start,end;
941c4762a1bSJed Brown   const PetscInt    *coveredPoints;
942c4762a1bSJed Brown   const PetscInt    *arr, *cone;
943c4762a1bSJed Brown   PetscInt          *newarr;
944c4762a1bSJed Brown   PetscErrorCode    ierr;
945c4762a1bSJed Brown 
946c4762a1bSJed Brown   PetscFunctionBegin;
947c4762a1bSJed Brown   ierr = ISGetLocalSize(is, &n);CHKERRQ(ierr);
948c4762a1bSJed Brown   ierr = PetscSectionGetStorageSize(section, &n1);CHKERRQ(ierr);
949c4762a1bSJed Brown   ierr = PetscSectionGetChart(section, &start, &end);CHKERRQ(ierr);
950c4762a1bSJed Brown   if (n != n1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "IS size = %D != %D = section storage size\n", n, n1);
951c4762a1bSJed Brown   ierr = ISGetIndices(is, &arr);CHKERRQ(ierr);
952c4762a1bSJed Brown   ierr = PetscMalloc1(end-start, &newarr);CHKERRQ(ierr);
953c4762a1bSJed Brown   for (q=start; q<end; q++) {
954c4762a1bSJed Brown     ierr = PetscSectionGetDof(section, q, &ncone);CHKERRQ(ierr);
955c4762a1bSJed Brown     ierr = PetscSectionGetOffset(section, q, &o);CHKERRQ(ierr);
956c4762a1bSJed Brown     cone = &arr[o];
957c4762a1bSJed Brown     if (ncone == 1) {
958c4762a1bSJed Brown       numCoveredPoints = 1;
959c4762a1bSJed Brown       p = cone[0];
960c4762a1bSJed Brown     } else {
961c4762a1bSJed Brown       PetscInt i;
962c4762a1bSJed Brown       p = PETSC_MAX_INT;
963c4762a1bSJed Brown       for (i=0; i<ncone; i++) if (cone[i] < 0) {p = -1; break;}
964c4762a1bSJed Brown       if (p >= 0) {
965c4762a1bSJed Brown         ierr = DMPlexGetJoin(dm, ncone, cone, &numCoveredPoints, &coveredPoints);CHKERRQ(ierr);
966c4762a1bSJed Brown         if (numCoveredPoints > 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "more than one covered points for section point %D",q);
967c4762a1bSJed Brown         if (numCoveredPoints) p = coveredPoints[0];
968c4762a1bSJed Brown         else                  p = -2;
969c4762a1bSJed Brown         ierr = DMPlexRestoreJoin(dm, ncone, cone, &numCoveredPoints, &coveredPoints);CHKERRQ(ierr);
970c4762a1bSJed Brown       }
971c4762a1bSJed Brown     }
972c4762a1bSJed Brown     newarr[q-start] = p;
973c4762a1bSJed Brown   }
974c4762a1bSJed Brown   ierr = ISRestoreIndices(is, &arr);CHKERRQ(ierr);
975c4762a1bSJed Brown   ierr = ISCreateGeneral(PETSC_COMM_SELF, end-start, newarr, PETSC_OWN_POINTER, newis);CHKERRQ(ierr);
976c4762a1bSJed Brown   PetscFunctionReturn(0);
977c4762a1bSJed Brown }
978c4762a1bSJed Brown 
979c4762a1bSJed Brown static PetscErrorCode DMPlexExpandedVerticesToFaces_Private(DM dm, IS boundary_expanded_is, PetscInt depth, PetscSection sections[], IS *boundary_is)
980c4762a1bSJed Brown {
981c4762a1bSJed Brown   PetscInt          d;
982c4762a1bSJed Brown   IS                is,newis;
983c4762a1bSJed Brown   PetscErrorCode    ierr;
984c4762a1bSJed Brown 
985c4762a1bSJed Brown   PetscFunctionBegin;
986c4762a1bSJed Brown   is = boundary_expanded_is;
987c4762a1bSJed Brown   ierr = PetscObjectReference((PetscObject)is);CHKERRQ(ierr);
988c4762a1bSJed Brown   for (d = 0; d < depth-1; ++d) {
989c4762a1bSJed Brown     ierr = DMPlexExpandedConesToFaces_Private(dm, is, sections[d], &newis);CHKERRQ(ierr);
990c4762a1bSJed Brown     ierr = ISDestroy(&is);CHKERRQ(ierr);
991c4762a1bSJed Brown     is = newis;
992c4762a1bSJed Brown   }
993c4762a1bSJed Brown   *boundary_is = is;
994c4762a1bSJed Brown   PetscFunctionReturn(0);
995c4762a1bSJed Brown }
996c4762a1bSJed Brown 
997c4762a1bSJed Brown #define CHKERRQI(incall,ierr) if (ierr) {incall = PETSC_FALSE; }
998c4762a1bSJed Brown 
999c4762a1bSJed Brown static PetscErrorCode DMLabelViewFromOptionsOnComm_Private(DMLabel label, const char optionname[], MPI_Comm comm)
1000c4762a1bSJed Brown {
1001c4762a1bSJed Brown   PetscErrorCode    ierr;
1002c4762a1bSJed Brown   PetscViewer       viewer;
1003c4762a1bSJed Brown   PetscBool         flg;
1004c4762a1bSJed Brown   static PetscBool  incall = PETSC_FALSE;
1005c4762a1bSJed Brown   PetscViewerFormat format;
1006c4762a1bSJed Brown 
1007c4762a1bSJed Brown   PetscFunctionBegin;
1008c4762a1bSJed Brown   if (incall) PetscFunctionReturn(0);
1009c4762a1bSJed Brown   incall = PETSC_TRUE;
1010c4762a1bSJed Brown   ierr   = PetscOptionsGetViewer(comm,((PetscObject)label)->options,((PetscObject)label)->prefix,optionname,&viewer,&format,&flg);CHKERRQI(incall,ierr);
1011c4762a1bSJed Brown   if (flg) {
1012c4762a1bSJed Brown     ierr = PetscViewerPushFormat(viewer,format);CHKERRQI(incall,ierr);
1013c4762a1bSJed Brown     ierr = DMLabelView(label, viewer);CHKERRQI(incall,ierr);
1014c4762a1bSJed Brown     ierr = PetscViewerPopFormat(viewer);CHKERRQI(incall,ierr);
1015c4762a1bSJed Brown     ierr = PetscViewerDestroy(&viewer);CHKERRQI(incall,ierr);
1016c4762a1bSJed Brown   }
1017c4762a1bSJed Brown   incall = PETSC_FALSE;
1018c4762a1bSJed Brown   PetscFunctionReturn(0);
1019c4762a1bSJed Brown }
1020c4762a1bSJed Brown 
1021c4762a1bSJed Brown /* TODO: this is hotfixing DMLabelGetStratumIS() - it should be fixed systematically instead */
1022c4762a1bSJed Brown PETSC_STATIC_INLINE PetscErrorCode DMLabelGetStratumISOnComm_Private(DMLabel label, PetscInt value, MPI_Comm comm, IS *is)
1023c4762a1bSJed Brown {
1024c4762a1bSJed Brown   IS                tmpis;
1025c4762a1bSJed Brown   PetscErrorCode    ierr;
1026c4762a1bSJed Brown 
1027c4762a1bSJed Brown   PetscFunctionBegin;
1028c4762a1bSJed Brown   ierr = DMLabelGetStratumIS(label, value, &tmpis);CHKERRQ(ierr);
1029c4762a1bSJed Brown   if (!tmpis) {ierr = ISCreateGeneral(PETSC_COMM_SELF, 0, NULL, PETSC_USE_POINTER, &tmpis);CHKERRQ(ierr);}
1030c4762a1bSJed Brown   ierr = ISOnComm(tmpis, comm, PETSC_COPY_VALUES, is);CHKERRQ(ierr);
1031c4762a1bSJed Brown   ierr = ISDestroy(&tmpis);CHKERRQ(ierr);
1032c4762a1bSJed Brown   PetscFunctionReturn(0);
1033c4762a1bSJed Brown }
1034c4762a1bSJed Brown 
1035c4762a1bSJed Brown /* currently only for simple PetscSection without fields or constraints */
1036c4762a1bSJed Brown static PetscErrorCode PetscSectionReplicate_Private(MPI_Comm comm, PetscMPIInt rootrank, PetscSection sec0, PetscSection *secout)
1037c4762a1bSJed Brown {
1038c4762a1bSJed Brown   PetscSection      sec;
1039c4762a1bSJed Brown   PetscInt          chart[2], p;
1040c4762a1bSJed Brown   PetscInt          *dofarr;
1041c4762a1bSJed Brown   PetscMPIInt       rank;
1042c4762a1bSJed Brown   PetscErrorCode    ierr;
1043c4762a1bSJed Brown 
1044c4762a1bSJed Brown   PetscFunctionBegin;
1045c4762a1bSJed Brown   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1046c4762a1bSJed Brown   if (rank == rootrank) {
1047c4762a1bSJed Brown     ierr = PetscSectionGetChart(sec0, &chart[0], &chart[1]);CHKERRQ(ierr);
1048c4762a1bSJed Brown   }
1049c4762a1bSJed Brown   ierr = MPI_Bcast(chart, 2, MPIU_INT, rootrank, comm);CHKERRQ(ierr);
1050c4762a1bSJed Brown   ierr = PetscMalloc1(chart[1]-chart[0], &dofarr);CHKERRQ(ierr);
1051c4762a1bSJed Brown   if (rank == rootrank) {
1052c4762a1bSJed Brown     for (p = chart[0]; p < chart[1]; p++) {
1053c4762a1bSJed Brown       ierr = PetscSectionGetDof(sec0, p, &dofarr[p-chart[0]]);CHKERRQ(ierr);
1054c4762a1bSJed Brown     }
1055c4762a1bSJed Brown   }
1056c4762a1bSJed Brown   ierr = MPI_Bcast(dofarr, chart[1]-chart[0], MPIU_INT, rootrank, comm);CHKERRQ(ierr);
1057c4762a1bSJed Brown   ierr = PetscSectionCreate(comm, &sec);CHKERRQ(ierr);
1058c4762a1bSJed Brown   ierr = PetscSectionSetChart(sec, chart[0], chart[1]);CHKERRQ(ierr);
1059c4762a1bSJed Brown   for (p = chart[0]; p < chart[1]; p++) {
1060c4762a1bSJed Brown     ierr = PetscSectionSetDof(sec, p, dofarr[p-chart[0]]);CHKERRQ(ierr);
1061c4762a1bSJed Brown   }
1062c4762a1bSJed Brown   ierr = PetscSectionSetUp(sec);CHKERRQ(ierr);
1063c4762a1bSJed Brown   ierr = PetscFree(dofarr);CHKERRQ(ierr);
1064c4762a1bSJed Brown   *secout = sec;
1065c4762a1bSJed Brown   PetscFunctionReturn(0);
1066c4762a1bSJed Brown }
1067c4762a1bSJed Brown 
1068c4762a1bSJed Brown static PetscErrorCode VecToPetscReal_Private(Vec vec, PetscReal *rvals[])
1069c4762a1bSJed Brown {
1070c4762a1bSJed Brown   PetscInt          n;
1071c4762a1bSJed Brown   const PetscScalar *svals;
1072c4762a1bSJed Brown   PetscErrorCode    ierr;
1073c4762a1bSJed Brown 
1074c4762a1bSJed Brown   PetscFunctionBegin;
1075c4762a1bSJed Brown   ierr = VecGetLocalSize(vec, &n);CHKERRQ(ierr);
1076c4762a1bSJed Brown   ierr = VecGetArrayRead(vec, &svals);CHKERRQ(ierr);
1077c4762a1bSJed Brown   ierr = PetscMalloc1(n, rvals);CHKERRQ(ierr);
1078c4762a1bSJed Brown #if defined(PETSC_USE_COMPLEX)
1079c4762a1bSJed Brown   {
1080c4762a1bSJed Brown     PetscInt i;
1081c4762a1bSJed Brown     for (i=0; i<n; i++) (*rvals)[i] = PetscRealPart(svals[i]);
1082c4762a1bSJed Brown   }
1083c4762a1bSJed Brown #else
1084c4762a1bSJed Brown   ierr = PetscMemcpy(*rvals, svals, n*sizeof(PetscReal));
1085c4762a1bSJed Brown #endif
1086c4762a1bSJed Brown   ierr = VecRestoreArrayRead(vec, &svals);CHKERRQ(ierr);
1087c4762a1bSJed Brown   PetscFunctionReturn(0);
1088c4762a1bSJed Brown }
1089c4762a1bSJed Brown 
1090c4762a1bSJed Brown static PetscErrorCode DMPlexExpandedVerticesCoordinatesToFaces_Private(DM ipdm, PortableBoundary bnd, IS *face_is)
1091c4762a1bSJed Brown {
1092c4762a1bSJed Brown   PetscInt            dim, ncoords, npoints;
1093c4762a1bSJed Brown   PetscReal           *rcoords;
1094c4762a1bSJed Brown   PetscInt            *points;
1095c4762a1bSJed Brown   IS                  faces_expanded_is;
1096c4762a1bSJed Brown   PetscErrorCode      ierr;
1097c4762a1bSJed Brown 
1098c4762a1bSJed Brown   PetscFunctionBegin;
1099c4762a1bSJed Brown   ierr = DMGetDimension(ipdm, &dim);CHKERRQ(ierr);
1100c4762a1bSJed Brown   ierr = VecGetLocalSize(bnd->coordinates, &ncoords);CHKERRQ(ierr);
1101c4762a1bSJed Brown   ierr = VecToPetscReal_Private(bnd->coordinates, &rcoords);CHKERRQ(ierr);
1102c4762a1bSJed Brown   npoints = ncoords / dim;
1103c4762a1bSJed Brown   ierr = PetscMalloc1(npoints, &points);CHKERRQ(ierr);
1104c4762a1bSJed Brown   ierr = DMPlexFindVertices(ipdm, npoints, rcoords, 0.0, points);CHKERRQ(ierr);
1105c4762a1bSJed Brown   ierr = ISCreateGeneral(PETSC_COMM_SELF, npoints, points, PETSC_OWN_POINTER, &faces_expanded_is);CHKERRQ(ierr);
1106c4762a1bSJed Brown   ierr = DMPlexExpandedVerticesToFaces_Private(ipdm, faces_expanded_is, bnd->depth, bnd->sections, face_is);CHKERRQ(ierr);
1107c4762a1bSJed Brown   ierr = PetscFree(rcoords);CHKERRQ(ierr);
1108c4762a1bSJed Brown   ierr = ISDestroy(&faces_expanded_is);CHKERRQ(ierr);
1109c4762a1bSJed Brown   PetscFunctionReturn(0);
1110c4762a1bSJed Brown }
1111c4762a1bSJed Brown 
1112c4762a1bSJed Brown /* hack disabling DMPlexOrientInterface() call in DMPlexInterpolate() via -dm_plex_interpolate_orient_interfaces option */
1113c4762a1bSJed Brown static PetscErrorCode DMPlexSetOrientInterface_Private(DM dm, PetscBool enable)
1114c4762a1bSJed Brown {
1115c4762a1bSJed Brown   PetscOptions      options = NULL;
1116c4762a1bSJed Brown   const char        *prefix = NULL;
1117c4762a1bSJed Brown   const char        opt[] = "-dm_plex_interpolate_orient_interfaces";
1118c4762a1bSJed Brown   char              prefix_opt[512];
1119c4762a1bSJed Brown   PetscBool         flg, set;
1120c4762a1bSJed Brown   static PetscBool  wasSetTrue = PETSC_FALSE;
1121c4762a1bSJed Brown   PetscErrorCode    ierr;
1122c4762a1bSJed Brown 
1123c4762a1bSJed Brown   PetscFunctionBegin;
1124c4762a1bSJed Brown   if (dm) {
1125c4762a1bSJed Brown     ierr = PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix);CHKERRQ(ierr);
1126c4762a1bSJed Brown     options = ((PetscObject)dm)->options;
1127c4762a1bSJed Brown   }
1128c4762a1bSJed Brown   ierr = PetscStrcpy(prefix_opt, "-");CHKERRQ(ierr);
1129c4762a1bSJed Brown   ierr = PetscStrlcat(prefix_opt, prefix, sizeof(prefix_opt));CHKERRQ(ierr);
1130c4762a1bSJed Brown   ierr = PetscStrlcat(prefix_opt, &opt[1], sizeof(prefix_opt));CHKERRQ(ierr);
1131c4762a1bSJed Brown   ierr = PetscOptionsGetBool(options, prefix, opt, &flg, &set);CHKERRQ(ierr);
1132c4762a1bSJed Brown   if (!enable) {
1133c4762a1bSJed Brown     if (set && flg) wasSetTrue = PETSC_TRUE;
1134c4762a1bSJed Brown     ierr = PetscOptionsSetValue(options, prefix_opt, "0");CHKERRQ(ierr);
1135c4762a1bSJed Brown   } else if (set && !flg) {
1136c4762a1bSJed Brown     if (wasSetTrue) {
1137c4762a1bSJed Brown       ierr = PetscOptionsSetValue(options, prefix_opt, "1");CHKERRQ(ierr);
1138c4762a1bSJed Brown     } else {
1139c4762a1bSJed Brown       /* default is PETSC_TRUE */
1140c4762a1bSJed Brown       ierr = PetscOptionsClearValue(options, prefix_opt);CHKERRQ(ierr);
1141c4762a1bSJed Brown     }
1142c4762a1bSJed Brown     wasSetTrue = PETSC_FALSE;
1143c4762a1bSJed Brown   }
114476bd3646SJed Brown   if (PetscDefined(USE_DEBUG)) {
1145c4762a1bSJed Brown     ierr = PetscOptionsGetBool(options, prefix, opt, &flg, &set);CHKERRQ(ierr);
1146c4762a1bSJed Brown     if (PetscUnlikely(set && flg != enable)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "PetscOptionsSetValue did not have the desired effect");
1147c4762a1bSJed Brown   }
1148c4762a1bSJed Brown   PetscFunctionReturn(0);
1149c4762a1bSJed Brown }
1150c4762a1bSJed Brown 
1151c4762a1bSJed Brown /* get coordinate description of the whole-domain boundary */
1152c4762a1bSJed Brown static PetscErrorCode DMPlexGetExpandedBoundary_Private(DM dm, PortableBoundary *boundary)
1153c4762a1bSJed Brown {
1154c4762a1bSJed Brown   PortableBoundary       bnd0, bnd;
1155c4762a1bSJed Brown   MPI_Comm               comm;
1156c4762a1bSJed Brown   DM                     idm;
1157c4762a1bSJed Brown   DMLabel                label;
1158c4762a1bSJed Brown   PetscInt               d;
1159c4762a1bSJed Brown   const char             boundaryName[] = "DMPlexDistributeInterpolateMarkInterface_boundary";
1160c4762a1bSJed Brown   IS                     boundary_is;
1161c4762a1bSJed Brown   IS                     *boundary_expanded_iss;
1162c4762a1bSJed Brown   PetscMPIInt            rootrank = 0;
1163c4762a1bSJed Brown   PetscMPIInt            rank, size;
1164c4762a1bSJed Brown   PetscInt               value = 1;
1165c4762a1bSJed Brown   DMPlexInterpolatedFlag intp;
1166c4762a1bSJed Brown   PetscBool              flg;
1167c4762a1bSJed Brown   PetscErrorCode         ierr;
1168c4762a1bSJed Brown 
1169c4762a1bSJed Brown   PetscFunctionBegin;
1170c4762a1bSJed Brown   ierr = PetscNew(&bnd);CHKERRQ(ierr);
1171c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
1172c4762a1bSJed Brown   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1173c4762a1bSJed Brown   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1174c4762a1bSJed Brown   ierr = DMPlexIsDistributed(dm, &flg);CHKERRQ(ierr);
1175c4762a1bSJed Brown   if (flg) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "serial DM (all points on one rank) needed");
1176c4762a1bSJed Brown 
1177c4762a1bSJed Brown   /* interpolate serial DM if not yet interpolated */
1178c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(dm, &intp);CHKERRQ(ierr);
1179c4762a1bSJed Brown   if (intp == DMPLEX_INTERPOLATED_FULL) {
1180c4762a1bSJed Brown     idm = dm;
1181c4762a1bSJed Brown     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
1182c4762a1bSJed Brown   } else {
1183c4762a1bSJed Brown     ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
1184c4762a1bSJed Brown     ierr = DMViewFromOptions(idm, NULL, "-idm_view");CHKERRQ(ierr);
1185c4762a1bSJed Brown   }
1186c4762a1bSJed Brown 
1187c4762a1bSJed Brown   /* mark whole-domain boundary of the serial DM */
1188c4762a1bSJed Brown   ierr = DMLabelCreate(PETSC_COMM_SELF, boundaryName, &label);CHKERRQ(ierr);
1189c4762a1bSJed Brown   ierr = DMAddLabel(idm, label);CHKERRQ(ierr);
1190c4762a1bSJed Brown   ierr = DMPlexMarkBoundaryFaces(idm, value, label);CHKERRQ(ierr);
1191c4762a1bSJed Brown   ierr = DMLabelViewFromOptionsOnComm_Private(label, "-idm_boundary_view", comm);CHKERRQ(ierr);
1192c4762a1bSJed Brown   ierr = DMLabelGetStratumIS(label, value, &boundary_is);CHKERRQ(ierr);
1193c4762a1bSJed Brown 
1194c4762a1bSJed Brown   /* translate to coordinates */
1195c4762a1bSJed Brown   ierr = PetscNew(&bnd0);CHKERRQ(ierr);
1196c4762a1bSJed Brown   ierr = DMGetCoordinatesLocalSetUp(idm);CHKERRQ(ierr);
1197c4762a1bSJed Brown   if (rank == rootrank) {
1198c4762a1bSJed Brown     ierr = DMPlexGetConeRecursive(idm, boundary_is, &bnd0->depth, &boundary_expanded_iss, &bnd0->sections);CHKERRQ(ierr);
1199c4762a1bSJed Brown     ierr = DMGetCoordinatesLocalTuple(dm, boundary_expanded_iss[0], NULL, &bnd0->coordinates);CHKERRQ(ierr);
1200c4762a1bSJed Brown     /* self-check */
1201c4762a1bSJed Brown     {
1202c4762a1bSJed Brown       IS is0;
1203c4762a1bSJed Brown       ierr = DMPlexExpandedVerticesCoordinatesToFaces_Private(idm, bnd0, &is0);CHKERRQ(ierr);
1204c4762a1bSJed Brown       ierr = ISEqual(is0, boundary_is, &flg);CHKERRQ(ierr);
1205c4762a1bSJed Brown       if (!flg) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "DMPlexExpandedVerticesCoordinatesToFaces_Private produced a wrong IS");
1206c4762a1bSJed Brown       ierr = ISDestroy(&is0);CHKERRQ(ierr);
1207c4762a1bSJed Brown     }
1208c4762a1bSJed Brown   } else {
1209c4762a1bSJed Brown     ierr = VecCreateSeq(PETSC_COMM_SELF, 0, &bnd0->coordinates);CHKERRQ(ierr);
1210c4762a1bSJed Brown   }
1211c4762a1bSJed Brown 
1212c4762a1bSJed Brown   {
1213c4762a1bSJed Brown     Vec         tmp;
1214c4762a1bSJed Brown     VecScatter  sc;
1215c4762a1bSJed Brown     IS          xis;
1216c4762a1bSJed Brown     PetscInt    n;
1217c4762a1bSJed Brown 
1218c4762a1bSJed Brown     /* just convert seq vectors to mpi vector */
1219c4762a1bSJed Brown     ierr = VecGetLocalSize(bnd0->coordinates, &n);CHKERRQ(ierr);
1220c4762a1bSJed Brown     ierr = MPI_Bcast(&n, 1, MPIU_INT, rootrank, comm);CHKERRQ(ierr);
1221c4762a1bSJed Brown     if (rank == rootrank) {
1222c4762a1bSJed Brown       ierr = VecCreateMPI(comm, n, n, &tmp);CHKERRQ(ierr);
1223c4762a1bSJed Brown     } else {
1224c4762a1bSJed Brown       ierr = VecCreateMPI(comm, 0, n, &tmp);CHKERRQ(ierr);
1225c4762a1bSJed Brown     }
1226c4762a1bSJed Brown     ierr = VecCopy(bnd0->coordinates, tmp);CHKERRQ(ierr);
1227c4762a1bSJed Brown     ierr = VecDestroy(&bnd0->coordinates);CHKERRQ(ierr);
1228c4762a1bSJed Brown     bnd0->coordinates = tmp;
1229c4762a1bSJed Brown 
1230c4762a1bSJed Brown     /* replicate coordinates from root rank to all ranks */
1231c4762a1bSJed Brown     ierr = VecCreateMPI(comm, n, n*size, &bnd->coordinates);CHKERRQ(ierr);
1232c4762a1bSJed Brown     ierr = ISCreateStride(comm, n, 0, 1, &xis);CHKERRQ(ierr);
1233c4762a1bSJed Brown     ierr = VecScatterCreate(bnd0->coordinates, xis, bnd->coordinates, NULL, &sc);CHKERRQ(ierr);
1234c4762a1bSJed Brown     ierr = VecScatterBegin(sc, bnd0->coordinates, bnd->coordinates, INSERT_VALUES, SCATTER_FORWARD);CHKERRQ(ierr);
1235c4762a1bSJed Brown     ierr = VecScatterEnd(  sc, bnd0->coordinates, bnd->coordinates, INSERT_VALUES, SCATTER_FORWARD);CHKERRQ(ierr);
1236c4762a1bSJed Brown     ierr = VecScatterDestroy(&sc);CHKERRQ(ierr);
1237c4762a1bSJed Brown     ierr = ISDestroy(&xis);CHKERRQ(ierr);
1238c4762a1bSJed Brown   }
1239c4762a1bSJed Brown   bnd->depth = bnd0->depth;
1240c4762a1bSJed Brown   ierr = MPI_Bcast(&bnd->depth, 1, MPIU_INT, rootrank, comm);CHKERRQ(ierr);
1241c4762a1bSJed Brown   ierr = PetscMalloc1(bnd->depth, &bnd->sections);CHKERRQ(ierr);
1242c4762a1bSJed Brown   for (d=0; d<bnd->depth; d++) {
1243c4762a1bSJed Brown     ierr = PetscSectionReplicate_Private(comm, rootrank, (rank == rootrank) ? bnd0->sections[d] : NULL, &bnd->sections[d]);CHKERRQ(ierr);
1244c4762a1bSJed Brown   }
1245c4762a1bSJed Brown 
1246c4762a1bSJed Brown   if (rank == rootrank) {
1247c4762a1bSJed Brown     ierr = DMPlexRestoreConeRecursive(idm, boundary_is, &bnd0->depth, &boundary_expanded_iss, &bnd0->sections);CHKERRQ(ierr);
1248c4762a1bSJed Brown   }
1249c4762a1bSJed Brown   ierr = PortableBoundaryDestroy(&bnd0);CHKERRQ(ierr);
1250c4762a1bSJed Brown   ierr = DMRemoveLabelBySelf(idm, &label, PETSC_TRUE);CHKERRQ(ierr);
1251c4762a1bSJed Brown   ierr = DMLabelDestroy(&label);CHKERRQ(ierr);
1252c4762a1bSJed Brown   ierr = ISDestroy(&boundary_is);CHKERRQ(ierr);
1253c4762a1bSJed Brown   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1254c4762a1bSJed Brown   *boundary = bnd;
1255c4762a1bSJed Brown   PetscFunctionReturn(0);
1256c4762a1bSJed Brown }
1257c4762a1bSJed Brown 
1258c4762a1bSJed Brown /* get faces of inter-partition interface */
1259c4762a1bSJed Brown static PetscErrorCode DMPlexGetInterfaceFaces_Private(DM ipdm, IS boundary_faces_is, IS *interface_faces_is)
1260c4762a1bSJed Brown {
1261c4762a1bSJed Brown   MPI_Comm               comm;
1262c4762a1bSJed Brown   DMLabel                label;
1263c4762a1bSJed Brown   IS                     part_boundary_faces_is;
1264c4762a1bSJed Brown   const char             partBoundaryName[] = "DMPlexDistributeInterpolateMarkInterface_partBoundary";
1265c4762a1bSJed Brown   PetscInt               value = 1;
1266c4762a1bSJed Brown   DMPlexInterpolatedFlag intp;
1267c4762a1bSJed Brown   PetscErrorCode         ierr;
1268c4762a1bSJed Brown 
1269c4762a1bSJed Brown   PetscFunctionBegin;
1270c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)ipdm, &comm);CHKERRQ(ierr);
1271c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(ipdm, &intp);CHKERRQ(ierr);
1272c4762a1bSJed Brown   if (intp != DMPLEX_INTERPOLATED_FULL) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "only for fully interpolated DMPlex");
1273c4762a1bSJed Brown 
1274c4762a1bSJed Brown   /* get ipdm partition boundary (partBoundary) */
1275c4762a1bSJed Brown   ierr = DMLabelCreate(PETSC_COMM_SELF, partBoundaryName, &label);CHKERRQ(ierr);
1276c4762a1bSJed Brown   ierr = DMAddLabel(ipdm, label);CHKERRQ(ierr);
1277c4762a1bSJed Brown   ierr = DMPlexMarkBoundaryFaces(ipdm, value, label);CHKERRQ(ierr);
1278c4762a1bSJed Brown   ierr = DMLabelViewFromOptionsOnComm_Private(label, "-ipdm_part_boundary_view", comm);CHKERRQ(ierr);
1279c4762a1bSJed Brown   ierr = DMLabelGetStratumISOnComm_Private(label, value, comm, &part_boundary_faces_is);CHKERRQ(ierr);
1280c4762a1bSJed Brown   ierr = DMRemoveLabelBySelf(ipdm, &label, PETSC_TRUE);CHKERRQ(ierr);
1281c4762a1bSJed Brown   ierr = DMLabelDestroy(&label);CHKERRQ(ierr);
1282c4762a1bSJed Brown 
1283c4762a1bSJed Brown   /* remove ipdm whole-domain boundary (boundary_faces_is) from ipdm partition boundary (part_boundary_faces_is), resulting just in inter-partition interface */
1284c4762a1bSJed Brown   ierr = ISDifference(part_boundary_faces_is,boundary_faces_is,interface_faces_is);CHKERRQ(ierr);
1285c4762a1bSJed Brown   ierr = ISDestroy(&part_boundary_faces_is);CHKERRQ(ierr);
1286c4762a1bSJed Brown   PetscFunctionReturn(0);
1287c4762a1bSJed Brown }
1288c4762a1bSJed Brown 
1289c4762a1bSJed Brown /* compute inter-partition interface including edges and vertices */
1290c4762a1bSJed Brown static PetscErrorCode DMPlexComputeCompleteInterface_Private(DM ipdm, IS interface_faces_is, IS *interface_is)
1291c4762a1bSJed Brown {
1292c4762a1bSJed Brown   DMLabel                label;
1293c4762a1bSJed Brown   PetscInt               value = 1;
1294c4762a1bSJed Brown   const char             interfaceName[] = "DMPlexDistributeInterpolateMarkInterface_interface";
1295c4762a1bSJed Brown   DMPlexInterpolatedFlag intp;
1296c4762a1bSJed Brown   MPI_Comm               comm;
1297c4762a1bSJed Brown   PetscErrorCode         ierr;
1298c4762a1bSJed Brown 
1299c4762a1bSJed Brown   PetscFunctionBegin;
1300c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)ipdm, &comm);CHKERRQ(ierr);
1301c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(ipdm, &intp);CHKERRQ(ierr);
1302c4762a1bSJed Brown   if (intp != DMPLEX_INTERPOLATED_FULL) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "only for fully interpolated DMPlex");
1303c4762a1bSJed Brown 
1304c4762a1bSJed Brown   ierr = DMLabelCreate(PETSC_COMM_SELF, interfaceName, &label);CHKERRQ(ierr);
1305c4762a1bSJed Brown   ierr = DMAddLabel(ipdm, label);CHKERRQ(ierr);
1306c4762a1bSJed Brown   ierr = DMLabelSetStratumIS(label, value, interface_faces_is);CHKERRQ(ierr);
1307c4762a1bSJed Brown   ierr = DMLabelViewFromOptionsOnComm_Private(label, "-interface_faces_view", comm);CHKERRQ(ierr);
1308c4762a1bSJed Brown   ierr = DMPlexLabelComplete(ipdm, label);CHKERRQ(ierr);
1309c4762a1bSJed Brown   ierr = DMLabelViewFromOptionsOnComm_Private(label, "-interface_view", comm);CHKERRQ(ierr);
1310c4762a1bSJed Brown   ierr = DMLabelGetStratumISOnComm_Private(label, value, comm, interface_is);CHKERRQ(ierr);
1311c4762a1bSJed Brown   ierr = PetscObjectSetName((PetscObject)*interface_is, "interface_is");CHKERRQ(ierr);
1312c4762a1bSJed Brown   ierr = ISViewFromOptions(*interface_is, NULL, "-interface_is_view");CHKERRQ(ierr);
1313c4762a1bSJed Brown   ierr = DMRemoveLabelBySelf(ipdm, &label, PETSC_TRUE);CHKERRQ(ierr);
1314c4762a1bSJed Brown   ierr = DMLabelDestroy(&label);CHKERRQ(ierr);
1315c4762a1bSJed Brown   PetscFunctionReturn(0);
1316c4762a1bSJed Brown }
1317c4762a1bSJed Brown 
1318c4762a1bSJed Brown static PetscErrorCode PointSFGetOutwardInterfacePoints(PetscSF sf, IS *is)
1319c4762a1bSJed Brown {
1320c4762a1bSJed Brown   PetscInt        n;
1321c4762a1bSJed Brown   const PetscInt  *arr;
1322c4762a1bSJed Brown   PetscErrorCode  ierr;
1323c4762a1bSJed Brown 
1324c4762a1bSJed Brown   PetscFunctionBegin;
1325c4762a1bSJed Brown   ierr = PetscSFGetGraph(sf, NULL, &n, &arr, NULL);CHKERRQ(ierr);
1326c4762a1bSJed Brown   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)sf), n, arr, PETSC_USE_POINTER, is);CHKERRQ(ierr);
1327c4762a1bSJed Brown   PetscFunctionReturn(0);
1328c4762a1bSJed Brown }
1329c4762a1bSJed Brown 
1330c4762a1bSJed Brown static PetscErrorCode PointSFGetInwardInterfacePoints(PetscSF sf, IS *is)
1331c4762a1bSJed Brown {
1332c4762a1bSJed Brown   PetscInt        n;
1333c4762a1bSJed Brown   const PetscInt  *rootdegree;
1334c4762a1bSJed Brown   PetscInt        *arr;
1335c4762a1bSJed Brown   PetscErrorCode  ierr;
1336c4762a1bSJed Brown 
1337c4762a1bSJed Brown   PetscFunctionBegin;
1338c4762a1bSJed Brown   ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
1339c4762a1bSJed Brown   ierr = PetscSFComputeDegreeBegin(sf, &rootdegree);CHKERRQ(ierr);
1340c4762a1bSJed Brown   ierr = PetscSFComputeDegreeEnd(sf, &rootdegree);CHKERRQ(ierr);
1341c4762a1bSJed Brown   ierr = PetscSFComputeMultiRootOriginalNumbering(sf, rootdegree, &n, &arr);CHKERRQ(ierr);
1342c4762a1bSJed Brown   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)sf), n, arr, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
1343c4762a1bSJed Brown   PetscFunctionReturn(0);
1344c4762a1bSJed Brown }
1345c4762a1bSJed Brown 
1346c4762a1bSJed Brown static PetscErrorCode PointSFGetInterfacePoints_Private(PetscSF pointSF, IS *is)
1347c4762a1bSJed Brown {
1348c4762a1bSJed Brown   IS pointSF_out_is, pointSF_in_is;
1349c4762a1bSJed Brown   PetscErrorCode  ierr;
1350c4762a1bSJed Brown 
1351c4762a1bSJed Brown   PetscFunctionBegin;
1352c4762a1bSJed Brown   ierr = PointSFGetOutwardInterfacePoints(pointSF, &pointSF_out_is);CHKERRQ(ierr);
1353c4762a1bSJed Brown   ierr = PointSFGetInwardInterfacePoints(pointSF, &pointSF_in_is);CHKERRQ(ierr);
1354c4762a1bSJed Brown   ierr = ISExpand(pointSF_out_is, pointSF_in_is, is);CHKERRQ(ierr);
1355c4762a1bSJed Brown   ierr = ISDestroy(&pointSF_out_is);CHKERRQ(ierr);
1356c4762a1bSJed Brown   ierr = ISDestroy(&pointSF_in_is);CHKERRQ(ierr);
1357c4762a1bSJed Brown   PetscFunctionReturn(0);
1358c4762a1bSJed Brown }
1359c4762a1bSJed Brown 
1360c4762a1bSJed Brown #define MYCHKERRQ(ierr) do {if (PetscUnlikely(ierr)) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "PointSF is wrong. Unable to show details!");} while (0)
1361c4762a1bSJed Brown 
1362c4762a1bSJed Brown static PetscErrorCode ViewPointsWithType_Internal(DM dm, IS pointsIS, PetscViewer v)
1363c4762a1bSJed Brown {
1364c4762a1bSJed Brown   DMLabel         label;
1365c4762a1bSJed Brown   PetscSection    coordsSection;
1366c4762a1bSJed Brown   Vec             coordsVec;
1367c4762a1bSJed Brown   PetscScalar     *coordsScalar;
1368c4762a1bSJed Brown   PetscInt        coneSize, depth, dim, i, p, npoints;
1369c4762a1bSJed Brown   const PetscInt  *points;
1370c4762a1bSJed Brown   PetscErrorCode  ierr;
1371c4762a1bSJed Brown 
1372c4762a1bSJed Brown   PetscFunctionBegin;
1373c4762a1bSJed Brown   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1374c4762a1bSJed Brown   ierr = DMGetCoordinateSection(dm, &coordsSection);CHKERRQ(ierr);
1375c4762a1bSJed Brown   ierr = DMGetCoordinatesLocal(dm, &coordsVec);CHKERRQ(ierr);
1376c4762a1bSJed Brown   ierr = VecGetArray(coordsVec, &coordsScalar);CHKERRQ(ierr);
1377c4762a1bSJed Brown   ierr = ISGetLocalSize(pointsIS, &npoints);CHKERRQ(ierr);
1378c4762a1bSJed Brown   ierr = ISGetIndices(pointsIS, &points);CHKERRQ(ierr);
1379c4762a1bSJed Brown   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
1380c4762a1bSJed Brown   ierr = PetscViewerASCIIPushTab(v);CHKERRQ(ierr);
1381c4762a1bSJed Brown   for (i=0; i<npoints; i++) {
1382c4762a1bSJed Brown     p = points[i];
1383c4762a1bSJed Brown     ierr = DMLabelGetValue(label, p, &depth);CHKERRQ(ierr);
1384c4762a1bSJed Brown     if (!depth) {
1385c4762a1bSJed Brown       PetscInt        c, n, o;
1386c4762a1bSJed Brown       PetscReal       coords[3];
1387c4762a1bSJed Brown       char            coordstr[128];
1388c4762a1bSJed Brown 
1389c4762a1bSJed Brown       ierr = PetscSectionGetDof(coordsSection, p, &n);CHKERRQ(ierr);
1390c4762a1bSJed Brown       ierr = PetscSectionGetOffset(coordsSection, p, &o);CHKERRQ(ierr);
1391c4762a1bSJed Brown       for (c=0; c<n; c++) coords[c] = PetscRealPart(coordsScalar[o+c]);
1392c4762a1bSJed Brown       ierr = coord2str(coordstr, sizeof(coordstr), n, coords, 1.0);CHKERRQ(ierr);
1393c4762a1bSJed Brown       ierr = PetscViewerASCIISynchronizedPrintf(v, "vertex %D w/ coordinates %s\n", p, coordstr);CHKERRQ(ierr);
1394c4762a1bSJed Brown     } else {
1395c4762a1bSJed Brown       char            entityType[16];
1396c4762a1bSJed Brown 
1397c4762a1bSJed Brown       switch (depth) {
1398c4762a1bSJed Brown         case 1: ierr = PetscStrcpy(entityType, "edge");CHKERRQ(ierr); break;
1399c4762a1bSJed Brown         case 2: ierr = PetscStrcpy(entityType, "face");CHKERRQ(ierr); break;
1400c4762a1bSJed Brown         case 3: ierr = PetscStrcpy(entityType, "cell");CHKERRQ(ierr); break;
1401c4762a1bSJed Brown         default: SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_SUP, "Only for depth <= 3");CHKERRQ(ierr);
1402c4762a1bSJed Brown       }
1403c4762a1bSJed Brown       if (depth == dim && dim < 3) {
1404c4762a1bSJed Brown         ierr = PetscStrlcat(entityType, " (cell)", sizeof(entityType));CHKERRQ(ierr);
1405c4762a1bSJed Brown       }
1406c4762a1bSJed Brown       ierr = PetscViewerASCIISynchronizedPrintf(v, "%s %D\n", entityType, p);CHKERRQ(ierr);
1407c4762a1bSJed Brown     }
1408c4762a1bSJed Brown     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1409c4762a1bSJed Brown     if (coneSize) {
1410c4762a1bSJed Brown       const PetscInt *cone;
1411c4762a1bSJed Brown       IS             coneIS;
1412c4762a1bSJed Brown 
1413c4762a1bSJed Brown       ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1414c4762a1bSJed Brown       ierr = ISCreateGeneral(PETSC_COMM_SELF, coneSize, cone, PETSC_USE_POINTER, &coneIS);CHKERRQ(ierr);
1415c4762a1bSJed Brown       ierr = ViewPointsWithType_Internal(dm, coneIS, v);CHKERRQ(ierr);
1416c4762a1bSJed Brown       ierr = ISDestroy(&coneIS);CHKERRQ(ierr);
1417c4762a1bSJed Brown     }
1418c4762a1bSJed Brown   }
1419c4762a1bSJed Brown   ierr = PetscViewerASCIIPopTab(v);CHKERRQ(ierr);
1420c4762a1bSJed Brown   ierr = VecRestoreArray(coordsVec, &coordsScalar);CHKERRQ(ierr);
1421c4762a1bSJed Brown   ierr = ISRestoreIndices(pointsIS, &points);CHKERRQ(ierr);
1422c4762a1bSJed Brown   PetscFunctionReturn(0);
1423c4762a1bSJed Brown }
1424c4762a1bSJed Brown 
1425c4762a1bSJed Brown static PetscErrorCode ViewPointsWithType(DM dm, IS points, PetscViewer v)
1426c4762a1bSJed Brown {
1427c4762a1bSJed Brown   PetscBool       flg;
1428c4762a1bSJed Brown   PetscInt        npoints;
1429c4762a1bSJed Brown   PetscMPIInt     rank;
1430c4762a1bSJed Brown   PetscErrorCode  ierr;
1431c4762a1bSJed Brown 
1432c4762a1bSJed Brown   PetscFunctionBegin;
1433c4762a1bSJed Brown   ierr = PetscObjectTypeCompare((PetscObject)v, PETSCVIEWERASCII, &flg);CHKERRQ(ierr);
1434c4762a1bSJed Brown   if (!flg) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_SUP, "Only for ASCII viewer");CHKERRQ(ierr);
1435c4762a1bSJed Brown   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)v), &rank);CHKERRQ(ierr);
1436c4762a1bSJed Brown   ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
1437c4762a1bSJed Brown   ierr = ISGetLocalSize(points, &npoints);CHKERRQ(ierr);
1438c4762a1bSJed Brown   if (npoints) {
1439c4762a1bSJed Brown     ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d] --------\n", rank);CHKERRQ(ierr);
1440c4762a1bSJed Brown     ierr = ViewPointsWithType_Internal(dm, points, v);CHKERRQ(ierr);
1441c4762a1bSJed Brown   }
1442c4762a1bSJed Brown   ierr = PetscViewerFlush(v);CHKERRQ(ierr);
1443c4762a1bSJed Brown   ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
1444c4762a1bSJed Brown   PetscFunctionReturn(0);
1445c4762a1bSJed Brown }
1446c4762a1bSJed Brown 
1447c4762a1bSJed Brown static PetscErrorCode DMPlexComparePointSFWithInterface_Private(DM ipdm, IS interface_is)
1448c4762a1bSJed Brown {
1449c4762a1bSJed Brown   PetscSF         pointsf;
1450c4762a1bSJed Brown   IS              pointsf_is;
1451c4762a1bSJed Brown   PetscBool       flg;
1452c4762a1bSJed Brown   MPI_Comm        comm;
1453c4762a1bSJed Brown   PetscMPIInt     size;
1454c4762a1bSJed Brown   PetscErrorCode  ierr;
1455c4762a1bSJed Brown 
1456c4762a1bSJed Brown   PetscFunctionBegin;
1457c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)ipdm, &comm);CHKERRQ(ierr);
1458c4762a1bSJed Brown   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1459c4762a1bSJed Brown   ierr = DMGetPointSF(ipdm, &pointsf);CHKERRQ(ierr);
1460c4762a1bSJed Brown   if (pointsf) {
1461c4762a1bSJed Brown     PetscInt nroots;
1462c4762a1bSJed Brown     ierr = PetscSFGetGraph(pointsf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
1463c4762a1bSJed Brown     if (nroots < 0) pointsf = NULL; /* uninitialized SF */
1464c4762a1bSJed Brown   }
1465c4762a1bSJed Brown   if (!pointsf) {
1466c4762a1bSJed Brown     PetscInt N=0;
1467c4762a1bSJed Brown     if (interface_is) {ierr = ISGetSize(interface_is, &N);CHKERRQ(ierr);}
1468c4762a1bSJed Brown     if (N) SETERRQ(comm, PETSC_ERR_PLIB, "interface_is should be NULL or empty for PointSF being NULL");
1469c4762a1bSJed Brown     PetscFunctionReturn(0);
1470c4762a1bSJed Brown   }
1471c4762a1bSJed Brown 
1472c4762a1bSJed Brown   /* get PointSF points as IS pointsf_is */
1473c4762a1bSJed Brown   ierr = PointSFGetInterfacePoints_Private(pointsf, &pointsf_is);CHKERRQ(ierr);
1474c4762a1bSJed Brown 
1475c4762a1bSJed Brown   /* compare pointsf_is with interface_is */
1476c4762a1bSJed Brown   ierr = ISEqual(interface_is, pointsf_is, &flg);CHKERRQ(ierr);
1477c4762a1bSJed Brown   ierr = MPI_Allreduce(MPI_IN_PLACE,&flg,1,MPIU_BOOL,MPI_LAND,comm);CHKERRQ(ierr);
1478c4762a1bSJed Brown   if (!flg) {
1479c4762a1bSJed Brown     IS pointsf_extra_is, pointsf_missing_is;
1480c4762a1bSJed Brown     PetscViewer errv = PETSC_VIEWER_STDERR_(comm);
1481c4762a1bSJed Brown     ierr = ISDifference(interface_is, pointsf_is, &pointsf_missing_is);MYCHKERRQ(ierr);
1482c4762a1bSJed Brown     ierr = ISDifference(pointsf_is, interface_is, &pointsf_extra_is);MYCHKERRQ(ierr);
1483c4762a1bSJed Brown     ierr = PetscViewerASCIIPrintf(errv, "Points missing in PointSF:\n");MYCHKERRQ(ierr);
1484c4762a1bSJed Brown     ierr = ViewPointsWithType(ipdm, pointsf_missing_is, errv);MYCHKERRQ(ierr);
1485c4762a1bSJed Brown     ierr = PetscViewerASCIIPrintf(errv, "Extra points in PointSF:\n");MYCHKERRQ(ierr);
1486c4762a1bSJed Brown     ierr = ViewPointsWithType(ipdm, pointsf_extra_is, errv);MYCHKERRQ(ierr);
1487c4762a1bSJed Brown     ierr = ISDestroy(&pointsf_extra_is);MYCHKERRQ(ierr);
1488c4762a1bSJed Brown     ierr = ISDestroy(&pointsf_missing_is);MYCHKERRQ(ierr);
1489c4762a1bSJed Brown     SETERRQ(comm, PETSC_ERR_PLIB, "PointSF is wrong! See details above.");
1490c4762a1bSJed Brown   }
1491c4762a1bSJed Brown   ierr = ISDestroy(&pointsf_is);CHKERRQ(ierr);
1492c4762a1bSJed Brown   PetscFunctionReturn(0);
1493c4762a1bSJed Brown }
1494c4762a1bSJed Brown 
1495c4762a1bSJed Brown /* remove faces & edges from label, leave just vertices */
1496c4762a1bSJed Brown static PetscErrorCode DMPlexISFilterVertices_Private(DM dm, IS points)
1497c4762a1bSJed Brown {
1498c4762a1bSJed Brown   PetscInt        vStart, vEnd;
1499c4762a1bSJed Brown   MPI_Comm        comm;
1500c4762a1bSJed Brown   PetscErrorCode  ierr;
1501c4762a1bSJed Brown 
1502c4762a1bSJed Brown   PetscFunctionBegin;
1503c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
1504c4762a1bSJed Brown   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1505c4762a1bSJed Brown   ierr = ISGeneralFilter(points, vStart, vEnd);CHKERRQ(ierr);
1506c4762a1bSJed Brown   PetscFunctionReturn(0);
1507c4762a1bSJed Brown }
1508c4762a1bSJed Brown 
1509c4762a1bSJed Brown /*
1510c4762a1bSJed Brown   DMPlexCheckPointSFHeavy - Thorough test that the PointSF after parallel DMPlexInterpolate() includes exactly all interface points.
1511c4762a1bSJed Brown 
1512c4762a1bSJed Brown   Collective
1513c4762a1bSJed Brown 
1514c4762a1bSJed Brown   Input Parameters:
1515c4762a1bSJed Brown . dm - The DMPlex object
1516c4762a1bSJed Brown 
1517c4762a1bSJed Brown   Notes:
1518c4762a1bSJed Brown   The input DMPlex must be serial (one partition has all points, the other partitions have no points).
1519c4762a1bSJed Brown   This is a heavy test which involves DMPlexInterpolate() if the input DM is not interpolated yet, and depends on having a representation of the whole-domain boundary (PortableBoundary), which can be obtained only by DMPlexGetExpandedBoundary_Private() (which involves DMPlexInterpolate() of a sequential DM).
1520c4762a1bSJed Brown   This is mainly intended for debugging/testing purposes.
1521c4762a1bSJed Brown 
1522c4762a1bSJed Brown   Algorithm:
1523c4762a1bSJed Brown   1. boundary faces of the serial version of the whole mesh are found using DMPlexMarkBoundaryFaces()
1524c4762a1bSJed Brown   2. boundary faces are translated into vertices using DMPlexGetConeRecursive() and these are translated into coordinates - this description (aka PortableBoundary) is completely independent of partitioning and point numbering
1525c4762a1bSJed Brown   3. the mesh is distributed or loaded in parallel
1526c4762a1bSJed Brown   4. boundary faces of the distributed mesh are reconstructed from PortableBoundary using DMPlexFindVertices()
1527c4762a1bSJed Brown   5. partition boundary faces of the parallel mesh are found using DMPlexMarkBoundaryFaces()
1528c4762a1bSJed Brown   6. partition interfaces are computed as set difference of partition boundary faces minus the reconstructed boundary
1529c4762a1bSJed Brown   7. check that interface covered by PointSF (union of inward and outward points) is equal to the partition interface for each rank, otherwise print the difference and throw an error
1530c4762a1bSJed Brown 
1531c4762a1bSJed Brown   Level: developer
1532c4762a1bSJed Brown 
1533c4762a1bSJed Brown .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
1534c4762a1bSJed Brown */
1535c4762a1bSJed Brown static PetscErrorCode DMPlexCheckPointSFHeavy(DM dm, PortableBoundary bnd)
1536c4762a1bSJed Brown {
1537c4762a1bSJed Brown   DM                     ipdm=NULL;
1538c4762a1bSJed Brown   IS                     boundary_faces_is, interface_faces_is, interface_is;
1539c4762a1bSJed Brown   DMPlexInterpolatedFlag intp;
1540c4762a1bSJed Brown   MPI_Comm               comm;
1541c4762a1bSJed Brown   PetscErrorCode         ierr;
1542c4762a1bSJed Brown 
1543c4762a1bSJed Brown   PetscFunctionBegin;
1544c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
1545c4762a1bSJed Brown 
1546c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(dm, &intp);CHKERRQ(ierr);
1547c4762a1bSJed Brown   if (intp == DMPLEX_INTERPOLATED_FULL) {
1548c4762a1bSJed Brown     ipdm = dm;
1549c4762a1bSJed Brown   } else {
1550c4762a1bSJed Brown     /* create temporary interpolated DM if input DM is not interpolated */
1551c4762a1bSJed Brown     ierr = DMPlexSetOrientInterface_Private(dm, PETSC_FALSE);CHKERRQ(ierr);
1552c4762a1bSJed Brown     ierr = DMPlexInterpolate(dm, &ipdm);CHKERRQ(ierr); /* with DMPlexOrientInterface_Internal() call skipped so that PointSF issues are left to DMPlexComparePointSFWithInterface_Private() below */
1553c4762a1bSJed Brown     ierr = DMPlexSetOrientInterface_Private(dm, PETSC_TRUE);CHKERRQ(ierr);
1554c4762a1bSJed Brown   }
1555c4762a1bSJed Brown   ierr = DMViewFromOptions(ipdm, NULL, "-ipdm_view");CHKERRQ(ierr);
1556c4762a1bSJed Brown 
1557c4762a1bSJed Brown   /* recover ipdm whole-domain boundary faces from the expanded vertices coordinates */
1558c4762a1bSJed Brown   ierr = DMPlexExpandedVerticesCoordinatesToFaces_Private(ipdm, bnd, &boundary_faces_is);CHKERRQ(ierr);
1559c4762a1bSJed Brown   /* get inter-partition interface faces (interface_faces_is)*/
1560c4762a1bSJed Brown   ierr = DMPlexGetInterfaceFaces_Private(ipdm, boundary_faces_is, &interface_faces_is);CHKERRQ(ierr);
1561c4762a1bSJed Brown   /* compute inter-partition interface including edges and vertices (interface_is) */
1562c4762a1bSJed Brown   ierr = DMPlexComputeCompleteInterface_Private(ipdm, interface_faces_is, &interface_is);CHKERRQ(ierr);
1563c4762a1bSJed Brown   /* destroy immediate ISs */
1564c4762a1bSJed Brown   ierr = ISDestroy(&boundary_faces_is);CHKERRQ(ierr);
1565c4762a1bSJed Brown   ierr = ISDestroy(&interface_faces_is);CHKERRQ(ierr);
1566c4762a1bSJed Brown 
1567c4762a1bSJed Brown   /* for uninterpolated case, keep just vertices in interface */
1568c4762a1bSJed Brown   if (!intp) {
1569c4762a1bSJed Brown     ierr = DMPlexISFilterVertices_Private(ipdm, interface_is);CHKERRQ(ierr);
1570c4762a1bSJed Brown     ierr = DMDestroy(&ipdm);CHKERRQ(ierr);
1571c4762a1bSJed Brown   }
1572c4762a1bSJed Brown 
1573c4762a1bSJed Brown   /* compare PointSF with the boundary reconstructed from coordinates */
1574c4762a1bSJed Brown   ierr = DMPlexComparePointSFWithInterface_Private(dm, interface_is);CHKERRQ(ierr);
1575c4762a1bSJed Brown   ierr = PetscPrintf(comm, "DMPlexCheckPointSFHeavy PASSED\n");CHKERRQ(ierr);
1576c4762a1bSJed Brown   ierr = ISDestroy(&interface_is);CHKERRQ(ierr);
1577c4762a1bSJed Brown   PetscFunctionReturn(0);
1578c4762a1bSJed Brown }
1579c4762a1bSJed Brown 
1580c4762a1bSJed Brown int main(int argc, char **argv)
1581c4762a1bSJed Brown {
1582c4762a1bSJed Brown   DM             dm;
1583c4762a1bSJed Brown   AppCtx         user;
1584c4762a1bSJed Brown   PetscErrorCode ierr;
1585c4762a1bSJed Brown 
1586c4762a1bSJed Brown   ierr = PetscInitialize(&argc, &argv, NULL, help);if (ierr) return ierr;
1587c4762a1bSJed Brown   ierr = PetscLogStageRegister("create",&stage[0]);CHKERRQ(ierr);
1588c4762a1bSJed Brown   ierr = PetscLogStageRegister("distribute",&stage[1]);CHKERRQ(ierr);
1589c4762a1bSJed Brown   ierr = PetscLogStageRegister("interpolate",&stage[2]);CHKERRQ(ierr);
1590c4762a1bSJed Brown   ierr = ProcessOptions(PETSC_COMM_WORLD, &user);CHKERRQ(ierr);
1591c4762a1bSJed Brown   ierr = CreateMesh(PETSC_COMM_WORLD, &user, &dm);CHKERRQ(ierr);
1592c4762a1bSJed Brown   if (user.nPointsToExpand) {
1593c4762a1bSJed Brown     ierr = TestExpandPoints(dm, &user);CHKERRQ(ierr);
1594c4762a1bSJed Brown   }
1595c4762a1bSJed Brown   if (user.ncoords) {
1596c4762a1bSJed Brown     ierr = ViewVerticesFromCoords(dm, user.ncoords/user.dim, user.coords, user.coordsTol, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
1597c4762a1bSJed Brown   }
1598c4762a1bSJed Brown   ierr = DMDestroy(&dm);CHKERRQ(ierr);
1599c4762a1bSJed Brown   ierr = PetscFinalize();
1600c4762a1bSJed Brown   return ierr;
1601c4762a1bSJed Brown }
1602c4762a1bSJed Brown 
1603c4762a1bSJed Brown /*TEST
1604c4762a1bSJed Brown 
1605c4762a1bSJed Brown   testset:
1606c4762a1bSJed Brown     nsize: 2
1607c4762a1bSJed Brown     args: -dm_view ascii::ascii_info_detail
1608c4762a1bSJed Brown     args: -dm_plex_check_all
1609c4762a1bSJed Brown     test:
1610c4762a1bSJed Brown       suffix: 1_tri_dist0
1611c4762a1bSJed Brown       args: -distribute 0 -interpolate {{none create}separate output}
1612c4762a1bSJed Brown     test:
1613c4762a1bSJed Brown       suffix: 1_tri_dist1
1614c4762a1bSJed Brown       args: -distribute 1 -interpolate {{none create after_distribute}separate output}
1615c4762a1bSJed Brown     test:
1616c4762a1bSJed Brown       suffix: 1_quad_dist0
1617c4762a1bSJed Brown       args: -cell_simplex 0 -distribute 0 -interpolate {{none create}separate output}
1618c4762a1bSJed Brown     test:
1619c4762a1bSJed Brown       suffix: 1_quad_dist1
1620c4762a1bSJed Brown       args: -cell_simplex 0 -distribute 1 -interpolate {{none create after_distribute}separate output}
1621c4762a1bSJed Brown     test:
1622c4762a1bSJed Brown       suffix: 1_1d_dist1
1623c4762a1bSJed Brown       args: -dim 1 -distribute 1
1624c4762a1bSJed Brown 
1625c4762a1bSJed Brown   testset:
1626c4762a1bSJed Brown     nsize: 3
1627c4762a1bSJed Brown     args: -testnum 1 -interpolate create
1628c4762a1bSJed Brown     args: -dm_plex_check_all
1629c4762a1bSJed Brown     test:
1630c4762a1bSJed Brown       suffix: 2
1631c4762a1bSJed Brown       args: -dm_view ascii::ascii_info_detail
1632c4762a1bSJed Brown     test:
1633c4762a1bSJed Brown       suffix: 2a
1634c4762a1bSJed Brown       args: -dm_plex_check_cones_conform_on_interfaces_verbose
1635c4762a1bSJed Brown     test:
1636c4762a1bSJed Brown       suffix: 2b
1637c4762a1bSJed Brown       args: -test_expand_points 0,1,2,5,6
1638c4762a1bSJed Brown     test:
1639c4762a1bSJed Brown       suffix: 2c
1640c4762a1bSJed Brown       args: -test_expand_points 0,1,2,5,6 -test_expand_points_empty
1641c4762a1bSJed Brown 
1642c4762a1bSJed Brown   testset:
1643c4762a1bSJed Brown     # the same as 1% for 3D
1644c4762a1bSJed Brown     nsize: 2
1645c4762a1bSJed Brown     args: -dim 3 -dm_view ascii::ascii_info_detail
1646c4762a1bSJed Brown     args: -dm_plex_check_all
1647c4762a1bSJed Brown     test:
1648c4762a1bSJed Brown       suffix: 4_tet_dist0
1649c4762a1bSJed Brown       args: -distribute 0 -interpolate {{none create}separate output}
1650c4762a1bSJed Brown     test:
1651c4762a1bSJed Brown       suffix: 4_tet_dist1
1652c4762a1bSJed Brown       args: -distribute 1 -interpolate {{none create after_distribute}separate output}
1653c4762a1bSJed Brown     test:
1654c4762a1bSJed Brown       suffix: 4_hex_dist0
1655c4762a1bSJed Brown       args: -cell_simplex 0 -distribute 0 -interpolate {{none create}separate output}
1656c4762a1bSJed Brown     test:
1657c4762a1bSJed Brown       suffix: 4_hex_dist1
1658c4762a1bSJed Brown       args: -cell_simplex 0 -distribute 1 -interpolate {{none create after_distribute}separate output}
1659c4762a1bSJed Brown 
1660c4762a1bSJed Brown   test:
1661c4762a1bSJed Brown     # the same as 4_tet_dist0 but test different initial orientations
1662c4762a1bSJed Brown     suffix: 4_tet_test_orient
1663c4762a1bSJed Brown     nsize: 2
1664c4762a1bSJed Brown     args: -dim 3 -distribute 0
1665c4762a1bSJed Brown     args: -dm_plex_check_all
1666c4762a1bSJed Brown     args: -rotate_interface_0 {{0 1 2 11 12 13}}
1667c4762a1bSJed Brown     args: -rotate_interface_1 {{0 1 2 11 12 13}}
1668c4762a1bSJed Brown 
1669c4762a1bSJed Brown   testset:
1670c4762a1bSJed Brown     requires: exodusii
1671c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/TwoQuads.exo
1672c4762a1bSJed Brown     args: -dm_view ascii::ascii_info_detail
1673c4762a1bSJed Brown     args: -dm_plex_check_all
1674c4762a1bSJed Brown     args: -custom_view
1675c4762a1bSJed Brown     test:
1676c4762a1bSJed Brown       suffix: 5_seq
1677c4762a1bSJed Brown       nsize: 1
1678c4762a1bSJed Brown       args: -distribute 0 -interpolate {{none create}separate output}
1679c4762a1bSJed Brown     test:
1680c4762a1bSJed Brown       suffix: 5_dist0
1681c4762a1bSJed Brown       nsize: 2
1682c4762a1bSJed Brown       args: -distribute 0 -interpolate {{none create}separate output}
1683c4762a1bSJed Brown     test:
1684c4762a1bSJed Brown       suffix: 5_dist1
1685c4762a1bSJed Brown       nsize: 2
1686c4762a1bSJed Brown       args: -distribute 1 -interpolate {{none create after_distribute}separate output}
1687c4762a1bSJed Brown 
1688c4762a1bSJed Brown   testset:
1689c4762a1bSJed Brown     nsize: {{1 2 4}}
1690c4762a1bSJed Brown     args: -use_generator
1691c4762a1bSJed Brown     args: -dm_plex_check_all
1692c4762a1bSJed Brown     args: -distribute -interpolate none
1693c4762a1bSJed Brown     test:
1694c4762a1bSJed Brown       suffix: 6_tri
1695c4762a1bSJed Brown       requires: triangle
1696c4762a1bSJed Brown       args: -faces {{2,2  1,3  7,4}} -cell_simplex 1 -dm_plex_generator triangle
1697c4762a1bSJed Brown     test:
1698c4762a1bSJed Brown       suffix: 6_quad
1699c4762a1bSJed Brown       args: -faces {{2,2  1,3  7,4}} -cell_simplex 0
1700c4762a1bSJed Brown     test:
1701c4762a1bSJed Brown       suffix: 6_tet
1702c4762a1bSJed Brown       requires: ctetgen
1703c4762a1bSJed Brown       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 1 -dm_plex_generator ctetgen
1704c4762a1bSJed Brown     test:
1705c4762a1bSJed Brown       suffix: 6_hex
1706c4762a1bSJed Brown       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 0
1707c4762a1bSJed Brown   testset:
1708c4762a1bSJed Brown     nsize: {{1 2 4}}
1709c4762a1bSJed Brown     args: -use_generator
1710c4762a1bSJed Brown     args: -dm_plex_check_all
1711c4762a1bSJed Brown     args: -distribute -interpolate create
1712c4762a1bSJed Brown     test:
1713c4762a1bSJed Brown       suffix: 6_int_tri
1714c4762a1bSJed Brown       requires: triangle
1715c4762a1bSJed Brown       args: -faces {{2,2  1,3  7,4}} -cell_simplex 1 -dm_plex_generator triangle
1716c4762a1bSJed Brown     test:
1717c4762a1bSJed Brown       suffix: 6_int_quad
1718c4762a1bSJed Brown       args: -faces {{2,2  1,3  7,4}} -cell_simplex 0
1719c4762a1bSJed Brown     test:
1720c4762a1bSJed Brown       suffix: 6_int_tet
1721c4762a1bSJed Brown       requires: ctetgen
1722c4762a1bSJed Brown       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 1 -dm_plex_generator ctetgen
1723c4762a1bSJed Brown     test:
1724c4762a1bSJed Brown       suffix: 6_int_hex
1725c4762a1bSJed Brown       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 0
1726c4762a1bSJed Brown   testset:
1727c4762a1bSJed Brown     nsize: {{2 4}}
1728c4762a1bSJed Brown     args: -use_generator
1729c4762a1bSJed Brown     args: -dm_plex_check_all
1730c4762a1bSJed Brown     args: -distribute -interpolate after_distribute
1731c4762a1bSJed Brown     test:
1732c4762a1bSJed Brown       suffix: 6_parint_tri
1733c4762a1bSJed Brown       requires: triangle
1734c4762a1bSJed Brown       args: -faces {{2,2  1,3  7,4}} -cell_simplex 1 -dm_plex_generator triangle
1735c4762a1bSJed Brown     test:
1736c4762a1bSJed Brown       suffix: 6_parint_quad
1737c4762a1bSJed Brown       args: -faces {{2,2  1,3  7,4}} -cell_simplex 0
1738c4762a1bSJed Brown     test:
1739c4762a1bSJed Brown       suffix: 6_parint_tet
1740c4762a1bSJed Brown       requires: ctetgen
1741c4762a1bSJed Brown       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 1 -dm_plex_generator ctetgen
1742c4762a1bSJed Brown     test:
1743c4762a1bSJed Brown       suffix: 6_parint_hex
1744c4762a1bSJed Brown       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 0
1745c4762a1bSJed Brown 
1746c4762a1bSJed Brown   testset: # 7 EXODUS
1747c4762a1bSJed Brown     requires: exodusii
1748c4762a1bSJed Brown     args: -dm_plex_check_all
1749c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/blockcylinder-50.exo
1750c4762a1bSJed Brown     args: -distribute
1751c4762a1bSJed Brown     test: # seq load, simple partitioner
1752c4762a1bSJed Brown       suffix: 7_exo
1753c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1754c4762a1bSJed Brown       args: -interpolate none
1755c4762a1bSJed Brown     test: # seq load, seq interpolation, simple partitioner
1756c4762a1bSJed Brown       suffix: 7_exo_int_simple
1757c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1758c4762a1bSJed Brown       args: -interpolate create
1759c4762a1bSJed Brown     test: # seq load, seq interpolation, metis partitioner
1760c4762a1bSJed Brown       suffix: 7_exo_int_metis
1761c4762a1bSJed Brown       requires: parmetis
1762c4762a1bSJed Brown       nsize: {{2 4 5}}
1763c4762a1bSJed Brown       args: -interpolate create
1764c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1765c4762a1bSJed Brown     test: # seq load, simple partitioner, par interpolation
1766c4762a1bSJed Brown       suffix: 7_exo_simple_int
1767c4762a1bSJed Brown       nsize: {{2 4 5}}
1768c4762a1bSJed Brown       args: -interpolate after_distribute
1769c4762a1bSJed Brown     test: # seq load, metis partitioner, par interpolation
1770c4762a1bSJed Brown       suffix: 7_exo_metis_int
1771c4762a1bSJed Brown       requires: parmetis
1772c4762a1bSJed Brown       nsize: {{2 4 5}}
1773c4762a1bSJed Brown       args: -interpolate after_distribute
1774c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1775c4762a1bSJed Brown 
1776c4762a1bSJed Brown   testset: # 7 HDF5 SEQUANTIAL LOAD
1777c4762a1bSJed Brown     requires: hdf5 !complex
1778c4762a1bSJed Brown     args: -dm_plex_check_all
1779c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/blockcylinder-50.h5 -dm_plex_create_from_hdf5_xdmf
1780c4762a1bSJed Brown     args: -dm_plex_hdf5_force_sequential
1781c4762a1bSJed Brown     args: -distribute
1782c4762a1bSJed Brown     test: # seq load, simple partitioner
1783c4762a1bSJed Brown       suffix: 7_seq_hdf5_simple
1784c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1785c4762a1bSJed Brown       args: -interpolate none
1786c4762a1bSJed Brown     test: # seq load, seq interpolation, simple partitioner
1787c4762a1bSJed Brown       suffix: 7_seq_hdf5_int_simple
1788c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1789c4762a1bSJed Brown       args: -interpolate after_create
1790c4762a1bSJed Brown     test: # seq load, seq interpolation, metis partitioner
1791c4762a1bSJed Brown       nsize: {{2 4 5}}
1792c4762a1bSJed Brown       suffix: 7_seq_hdf5_int_metis
1793c4762a1bSJed Brown       requires: parmetis
1794c4762a1bSJed Brown       args: -interpolate after_create
1795c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1796c4762a1bSJed Brown     test: # seq load, simple partitioner, par interpolation
1797c4762a1bSJed Brown       suffix: 7_seq_hdf5_simple_int
1798c4762a1bSJed Brown       nsize: {{2 4 5}}
1799c4762a1bSJed Brown       args: -interpolate after_distribute
1800c4762a1bSJed Brown     test: # seq load, metis partitioner, par interpolation
1801c4762a1bSJed Brown       nsize: {{2 4 5}}
1802c4762a1bSJed Brown       suffix: 7_seq_hdf5_metis_int
1803c4762a1bSJed Brown       requires: parmetis
1804c4762a1bSJed Brown       args: -interpolate after_distribute
1805c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1806c4762a1bSJed Brown 
1807c4762a1bSJed Brown   testset: # 7 HDF5 PARALLEL LOAD
1808c4762a1bSJed Brown     requires: hdf5 !complex
1809c4762a1bSJed Brown     nsize: {{2 4 5}}
1810c4762a1bSJed Brown     args: -dm_plex_check_all
1811c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/blockcylinder-50.h5 -dm_plex_create_from_hdf5_xdmf
1812c4762a1bSJed Brown     test: # par load
1813c4762a1bSJed Brown       suffix: 7_par_hdf5
1814c4762a1bSJed Brown       args: -interpolate none
1815c4762a1bSJed Brown     test: # par load, par interpolation
1816c4762a1bSJed Brown       suffix: 7_par_hdf5_int
1817c4762a1bSJed Brown       args: -interpolate after_create
1818c4762a1bSJed Brown     test: # par load, parmetis repartitioner
1819c4762a1bSJed Brown       TODO: Parallel partitioning of uninterpolated meshes not supported
1820c4762a1bSJed Brown       suffix: 7_par_hdf5_parmetis
1821c4762a1bSJed Brown       requires: parmetis
1822c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1823c4762a1bSJed Brown       args: -interpolate none
1824c4762a1bSJed Brown     test: # par load, par interpolation, parmetis repartitioner
1825c4762a1bSJed Brown       suffix: 7_par_hdf5_int_parmetis
1826c4762a1bSJed Brown       requires: parmetis
1827c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1828c4762a1bSJed Brown       args: -interpolate after_create
1829c4762a1bSJed Brown     test: # par load, parmetis partitioner, par interpolation
1830c4762a1bSJed Brown       TODO: Parallel partitioning of uninterpolated meshes not supported
1831c4762a1bSJed Brown       suffix: 7_par_hdf5_parmetis_int
1832c4762a1bSJed Brown       requires: parmetis
1833c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1834c4762a1bSJed Brown       args: -interpolate after_distribute
1835c4762a1bSJed Brown 
1836c4762a1bSJed Brown     test:
1837c4762a1bSJed Brown       suffix: 7_hdf5_hierarch
1838c4762a1bSJed Brown       requires: hdf5 ptscotch !complex
1839c4762a1bSJed Brown       nsize: {{2 3 4}separate output}
1840c4762a1bSJed Brown       args: -distribute
1841c4762a1bSJed Brown       args: -interpolate after_create
1842c4762a1bSJed Brown       args: -petscpartitioner_type matpartitioning -petscpartitioner_view ::ascii_info
1843c4762a1bSJed Brown       args: -mat_partitioning_type hierarch -mat_partitioning_hierarchical_nfineparts 2
1844c4762a1bSJed Brown       args: -mat_partitioning_hierarchical_coarseparttype ptscotch -mat_partitioning_hierarchical_fineparttype ptscotch
1845c4762a1bSJed Brown 
1846c4762a1bSJed Brown   test:
1847c4762a1bSJed Brown     suffix: 8
1848c4762a1bSJed Brown     requires: hdf5 !complex
1849c4762a1bSJed Brown     nsize: 4
1850c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/blockcylinder-50.h5 -dm_plex_create_from_hdf5_xdmf
1851c4762a1bSJed Brown     args: -distribute 0 -interpolate after_create
1852c4762a1bSJed Brown     args: -view_vertices_from_coords 0.,1.,0.,-0.5,1.,0.,0.583,-0.644,0.,-2.,-2.,-2. -view_vertices_from_coords_tol 1e-3
1853c4762a1bSJed Brown     args: -dm_plex_check_all
1854c4762a1bSJed Brown     args: -custom_view
1855c4762a1bSJed Brown 
1856c4762a1bSJed Brown   testset: # 9 HDF5 SEQUANTIAL LOAD
1857c4762a1bSJed Brown     requires: hdf5 !complex datafilespath
1858c4762a1bSJed Brown     args: -dm_plex_check_all
1859c4762a1bSJed Brown     args: -filename ${DATAFILESPATH}/meshes/cube-hexahedra-refined.h5 -dm_plex_create_from_hdf5_xdmf -dm_plex_hdf5_topology_path /cells -dm_plex_hdf5_geometry_path /coordinates
1860c4762a1bSJed Brown     args: -dm_plex_hdf5_force_sequential
1861c4762a1bSJed Brown     args: -distribute
1862c4762a1bSJed Brown     test: # seq load, simple partitioner
1863c4762a1bSJed Brown       suffix: 9_seq_hdf5_simple
1864c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1865c4762a1bSJed Brown       args: -interpolate none
1866c4762a1bSJed Brown     test: # seq load, seq interpolation, simple partitioner
1867c4762a1bSJed Brown       suffix: 9_seq_hdf5_int_simple
1868c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1869c4762a1bSJed Brown       args: -interpolate after_create
1870c4762a1bSJed Brown     test: # seq load, seq interpolation, metis partitioner
1871c4762a1bSJed Brown       nsize: {{2 4 5}}
1872c4762a1bSJed Brown       suffix: 9_seq_hdf5_int_metis
1873c4762a1bSJed Brown       requires: parmetis
1874c4762a1bSJed Brown       args: -interpolate after_create
1875c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1876c4762a1bSJed Brown     test: # seq load, simple partitioner, par interpolation
1877c4762a1bSJed Brown       suffix: 9_seq_hdf5_simple_int
1878c4762a1bSJed Brown       nsize: {{2 4 5}}
1879c4762a1bSJed Brown       args: -interpolate after_distribute
1880c4762a1bSJed Brown     test: # seq load, simple partitioner, par interpolation
1881c4762a1bSJed Brown       # This is like 9_seq_hdf5_simple_int but testing error output of DMPlexCheckPointSFHeavy().
1882c4762a1bSJed Brown       # Once 9_seq_hdf5_simple_int gets fixed, this one gets broken.
1883c4762a1bSJed Brown       # We can then provide an intentionally broken mesh instead.
1884c4762a1bSJed Brown       TODO: This test is broken because PointSF is fixed.
1885c4762a1bSJed Brown       suffix: 9_seq_hdf5_simple_int_err
1886c4762a1bSJed Brown       nsize: 4
1887c4762a1bSJed Brown       args: -interpolate after_distribute
1888c4762a1bSJed Brown       filter: sed -e "/PETSC ERROR/,$$d"
1889c4762a1bSJed Brown     test: # seq load, metis partitioner, par interpolation
1890c4762a1bSJed Brown       nsize: {{2 4 5}}
1891c4762a1bSJed Brown       suffix: 9_seq_hdf5_metis_int
1892c4762a1bSJed Brown       requires: parmetis
1893c4762a1bSJed Brown       args: -interpolate after_distribute
1894c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1895c4762a1bSJed Brown 
1896c4762a1bSJed Brown   testset: # 9 HDF5 PARALLEL LOAD
1897c4762a1bSJed Brown     requires: hdf5 !complex datafilespath
1898c4762a1bSJed Brown     nsize: {{2 4 5}}
1899c4762a1bSJed Brown     args: -dm_plex_check_all
1900c4762a1bSJed Brown     args: -filename ${DATAFILESPATH}/meshes/cube-hexahedra-refined.h5 -dm_plex_create_from_hdf5_xdmf -dm_plex_hdf5_topology_path /cells -dm_plex_hdf5_geometry_path /coordinates
1901c4762a1bSJed Brown     test: # par load
1902c4762a1bSJed Brown       suffix: 9_par_hdf5
1903c4762a1bSJed Brown       args: -interpolate none
1904c4762a1bSJed Brown     test: # par load, par interpolation
1905c4762a1bSJed Brown       suffix: 9_par_hdf5_int
1906c4762a1bSJed Brown       args: -interpolate after_create
1907c4762a1bSJed Brown     test: # par load, parmetis repartitioner
1908c4762a1bSJed Brown       TODO: Parallel partitioning of uninterpolated meshes not supported
1909c4762a1bSJed Brown       suffix: 9_par_hdf5_parmetis
1910c4762a1bSJed Brown       requires: parmetis
1911c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1912c4762a1bSJed Brown       args: -interpolate none
1913c4762a1bSJed Brown     test: # par load, par interpolation, parmetis repartitioner
1914c4762a1bSJed Brown       suffix: 9_par_hdf5_int_parmetis
1915c4762a1bSJed Brown       requires: parmetis
1916c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1917c4762a1bSJed Brown       args: -interpolate after_create
1918c4762a1bSJed Brown     test: # par load, parmetis partitioner, par interpolation
1919c4762a1bSJed Brown       TODO: Parallel partitioning of uninterpolated meshes not supported
1920c4762a1bSJed Brown       suffix: 9_par_hdf5_parmetis_int
1921c4762a1bSJed Brown       requires: parmetis
1922c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1923c4762a1bSJed Brown       args: -interpolate after_distribute
1924c4762a1bSJed Brown 
1925c4762a1bSJed Brown   testset: # 10 HDF5 PARALLEL LOAD
1926c4762a1bSJed Brown     requires: hdf5 !complex datafilespath
1927c4762a1bSJed Brown     nsize: {{2 4 7}}
1928c4762a1bSJed Brown     args: -dm_plex_check_all
1929c4762a1bSJed Brown     args: -filename ${DATAFILESPATH}/meshes/cube-hexahedra-refined2.h5 -dm_plex_create_from_hdf5_xdmf -dm_plex_hdf5_topology_path /topo -dm_plex_hdf5_geometry_path /geom
1930c4762a1bSJed Brown     test: # par load, par interpolation
1931c4762a1bSJed Brown       suffix: 10_par_hdf5_int
1932c4762a1bSJed Brown       args: -interpolate after_create
1933c4762a1bSJed Brown TEST*/
1934