xref: /petsc/src/dm/impls/plex/tests/ex18.c (revision 71bbd31f057c74b2c85463d0d1a685d596693375) 
1c4762a1bSJed Brown static char help[] = "Tests for parallel mesh loading and parallel topological interpolation\n\n";
2c4762a1bSJed Brown 
3c4762a1bSJed Brown #include <petsc/private/dmpleximpl.h>
4c4762a1bSJed Brown /* List of test meshes
5c4762a1bSJed Brown 
6c4762a1bSJed Brown Network
7c4762a1bSJed Brown -------
8c4762a1bSJed Brown Test 0 (2 ranks):
9c4762a1bSJed Brown 
10c4762a1bSJed Brown network=0:
11c4762a1bSJed Brown ---------
12c4762a1bSJed Brown   cell 0   cell 1   cell 2          nCells-1       (edge)
13c4762a1bSJed Brown 0 ------ 1 ------ 2 ------ 3 -- -- v --  -- nCells (vertex)
14c4762a1bSJed Brown 
15c4762a1bSJed Brown   vertex distribution:
16c4762a1bSJed Brown     rank 0: 0 1
17c4762a1bSJed Brown     rank 1: 2 3 ... nCells
18c4762a1bSJed Brown   cell(edge) distribution:
19c4762a1bSJed Brown     rank 0: 0 1
20c4762a1bSJed Brown     rank 1: 2 ... nCells-1
21c4762a1bSJed Brown 
22c4762a1bSJed Brown network=1:
23c4762a1bSJed Brown ---------
24c4762a1bSJed Brown                v2
25c4762a1bSJed Brown                 ^
26c4762a1bSJed Brown                 |
27c4762a1bSJed Brown                cell 2
28c4762a1bSJed Brown                 |
29c4762a1bSJed Brown  v0 --cell 0--> v3--cell 1--> v1
30c4762a1bSJed Brown 
31c4762a1bSJed Brown   vertex distribution:
32c4762a1bSJed Brown     rank 0: 0 1 3
33c4762a1bSJed Brown     rank 1: 2
34c4762a1bSJed Brown   cell(edge) distribution:
35c4762a1bSJed Brown     rank 0: 0 1
36c4762a1bSJed Brown     rank 1: 2
37c4762a1bSJed Brown 
38c4762a1bSJed Brown   example:
39c4762a1bSJed Brown     mpiexec -n 2 ./ex18 -distribute 1 -dim 1 -orig_dm_view -dist_dm_view -dist_dm_view -petscpartitioner_type parmetis -ncells 50
40c4762a1bSJed Brown 
41c4762a1bSJed Brown Triangle
42c4762a1bSJed Brown --------
43c4762a1bSJed Brown Test 0 (2 ranks):
44c4762a1bSJed Brown Two triangles sharing a face
45c4762a1bSJed Brown 
46c4762a1bSJed Brown         2
47c4762a1bSJed Brown       / | \
48c4762a1bSJed Brown      /  |  \
49c4762a1bSJed Brown     /   |   \
50c4762a1bSJed Brown    0  0 | 1  3
51c4762a1bSJed Brown     \   |   /
52c4762a1bSJed Brown      \  |  /
53c4762a1bSJed Brown       \ | /
54c4762a1bSJed Brown         1
55c4762a1bSJed Brown 
56c4762a1bSJed Brown   vertex distribution:
57c4762a1bSJed Brown     rank 0: 0 1
58c4762a1bSJed Brown     rank 1: 2 3
59c4762a1bSJed Brown   cell distribution:
60c4762a1bSJed Brown     rank 0: 0
61c4762a1bSJed Brown     rank 1: 1
62c4762a1bSJed Brown 
63c4762a1bSJed Brown Test 1 (3 ranks):
64c4762a1bSJed Brown Four triangles partitioned across 3 ranks
65c4762a1bSJed Brown 
66c4762a1bSJed Brown    0 _______ 3
67c4762a1bSJed Brown    | \     / |
68c4762a1bSJed Brown    |  \ 1 /  |
69c4762a1bSJed Brown    |   \ /   |
70c4762a1bSJed Brown    | 0  2  2 |
71c4762a1bSJed Brown    |   / \   |
72c4762a1bSJed Brown    |  / 3 \  |
73c4762a1bSJed Brown    | /     \ |
74c4762a1bSJed Brown    1 ------- 4
75c4762a1bSJed Brown 
76c4762a1bSJed Brown   vertex distribution:
77c4762a1bSJed Brown     rank 0: 0 1
78c4762a1bSJed Brown     rank 1: 2 3
79c4762a1bSJed Brown     rank 2: 4
80c4762a1bSJed Brown   cell distribution:
81c4762a1bSJed Brown     rank 0: 0
82c4762a1bSJed Brown     rank 1: 1
83c4762a1bSJed Brown     rank 2: 2 3
84c4762a1bSJed Brown 
85c4762a1bSJed Brown Test 2 (3 ranks):
86c4762a1bSJed Brown Four triangles partitioned across 3 ranks
87c4762a1bSJed Brown 
88c4762a1bSJed Brown    1 _______ 3
89c4762a1bSJed Brown    | \     / |
90c4762a1bSJed Brown    |  \ 1 /  |
91c4762a1bSJed Brown    |   \ /   |
92c4762a1bSJed Brown    | 0  0  2 |
93c4762a1bSJed Brown    |   / \   |
94c4762a1bSJed Brown    |  / 3 \  |
95c4762a1bSJed Brown    | /     \ |
96c4762a1bSJed Brown    2 ------- 4
97c4762a1bSJed Brown 
98c4762a1bSJed Brown   vertex distribution:
99c4762a1bSJed Brown     rank 0: 0 1
100c4762a1bSJed Brown     rank 1: 2 3
101c4762a1bSJed Brown     rank 2: 4
102c4762a1bSJed Brown   cell distribution:
103c4762a1bSJed Brown     rank 0: 0
104c4762a1bSJed Brown     rank 1: 1
105c4762a1bSJed Brown     rank 2: 2 3
106c4762a1bSJed Brown 
107c4762a1bSJed Brown Tetrahedron
108c4762a1bSJed Brown -----------
109c4762a1bSJed Brown Test 0:
110c4762a1bSJed Brown Two tets sharing a face
111c4762a1bSJed Brown 
112c4762a1bSJed Brown  cell   3 _______    cell
113c4762a1bSJed Brown  0    / | \      \   1
114c4762a1bSJed Brown      /  |  \      \
115c4762a1bSJed Brown     /   |   \      \
116c4762a1bSJed Brown    0----|----4-----2
117c4762a1bSJed Brown     \   |   /      /
118c4762a1bSJed Brown      \  |  /      /
119c4762a1bSJed Brown       \ | /      /
120c4762a1bSJed Brown         1-------
121c4762a1bSJed Brown    y
122c4762a1bSJed Brown    | x
123c4762a1bSJed Brown    |/
124c4762a1bSJed Brown    *----z
125c4762a1bSJed Brown 
126c4762a1bSJed Brown   vertex distribution:
127c4762a1bSJed Brown     rank 0: 0 1
128c4762a1bSJed Brown     rank 1: 2 3 4
129c4762a1bSJed Brown   cell distribution:
130c4762a1bSJed Brown     rank 0: 0
131c4762a1bSJed Brown     rank 1: 1
132c4762a1bSJed Brown 
133c4762a1bSJed Brown Quadrilateral
134c4762a1bSJed Brown -------------
135c4762a1bSJed Brown Test 0 (2 ranks):
136c4762a1bSJed Brown Two quads sharing a face
137c4762a1bSJed Brown 
138c4762a1bSJed Brown    3-------2-------5
139c4762a1bSJed Brown    |       |       |
140c4762a1bSJed Brown    |   0   |   1   |
141c4762a1bSJed Brown    |       |       |
142c4762a1bSJed Brown    0-------1-------4
143c4762a1bSJed Brown 
144c4762a1bSJed Brown   vertex distribution:
145c4762a1bSJed Brown     rank 0: 0 1 2
146c4762a1bSJed Brown     rank 1: 3 4 5
147c4762a1bSJed Brown   cell distribution:
148c4762a1bSJed Brown     rank 0: 0
149c4762a1bSJed Brown     rank 1: 1
150c4762a1bSJed Brown 
151c4762a1bSJed Brown TODO Test 1:
152c4762a1bSJed Brown A quad and a triangle sharing a face
153c4762a1bSJed Brown 
154c4762a1bSJed Brown    5-------4
155c4762a1bSJed Brown    |       | \
156c4762a1bSJed Brown    |   0   |  \
157c4762a1bSJed Brown    |       | 1 \
158c4762a1bSJed Brown    2-------3----6
159c4762a1bSJed Brown 
160c4762a1bSJed Brown Hexahedron
161c4762a1bSJed Brown ----------
162c4762a1bSJed Brown Test 0 (2 ranks):
163c4762a1bSJed Brown Two hexes sharing a face
164c4762a1bSJed Brown 
165c4762a1bSJed Brown cell   7-------------6-------------11 cell
166c4762a1bSJed Brown 0     /|            /|            /|     1
167c4762a1bSJed Brown      / |   F1      / |   F7      / |
168c4762a1bSJed Brown     /  |          /  |          /  |
169c4762a1bSJed Brown    4-------------5-------------10  |
170c4762a1bSJed Brown    |   |     F4  |   |     F10 |   |
171c4762a1bSJed Brown    |   |         |   |         |   |
172c4762a1bSJed Brown    |F5 |         |F3 |         |F9 |
173c4762a1bSJed Brown    |   |  F2     |   |   F8    |   |
174c4762a1bSJed Brown    |   3---------|---2---------|---9
175c4762a1bSJed Brown    |  /          |  /          |  /
176c4762a1bSJed Brown    | /   F0      | /    F6     | /
177c4762a1bSJed Brown    |/            |/            |/
178c4762a1bSJed Brown    0-------------1-------------8
179c4762a1bSJed Brown 
180c4762a1bSJed Brown   vertex distribution:
181c4762a1bSJed Brown     rank 0: 0 1 2 3 4 5
182c4762a1bSJed Brown     rank 1: 6 7 8 9 10 11
183c4762a1bSJed Brown   cell distribution:
184c4762a1bSJed Brown     rank 0: 0
185c4762a1bSJed Brown     rank 1: 1
186c4762a1bSJed Brown 
187c4762a1bSJed Brown */
188c4762a1bSJed Brown 
189c4762a1bSJed Brown typedef enum {NONE, CREATE, AFTER_CREATE, AFTER_DISTRIBUTE} InterpType;
190c4762a1bSJed Brown 
191c4762a1bSJed Brown typedef struct {
192c4762a1bSJed Brown   PetscInt   debug;                        /* The debugging level */
193c4762a1bSJed Brown   PetscInt   testNum;                      /* Indicates the mesh to create */
194c4762a1bSJed Brown   PetscInt   dim;                          /* The topological mesh dimension */
195c4762a1bSJed Brown   PetscBool  cellSimplex;                  /* Use simplices or hexes */
196c4762a1bSJed Brown   PetscBool  distribute;                   /* Distribute the mesh */
197c4762a1bSJed Brown   InterpType interpolate;                  /* Interpolate the mesh before or after DMPlexDistribute() */
198c4762a1bSJed Brown   PetscBool  useGenerator;                 /* Construct mesh with a mesh generator */
199c4762a1bSJed Brown   PetscBool  testOrientIF;                 /* Test for different original interface orientations */
200c4762a1bSJed Brown   PetscBool  testHeavy;                    /* Run the heavy PointSF test */
201c4762a1bSJed Brown   PetscBool  customView;                   /* Show results of DMPlexIsInterpolated() etc. */
202c4762a1bSJed Brown   PetscInt   ornt[2];                      /* Orientation of interface on rank 0 and rank 1 */
203c4762a1bSJed Brown   PetscInt   faces[3];                     /* Number of faces per dimension for generator */
204c4762a1bSJed Brown   PetscReal  coords[128];
205c4762a1bSJed Brown   PetscReal  coordsTol;
206c4762a1bSJed Brown   PetscInt   ncoords;
207c4762a1bSJed Brown   PetscInt   pointsToExpand[128];
208c4762a1bSJed Brown   PetscInt   nPointsToExpand;
209c4762a1bSJed Brown   PetscBool  testExpandPointsEmpty;
210c4762a1bSJed Brown   char       filename[PETSC_MAX_PATH_LEN]; /* Import mesh from file */
211c4762a1bSJed Brown } AppCtx;
212c4762a1bSJed Brown 
213c4762a1bSJed Brown struct _n_PortableBoundary {
214c4762a1bSJed Brown   Vec coordinates;
215c4762a1bSJed Brown   PetscInt depth;
216c4762a1bSJed Brown   PetscSection *sections;
217c4762a1bSJed Brown };
218c4762a1bSJed Brown typedef struct _n_PortableBoundary * PortableBoundary;
219c4762a1bSJed Brown 
220956f8c0dSBarry Smith #if defined(PETSC_USE_LOG)
221c4762a1bSJed Brown static PetscLogStage  stage[3];
222956f8c0dSBarry Smith #endif
223c4762a1bSJed Brown 
224c4762a1bSJed Brown static PetscErrorCode DMPlexCheckPointSFHeavy(DM, PortableBoundary);
225c4762a1bSJed Brown static PetscErrorCode DMPlexSetOrientInterface_Private(DM,PetscBool);
226c4762a1bSJed Brown static PetscErrorCode DMPlexGetExpandedBoundary_Private(DM, PortableBoundary *);
227c4762a1bSJed Brown static PetscErrorCode DMPlexExpandedConesToFaces_Private(DM, IS, PetscSection, IS *);
228c4762a1bSJed Brown 
229c4762a1bSJed Brown static PetscErrorCode PortableBoundaryDestroy(PortableBoundary *bnd)
230c4762a1bSJed Brown {
231c4762a1bSJed Brown   PetscInt       d;
232c4762a1bSJed Brown   PetscErrorCode ierr;
233c4762a1bSJed Brown 
234c4762a1bSJed Brown   PetscFunctionBegin;
235c4762a1bSJed Brown   if (!*bnd) PetscFunctionReturn(0);
236c4762a1bSJed Brown   ierr = VecDestroy(&(*bnd)->coordinates);CHKERRQ(ierr);
237c4762a1bSJed Brown   for (d=0; d < (*bnd)->depth; d++) {
238c4762a1bSJed Brown     ierr = PetscSectionDestroy(&(*bnd)->sections[d]);CHKERRQ(ierr);
239c4762a1bSJed Brown   }
240c4762a1bSJed Brown   ierr = PetscFree((*bnd)->sections);CHKERRQ(ierr);
241c4762a1bSJed Brown   ierr = PetscFree(*bnd);CHKERRQ(ierr);
242c4762a1bSJed Brown   PetscFunctionReturn(0);
243c4762a1bSJed Brown }
244c4762a1bSJed Brown 
245c4762a1bSJed Brown static PetscErrorCode ProcessOptions(MPI_Comm comm, AppCtx *options)
246c4762a1bSJed Brown {
247c4762a1bSJed Brown   const char    *interpTypes[4]  = {"none", "create", "after_create", "after_distribute"};
248c4762a1bSJed Brown   PetscInt       interp=NONE, dim;
249c4762a1bSJed Brown   PetscBool      flg1, flg2;
250c4762a1bSJed Brown   PetscErrorCode ierr;
251c4762a1bSJed Brown 
252c4762a1bSJed Brown   PetscFunctionBegin;
253c4762a1bSJed Brown   options->debug        = 0;
254c4762a1bSJed Brown   options->testNum      = 0;
255c4762a1bSJed Brown   options->dim          = 2;
256c4762a1bSJed Brown   options->cellSimplex  = PETSC_TRUE;
257c4762a1bSJed Brown   options->distribute   = PETSC_FALSE;
258c4762a1bSJed Brown   options->interpolate  = NONE;
259c4762a1bSJed Brown   options->useGenerator = PETSC_FALSE;
260c4762a1bSJed Brown   options->testOrientIF = PETSC_FALSE;
261c4762a1bSJed Brown   options->testHeavy    = PETSC_TRUE;
262c4762a1bSJed Brown   options->customView   = PETSC_FALSE;
263c4762a1bSJed Brown   options->testExpandPointsEmpty = PETSC_FALSE;
264c4762a1bSJed Brown   options->ornt[0]      = 0;
265c4762a1bSJed Brown   options->ornt[1]      = 0;
266c4762a1bSJed Brown   options->faces[0]     = 2;
267c4762a1bSJed Brown   options->faces[1]     = 2;
268c4762a1bSJed Brown   options->faces[2]     = 2;
269c4762a1bSJed Brown   options->filename[0]  = '\0';
270c4762a1bSJed Brown   options->coordsTol    = PETSC_DEFAULT;
271c4762a1bSJed Brown 
272c4762a1bSJed Brown   ierr = PetscOptionsBegin(comm, "", "Meshing Interpolation Test Options", "DMPLEX");CHKERRQ(ierr);
273c4762a1bSJed Brown   ierr = PetscOptionsBoundedInt("-debug", "The debugging level", "ex18.c", options->debug, &options->debug, NULL,0);CHKERRQ(ierr);
274c4762a1bSJed Brown   ierr = PetscOptionsBoundedInt("-testnum", "The mesh to create", "ex18.c", options->testNum, &options->testNum, NULL,0);CHKERRQ(ierr);
275c4762a1bSJed Brown   ierr = PetscOptionsBool("-cell_simplex", "Generate simplices if true, otherwise hexes", "ex18.c", options->cellSimplex, &options->cellSimplex, NULL);CHKERRQ(ierr);
276c4762a1bSJed Brown   ierr = PetscOptionsBool("-distribute", "Distribute the mesh", "ex18.c", options->distribute, &options->distribute, NULL);CHKERRQ(ierr);
277c4762a1bSJed Brown   ierr = PetscOptionsEList("-interpolate", "Type of mesh interpolation (none, create, after_create, after_distribute)", "ex18.c", interpTypes, 4, interpTypes[options->interpolate], &interp, NULL);CHKERRQ(ierr);
278c4762a1bSJed Brown   options->interpolate = (InterpType) interp;
279c4762a1bSJed Brown   if (!options->distribute && options->interpolate == AFTER_DISTRIBUTE) SETERRQ(comm, PETSC_ERR_SUP, "-interpolate after_distribute  needs  -distribute 1");
280c4762a1bSJed Brown   ierr = PetscOptionsBool("-use_generator", "Use a mesh generator to build the mesh", "ex18.c", options->useGenerator, &options->useGenerator, NULL);CHKERRQ(ierr);
281c4762a1bSJed Brown   options->ncoords = 128;
282c4762a1bSJed Brown   ierr = PetscOptionsRealArray("-view_vertices_from_coords", "Print DAG points corresponding to vertices with given coordinates", "ex18.c", options->coords, &options->ncoords, NULL);CHKERRQ(ierr);
283c4762a1bSJed Brown   ierr = PetscOptionsReal("-view_vertices_from_coords_tol", "Tolerance for -view_vertices_from_coords", "ex18.c", options->coordsTol, &options->coordsTol, NULL);CHKERRQ(ierr);
284c4762a1bSJed Brown   options->nPointsToExpand = 128;
285c4762a1bSJed Brown   ierr = PetscOptionsIntArray("-test_expand_points", "Expand given array of DAG point using DMPlexGetConeRecursive() and print results", "ex18.c", options->pointsToExpand, &options->nPointsToExpand, NULL);CHKERRQ(ierr);
286c4762a1bSJed Brown   if (options->nPointsToExpand) {
287c4762a1bSJed Brown     ierr = PetscOptionsBool("-test_expand_points_empty", "For -test_expand_points, rank 0 will have empty input array", "ex18.c", options->testExpandPointsEmpty, &options->testExpandPointsEmpty, NULL);CHKERRQ(ierr);
288c4762a1bSJed Brown   }
289c4762a1bSJed Brown   ierr = PetscOptionsBool("-test_heavy", "Run the heavy PointSF test", "ex18.c", options->testHeavy, &options->testHeavy, NULL);CHKERRQ(ierr);
290c4762a1bSJed Brown   ierr = PetscOptionsBool("-custom_view", "Custom DMPlex view", "ex18.c", options->customView, &options->customView, NULL);CHKERRQ(ierr);
291c4762a1bSJed Brown   ierr = PetscOptionsRangeInt("-dim", "The topological mesh dimension", "ex18.c", options->dim, &options->dim, &flg1,1,3);CHKERRQ(ierr);
292c4762a1bSJed Brown   dim = 3;
293c4762a1bSJed Brown   ierr = PetscOptionsIntArray("-faces", "Number of faces per dimension", "ex18.c", options->faces, &dim, &flg2);CHKERRQ(ierr);
294c4762a1bSJed Brown   if (flg2) {
295c4762a1bSJed Brown     if (flg1 && dim != options->dim) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "specified -dim %D is not equal to length %D of -faces (note that -dim can be omitted)", options->dim, dim);
296c4762a1bSJed Brown     options->dim = dim;
297c4762a1bSJed Brown   }
298589a23caSBarry Smith   ierr = PetscOptionsString("-filename", "The mesh file", "ex18.c", options->filename, options->filename, sizeof(options->filename), NULL);CHKERRQ(ierr);
299c4762a1bSJed Brown   ierr = PetscOptionsBoundedInt("-rotate_interface_0", "Rotation (relative orientation) of interface on rank 0; implies -interpolate create -distribute 0", "ex18.c", options->ornt[0], &options->ornt[0], &options->testOrientIF,0);CHKERRQ(ierr);
300c4762a1bSJed Brown   ierr = PetscOptionsBoundedInt("-rotate_interface_1", "Rotation (relative orientation) of interface on rank 1; implies -interpolate create -distribute 0", "ex18.c", options->ornt[1], &options->ornt[1], &flg2,0);CHKERRQ(ierr);
301c4762a1bSJed Brown   if (flg2 != options->testOrientIF) SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "neither or both -rotate_interface_0 -rotate_interface_1 must be set");
302c4762a1bSJed Brown   if (options->testOrientIF) {
303c4762a1bSJed Brown     PetscInt i;
304c4762a1bSJed Brown     for (i=0; i<2; i++) {
305c4762a1bSJed Brown       if (options->ornt[i] >= 10) options->ornt[i] = -(options->ornt[i]-10);  /* 11 12 13 become -1 -2 -3 */
306c4762a1bSJed Brown     }
307c4762a1bSJed Brown     options->filename[0]  = 0;
308c4762a1bSJed Brown     options->useGenerator = PETSC_FALSE;
309c4762a1bSJed Brown     options->dim          = 3;
310c4762a1bSJed Brown     options->cellSimplex  = PETSC_TRUE;
311c4762a1bSJed Brown     options->interpolate  = CREATE;
312c4762a1bSJed Brown     options->distribute   = PETSC_FALSE;
313c4762a1bSJed Brown   }
3141e1ea65dSPierre Jolivet   ierr = PetscOptionsEnd();CHKERRQ(ierr);
315c4762a1bSJed Brown   PetscFunctionReturn(0);
316c4762a1bSJed Brown }
317c4762a1bSJed Brown 
318c4762a1bSJed Brown static PetscErrorCode CreateMesh_1D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
319c4762a1bSJed Brown {
320c4762a1bSJed Brown   PetscInt       testNum = user->testNum;
321c4762a1bSJed Brown   PetscMPIInt    rank,size;
322c4762a1bSJed Brown   PetscErrorCode ierr;
323c4762a1bSJed Brown   PetscInt       numCorners=2,i;
324c4762a1bSJed Brown   PetscInt       numCells,numVertices,network;
325a4a685f2SJacob Faibussowitsch   PetscInt       *cells;
326c4762a1bSJed Brown   PetscReal      *coords;
327c4762a1bSJed Brown 
328c4762a1bSJed Brown   PetscFunctionBegin;
329ffc4695bSBarry Smith   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
330ffc4695bSBarry Smith   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
331c4762a1bSJed Brown   if (size > 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for <=2 processes",testNum);
332c4762a1bSJed Brown 
333c4762a1bSJed Brown   numCells = 3;
334c4762a1bSJed Brown   ierr = PetscOptionsGetInt(NULL, NULL, "-ncells", &numCells, NULL);CHKERRQ(ierr);
335c4762a1bSJed Brown   if (numCells < 3) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test ncells must >=3",numCells);
336c4762a1bSJed Brown 
337c4762a1bSJed Brown   if (size == 1) {
338c4762a1bSJed Brown     numVertices = numCells + 1;
339*71bbd31fSVaclav Hapla     ierr = PetscMalloc2(2*numCells,&cells,2*numVertices,&coords);CHKERRQ(ierr);
340c4762a1bSJed Brown     for (i=0; i<numCells; i++) {
341c4762a1bSJed Brown       cells[2*i] = i; cells[2*i+1] = i + 1;
342*71bbd31fSVaclav Hapla       coords[2*i] = i; coords[2*i+1] = i + 1;
343c4762a1bSJed Brown     }
344c4762a1bSJed Brown 
345*71bbd31fSVaclav Hapla     ierr = DMPlexCreateFromCellListPetsc(comm, user->dim, numCells, numVertices, numCorners, PETSC_FALSE, cells, user->dim, coords, dm);CHKERRQ(ierr);
346c4762a1bSJed Brown     ierr = PetscFree2(cells,coords);CHKERRQ(ierr);
347c4762a1bSJed Brown     PetscFunctionReturn(0);
348c4762a1bSJed Brown   }
349c4762a1bSJed Brown 
350c4762a1bSJed Brown   network = 0;
351c4762a1bSJed Brown   ierr = PetscOptionsGetInt(NULL, NULL, "-network_case", &network, NULL);CHKERRQ(ierr);
352c4762a1bSJed Brown   if (network == 0) {
353c4762a1bSJed Brown     switch (rank) {
354c4762a1bSJed Brown     case 0:
355c4762a1bSJed Brown     {
356c4762a1bSJed Brown       numCells    = 2;
357c4762a1bSJed Brown       numVertices = numCells;
358c4762a1bSJed Brown       ierr = PetscMalloc2(2*numCells,&cells,2*numCells,&coords);CHKERRQ(ierr);
359c4762a1bSJed Brown       cells[0] = 0; cells[1] = 1;
360c4762a1bSJed Brown       cells[2] = 1; cells[3] = 2;
361c4762a1bSJed Brown       coords[0] = 0.; coords[1] = 1.;
362c4762a1bSJed Brown       coords[2] = 1.; coords[3] = 2.;
363c4762a1bSJed Brown     }
364c4762a1bSJed Brown     break;
365c4762a1bSJed Brown     case 1:
366c4762a1bSJed Brown     {
367c4762a1bSJed Brown       numCells    -= 2;
368c4762a1bSJed Brown       numVertices = numCells + 1;
369c4762a1bSJed Brown       ierr = PetscMalloc2(2*numCells,&cells,2*numCells,&coords);CHKERRQ(ierr);
370c4762a1bSJed Brown       for (i=0; i<numCells; i++) {
371c4762a1bSJed Brown         cells[2*i] = 2+i; cells[2*i+1] = 2 + i + 1;
372c4762a1bSJed Brown         coords[2*i] = 2+i; coords[2*i+1] = 2 + i + 1;
373c4762a1bSJed Brown       }
374c4762a1bSJed Brown     }
375c4762a1bSJed Brown     break;
376c4762a1bSJed Brown     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
377c4762a1bSJed Brown     }
378c4762a1bSJed Brown   } else { /* network_case = 1 */
379c4762a1bSJed Brown     /* ----------------------- */
380c4762a1bSJed Brown     switch (rank) {
381c4762a1bSJed Brown     case 0:
382c4762a1bSJed Brown     {
383c4762a1bSJed Brown       numCells    = 2;
384c4762a1bSJed Brown       numVertices = 3;
385c4762a1bSJed Brown       ierr = PetscMalloc2(2*numCells,&cells,2*numCells,&coords);CHKERRQ(ierr);
386c4762a1bSJed Brown       cells[0] = 0; cells[1] = 3;
387c4762a1bSJed Brown       cells[2] = 3; cells[3] = 1;
388c4762a1bSJed Brown     }
389c4762a1bSJed Brown     break;
390c4762a1bSJed Brown     case 1:
391c4762a1bSJed Brown     {
392c4762a1bSJed Brown       numCells    = 1;
393c4762a1bSJed Brown       numVertices = 1;
394c4762a1bSJed Brown       ierr = PetscMalloc2(2*numCells,&cells,2*numCells,&coords);CHKERRQ(ierr);
395c4762a1bSJed Brown       cells[0] = 3; cells[1] = 2;
396c4762a1bSJed Brown     }
397c4762a1bSJed Brown     break;
398c4762a1bSJed Brown     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
399c4762a1bSJed Brown     }
400c4762a1bSJed Brown   }
401be8c289dSNicolas Barral   ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, PETSC_FALSE, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
402c4762a1bSJed Brown   ierr = PetscFree2(cells,coords);CHKERRQ(ierr);
403c4762a1bSJed Brown   PetscFunctionReturn(0);
404c4762a1bSJed Brown }
405c4762a1bSJed Brown 
406c4762a1bSJed Brown static PetscErrorCode CreateSimplex_2D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
407c4762a1bSJed Brown {
408c4762a1bSJed Brown   PetscInt       testNum = user->testNum, p;
409c4762a1bSJed Brown   PetscMPIInt    rank, size;
410c4762a1bSJed Brown   PetscErrorCode ierr;
411c4762a1bSJed Brown 
412c4762a1bSJed Brown   PetscFunctionBegin;
413ffc4695bSBarry Smith   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
414ffc4695bSBarry Smith   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
415c4762a1bSJed Brown   switch (testNum) {
416c4762a1bSJed Brown   case 0:
417c4762a1bSJed Brown     if (size != 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 2 processes", testNum);
418c4762a1bSJed Brown     switch (rank) {
419c4762a1bSJed Brown       case 0:
420c4762a1bSJed Brown       {
421c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
422a4a685f2SJacob Faibussowitsch         const PetscInt cells[3]  = {0, 1, 2};
423c4762a1bSJed Brown         PetscReal      coords[4] = {-0.5, 0.5, 0.0, 0.0};
424c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
425c4762a1bSJed Brown 
426be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
427c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
428c4762a1bSJed Brown       }
429c4762a1bSJed Brown       break;
430c4762a1bSJed Brown       case 1:
431c4762a1bSJed Brown       {
432c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
433a4a685f2SJacob Faibussowitsch         const PetscInt cells[3]  = {1, 3, 2};
434c4762a1bSJed Brown         PetscReal      coords[4] = {0.0, 1.0, 0.5, 0.5};
435c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
436c4762a1bSJed Brown 
437be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
438c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
439c4762a1bSJed Brown       }
440c4762a1bSJed Brown       break;
441c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
442c4762a1bSJed Brown     }
443c4762a1bSJed Brown     break;
444c4762a1bSJed Brown   case 1:
445c4762a1bSJed Brown     if (size != 3) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 3 processes", testNum);
446c4762a1bSJed Brown     switch (rank) {
447c4762a1bSJed Brown       case 0:
448c4762a1bSJed Brown       {
449c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
450a4a685f2SJacob Faibussowitsch         const PetscInt cells[3]  = {0, 1, 2};
451c4762a1bSJed Brown         PetscReal      coords[4] = {0.0, 1.0, 0.0, 0.0};
452c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
453c4762a1bSJed Brown 
454be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
455c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
456c4762a1bSJed Brown       }
457c4762a1bSJed Brown       break;
458c4762a1bSJed Brown       case 1:
459c4762a1bSJed Brown       {
460c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
461a4a685f2SJacob Faibussowitsch         const PetscInt cells[3]  = {0, 2, 3};
462c4762a1bSJed Brown         PetscReal      coords[4] = {0.5, 0.5, 1.0, 1.0};
463c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
464c4762a1bSJed Brown 
465be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
466c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
467c4762a1bSJed Brown       }
468c4762a1bSJed Brown       break;
469c4762a1bSJed Brown       case 2:
470c4762a1bSJed Brown       {
471c4762a1bSJed Brown         const PetscInt numCells  = 2, numVertices = 1, numCorners = 3;
472a4a685f2SJacob Faibussowitsch         const PetscInt cells[6]  = {2, 4, 3, 2, 1, 4};
473c4762a1bSJed Brown         PetscReal      coords[2] = {1.0, 0.0};
474c4762a1bSJed Brown         PetscInt       markerPoints[10] = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
475c4762a1bSJed Brown 
476be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
477c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
478c4762a1bSJed Brown       }
479c4762a1bSJed Brown       break;
480c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
481c4762a1bSJed Brown     }
482c4762a1bSJed Brown     break;
483c4762a1bSJed Brown   case 2:
484c4762a1bSJed Brown     if (size != 3) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 3 processes", testNum);
485c4762a1bSJed Brown     switch (rank) {
486c4762a1bSJed Brown       case 0:
487c4762a1bSJed Brown       {
488c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
489a4a685f2SJacob Faibussowitsch         const PetscInt cells[3]  = {1, 2, 0};
490c4762a1bSJed Brown         PetscReal      coords[4] = {0.5, 0.5, 0.0, 1.0};
491c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
492c4762a1bSJed Brown 
493be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
494c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
495c4762a1bSJed Brown       }
496c4762a1bSJed Brown       break;
497c4762a1bSJed Brown       case 1:
498c4762a1bSJed Brown       {
499c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 3;
500a4a685f2SJacob Faibussowitsch         const PetscInt cells[3]  = {1, 0, 3};
501c4762a1bSJed Brown         PetscReal      coords[4] = {0.0, 0.0, 1.0, 1.0};
502c4762a1bSJed Brown         PetscInt       markerPoints[6] = {1, 1, 2, 1, 3, 1};
503c4762a1bSJed Brown 
504be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
505c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
506c4762a1bSJed Brown       }
507c4762a1bSJed Brown       break;
508c4762a1bSJed Brown       case 2:
509c4762a1bSJed Brown       {
510c4762a1bSJed Brown         const PetscInt numCells  = 2, numVertices = 1, numCorners = 3;
511a4a685f2SJacob Faibussowitsch         const PetscInt cells[6]  = {0, 4, 3, 0, 2, 4};
512c4762a1bSJed Brown         PetscReal      coords[2] = {1.0, 0.0};
513c4762a1bSJed Brown         PetscInt       markerPoints[10] = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
514c4762a1bSJed Brown 
515be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
516c4762a1bSJed Brown         for (p = 0; p < 3; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
517c4762a1bSJed Brown       }
518c4762a1bSJed Brown       break;
519c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
520c4762a1bSJed Brown     }
521c4762a1bSJed Brown     break;
522c4762a1bSJed Brown   default: SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh %D", testNum);
523c4762a1bSJed Brown   }
524c4762a1bSJed Brown   PetscFunctionReturn(0);
525c4762a1bSJed Brown }
526c4762a1bSJed Brown 
527c4762a1bSJed Brown static PetscErrorCode CreateSimplex_3D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
528c4762a1bSJed Brown {
529c4762a1bSJed Brown   PetscInt       testNum = user->testNum, p;
530c4762a1bSJed Brown   PetscMPIInt    rank, size;
531c4762a1bSJed Brown   PetscErrorCode ierr;
532c4762a1bSJed Brown 
533c4762a1bSJed Brown   PetscFunctionBegin;
534ffc4695bSBarry Smith   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
535ffc4695bSBarry Smith   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
536c4762a1bSJed Brown   switch (testNum) {
537c4762a1bSJed Brown   case 0:
538c4762a1bSJed Brown     if (size != 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 2 processes", testNum);
539c4762a1bSJed Brown     switch (rank) {
540c4762a1bSJed Brown       case 0:
541c4762a1bSJed Brown       {
542c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 2, numCorners = 4;
543a4a685f2SJacob Faibussowitsch         const PetscInt cells[4]  = {0, 2, 1, 3};
544c4762a1bSJed Brown         PetscReal      coords[6] = {0.0, 0.0, -0.5,  0.0, -0.5, 0.0};
545c4762a1bSJed Brown         PetscInt       markerPoints[8] = {1, 1, 2, 1, 3, 1, 4, 1};
546c4762a1bSJed Brown 
547be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
548c4762a1bSJed Brown         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
549c4762a1bSJed Brown       }
550c4762a1bSJed Brown       break;
551c4762a1bSJed Brown       case 1:
552c4762a1bSJed Brown       {
553c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 3, numCorners = 4;
554a4a685f2SJacob Faibussowitsch         const PetscInt cells[4]  = {1, 2, 4, 3};
555c4762a1bSJed Brown         PetscReal      coords[9] = {1.0, 0.0, 0.0,  0.0, 0.5, 0.0,  0.0, 0.0, 0.5};
556c4762a1bSJed Brown         PetscInt       markerPoints[8] = {1, 1, 2, 1, 3, 1, 4, 1};
557c4762a1bSJed Brown 
558be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
559c4762a1bSJed Brown         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
560c4762a1bSJed Brown       }
561c4762a1bSJed Brown       break;
562c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
563c4762a1bSJed Brown     }
564c4762a1bSJed Brown     break;
565c4762a1bSJed Brown   default: SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh %D", testNum);
566c4762a1bSJed Brown   }
567c4762a1bSJed Brown   if (user->testOrientIF) {
568c4762a1bSJed Brown     PetscInt ifp[] = {8, 6};
569c4762a1bSJed Brown 
570c4762a1bSJed Brown     ierr = PetscObjectSetName((PetscObject) *dm, "Mesh before orientation");CHKERRQ(ierr);
571c4762a1bSJed Brown     ierr = DMViewFromOptions(*dm, NULL, "-before_orientation_dm_view");CHKERRQ(ierr);
572c4762a1bSJed Brown     /* rotate interface face ifp[rank] by given orientation ornt[rank] */
573b5a892a1SMatthew G. Knepley     ierr = DMPlexOrientPoint(*dm, ifp[rank], user->ornt[rank]);CHKERRQ(ierr);
574b5a892a1SMatthew G. Knepley     ierr = DMViewFromOptions(*dm, NULL, "-before_orientation_dm_view");CHKERRQ(ierr);
575c4762a1bSJed Brown     ierr = DMPlexCheckFaces(*dm, 0);CHKERRQ(ierr);
576c4762a1bSJed Brown     ierr = DMPlexOrientInterface_Internal(*dm);CHKERRQ(ierr);
577c4762a1bSJed Brown     ierr = PetscPrintf(comm, "Orientation test PASSED\n");CHKERRQ(ierr);
578c4762a1bSJed Brown   }
579c4762a1bSJed Brown   PetscFunctionReturn(0);
580c4762a1bSJed Brown }
581c4762a1bSJed Brown 
582c4762a1bSJed Brown static PetscErrorCode CreateQuad_2D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
583c4762a1bSJed Brown {
584c4762a1bSJed Brown   PetscInt       testNum = user->testNum, p;
585c4762a1bSJed Brown   PetscMPIInt    rank, size;
586c4762a1bSJed Brown   PetscErrorCode ierr;
587c4762a1bSJed Brown 
588c4762a1bSJed Brown   PetscFunctionBegin;
589ffc4695bSBarry Smith   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
590ffc4695bSBarry Smith   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
591c4762a1bSJed Brown   switch (testNum) {
592c4762a1bSJed Brown   case 0:
593c4762a1bSJed Brown     if (size != 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 2 processes", testNum);
594c4762a1bSJed Brown     switch (rank) {
595c4762a1bSJed Brown       case 0:
596c4762a1bSJed Brown       {
597c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 3, numCorners = 4;
598a4a685f2SJacob Faibussowitsch         const PetscInt cells[4]  = {0, 1, 2, 3};
599c4762a1bSJed Brown         PetscReal      coords[6] = {-0.5, 0.0, 0.0, 0.0, 0.0, 1.0};
600c4762a1bSJed Brown         PetscInt       markerPoints[4*2] = {1, 1, 2, 1, 3, 1, 4, 1};
601c4762a1bSJed Brown 
602be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
603c4762a1bSJed Brown         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
604c4762a1bSJed Brown       }
605c4762a1bSJed Brown       break;
606c4762a1bSJed Brown       case 1:
607c4762a1bSJed Brown       {
608c4762a1bSJed Brown         const PetscInt numCells  = 1, numVertices = 3, numCorners = 4;
609a4a685f2SJacob Faibussowitsch         const PetscInt cells[4]  = {1, 4, 5, 2};
610c4762a1bSJed Brown         PetscReal      coords[6] = {-0.5, 1.0, 0.5, 0.0, 0.5, 1.0};
611c4762a1bSJed Brown         PetscInt       markerPoints[4*2] = {1, 1, 2, 1, 3, 1, 4, 1};
612c4762a1bSJed Brown 
613be8c289dSNicolas Barral         ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
614c4762a1bSJed Brown         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
615c4762a1bSJed Brown       }
616c4762a1bSJed Brown       break;
617c4762a1bSJed Brown       default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
618c4762a1bSJed Brown     }
619c4762a1bSJed Brown     break;
620c4762a1bSJed Brown   default: SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh %D", testNum);
621c4762a1bSJed Brown   }
622c4762a1bSJed Brown   PetscFunctionReturn(0);
623c4762a1bSJed Brown }
624c4762a1bSJed Brown 
625c4762a1bSJed Brown static PetscErrorCode CreateHex_3D(MPI_Comm comm, PetscBool interpolate, AppCtx *user, DM *dm)
626c4762a1bSJed Brown {
627c4762a1bSJed Brown   PetscInt       testNum = user->testNum, p;
628c4762a1bSJed Brown   PetscMPIInt    rank, size;
629c4762a1bSJed Brown   PetscErrorCode ierr;
630c4762a1bSJed Brown 
631c4762a1bSJed Brown   PetscFunctionBegin;
632ffc4695bSBarry Smith   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
633ffc4695bSBarry Smith   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
634c4762a1bSJed Brown   switch (testNum) {
635c4762a1bSJed Brown   case 0:
636c4762a1bSJed Brown     if (size != 2) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Test mesh %d only for 2 processes", testNum);
637c4762a1bSJed Brown     switch (rank) {
638c4762a1bSJed Brown     case 0:
639c4762a1bSJed Brown     {
640c4762a1bSJed Brown       const PetscInt numCells  = 1, numVertices = 6, numCorners = 8;
641a4a685f2SJacob Faibussowitsch       const PetscInt cells[8]  = {0, 3, 2, 1, 4, 5, 6, 7};
642c4762a1bSJed Brown       PetscReal      coords[6*3] = {-0.5,0.0,0.0, 0.0,0.0,0.0, 0.0,1.0,0.0, -0.5,1.0,0.0, -0.5,0.0,1.0, 0.0,0.0,1.0};
643c4762a1bSJed Brown       PetscInt       markerPoints[8*2] = {2,1,3,1,4,1,5,1,6,1,7,1,8,1,9,1};
644c4762a1bSJed Brown 
645be8c289dSNicolas Barral       ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
646c4762a1bSJed Brown       for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
647c4762a1bSJed Brown     }
648c4762a1bSJed Brown     break;
649c4762a1bSJed Brown     case 1:
650c4762a1bSJed Brown     {
651c4762a1bSJed Brown       const PetscInt numCells  = 1, numVertices = 6, numCorners = 8;
652a4a685f2SJacob Faibussowitsch       const PetscInt cells[8]  = {1, 2, 9, 8, 5, 10, 11, 6};
653c4762a1bSJed Brown       PetscReal      coords[6*3] = {0.0,1.0,1.0, -0.5,1.0,1.0, 0.5,0.0,0.0, 0.5,1.0,0.0, 0.5,0.0,1.0,  0.5,1.0,1.0};
654c4762a1bSJed Brown       PetscInt       markerPoints[8*2] = {2,1,3,1,4,1,5,1,6,1,7,1,8,1,9,1};
655c4762a1bSJed Brown 
656be8c289dSNicolas Barral       ierr = DMPlexCreateFromCellListParallelPetsc(comm, user->dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, cells, user->dim, coords, NULL, NULL, dm);CHKERRQ(ierr);
657c4762a1bSJed Brown       for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(*dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
658c4762a1bSJed Brown     }
659c4762a1bSJed Brown     break;
660c4762a1bSJed Brown     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh for rank %d", rank);
661c4762a1bSJed Brown     }
662c4762a1bSJed Brown   break;
663c4762a1bSJed Brown   default: SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "No test mesh %D", testNum);
664c4762a1bSJed Brown   }
665c4762a1bSJed Brown   PetscFunctionReturn(0);
666c4762a1bSJed Brown }
667c4762a1bSJed Brown 
668c4762a1bSJed Brown static PetscErrorCode CustomView(DM dm, PetscViewer v)
669c4762a1bSJed Brown {
670c4762a1bSJed Brown   DMPlexInterpolatedFlag interpolated;
671c4762a1bSJed Brown   PetscBool              distributed;
672c4762a1bSJed Brown   PetscErrorCode         ierr;
673c4762a1bSJed Brown 
674c4762a1bSJed Brown   PetscFunctionBegin;
675c4762a1bSJed Brown   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
676c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(dm, &interpolated);CHKERRQ(ierr);
677c4762a1bSJed Brown   ierr = PetscViewerASCIIPrintf(v, "DMPlexIsDistributed: %s\n", PetscBools[distributed]);CHKERRQ(ierr);
678c4762a1bSJed Brown   ierr = PetscViewerASCIIPrintf(v, "DMPlexIsInterpolatedCollective: %s\n", DMPlexInterpolatedFlags[interpolated]);CHKERRQ(ierr);
679c4762a1bSJed Brown   PetscFunctionReturn(0);
680c4762a1bSJed Brown }
681c4762a1bSJed Brown 
682c4762a1bSJed Brown static PetscErrorCode CreateMeshFromFile(MPI_Comm comm, AppCtx *user, DM *dm, DM *serialDM)
683c4762a1bSJed Brown {
684c4762a1bSJed Brown   const char    *filename       = user->filename;
685c4762a1bSJed Brown   PetscBool      testHeavy      = user->testHeavy;
686c4762a1bSJed Brown   PetscBool      interpCreate   = user->interpolate == CREATE ? PETSC_TRUE : PETSC_FALSE;
687c4762a1bSJed Brown   PetscBool      distributed    = PETSC_FALSE;
688c4762a1bSJed Brown   PetscErrorCode ierr;
689c4762a1bSJed Brown 
690c4762a1bSJed Brown   PetscFunctionBegin;
691c4762a1bSJed Brown   *serialDM = NULL;
692c4762a1bSJed Brown   if (testHeavy && interpCreate) {ierr = DMPlexSetOrientInterface_Private(NULL, PETSC_FALSE);CHKERRQ(ierr);}
693c4762a1bSJed Brown   ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr);
694cd7e8a5eSksagiyam   ierr = DMPlexCreateFromFile(comm, filename, "ex18_plex", interpCreate, dm);CHKERRQ(ierr); /* with DMPlexOrientInterface_Internal() call skipped so that PointSF issues are left to DMPlexCheckPointSFHeavy() */
695c4762a1bSJed Brown   ierr = PetscLogStagePop();CHKERRQ(ierr);
696c4762a1bSJed Brown   if (testHeavy && interpCreate) {ierr = DMPlexSetOrientInterface_Private(NULL, PETSC_TRUE);CHKERRQ(ierr);}
697c4762a1bSJed Brown   ierr = DMPlexIsDistributed(*dm, &distributed);CHKERRQ(ierr);
698c4762a1bSJed Brown   ierr = PetscPrintf(comm, "DMPlexCreateFromFile produced %s mesh.\n", distributed ? "distributed" : "serial");CHKERRQ(ierr);
699c4762a1bSJed Brown   if (testHeavy && distributed) {
700c4762a1bSJed Brown     ierr = PetscOptionsSetValue(NULL, "-dm_plex_hdf5_force_sequential", NULL);CHKERRQ(ierr);
701cd7e8a5eSksagiyam     ierr = DMPlexCreateFromFile(comm, filename, "ex18_plex", interpCreate, serialDM);CHKERRQ(ierr);
702c4762a1bSJed Brown     ierr = DMPlexIsDistributed(*serialDM, &distributed);CHKERRQ(ierr);
703c4762a1bSJed Brown     if (distributed) SETERRQ(comm, PETSC_ERR_PLIB, "unable to create a serial DM from file");
704c4762a1bSJed Brown   }
705c4762a1bSJed Brown   ierr = DMGetDimension(*dm, &user->dim);CHKERRQ(ierr);
706c4762a1bSJed Brown   PetscFunctionReturn(0);
707c4762a1bSJed Brown }
708c4762a1bSJed Brown 
709c4762a1bSJed Brown static PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
710c4762a1bSJed Brown {
711c4762a1bSJed Brown   PetscPartitioner part;
712c4762a1bSJed Brown   PortableBoundary boundary     = NULL;
713c4762a1bSJed Brown   DM             serialDM       = NULL;
714c4762a1bSJed Brown   PetscBool      cellSimplex    = user->cellSimplex;
715c4762a1bSJed Brown   PetscBool      useGenerator   = user->useGenerator;
716c4762a1bSJed Brown   PetscBool      interpCreate   = user->interpolate == CREATE ? PETSC_TRUE : PETSC_FALSE;
717c4762a1bSJed Brown   PetscBool      interpSerial   = user->interpolate == AFTER_CREATE ? PETSC_TRUE : PETSC_FALSE;
718c4762a1bSJed Brown   PetscBool      interpParallel = user->interpolate == AFTER_DISTRIBUTE ? PETSC_TRUE : PETSC_FALSE;
719c4762a1bSJed Brown   PetscBool      testHeavy      = user->testHeavy;
720c4762a1bSJed Brown   PetscMPIInt    rank;
721c4762a1bSJed Brown   PetscErrorCode ierr;
722c4762a1bSJed Brown 
723c4762a1bSJed Brown   PetscFunctionBegin;
724ffc4695bSBarry Smith   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
725c4762a1bSJed Brown   if (user->filename[0]) {
726c4762a1bSJed Brown     ierr = CreateMeshFromFile(comm, user, dm, &serialDM);CHKERRQ(ierr);
727c4762a1bSJed Brown   } else if (useGenerator) {
728c4762a1bSJed Brown     ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr);
729c4762a1bSJed Brown     ierr = DMPlexCreateBoxMesh(comm, user->dim, cellSimplex, user->faces, NULL, NULL, NULL, interpCreate, dm);CHKERRQ(ierr);
730c4762a1bSJed Brown     ierr = PetscLogStagePop();CHKERRQ(ierr);
731c4762a1bSJed Brown   } else {
732c4762a1bSJed Brown     ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr);
733c4762a1bSJed Brown     switch (user->dim) {
734c4762a1bSJed Brown     case 1:
735c4762a1bSJed Brown       ierr = CreateMesh_1D(comm, interpCreate, user, dm);CHKERRQ(ierr);
736c4762a1bSJed Brown       break;
737c4762a1bSJed Brown     case 2:
738c4762a1bSJed Brown       if (cellSimplex) {
739c4762a1bSJed Brown         ierr = CreateSimplex_2D(comm, interpCreate, user, dm);CHKERRQ(ierr);
740c4762a1bSJed Brown       } else {
741c4762a1bSJed Brown         ierr = CreateQuad_2D(comm, interpCreate, user, dm);CHKERRQ(ierr);
742c4762a1bSJed Brown       }
743c4762a1bSJed Brown       break;
744c4762a1bSJed Brown     case 3:
745c4762a1bSJed Brown       if (cellSimplex) {
746c4762a1bSJed Brown         ierr = CreateSimplex_3D(comm, interpCreate, user, dm);CHKERRQ(ierr);
747c4762a1bSJed Brown       } else {
748c4762a1bSJed Brown         ierr = CreateHex_3D(comm, interpCreate, user, dm);CHKERRQ(ierr);
749c4762a1bSJed Brown       }
750c4762a1bSJed Brown       break;
751c4762a1bSJed Brown     default:
752c4762a1bSJed Brown       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %D", user->dim);
753c4762a1bSJed Brown     }
754c4762a1bSJed Brown     ierr = PetscLogStagePop();CHKERRQ(ierr);
755c4762a1bSJed Brown   }
756c4762a1bSJed Brown   if (user->ncoords % user->dim) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "length of coordinates array %D must be divisable by spatial dimension %D", user->ncoords, user->dim);
757c4762a1bSJed Brown   ierr = PetscObjectSetName((PetscObject) *dm, "Original Mesh");CHKERRQ(ierr);
758c4762a1bSJed Brown   ierr = DMViewFromOptions(*dm, NULL, "-orig_dm_view");CHKERRQ(ierr);
759c4762a1bSJed Brown 
760c4762a1bSJed Brown   if (interpSerial) {
761c4762a1bSJed Brown     DM idm;
762c4762a1bSJed Brown 
763c4762a1bSJed Brown     if (testHeavy) {ierr = DMPlexSetOrientInterface_Private(*dm, PETSC_FALSE);CHKERRQ(ierr);}
764c4762a1bSJed Brown     ierr = PetscLogStagePush(stage[2]);CHKERRQ(ierr);
765c4762a1bSJed Brown     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr); /* with DMPlexOrientInterface_Internal() call skipped so that PointSF issues are left to DMPlexCheckPointSFHeavy() */
766c4762a1bSJed Brown     ierr = PetscLogStagePop();CHKERRQ(ierr);
767c4762a1bSJed Brown     if (testHeavy) {ierr = DMPlexSetOrientInterface_Private(*dm, PETSC_TRUE);CHKERRQ(ierr);}
768c4762a1bSJed Brown     ierr = DMDestroy(dm);CHKERRQ(ierr);
769c4762a1bSJed Brown     *dm = idm;
770c4762a1bSJed Brown     ierr = PetscObjectSetName((PetscObject) *dm, "Interpolated Mesh");CHKERRQ(ierr);
771c4762a1bSJed Brown     ierr = DMViewFromOptions(*dm, NULL, "-intp_dm_view");CHKERRQ(ierr);
772c4762a1bSJed Brown   }
773c4762a1bSJed Brown 
774c4762a1bSJed Brown   /* Set partitioner options */
775c4762a1bSJed Brown   ierr = DMPlexGetPartitioner(*dm, &part);CHKERRQ(ierr);
776c4762a1bSJed Brown   if (part) {
777c4762a1bSJed Brown     ierr = PetscPartitionerSetType(part, PETSCPARTITIONERSIMPLE);CHKERRQ(ierr);
778c4762a1bSJed Brown     ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
779c4762a1bSJed Brown   }
780c4762a1bSJed Brown 
781c4762a1bSJed Brown   if (user->customView) {ierr = CustomView(*dm, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);}
782c4762a1bSJed Brown   if (testHeavy) {
783c4762a1bSJed Brown     PetscBool distributed;
784c4762a1bSJed Brown 
785c4762a1bSJed Brown     ierr = DMPlexIsDistributed(*dm, &distributed);CHKERRQ(ierr);
786c4762a1bSJed Brown     if (!serialDM && !distributed) {
787c4762a1bSJed Brown       serialDM = *dm;
788c4762a1bSJed Brown       ierr = PetscObjectReference((PetscObject)*dm);CHKERRQ(ierr);
789c4762a1bSJed Brown     }
790c4762a1bSJed Brown     if (serialDM) {
791c4762a1bSJed Brown       ierr = DMPlexGetExpandedBoundary_Private(serialDM, &boundary);CHKERRQ(ierr);
792c4762a1bSJed Brown     }
793c4762a1bSJed Brown     if (boundary) {
794c4762a1bSJed Brown       /* check DM which has been created in parallel and already interpolated */
795c4762a1bSJed Brown       ierr = DMPlexCheckPointSFHeavy(*dm, boundary);CHKERRQ(ierr);
796c4762a1bSJed Brown     }
797c4762a1bSJed Brown     /* Orient interface because it could be deliberately skipped above. It is idempotent. */
798c4762a1bSJed Brown     ierr = DMPlexOrientInterface_Internal(*dm);CHKERRQ(ierr);
799c4762a1bSJed Brown   }
800c4762a1bSJed Brown   if (user->distribute) {
801c4762a1bSJed Brown     DM               pdm = NULL;
802c4762a1bSJed Brown 
803c4762a1bSJed Brown     /* Redistribute mesh over processes using that partitioner */
804c4762a1bSJed Brown     ierr = PetscLogStagePush(stage[1]);CHKERRQ(ierr);
805c4762a1bSJed Brown     ierr = DMPlexDistribute(*dm, 0, NULL, &pdm);CHKERRQ(ierr);
806c4762a1bSJed Brown     ierr = PetscLogStagePop();CHKERRQ(ierr);
807c4762a1bSJed Brown     if (pdm) {
808c4762a1bSJed Brown       ierr = DMDestroy(dm);CHKERRQ(ierr);
809c4762a1bSJed Brown       *dm  = pdm;
810c4762a1bSJed Brown       ierr = PetscObjectSetName((PetscObject) *dm, "Redistributed Mesh");CHKERRQ(ierr);
811c4762a1bSJed Brown       ierr = DMViewFromOptions(*dm, NULL, "-dist_dm_view");CHKERRQ(ierr);
812c4762a1bSJed Brown     }
813c4762a1bSJed Brown 
814c4762a1bSJed Brown     if (interpParallel) {
815c4762a1bSJed Brown       DM idm;
816c4762a1bSJed Brown 
817c4762a1bSJed Brown       if (testHeavy) {ierr = DMPlexSetOrientInterface_Private(*dm, PETSC_FALSE);CHKERRQ(ierr);}
818c4762a1bSJed Brown       ierr = PetscLogStagePush(stage[2]);CHKERRQ(ierr);
819c4762a1bSJed Brown       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr); /* with DMPlexOrientInterface_Internal() call skipped so that PointSF issues are left to DMPlexCheckPointSFHeavy() */
820c4762a1bSJed Brown       ierr = PetscLogStagePop();CHKERRQ(ierr);
821c4762a1bSJed Brown       if (testHeavy) {ierr = DMPlexSetOrientInterface_Private(*dm, PETSC_TRUE);CHKERRQ(ierr);}
822c4762a1bSJed Brown       ierr = DMDestroy(dm);CHKERRQ(ierr);
823c4762a1bSJed Brown       *dm = idm;
824c4762a1bSJed Brown       ierr = PetscObjectSetName((PetscObject) *dm, "Interpolated Redistributed Mesh");CHKERRQ(ierr);
825c4762a1bSJed Brown       ierr = DMViewFromOptions(*dm, NULL, "-intp_dm_view");CHKERRQ(ierr);
826c4762a1bSJed Brown     }
827c4762a1bSJed Brown   }
828c4762a1bSJed Brown   if (testHeavy) {
829c4762a1bSJed Brown     if (boundary) {
830c4762a1bSJed Brown       ierr = DMPlexCheckPointSFHeavy(*dm, boundary);CHKERRQ(ierr);
831c4762a1bSJed Brown     }
832c4762a1bSJed Brown     /* Orient interface because it could be deliberately skipped above. It is idempotent. */
833c4762a1bSJed Brown     ierr = DMPlexOrientInterface_Internal(*dm);CHKERRQ(ierr);
834c4762a1bSJed Brown   }
835c4762a1bSJed Brown 
836c4762a1bSJed Brown   ierr = PetscObjectSetName((PetscObject) *dm, "Parallel Mesh");CHKERRQ(ierr);
837c4762a1bSJed Brown   ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
838c4762a1bSJed Brown   ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
839c4762a1bSJed Brown 
840c4762a1bSJed Brown   if (user->customView) {ierr = CustomView(*dm, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);}
841c4762a1bSJed Brown   ierr = DMDestroy(&serialDM);CHKERRQ(ierr);
842c4762a1bSJed Brown   ierr = PortableBoundaryDestroy(&boundary);CHKERRQ(ierr);
843c4762a1bSJed Brown   PetscFunctionReturn(0);
844c4762a1bSJed Brown }
845c4762a1bSJed Brown 
846c4762a1bSJed Brown PETSC_STATIC_INLINE PetscErrorCode coord2str(char buf[], size_t len, PetscInt dim, PetscReal *coords, PetscReal tol)
847c4762a1bSJed Brown {
848c4762a1bSJed Brown   PetscErrorCode ierr;
849c4762a1bSJed Brown 
850c4762a1bSJed Brown   PetscFunctionBegin;
851c4762a1bSJed Brown   if (dim > 3) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "dim must be less than or equal 3");
852c4762a1bSJed Brown   if (tol >= 1e-3) {
853c4762a1bSJed Brown     switch (dim) {
854c4762a1bSJed Brown       case 1: ierr = PetscSNPrintf(buf,len,"(%12.3f)",(double)coords[0]);CHKERRQ(ierr);
855c4762a1bSJed Brown       case 2: ierr = PetscSNPrintf(buf,len,"(%12.3f, %12.3f)",(double)coords[0],(double)coords[1]);CHKERRQ(ierr);
856c4762a1bSJed Brown       case 3: ierr = PetscSNPrintf(buf,len,"(%12.3f, %12.3f, %12.3f)",(double)coords[0],(double)coords[1],(double)coords[2]);CHKERRQ(ierr);
857c4762a1bSJed Brown     }
858c4762a1bSJed Brown   } else {
859c4762a1bSJed Brown     switch (dim) {
860c4762a1bSJed Brown       case 1: ierr = PetscSNPrintf(buf,len,"(%12.6f)",(double)coords[0]);CHKERRQ(ierr);
861c4762a1bSJed Brown       case 2: ierr = PetscSNPrintf(buf,len,"(%12.6f, %12.6f)",(double)coords[0],(double)coords[1]);CHKERRQ(ierr);
862c4762a1bSJed Brown       case 3: ierr = PetscSNPrintf(buf,len,"(%12.6f, %12.6f, %12.6f)",(double)coords[0],(double)coords[1],(double)coords[2]);CHKERRQ(ierr);
863c4762a1bSJed Brown     }
864c4762a1bSJed Brown   }
865c4762a1bSJed Brown   PetscFunctionReturn(0);
866c4762a1bSJed Brown }
867c4762a1bSJed Brown 
868c4762a1bSJed Brown static PetscErrorCode ViewVerticesFromCoords(DM dm, PetscInt npoints, PetscReal coords[], PetscReal tol, PetscViewer viewer)
869c4762a1bSJed Brown {
870c4762a1bSJed Brown   PetscInt       dim, i;
871c4762a1bSJed Brown   PetscInt       *points;
872c4762a1bSJed Brown   char           coordstr[128];
873c4762a1bSJed Brown   MPI_Comm       comm;
874c4762a1bSJed Brown   PetscMPIInt    rank;
875c4762a1bSJed Brown   PetscErrorCode ierr;
876c4762a1bSJed Brown 
877c4762a1bSJed Brown   PetscFunctionBegin;
878c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
879ffc4695bSBarry Smith   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
880c4762a1bSJed Brown   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
881c4762a1bSJed Brown   ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
882c4762a1bSJed Brown   ierr = PetscMalloc1(npoints, &points);CHKERRQ(ierr);
883c4762a1bSJed Brown   ierr = DMPlexFindVertices(dm, npoints, coords, tol, points);CHKERRQ(ierr);
884c4762a1bSJed Brown   for (i=0; i < npoints; i++) {
885c4762a1bSJed Brown     ierr = coord2str(coordstr, sizeof(coordstr), dim, &coords[i*dim], tol);CHKERRQ(ierr);
886dd400576SPatrick Sanan     if (rank == 0 && i) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, "-----\n");CHKERRQ(ierr);}
887c4762a1bSJed Brown     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] %s --> points[%D] = %D\n", rank, coordstr, i, points[i]);CHKERRQ(ierr);
888c4762a1bSJed Brown     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
889c4762a1bSJed Brown   }
890c4762a1bSJed Brown   ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
891c4762a1bSJed Brown   ierr = PetscFree(points);CHKERRQ(ierr);
892c4762a1bSJed Brown   PetscFunctionReturn(0);
893c4762a1bSJed Brown }
894c4762a1bSJed Brown 
895c4762a1bSJed Brown static PetscErrorCode TestExpandPoints(DM dm, AppCtx *user)
896c4762a1bSJed Brown {
897c4762a1bSJed Brown   IS                is;
898c4762a1bSJed Brown   PetscSection      *sects;
899c4762a1bSJed Brown   IS                *iss;
900c4762a1bSJed Brown   PetscInt          d,depth;
901c4762a1bSJed Brown   PetscMPIInt       rank;
902c4762a1bSJed Brown   PetscErrorCode    ierr;
903c4762a1bSJed Brown   PetscViewer       viewer=PETSC_VIEWER_STDOUT_WORLD, sviewer;
904c4762a1bSJed Brown 
905c4762a1bSJed Brown   PetscFunctionBegin;
906ffc4695bSBarry Smith   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm),&rank);CHKERRMPI(ierr);
907dd400576SPatrick Sanan   if (user->testExpandPointsEmpty && rank == 0) {
908c4762a1bSJed Brown     ierr = ISCreateGeneral(PETSC_COMM_SELF, 0, NULL, PETSC_USE_POINTER, &is);CHKERRQ(ierr);
909c4762a1bSJed Brown   } else {
910c4762a1bSJed Brown     ierr = ISCreateGeneral(PETSC_COMM_SELF, user->nPointsToExpand, user->pointsToExpand, PETSC_USE_POINTER, &is);CHKERRQ(ierr);
911c4762a1bSJed Brown   }
912c4762a1bSJed Brown   ierr = DMPlexGetConeRecursive(dm, is, &depth, &iss, &sects);CHKERRQ(ierr);
913c4762a1bSJed Brown   ierr = PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
914c4762a1bSJed Brown   ierr = PetscViewerASCIIPrintf(sviewer, "[%d] ==========================\n",rank);CHKERRQ(ierr);
915c4762a1bSJed Brown   for (d=depth-1; d>=0; d--) {
916c4762a1bSJed Brown     IS          checkIS;
917c4762a1bSJed Brown     PetscBool   flg;
918c4762a1bSJed Brown 
919c4762a1bSJed Brown     ierr = PetscViewerASCIIPrintf(sviewer, "depth %D ---------------\n",d);CHKERRQ(ierr);
920c4762a1bSJed Brown     ierr = PetscSectionView(sects[d], sviewer);CHKERRQ(ierr);
921c4762a1bSJed Brown     ierr = ISView(iss[d], sviewer);CHKERRQ(ierr);
922c4762a1bSJed Brown     /* check reverse operation */
923c4762a1bSJed Brown     if (d < depth-1) {
924c4762a1bSJed Brown       ierr = DMPlexExpandedConesToFaces_Private(dm, iss[d], sects[d], &checkIS);CHKERRQ(ierr);
925c4762a1bSJed Brown       ierr = ISEqualUnsorted(checkIS, iss[d+1], &flg);CHKERRQ(ierr);
926c4762a1bSJed Brown       if (!flg) SETERRQ(PetscObjectComm((PetscObject) checkIS), PETSC_ERR_PLIB, "DMPlexExpandedConesToFaces_Private produced wrong IS");
927c4762a1bSJed Brown       ierr = ISDestroy(&checkIS);CHKERRQ(ierr);
928c4762a1bSJed Brown     }
929c4762a1bSJed Brown   }
930c4762a1bSJed Brown   ierr = PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
931c4762a1bSJed Brown   ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
932c4762a1bSJed Brown   ierr = DMPlexRestoreConeRecursive(dm, is, &depth, &iss, &sects);CHKERRQ(ierr);
933c4762a1bSJed Brown   ierr = ISDestroy(&is);CHKERRQ(ierr);
934c4762a1bSJed Brown   PetscFunctionReturn(0);
935c4762a1bSJed Brown }
936c4762a1bSJed Brown 
937c4762a1bSJed Brown static PetscErrorCode DMPlexExpandedConesToFaces_Private(DM dm, IS is, PetscSection section, IS *newis)
938c4762a1bSJed Brown {
939c4762a1bSJed Brown   PetscInt          n,n1,ncone,numCoveredPoints,o,p,q,start,end;
940c4762a1bSJed Brown   const PetscInt    *coveredPoints;
941c4762a1bSJed Brown   const PetscInt    *arr, *cone;
942c4762a1bSJed Brown   PetscInt          *newarr;
943c4762a1bSJed Brown   PetscErrorCode    ierr;
944c4762a1bSJed Brown 
945c4762a1bSJed Brown   PetscFunctionBegin;
946c4762a1bSJed Brown   ierr = ISGetLocalSize(is, &n);CHKERRQ(ierr);
947c4762a1bSJed Brown   ierr = PetscSectionGetStorageSize(section, &n1);CHKERRQ(ierr);
948c4762a1bSJed Brown   ierr = PetscSectionGetChart(section, &start, &end);CHKERRQ(ierr);
949c4762a1bSJed Brown   if (n != n1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "IS size = %D != %D = section storage size\n", n, n1);
950c4762a1bSJed Brown   ierr = ISGetIndices(is, &arr);CHKERRQ(ierr);
951c4762a1bSJed Brown   ierr = PetscMalloc1(end-start, &newarr);CHKERRQ(ierr);
952c4762a1bSJed Brown   for (q=start; q<end; q++) {
953c4762a1bSJed Brown     ierr = PetscSectionGetDof(section, q, &ncone);CHKERRQ(ierr);
954c4762a1bSJed Brown     ierr = PetscSectionGetOffset(section, q, &o);CHKERRQ(ierr);
955c4762a1bSJed Brown     cone = &arr[o];
956c4762a1bSJed Brown     if (ncone == 1) {
957c4762a1bSJed Brown       numCoveredPoints = 1;
958c4762a1bSJed Brown       p = cone[0];
959c4762a1bSJed Brown     } else {
960c4762a1bSJed Brown       PetscInt i;
961c4762a1bSJed Brown       p = PETSC_MAX_INT;
962c4762a1bSJed Brown       for (i=0; i<ncone; i++) if (cone[i] < 0) {p = -1; break;}
963c4762a1bSJed Brown       if (p >= 0) {
964c4762a1bSJed Brown         ierr = DMPlexGetJoin(dm, ncone, cone, &numCoveredPoints, &coveredPoints);CHKERRQ(ierr);
965c4762a1bSJed Brown         if (numCoveredPoints > 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "more than one covered points for section point %D",q);
966c4762a1bSJed Brown         if (numCoveredPoints) p = coveredPoints[0];
967c4762a1bSJed Brown         else                  p = -2;
968c4762a1bSJed Brown         ierr = DMPlexRestoreJoin(dm, ncone, cone, &numCoveredPoints, &coveredPoints);CHKERRQ(ierr);
969c4762a1bSJed Brown       }
970c4762a1bSJed Brown     }
971c4762a1bSJed Brown     newarr[q-start] = p;
972c4762a1bSJed Brown   }
973c4762a1bSJed Brown   ierr = ISRestoreIndices(is, &arr);CHKERRQ(ierr);
974c4762a1bSJed Brown   ierr = ISCreateGeneral(PETSC_COMM_SELF, end-start, newarr, PETSC_OWN_POINTER, newis);CHKERRQ(ierr);
975c4762a1bSJed Brown   PetscFunctionReturn(0);
976c4762a1bSJed Brown }
977c4762a1bSJed Brown 
978c4762a1bSJed Brown static PetscErrorCode DMPlexExpandedVerticesToFaces_Private(DM dm, IS boundary_expanded_is, PetscInt depth, PetscSection sections[], IS *boundary_is)
979c4762a1bSJed Brown {
980c4762a1bSJed Brown   PetscInt          d;
981c4762a1bSJed Brown   IS                is,newis;
982c4762a1bSJed Brown   PetscErrorCode    ierr;
983c4762a1bSJed Brown 
984c4762a1bSJed Brown   PetscFunctionBegin;
985c4762a1bSJed Brown   is = boundary_expanded_is;
986c4762a1bSJed Brown   ierr = PetscObjectReference((PetscObject)is);CHKERRQ(ierr);
987c4762a1bSJed Brown   for (d = 0; d < depth-1; ++d) {
988c4762a1bSJed Brown     ierr = DMPlexExpandedConesToFaces_Private(dm, is, sections[d], &newis);CHKERRQ(ierr);
989c4762a1bSJed Brown     ierr = ISDestroy(&is);CHKERRQ(ierr);
990c4762a1bSJed Brown     is = newis;
991c4762a1bSJed Brown   }
992c4762a1bSJed Brown   *boundary_is = is;
993c4762a1bSJed Brown   PetscFunctionReturn(0);
994c4762a1bSJed Brown }
995c4762a1bSJed Brown 
996c4762a1bSJed Brown #define CHKERRQI(incall,ierr) if (ierr) {incall = PETSC_FALSE; }
997c4762a1bSJed Brown 
998c4762a1bSJed Brown static PetscErrorCode DMLabelViewFromOptionsOnComm_Private(DMLabel label, const char optionname[], MPI_Comm comm)
999c4762a1bSJed Brown {
1000c4762a1bSJed Brown   PetscErrorCode    ierr;
1001c4762a1bSJed Brown   PetscViewer       viewer;
1002c4762a1bSJed Brown   PetscBool         flg;
1003c4762a1bSJed Brown   static PetscBool  incall = PETSC_FALSE;
1004c4762a1bSJed Brown   PetscViewerFormat format;
1005c4762a1bSJed Brown 
1006c4762a1bSJed Brown   PetscFunctionBegin;
1007c4762a1bSJed Brown   if (incall) PetscFunctionReturn(0);
1008c4762a1bSJed Brown   incall = PETSC_TRUE;
1009c4762a1bSJed Brown   ierr   = PetscOptionsGetViewer(comm,((PetscObject)label)->options,((PetscObject)label)->prefix,optionname,&viewer,&format,&flg);CHKERRQI(incall,ierr);
1010c4762a1bSJed Brown   if (flg) {
1011c4762a1bSJed Brown     ierr = PetscViewerPushFormat(viewer,format);CHKERRQI(incall,ierr);
1012c4762a1bSJed Brown     ierr = DMLabelView(label, viewer);CHKERRQI(incall,ierr);
1013c4762a1bSJed Brown     ierr = PetscViewerPopFormat(viewer);CHKERRQI(incall,ierr);
1014c4762a1bSJed Brown     ierr = PetscViewerDestroy(&viewer);CHKERRQI(incall,ierr);
1015c4762a1bSJed Brown   }
1016c4762a1bSJed Brown   incall = PETSC_FALSE;
1017c4762a1bSJed Brown   PetscFunctionReturn(0);
1018c4762a1bSJed Brown }
1019c4762a1bSJed Brown 
1020c4762a1bSJed Brown /* TODO: this is hotfixing DMLabelGetStratumIS() - it should be fixed systematically instead */
1021c4762a1bSJed Brown PETSC_STATIC_INLINE PetscErrorCode DMLabelGetStratumISOnComm_Private(DMLabel label, PetscInt value, MPI_Comm comm, IS *is)
1022c4762a1bSJed Brown {
1023c4762a1bSJed Brown   IS                tmpis;
1024c4762a1bSJed Brown   PetscErrorCode    ierr;
1025c4762a1bSJed Brown 
1026c4762a1bSJed Brown   PetscFunctionBegin;
1027c4762a1bSJed Brown   ierr = DMLabelGetStratumIS(label, value, &tmpis);CHKERRQ(ierr);
1028c4762a1bSJed Brown   if (!tmpis) {ierr = ISCreateGeneral(PETSC_COMM_SELF, 0, NULL, PETSC_USE_POINTER, &tmpis);CHKERRQ(ierr);}
1029c4762a1bSJed Brown   ierr = ISOnComm(tmpis, comm, PETSC_COPY_VALUES, is);CHKERRQ(ierr);
1030c4762a1bSJed Brown   ierr = ISDestroy(&tmpis);CHKERRQ(ierr);
1031c4762a1bSJed Brown   PetscFunctionReturn(0);
1032c4762a1bSJed Brown }
1033c4762a1bSJed Brown 
1034c4762a1bSJed Brown /* currently only for simple PetscSection without fields or constraints */
1035c4762a1bSJed Brown static PetscErrorCode PetscSectionReplicate_Private(MPI_Comm comm, PetscMPIInt rootrank, PetscSection sec0, PetscSection *secout)
1036c4762a1bSJed Brown {
1037c4762a1bSJed Brown   PetscSection      sec;
1038c4762a1bSJed Brown   PetscInt          chart[2], p;
1039c4762a1bSJed Brown   PetscInt          *dofarr;
1040c4762a1bSJed Brown   PetscMPIInt       rank;
1041c4762a1bSJed Brown   PetscErrorCode    ierr;
1042c4762a1bSJed Brown 
1043c4762a1bSJed Brown   PetscFunctionBegin;
1044ffc4695bSBarry Smith   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1045c4762a1bSJed Brown   if (rank == rootrank) {
1046c4762a1bSJed Brown     ierr = PetscSectionGetChart(sec0, &chart[0], &chart[1]);CHKERRQ(ierr);
1047c4762a1bSJed Brown   }
1048ffc4695bSBarry Smith   ierr = MPI_Bcast(chart, 2, MPIU_INT, rootrank, comm);CHKERRMPI(ierr);
1049c4762a1bSJed Brown   ierr = PetscMalloc1(chart[1]-chart[0], &dofarr);CHKERRQ(ierr);
1050c4762a1bSJed Brown   if (rank == rootrank) {
1051c4762a1bSJed Brown     for (p = chart[0]; p < chart[1]; p++) {
1052c4762a1bSJed Brown       ierr = PetscSectionGetDof(sec0, p, &dofarr[p-chart[0]]);CHKERRQ(ierr);
1053c4762a1bSJed Brown     }
1054c4762a1bSJed Brown   }
1055ffc4695bSBarry Smith   ierr = MPI_Bcast(dofarr, chart[1]-chart[0], MPIU_INT, rootrank, comm);CHKERRMPI(ierr);
1056c4762a1bSJed Brown   ierr = PetscSectionCreate(comm, &sec);CHKERRQ(ierr);
1057c4762a1bSJed Brown   ierr = PetscSectionSetChart(sec, chart[0], chart[1]);CHKERRQ(ierr);
1058c4762a1bSJed Brown   for (p = chart[0]; p < chart[1]; p++) {
1059c4762a1bSJed Brown     ierr = PetscSectionSetDof(sec, p, dofarr[p-chart[0]]);CHKERRQ(ierr);
1060c4762a1bSJed Brown   }
1061c4762a1bSJed Brown   ierr = PetscSectionSetUp(sec);CHKERRQ(ierr);
1062c4762a1bSJed Brown   ierr = PetscFree(dofarr);CHKERRQ(ierr);
1063c4762a1bSJed Brown   *secout = sec;
1064c4762a1bSJed Brown   PetscFunctionReturn(0);
1065c4762a1bSJed Brown }
1066c4762a1bSJed Brown 
1067c4762a1bSJed Brown static PetscErrorCode VecToPetscReal_Private(Vec vec, PetscReal *rvals[])
1068c4762a1bSJed Brown {
1069c4762a1bSJed Brown   PetscInt          n;
1070c4762a1bSJed Brown   const PetscScalar *svals;
1071c4762a1bSJed Brown   PetscErrorCode    ierr;
1072c4762a1bSJed Brown 
1073c4762a1bSJed Brown   PetscFunctionBegin;
1074c4762a1bSJed Brown   ierr = VecGetLocalSize(vec, &n);CHKERRQ(ierr);
1075c4762a1bSJed Brown   ierr = VecGetArrayRead(vec, &svals);CHKERRQ(ierr);
1076c4762a1bSJed Brown   ierr = PetscMalloc1(n, rvals);CHKERRQ(ierr);
1077c4762a1bSJed Brown #if defined(PETSC_USE_COMPLEX)
1078c4762a1bSJed Brown   {
1079c4762a1bSJed Brown     PetscInt i;
1080c4762a1bSJed Brown     for (i=0; i<n; i++) (*rvals)[i] = PetscRealPart(svals[i]);
1081c4762a1bSJed Brown   }
1082c4762a1bSJed Brown #else
10831e1ea65dSPierre Jolivet   ierr = PetscMemcpy(*rvals, svals, n*sizeof(PetscReal));CHKERRQ(ierr);
1084c4762a1bSJed Brown #endif
1085c4762a1bSJed Brown   ierr = VecRestoreArrayRead(vec, &svals);CHKERRQ(ierr);
1086c4762a1bSJed Brown   PetscFunctionReturn(0);
1087c4762a1bSJed Brown }
1088c4762a1bSJed Brown 
1089c4762a1bSJed Brown static PetscErrorCode DMPlexExpandedVerticesCoordinatesToFaces_Private(DM ipdm, PortableBoundary bnd, IS *face_is)
1090c4762a1bSJed Brown {
1091c4762a1bSJed Brown   PetscInt            dim, ncoords, npoints;
1092c4762a1bSJed Brown   PetscReal           *rcoords;
1093c4762a1bSJed Brown   PetscInt            *points;
1094c4762a1bSJed Brown   IS                  faces_expanded_is;
1095c4762a1bSJed Brown   PetscErrorCode      ierr;
1096c4762a1bSJed Brown 
1097c4762a1bSJed Brown   PetscFunctionBegin;
1098c4762a1bSJed Brown   ierr = DMGetDimension(ipdm, &dim);CHKERRQ(ierr);
1099c4762a1bSJed Brown   ierr = VecGetLocalSize(bnd->coordinates, &ncoords);CHKERRQ(ierr);
1100c4762a1bSJed Brown   ierr = VecToPetscReal_Private(bnd->coordinates, &rcoords);CHKERRQ(ierr);
1101c4762a1bSJed Brown   npoints = ncoords / dim;
1102c4762a1bSJed Brown   ierr = PetscMalloc1(npoints, &points);CHKERRQ(ierr);
1103c4762a1bSJed Brown   ierr = DMPlexFindVertices(ipdm, npoints, rcoords, 0.0, points);CHKERRQ(ierr);
1104c4762a1bSJed Brown   ierr = ISCreateGeneral(PETSC_COMM_SELF, npoints, points, PETSC_OWN_POINTER, &faces_expanded_is);CHKERRQ(ierr);
1105c4762a1bSJed Brown   ierr = DMPlexExpandedVerticesToFaces_Private(ipdm, faces_expanded_is, bnd->depth, bnd->sections, face_is);CHKERRQ(ierr);
1106c4762a1bSJed Brown   ierr = PetscFree(rcoords);CHKERRQ(ierr);
1107c4762a1bSJed Brown   ierr = ISDestroy(&faces_expanded_is);CHKERRQ(ierr);
1108c4762a1bSJed Brown   PetscFunctionReturn(0);
1109c4762a1bSJed Brown }
1110c4762a1bSJed Brown 
1111c4762a1bSJed Brown /* hack disabling DMPlexOrientInterface() call in DMPlexInterpolate() via -dm_plex_interpolate_orient_interfaces option */
1112c4762a1bSJed Brown static PetscErrorCode DMPlexSetOrientInterface_Private(DM dm, PetscBool enable)
1113c4762a1bSJed Brown {
1114c4762a1bSJed Brown   PetscOptions      options = NULL;
1115c4762a1bSJed Brown   const char        *prefix = NULL;
1116c4762a1bSJed Brown   const char        opt[] = "-dm_plex_interpolate_orient_interfaces";
1117c4762a1bSJed Brown   char              prefix_opt[512];
1118c4762a1bSJed Brown   PetscBool         flg, set;
1119c4762a1bSJed Brown   static PetscBool  wasSetTrue = PETSC_FALSE;
1120c4762a1bSJed Brown   PetscErrorCode    ierr;
1121c4762a1bSJed Brown 
1122c4762a1bSJed Brown   PetscFunctionBegin;
1123c4762a1bSJed Brown   if (dm) {
1124c4762a1bSJed Brown     ierr = PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix);CHKERRQ(ierr);
1125c4762a1bSJed Brown     options = ((PetscObject)dm)->options;
1126c4762a1bSJed Brown   }
1127c4762a1bSJed Brown   ierr = PetscStrcpy(prefix_opt, "-");CHKERRQ(ierr);
1128c4762a1bSJed Brown   ierr = PetscStrlcat(prefix_opt, prefix, sizeof(prefix_opt));CHKERRQ(ierr);
1129c4762a1bSJed Brown   ierr = PetscStrlcat(prefix_opt, &opt[1], sizeof(prefix_opt));CHKERRQ(ierr);
1130c4762a1bSJed Brown   ierr = PetscOptionsGetBool(options, prefix, opt, &flg, &set);CHKERRQ(ierr);
1131c4762a1bSJed Brown   if (!enable) {
1132c4762a1bSJed Brown     if (set && flg) wasSetTrue = PETSC_TRUE;
1133c4762a1bSJed Brown     ierr = PetscOptionsSetValue(options, prefix_opt, "0");CHKERRQ(ierr);
1134c4762a1bSJed Brown   } else if (set && !flg) {
1135c4762a1bSJed Brown     if (wasSetTrue) {
1136c4762a1bSJed Brown       ierr = PetscOptionsSetValue(options, prefix_opt, "1");CHKERRQ(ierr);
1137c4762a1bSJed Brown     } else {
1138c4762a1bSJed Brown       /* default is PETSC_TRUE */
1139c4762a1bSJed Brown       ierr = PetscOptionsClearValue(options, prefix_opt);CHKERRQ(ierr);
1140c4762a1bSJed Brown     }
1141c4762a1bSJed Brown     wasSetTrue = PETSC_FALSE;
1142c4762a1bSJed Brown   }
114376bd3646SJed Brown   if (PetscDefined(USE_DEBUG)) {
1144c4762a1bSJed Brown     ierr = PetscOptionsGetBool(options, prefix, opt, &flg, &set);CHKERRQ(ierr);
1145c4762a1bSJed Brown     if (PetscUnlikely(set && flg != enable)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "PetscOptionsSetValue did not have the desired effect");
1146c4762a1bSJed Brown   }
1147c4762a1bSJed Brown   PetscFunctionReturn(0);
1148c4762a1bSJed Brown }
1149c4762a1bSJed Brown 
1150c4762a1bSJed Brown /* get coordinate description of the whole-domain boundary */
1151c4762a1bSJed Brown static PetscErrorCode DMPlexGetExpandedBoundary_Private(DM dm, PortableBoundary *boundary)
1152c4762a1bSJed Brown {
1153c4762a1bSJed Brown   PortableBoundary       bnd0, bnd;
1154c4762a1bSJed Brown   MPI_Comm               comm;
1155c4762a1bSJed Brown   DM                     idm;
1156c4762a1bSJed Brown   DMLabel                label;
1157c4762a1bSJed Brown   PetscInt               d;
1158c4762a1bSJed Brown   const char             boundaryName[] = "DMPlexDistributeInterpolateMarkInterface_boundary";
1159c4762a1bSJed Brown   IS                     boundary_is;
1160c4762a1bSJed Brown   IS                     *boundary_expanded_iss;
1161c4762a1bSJed Brown   PetscMPIInt            rootrank = 0;
1162c4762a1bSJed Brown   PetscMPIInt            rank, size;
1163c4762a1bSJed Brown   PetscInt               value = 1;
1164c4762a1bSJed Brown   DMPlexInterpolatedFlag intp;
1165c4762a1bSJed Brown   PetscBool              flg;
1166c4762a1bSJed Brown   PetscErrorCode         ierr;
1167c4762a1bSJed Brown 
1168c4762a1bSJed Brown   PetscFunctionBegin;
1169c4762a1bSJed Brown   ierr = PetscNew(&bnd);CHKERRQ(ierr);
1170c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
1171ffc4695bSBarry Smith   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1172ffc4695bSBarry Smith   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1173c4762a1bSJed Brown   ierr = DMPlexIsDistributed(dm, &flg);CHKERRQ(ierr);
1174c4762a1bSJed Brown   if (flg) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "serial DM (all points on one rank) needed");
1175c4762a1bSJed Brown 
1176c4762a1bSJed Brown   /* interpolate serial DM if not yet interpolated */
1177c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(dm, &intp);CHKERRQ(ierr);
1178c4762a1bSJed Brown   if (intp == DMPLEX_INTERPOLATED_FULL) {
1179c4762a1bSJed Brown     idm = dm;
1180c4762a1bSJed Brown     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
1181c4762a1bSJed Brown   } else {
1182c4762a1bSJed Brown     ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
1183c4762a1bSJed Brown     ierr = DMViewFromOptions(idm, NULL, "-idm_view");CHKERRQ(ierr);
1184c4762a1bSJed Brown   }
1185c4762a1bSJed Brown 
1186c4762a1bSJed Brown   /* mark whole-domain boundary of the serial DM */
1187c4762a1bSJed Brown   ierr = DMLabelCreate(PETSC_COMM_SELF, boundaryName, &label);CHKERRQ(ierr);
1188c4762a1bSJed Brown   ierr = DMAddLabel(idm, label);CHKERRQ(ierr);
1189c4762a1bSJed Brown   ierr = DMPlexMarkBoundaryFaces(idm, value, label);CHKERRQ(ierr);
1190c4762a1bSJed Brown   ierr = DMLabelViewFromOptionsOnComm_Private(label, "-idm_boundary_view", comm);CHKERRQ(ierr);
1191c4762a1bSJed Brown   ierr = DMLabelGetStratumIS(label, value, &boundary_is);CHKERRQ(ierr);
1192c4762a1bSJed Brown 
1193c4762a1bSJed Brown   /* translate to coordinates */
1194c4762a1bSJed Brown   ierr = PetscNew(&bnd0);CHKERRQ(ierr);
1195c4762a1bSJed Brown   ierr = DMGetCoordinatesLocalSetUp(idm);CHKERRQ(ierr);
1196c4762a1bSJed Brown   if (rank == rootrank) {
1197c4762a1bSJed Brown     ierr = DMPlexGetConeRecursive(idm, boundary_is, &bnd0->depth, &boundary_expanded_iss, &bnd0->sections);CHKERRQ(ierr);
1198c4762a1bSJed Brown     ierr = DMGetCoordinatesLocalTuple(dm, boundary_expanded_iss[0], NULL, &bnd0->coordinates);CHKERRQ(ierr);
1199c4762a1bSJed Brown     /* self-check */
1200c4762a1bSJed Brown     {
1201c4762a1bSJed Brown       IS is0;
1202c4762a1bSJed Brown       ierr = DMPlexExpandedVerticesCoordinatesToFaces_Private(idm, bnd0, &is0);CHKERRQ(ierr);
1203c4762a1bSJed Brown       ierr = ISEqual(is0, boundary_is, &flg);CHKERRQ(ierr);
1204c4762a1bSJed Brown       if (!flg) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "DMPlexExpandedVerticesCoordinatesToFaces_Private produced a wrong IS");
1205c4762a1bSJed Brown       ierr = ISDestroy(&is0);CHKERRQ(ierr);
1206c4762a1bSJed Brown     }
1207c4762a1bSJed Brown   } else {
1208c4762a1bSJed Brown     ierr = VecCreateSeq(PETSC_COMM_SELF, 0, &bnd0->coordinates);CHKERRQ(ierr);
1209c4762a1bSJed Brown   }
1210c4762a1bSJed Brown 
1211c4762a1bSJed Brown   {
1212c4762a1bSJed Brown     Vec         tmp;
1213c4762a1bSJed Brown     VecScatter  sc;
1214c4762a1bSJed Brown     IS          xis;
1215c4762a1bSJed Brown     PetscInt    n;
1216c4762a1bSJed Brown 
1217c4762a1bSJed Brown     /* just convert seq vectors to mpi vector */
1218c4762a1bSJed Brown     ierr = VecGetLocalSize(bnd0->coordinates, &n);CHKERRQ(ierr);
1219ffc4695bSBarry Smith     ierr = MPI_Bcast(&n, 1, MPIU_INT, rootrank, comm);CHKERRMPI(ierr);
1220c4762a1bSJed Brown     if (rank == rootrank) {
1221c4762a1bSJed Brown       ierr = VecCreateMPI(comm, n, n, &tmp);CHKERRQ(ierr);
1222c4762a1bSJed Brown     } else {
1223c4762a1bSJed Brown       ierr = VecCreateMPI(comm, 0, n, &tmp);CHKERRQ(ierr);
1224c4762a1bSJed Brown     }
1225c4762a1bSJed Brown     ierr = VecCopy(bnd0->coordinates, tmp);CHKERRQ(ierr);
1226c4762a1bSJed Brown     ierr = VecDestroy(&bnd0->coordinates);CHKERRQ(ierr);
1227c4762a1bSJed Brown     bnd0->coordinates = tmp;
1228c4762a1bSJed Brown 
1229c4762a1bSJed Brown     /* replicate coordinates from root rank to all ranks */
1230c4762a1bSJed Brown     ierr = VecCreateMPI(comm, n, n*size, &bnd->coordinates);CHKERRQ(ierr);
1231c4762a1bSJed Brown     ierr = ISCreateStride(comm, n, 0, 1, &xis);CHKERRQ(ierr);
1232c4762a1bSJed Brown     ierr = VecScatterCreate(bnd0->coordinates, xis, bnd->coordinates, NULL, &sc);CHKERRQ(ierr);
1233c4762a1bSJed Brown     ierr = VecScatterBegin(sc, bnd0->coordinates, bnd->coordinates, INSERT_VALUES, SCATTER_FORWARD);CHKERRQ(ierr);
1234c4762a1bSJed Brown     ierr = VecScatterEnd(  sc, bnd0->coordinates, bnd->coordinates, INSERT_VALUES, SCATTER_FORWARD);CHKERRQ(ierr);
1235c4762a1bSJed Brown     ierr = VecScatterDestroy(&sc);CHKERRQ(ierr);
1236c4762a1bSJed Brown     ierr = ISDestroy(&xis);CHKERRQ(ierr);
1237c4762a1bSJed Brown   }
1238c4762a1bSJed Brown   bnd->depth = bnd0->depth;
1239ffc4695bSBarry Smith   ierr = MPI_Bcast(&bnd->depth, 1, MPIU_INT, rootrank, comm);CHKERRMPI(ierr);
1240c4762a1bSJed Brown   ierr = PetscMalloc1(bnd->depth, &bnd->sections);CHKERRQ(ierr);
1241c4762a1bSJed Brown   for (d=0; d<bnd->depth; d++) {
1242c4762a1bSJed Brown     ierr = PetscSectionReplicate_Private(comm, rootrank, (rank == rootrank) ? bnd0->sections[d] : NULL, &bnd->sections[d]);CHKERRQ(ierr);
1243c4762a1bSJed Brown   }
1244c4762a1bSJed Brown 
1245c4762a1bSJed Brown   if (rank == rootrank) {
1246c4762a1bSJed Brown     ierr = DMPlexRestoreConeRecursive(idm, boundary_is, &bnd0->depth, &boundary_expanded_iss, &bnd0->sections);CHKERRQ(ierr);
1247c4762a1bSJed Brown   }
1248c4762a1bSJed Brown   ierr = PortableBoundaryDestroy(&bnd0);CHKERRQ(ierr);
1249c4762a1bSJed Brown   ierr = DMRemoveLabelBySelf(idm, &label, PETSC_TRUE);CHKERRQ(ierr);
1250c4762a1bSJed Brown   ierr = DMLabelDestroy(&label);CHKERRQ(ierr);
1251c4762a1bSJed Brown   ierr = ISDestroy(&boundary_is);CHKERRQ(ierr);
1252c4762a1bSJed Brown   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1253c4762a1bSJed Brown   *boundary = bnd;
1254c4762a1bSJed Brown   PetscFunctionReturn(0);
1255c4762a1bSJed Brown }
1256c4762a1bSJed Brown 
1257c4762a1bSJed Brown /* get faces of inter-partition interface */
1258c4762a1bSJed Brown static PetscErrorCode DMPlexGetInterfaceFaces_Private(DM ipdm, IS boundary_faces_is, IS *interface_faces_is)
1259c4762a1bSJed Brown {
1260c4762a1bSJed Brown   MPI_Comm               comm;
1261c4762a1bSJed Brown   DMLabel                label;
1262c4762a1bSJed Brown   IS                     part_boundary_faces_is;
1263c4762a1bSJed Brown   const char             partBoundaryName[] = "DMPlexDistributeInterpolateMarkInterface_partBoundary";
1264c4762a1bSJed Brown   PetscInt               value = 1;
1265c4762a1bSJed Brown   DMPlexInterpolatedFlag intp;
1266c4762a1bSJed Brown   PetscErrorCode         ierr;
1267c4762a1bSJed Brown 
1268c4762a1bSJed Brown   PetscFunctionBegin;
1269c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)ipdm, &comm);CHKERRQ(ierr);
1270c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(ipdm, &intp);CHKERRQ(ierr);
1271c4762a1bSJed Brown   if (intp != DMPLEX_INTERPOLATED_FULL) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "only for fully interpolated DMPlex");
1272c4762a1bSJed Brown 
1273c4762a1bSJed Brown   /* get ipdm partition boundary (partBoundary) */
1274c4762a1bSJed Brown   ierr = DMLabelCreate(PETSC_COMM_SELF, partBoundaryName, &label);CHKERRQ(ierr);
1275c4762a1bSJed Brown   ierr = DMAddLabel(ipdm, label);CHKERRQ(ierr);
1276c4762a1bSJed Brown   ierr = DMPlexMarkBoundaryFaces(ipdm, value, label);CHKERRQ(ierr);
1277c4762a1bSJed Brown   ierr = DMLabelViewFromOptionsOnComm_Private(label, "-ipdm_part_boundary_view", comm);CHKERRQ(ierr);
1278c4762a1bSJed Brown   ierr = DMLabelGetStratumISOnComm_Private(label, value, comm, &part_boundary_faces_is);CHKERRQ(ierr);
1279c4762a1bSJed Brown   ierr = DMRemoveLabelBySelf(ipdm, &label, PETSC_TRUE);CHKERRQ(ierr);
1280c4762a1bSJed Brown   ierr = DMLabelDestroy(&label);CHKERRQ(ierr);
1281c4762a1bSJed Brown 
1282c4762a1bSJed Brown   /* remove ipdm whole-domain boundary (boundary_faces_is) from ipdm partition boundary (part_boundary_faces_is), resulting just in inter-partition interface */
1283c4762a1bSJed Brown   ierr = ISDifference(part_boundary_faces_is,boundary_faces_is,interface_faces_is);CHKERRQ(ierr);
1284c4762a1bSJed Brown   ierr = ISDestroy(&part_boundary_faces_is);CHKERRQ(ierr);
1285c4762a1bSJed Brown   PetscFunctionReturn(0);
1286c4762a1bSJed Brown }
1287c4762a1bSJed Brown 
1288c4762a1bSJed Brown /* compute inter-partition interface including edges and vertices */
1289c4762a1bSJed Brown static PetscErrorCode DMPlexComputeCompleteInterface_Private(DM ipdm, IS interface_faces_is, IS *interface_is)
1290c4762a1bSJed Brown {
1291c4762a1bSJed Brown   DMLabel                label;
1292c4762a1bSJed Brown   PetscInt               value = 1;
1293c4762a1bSJed Brown   const char             interfaceName[] = "DMPlexDistributeInterpolateMarkInterface_interface";
1294c4762a1bSJed Brown   DMPlexInterpolatedFlag intp;
1295c4762a1bSJed Brown   MPI_Comm               comm;
1296c4762a1bSJed Brown   PetscErrorCode         ierr;
1297c4762a1bSJed Brown 
1298c4762a1bSJed Brown   PetscFunctionBegin;
1299c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)ipdm, &comm);CHKERRQ(ierr);
1300c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(ipdm, &intp);CHKERRQ(ierr);
1301c4762a1bSJed Brown   if (intp != DMPLEX_INTERPOLATED_FULL) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "only for fully interpolated DMPlex");
1302c4762a1bSJed Brown 
1303c4762a1bSJed Brown   ierr = DMLabelCreate(PETSC_COMM_SELF, interfaceName, &label);CHKERRQ(ierr);
1304c4762a1bSJed Brown   ierr = DMAddLabel(ipdm, label);CHKERRQ(ierr);
1305c4762a1bSJed Brown   ierr = DMLabelSetStratumIS(label, value, interface_faces_is);CHKERRQ(ierr);
1306c4762a1bSJed Brown   ierr = DMLabelViewFromOptionsOnComm_Private(label, "-interface_faces_view", comm);CHKERRQ(ierr);
1307c4762a1bSJed Brown   ierr = DMPlexLabelComplete(ipdm, label);CHKERRQ(ierr);
1308c4762a1bSJed Brown   ierr = DMLabelViewFromOptionsOnComm_Private(label, "-interface_view", comm);CHKERRQ(ierr);
1309c4762a1bSJed Brown   ierr = DMLabelGetStratumISOnComm_Private(label, value, comm, interface_is);CHKERRQ(ierr);
1310c4762a1bSJed Brown   ierr = PetscObjectSetName((PetscObject)*interface_is, "interface_is");CHKERRQ(ierr);
1311c4762a1bSJed Brown   ierr = ISViewFromOptions(*interface_is, NULL, "-interface_is_view");CHKERRQ(ierr);
1312c4762a1bSJed Brown   ierr = DMRemoveLabelBySelf(ipdm, &label, PETSC_TRUE);CHKERRQ(ierr);
1313c4762a1bSJed Brown   ierr = DMLabelDestroy(&label);CHKERRQ(ierr);
1314c4762a1bSJed Brown   PetscFunctionReturn(0);
1315c4762a1bSJed Brown }
1316c4762a1bSJed Brown 
1317c4762a1bSJed Brown static PetscErrorCode PointSFGetOutwardInterfacePoints(PetscSF sf, IS *is)
1318c4762a1bSJed Brown {
1319c4762a1bSJed Brown   PetscInt        n;
1320c4762a1bSJed Brown   const PetscInt  *arr;
1321c4762a1bSJed Brown   PetscErrorCode  ierr;
1322c4762a1bSJed Brown 
1323c4762a1bSJed Brown   PetscFunctionBegin;
1324c4762a1bSJed Brown   ierr = PetscSFGetGraph(sf, NULL, &n, &arr, NULL);CHKERRQ(ierr);
1325c4762a1bSJed Brown   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)sf), n, arr, PETSC_USE_POINTER, is);CHKERRQ(ierr);
1326c4762a1bSJed Brown   PetscFunctionReturn(0);
1327c4762a1bSJed Brown }
1328c4762a1bSJed Brown 
1329c4762a1bSJed Brown static PetscErrorCode PointSFGetInwardInterfacePoints(PetscSF sf, IS *is)
1330c4762a1bSJed Brown {
1331c4762a1bSJed Brown   PetscInt        n;
1332c4762a1bSJed Brown   const PetscInt  *rootdegree;
1333c4762a1bSJed Brown   PetscInt        *arr;
1334c4762a1bSJed Brown   PetscErrorCode  ierr;
1335c4762a1bSJed Brown 
1336c4762a1bSJed Brown   PetscFunctionBegin;
1337c4762a1bSJed Brown   ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
1338c4762a1bSJed Brown   ierr = PetscSFComputeDegreeBegin(sf, &rootdegree);CHKERRQ(ierr);
1339c4762a1bSJed Brown   ierr = PetscSFComputeDegreeEnd(sf, &rootdegree);CHKERRQ(ierr);
1340c4762a1bSJed Brown   ierr = PetscSFComputeMultiRootOriginalNumbering(sf, rootdegree, &n, &arr);CHKERRQ(ierr);
1341c4762a1bSJed Brown   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)sf), n, arr, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
1342c4762a1bSJed Brown   PetscFunctionReturn(0);
1343c4762a1bSJed Brown }
1344c4762a1bSJed Brown 
1345c4762a1bSJed Brown static PetscErrorCode PointSFGetInterfacePoints_Private(PetscSF pointSF, IS *is)
1346c4762a1bSJed Brown {
1347c4762a1bSJed Brown   IS pointSF_out_is, pointSF_in_is;
1348c4762a1bSJed Brown   PetscErrorCode  ierr;
1349c4762a1bSJed Brown 
1350c4762a1bSJed Brown   PetscFunctionBegin;
1351c4762a1bSJed Brown   ierr = PointSFGetOutwardInterfacePoints(pointSF, &pointSF_out_is);CHKERRQ(ierr);
1352c4762a1bSJed Brown   ierr = PointSFGetInwardInterfacePoints(pointSF, &pointSF_in_is);CHKERRQ(ierr);
1353c4762a1bSJed Brown   ierr = ISExpand(pointSF_out_is, pointSF_in_is, is);CHKERRQ(ierr);
1354c4762a1bSJed Brown   ierr = ISDestroy(&pointSF_out_is);CHKERRQ(ierr);
1355c4762a1bSJed Brown   ierr = ISDestroy(&pointSF_in_is);CHKERRQ(ierr);
1356c4762a1bSJed Brown   PetscFunctionReturn(0);
1357c4762a1bSJed Brown }
1358c4762a1bSJed Brown 
13592e58f0efSBarry Smith #define CHKERRMY(ierr) do {if (PetscUnlikely(ierr)) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "PointSF is wrong. Unable to show details!");} while (0)
1360c4762a1bSJed Brown 
1361c4762a1bSJed Brown static PetscErrorCode ViewPointsWithType_Internal(DM dm, IS pointsIS, PetscViewer v)
1362c4762a1bSJed Brown {
1363c4762a1bSJed Brown   DMLabel         label;
1364c4762a1bSJed Brown   PetscSection    coordsSection;
1365c4762a1bSJed Brown   Vec             coordsVec;
1366c4762a1bSJed Brown   PetscScalar     *coordsScalar;
1367c4762a1bSJed Brown   PetscInt        coneSize, depth, dim, i, p, npoints;
1368c4762a1bSJed Brown   const PetscInt  *points;
1369c4762a1bSJed Brown   PetscErrorCode  ierr;
1370c4762a1bSJed Brown 
1371c4762a1bSJed Brown   PetscFunctionBegin;
1372c4762a1bSJed Brown   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1373c4762a1bSJed Brown   ierr = DMGetCoordinateSection(dm, &coordsSection);CHKERRQ(ierr);
1374c4762a1bSJed Brown   ierr = DMGetCoordinatesLocal(dm, &coordsVec);CHKERRQ(ierr);
1375c4762a1bSJed Brown   ierr = VecGetArray(coordsVec, &coordsScalar);CHKERRQ(ierr);
1376c4762a1bSJed Brown   ierr = ISGetLocalSize(pointsIS, &npoints);CHKERRQ(ierr);
1377c4762a1bSJed Brown   ierr = ISGetIndices(pointsIS, &points);CHKERRQ(ierr);
1378c4762a1bSJed Brown   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
1379c4762a1bSJed Brown   ierr = PetscViewerASCIIPushTab(v);CHKERRQ(ierr);
1380c4762a1bSJed Brown   for (i=0; i<npoints; i++) {
1381c4762a1bSJed Brown     p = points[i];
1382c4762a1bSJed Brown     ierr = DMLabelGetValue(label, p, &depth);CHKERRQ(ierr);
1383c4762a1bSJed Brown     if (!depth) {
1384c4762a1bSJed Brown       PetscInt        c, n, o;
1385c4762a1bSJed Brown       PetscReal       coords[3];
1386c4762a1bSJed Brown       char            coordstr[128];
1387c4762a1bSJed Brown 
1388c4762a1bSJed Brown       ierr = PetscSectionGetDof(coordsSection, p, &n);CHKERRQ(ierr);
1389c4762a1bSJed Brown       ierr = PetscSectionGetOffset(coordsSection, p, &o);CHKERRQ(ierr);
1390c4762a1bSJed Brown       for (c=0; c<n; c++) coords[c] = PetscRealPart(coordsScalar[o+c]);
1391c4762a1bSJed Brown       ierr = coord2str(coordstr, sizeof(coordstr), n, coords, 1.0);CHKERRQ(ierr);
1392c4762a1bSJed Brown       ierr = PetscViewerASCIISynchronizedPrintf(v, "vertex %D w/ coordinates %s\n", p, coordstr);CHKERRQ(ierr);
1393c4762a1bSJed Brown     } else {
1394c4762a1bSJed Brown       char            entityType[16];
1395c4762a1bSJed Brown 
1396c4762a1bSJed Brown       switch (depth) {
1397c4762a1bSJed Brown         case 1: ierr = PetscStrcpy(entityType, "edge");CHKERRQ(ierr); break;
1398c4762a1bSJed Brown         case 2: ierr = PetscStrcpy(entityType, "face");CHKERRQ(ierr); break;
1399c4762a1bSJed Brown         case 3: ierr = PetscStrcpy(entityType, "cell");CHKERRQ(ierr); break;
14002a27bf02SStefano Zampini         default: SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_SUP, "Only for depth <= 3");
1401c4762a1bSJed Brown       }
1402c4762a1bSJed Brown       if (depth == dim && dim < 3) {
1403c4762a1bSJed Brown         ierr = PetscStrlcat(entityType, " (cell)", sizeof(entityType));CHKERRQ(ierr);
1404c4762a1bSJed Brown       }
1405c4762a1bSJed Brown       ierr = PetscViewerASCIISynchronizedPrintf(v, "%s %D\n", entityType, p);CHKERRQ(ierr);
1406c4762a1bSJed Brown     }
1407c4762a1bSJed Brown     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1408c4762a1bSJed Brown     if (coneSize) {
1409c4762a1bSJed Brown       const PetscInt *cone;
1410c4762a1bSJed Brown       IS             coneIS;
1411c4762a1bSJed Brown 
1412c4762a1bSJed Brown       ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1413c4762a1bSJed Brown       ierr = ISCreateGeneral(PETSC_COMM_SELF, coneSize, cone, PETSC_USE_POINTER, &coneIS);CHKERRQ(ierr);
1414c4762a1bSJed Brown       ierr = ViewPointsWithType_Internal(dm, coneIS, v);CHKERRQ(ierr);
1415c4762a1bSJed Brown       ierr = ISDestroy(&coneIS);CHKERRQ(ierr);
1416c4762a1bSJed Brown     }
1417c4762a1bSJed Brown   }
1418c4762a1bSJed Brown   ierr = PetscViewerASCIIPopTab(v);CHKERRQ(ierr);
1419c4762a1bSJed Brown   ierr = VecRestoreArray(coordsVec, &coordsScalar);CHKERRQ(ierr);
1420c4762a1bSJed Brown   ierr = ISRestoreIndices(pointsIS, &points);CHKERRQ(ierr);
1421c4762a1bSJed Brown   PetscFunctionReturn(0);
1422c4762a1bSJed Brown }
1423c4762a1bSJed Brown 
1424c4762a1bSJed Brown static PetscErrorCode ViewPointsWithType(DM dm, IS points, PetscViewer v)
1425c4762a1bSJed Brown {
1426c4762a1bSJed Brown   PetscBool       flg;
1427c4762a1bSJed Brown   PetscInt        npoints;
1428c4762a1bSJed Brown   PetscMPIInt     rank;
1429c4762a1bSJed Brown   PetscErrorCode  ierr;
1430c4762a1bSJed Brown 
1431c4762a1bSJed Brown   PetscFunctionBegin;
1432c4762a1bSJed Brown   ierr = PetscObjectTypeCompare((PetscObject)v, PETSCVIEWERASCII, &flg);CHKERRQ(ierr);
14332758c9b9SBarry Smith   if (!flg) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_SUP, "Only for ASCII viewer");
1434ffc4695bSBarry Smith   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)v), &rank);CHKERRMPI(ierr);
1435c4762a1bSJed Brown   ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
1436c4762a1bSJed Brown   ierr = ISGetLocalSize(points, &npoints);CHKERRQ(ierr);
1437c4762a1bSJed Brown   if (npoints) {
1438c4762a1bSJed Brown     ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d] --------\n", rank);CHKERRQ(ierr);
1439c4762a1bSJed Brown     ierr = ViewPointsWithType_Internal(dm, points, v);CHKERRQ(ierr);
1440c4762a1bSJed Brown   }
1441c4762a1bSJed Brown   ierr = PetscViewerFlush(v);CHKERRQ(ierr);
1442c4762a1bSJed Brown   ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
1443c4762a1bSJed Brown   PetscFunctionReturn(0);
1444c4762a1bSJed Brown }
1445c4762a1bSJed Brown 
1446c4762a1bSJed Brown static PetscErrorCode DMPlexComparePointSFWithInterface_Private(DM ipdm, IS interface_is)
1447c4762a1bSJed Brown {
1448c4762a1bSJed Brown   PetscSF         pointsf;
1449c4762a1bSJed Brown   IS              pointsf_is;
1450c4762a1bSJed Brown   PetscBool       flg;
1451c4762a1bSJed Brown   MPI_Comm        comm;
1452c4762a1bSJed Brown   PetscMPIInt     size;
1453c4762a1bSJed Brown   PetscErrorCode  ierr;
1454c4762a1bSJed Brown 
1455c4762a1bSJed Brown   PetscFunctionBegin;
1456c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)ipdm, &comm);CHKERRQ(ierr);
1457ffc4695bSBarry Smith   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1458c4762a1bSJed Brown   ierr = DMGetPointSF(ipdm, &pointsf);CHKERRQ(ierr);
1459c4762a1bSJed Brown   if (pointsf) {
1460c4762a1bSJed Brown     PetscInt nroots;
1461c4762a1bSJed Brown     ierr = PetscSFGetGraph(pointsf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
1462c4762a1bSJed Brown     if (nroots < 0) pointsf = NULL; /* uninitialized SF */
1463c4762a1bSJed Brown   }
1464c4762a1bSJed Brown   if (!pointsf) {
1465c4762a1bSJed Brown     PetscInt N=0;
1466c4762a1bSJed Brown     if (interface_is) {ierr = ISGetSize(interface_is, &N);CHKERRQ(ierr);}
1467c4762a1bSJed Brown     if (N) SETERRQ(comm, PETSC_ERR_PLIB, "interface_is should be NULL or empty for PointSF being NULL");
1468c4762a1bSJed Brown     PetscFunctionReturn(0);
1469c4762a1bSJed Brown   }
1470c4762a1bSJed Brown 
1471c4762a1bSJed Brown   /* get PointSF points as IS pointsf_is */
1472c4762a1bSJed Brown   ierr = PointSFGetInterfacePoints_Private(pointsf, &pointsf_is);CHKERRQ(ierr);
1473c4762a1bSJed Brown 
1474c4762a1bSJed Brown   /* compare pointsf_is with interface_is */
1475c4762a1bSJed Brown   ierr = ISEqual(interface_is, pointsf_is, &flg);CHKERRQ(ierr);
1476ffc4695bSBarry Smith   ierr = MPI_Allreduce(MPI_IN_PLACE,&flg,1,MPIU_BOOL,MPI_LAND,comm);CHKERRMPI(ierr);
1477c4762a1bSJed Brown   if (!flg) {
1478c4762a1bSJed Brown     IS pointsf_extra_is, pointsf_missing_is;
1479c4762a1bSJed Brown     PetscViewer errv = PETSC_VIEWER_STDERR_(comm);
14802e58f0efSBarry Smith     ierr = ISDifference(interface_is, pointsf_is, &pointsf_missing_is);CHKERRMY(ierr);
14812e58f0efSBarry Smith     ierr = ISDifference(pointsf_is, interface_is, &pointsf_extra_is);CHKERRMY(ierr);
14822e58f0efSBarry Smith     ierr = PetscViewerASCIIPrintf(errv, "Points missing in PointSF:\n");CHKERRMY(ierr);
14832e58f0efSBarry Smith     ierr = ViewPointsWithType(ipdm, pointsf_missing_is, errv);CHKERRMY(ierr);
14842e58f0efSBarry Smith     ierr = PetscViewerASCIIPrintf(errv, "Extra points in PointSF:\n");CHKERRMY(ierr);
14852e58f0efSBarry Smith     ierr = ViewPointsWithType(ipdm, pointsf_extra_is, errv);CHKERRMY(ierr);
14862e58f0efSBarry Smith     ierr = ISDestroy(&pointsf_extra_is);CHKERRMY(ierr);
14872e58f0efSBarry Smith     ierr = ISDestroy(&pointsf_missing_is);CHKERRMY(ierr);
1488c4762a1bSJed Brown     SETERRQ(comm, PETSC_ERR_PLIB, "PointSF is wrong! See details above.");
1489c4762a1bSJed Brown   }
1490c4762a1bSJed Brown   ierr = ISDestroy(&pointsf_is);CHKERRQ(ierr);
1491c4762a1bSJed Brown   PetscFunctionReturn(0);
1492c4762a1bSJed Brown }
1493c4762a1bSJed Brown 
1494c4762a1bSJed Brown /* remove faces & edges from label, leave just vertices */
1495c4762a1bSJed Brown static PetscErrorCode DMPlexISFilterVertices_Private(DM dm, IS points)
1496c4762a1bSJed Brown {
1497c4762a1bSJed Brown   PetscInt        vStart, vEnd;
1498c4762a1bSJed Brown   MPI_Comm        comm;
1499c4762a1bSJed Brown   PetscErrorCode  ierr;
1500c4762a1bSJed Brown 
1501c4762a1bSJed Brown   PetscFunctionBegin;
1502c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
1503c4762a1bSJed Brown   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1504c4762a1bSJed Brown   ierr = ISGeneralFilter(points, vStart, vEnd);CHKERRQ(ierr);
1505c4762a1bSJed Brown   PetscFunctionReturn(0);
1506c4762a1bSJed Brown }
1507c4762a1bSJed Brown 
1508c4762a1bSJed Brown /*
15092e58f0efSBarry Smith   DMPlexCheckPointSFHeavy - Thoroughly test that the PointSF after parallel DMPlexInterpolate() includes exactly all interface points.
1510c4762a1bSJed Brown 
1511c4762a1bSJed Brown   Collective
1512c4762a1bSJed Brown 
1513c4762a1bSJed Brown   Input Parameters:
1514c4762a1bSJed Brown . dm - The DMPlex object
1515c4762a1bSJed Brown 
1516c4762a1bSJed Brown   Notes:
1517c4762a1bSJed Brown   The input DMPlex must be serial (one partition has all points, the other partitions have no points).
1518c4762a1bSJed Brown   This is a heavy test which involves DMPlexInterpolate() if the input DM is not interpolated yet, and depends on having a representation of the whole-domain boundary (PortableBoundary), which can be obtained only by DMPlexGetExpandedBoundary_Private() (which involves DMPlexInterpolate() of a sequential DM).
1519c4762a1bSJed Brown   This is mainly intended for debugging/testing purposes.
1520c4762a1bSJed Brown 
1521c4762a1bSJed Brown   Algorithm:
1522c4762a1bSJed Brown   1. boundary faces of the serial version of the whole mesh are found using DMPlexMarkBoundaryFaces()
1523c4762a1bSJed Brown   2. boundary faces are translated into vertices using DMPlexGetConeRecursive() and these are translated into coordinates - this description (aka PortableBoundary) is completely independent of partitioning and point numbering
1524c4762a1bSJed Brown   3. the mesh is distributed or loaded in parallel
1525c4762a1bSJed Brown   4. boundary faces of the distributed mesh are reconstructed from PortableBoundary using DMPlexFindVertices()
1526c4762a1bSJed Brown   5. partition boundary faces of the parallel mesh are found using DMPlexMarkBoundaryFaces()
1527c4762a1bSJed Brown   6. partition interfaces are computed as set difference of partition boundary faces minus the reconstructed boundary
1528c4762a1bSJed Brown   7. check that interface covered by PointSF (union of inward and outward points) is equal to the partition interface for each rank, otherwise print the difference and throw an error
1529c4762a1bSJed Brown 
1530c4762a1bSJed Brown   Level: developer
1531c4762a1bSJed Brown 
1532c4762a1bSJed Brown .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
1533c4762a1bSJed Brown */
1534c4762a1bSJed Brown static PetscErrorCode DMPlexCheckPointSFHeavy(DM dm, PortableBoundary bnd)
1535c4762a1bSJed Brown {
1536c4762a1bSJed Brown   DM                     ipdm=NULL;
1537c4762a1bSJed Brown   IS                     boundary_faces_is, interface_faces_is, interface_is;
1538c4762a1bSJed Brown   DMPlexInterpolatedFlag intp;
1539c4762a1bSJed Brown   MPI_Comm               comm;
1540c4762a1bSJed Brown   PetscErrorCode         ierr;
1541c4762a1bSJed Brown 
1542c4762a1bSJed Brown   PetscFunctionBegin;
1543c4762a1bSJed Brown   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
1544c4762a1bSJed Brown 
1545c4762a1bSJed Brown   ierr = DMPlexIsInterpolatedCollective(dm, &intp);CHKERRQ(ierr);
1546c4762a1bSJed Brown   if (intp == DMPLEX_INTERPOLATED_FULL) {
1547c4762a1bSJed Brown     ipdm = dm;
1548c4762a1bSJed Brown   } else {
1549c4762a1bSJed Brown     /* create temporary interpolated DM if input DM is not interpolated */
1550c4762a1bSJed Brown     ierr = DMPlexSetOrientInterface_Private(dm, PETSC_FALSE);CHKERRQ(ierr);
1551c4762a1bSJed Brown     ierr = DMPlexInterpolate(dm, &ipdm);CHKERRQ(ierr); /* with DMPlexOrientInterface_Internal() call skipped so that PointSF issues are left to DMPlexComparePointSFWithInterface_Private() below */
1552c4762a1bSJed Brown     ierr = DMPlexSetOrientInterface_Private(dm, PETSC_TRUE);CHKERRQ(ierr);
1553c4762a1bSJed Brown   }
1554c4762a1bSJed Brown   ierr = DMViewFromOptions(ipdm, NULL, "-ipdm_view");CHKERRQ(ierr);
1555c4762a1bSJed Brown 
1556c4762a1bSJed Brown   /* recover ipdm whole-domain boundary faces from the expanded vertices coordinates */
1557c4762a1bSJed Brown   ierr = DMPlexExpandedVerticesCoordinatesToFaces_Private(ipdm, bnd, &boundary_faces_is);CHKERRQ(ierr);
1558c4762a1bSJed Brown   /* get inter-partition interface faces (interface_faces_is)*/
1559c4762a1bSJed Brown   ierr = DMPlexGetInterfaceFaces_Private(ipdm, boundary_faces_is, &interface_faces_is);CHKERRQ(ierr);
1560c4762a1bSJed Brown   /* compute inter-partition interface including edges and vertices (interface_is) */
1561c4762a1bSJed Brown   ierr = DMPlexComputeCompleteInterface_Private(ipdm, interface_faces_is, &interface_is);CHKERRQ(ierr);
1562c4762a1bSJed Brown   /* destroy immediate ISs */
1563c4762a1bSJed Brown   ierr = ISDestroy(&boundary_faces_is);CHKERRQ(ierr);
1564c4762a1bSJed Brown   ierr = ISDestroy(&interface_faces_is);CHKERRQ(ierr);
1565c4762a1bSJed Brown 
1566c4762a1bSJed Brown   /* for uninterpolated case, keep just vertices in interface */
1567c4762a1bSJed Brown   if (!intp) {
1568c4762a1bSJed Brown     ierr = DMPlexISFilterVertices_Private(ipdm, interface_is);CHKERRQ(ierr);
1569c4762a1bSJed Brown     ierr = DMDestroy(&ipdm);CHKERRQ(ierr);
1570c4762a1bSJed Brown   }
1571c4762a1bSJed Brown 
1572c4762a1bSJed Brown   /* compare PointSF with the boundary reconstructed from coordinates */
1573c4762a1bSJed Brown   ierr = DMPlexComparePointSFWithInterface_Private(dm, interface_is);CHKERRQ(ierr);
1574c4762a1bSJed Brown   ierr = PetscPrintf(comm, "DMPlexCheckPointSFHeavy PASSED\n");CHKERRQ(ierr);
1575c4762a1bSJed Brown   ierr = ISDestroy(&interface_is);CHKERRQ(ierr);
1576c4762a1bSJed Brown   PetscFunctionReturn(0);
1577c4762a1bSJed Brown }
1578c4762a1bSJed Brown 
1579c4762a1bSJed Brown int main(int argc, char **argv)
1580c4762a1bSJed Brown {
1581c4762a1bSJed Brown   DM             dm;
1582c4762a1bSJed Brown   AppCtx         user;
1583c4762a1bSJed Brown   PetscErrorCode ierr;
1584c4762a1bSJed Brown 
1585c4762a1bSJed Brown   ierr = PetscInitialize(&argc, &argv, NULL, help);if (ierr) return ierr;
1586c4762a1bSJed Brown   ierr = PetscLogStageRegister("create",&stage[0]);CHKERRQ(ierr);
1587c4762a1bSJed Brown   ierr = PetscLogStageRegister("distribute",&stage[1]);CHKERRQ(ierr);
1588c4762a1bSJed Brown   ierr = PetscLogStageRegister("interpolate",&stage[2]);CHKERRQ(ierr);
1589c4762a1bSJed Brown   ierr = ProcessOptions(PETSC_COMM_WORLD, &user);CHKERRQ(ierr);
1590c4762a1bSJed Brown   ierr = CreateMesh(PETSC_COMM_WORLD, &user, &dm);CHKERRQ(ierr);
1591c4762a1bSJed Brown   if (user.nPointsToExpand) {
1592c4762a1bSJed Brown     ierr = TestExpandPoints(dm, &user);CHKERRQ(ierr);
1593c4762a1bSJed Brown   }
1594c4762a1bSJed Brown   if (user.ncoords) {
1595c4762a1bSJed Brown     ierr = ViewVerticesFromCoords(dm, user.ncoords/user.dim, user.coords, user.coordsTol, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
1596c4762a1bSJed Brown   }
1597c4762a1bSJed Brown   ierr = DMDestroy(&dm);CHKERRQ(ierr);
1598c4762a1bSJed Brown   ierr = PetscFinalize();
1599c4762a1bSJed Brown   return ierr;
1600c4762a1bSJed Brown }
1601c4762a1bSJed Brown 
1602c4762a1bSJed Brown /*TEST
1603c4762a1bSJed Brown 
1604c4762a1bSJed Brown   testset:
1605c4762a1bSJed Brown     nsize: 2
1606c4762a1bSJed Brown     args: -dm_view ascii::ascii_info_detail
1607c4762a1bSJed Brown     args: -dm_plex_check_all
1608c4762a1bSJed Brown     test:
1609c4762a1bSJed Brown       suffix: 1_tri_dist0
1610c4762a1bSJed Brown       args: -distribute 0 -interpolate {{none create}separate output}
1611c4762a1bSJed Brown     test:
1612c4762a1bSJed Brown       suffix: 1_tri_dist1
1613c4762a1bSJed Brown       args: -distribute 1 -interpolate {{none create after_distribute}separate output}
1614c4762a1bSJed Brown     test:
1615c4762a1bSJed Brown       suffix: 1_quad_dist0
1616c4762a1bSJed Brown       args: -cell_simplex 0 -distribute 0 -interpolate {{none create}separate output}
1617c4762a1bSJed Brown     test:
1618c4762a1bSJed Brown       suffix: 1_quad_dist1
1619c4762a1bSJed Brown       args: -cell_simplex 0 -distribute 1 -interpolate {{none create after_distribute}separate output}
1620c4762a1bSJed Brown     test:
1621c4762a1bSJed Brown       suffix: 1_1d_dist1
1622c4762a1bSJed Brown       args: -dim 1 -distribute 1
1623c4762a1bSJed Brown 
1624c4762a1bSJed Brown   testset:
1625c4762a1bSJed Brown     nsize: 3
1626c4762a1bSJed Brown     args: -testnum 1 -interpolate create
1627c4762a1bSJed Brown     args: -dm_plex_check_all
1628c4762a1bSJed Brown     test:
1629c4762a1bSJed Brown       suffix: 2
1630c4762a1bSJed Brown       args: -dm_view ascii::ascii_info_detail
1631c4762a1bSJed Brown     test:
1632c4762a1bSJed Brown       suffix: 2a
1633c4762a1bSJed Brown       args: -dm_plex_check_cones_conform_on_interfaces_verbose
1634c4762a1bSJed Brown     test:
1635c4762a1bSJed Brown       suffix: 2b
1636c4762a1bSJed Brown       args: -test_expand_points 0,1,2,5,6
1637c4762a1bSJed Brown     test:
1638c4762a1bSJed Brown       suffix: 2c
1639c4762a1bSJed Brown       args: -test_expand_points 0,1,2,5,6 -test_expand_points_empty
1640c4762a1bSJed Brown 
1641c4762a1bSJed Brown   testset:
1642c4762a1bSJed Brown     # the same as 1% for 3D
1643c4762a1bSJed Brown     nsize: 2
1644c4762a1bSJed Brown     args: -dim 3 -dm_view ascii::ascii_info_detail
1645c4762a1bSJed Brown     args: -dm_plex_check_all
1646c4762a1bSJed Brown     test:
1647c4762a1bSJed Brown       suffix: 4_tet_dist0
1648c4762a1bSJed Brown       args: -distribute 0 -interpolate {{none create}separate output}
1649c4762a1bSJed Brown     test:
1650c4762a1bSJed Brown       suffix: 4_tet_dist1
1651c4762a1bSJed Brown       args: -distribute 1 -interpolate {{none create after_distribute}separate output}
1652c4762a1bSJed Brown     test:
1653c4762a1bSJed Brown       suffix: 4_hex_dist0
1654c4762a1bSJed Brown       args: -cell_simplex 0 -distribute 0 -interpolate {{none create}separate output}
1655c4762a1bSJed Brown     test:
1656c4762a1bSJed Brown       suffix: 4_hex_dist1
1657c4762a1bSJed Brown       args: -cell_simplex 0 -distribute 1 -interpolate {{none create after_distribute}separate output}
1658c4762a1bSJed Brown 
1659c4762a1bSJed Brown   test:
1660c4762a1bSJed Brown     # the same as 4_tet_dist0 but test different initial orientations
1661c4762a1bSJed Brown     suffix: 4_tet_test_orient
1662c4762a1bSJed Brown     nsize: 2
1663c4762a1bSJed Brown     args: -dim 3 -distribute 0
1664c4762a1bSJed Brown     args: -dm_plex_check_all
1665c4762a1bSJed Brown     args: -rotate_interface_0 {{0 1 2 11 12 13}}
1666c4762a1bSJed Brown     args: -rotate_interface_1 {{0 1 2 11 12 13}}
1667c4762a1bSJed Brown 
1668c4762a1bSJed Brown   testset:
1669c4762a1bSJed Brown     requires: exodusii
1670c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/TwoQuads.exo
1671c4762a1bSJed Brown     args: -dm_view ascii::ascii_info_detail
1672c4762a1bSJed Brown     args: -dm_plex_check_all
1673c4762a1bSJed Brown     args: -custom_view
1674c4762a1bSJed Brown     test:
1675c4762a1bSJed Brown       suffix: 5_seq
1676c4762a1bSJed Brown       nsize: 1
1677c4762a1bSJed Brown       args: -distribute 0 -interpolate {{none create}separate output}
1678c4762a1bSJed Brown     test:
167930602db0SMatthew G. Knepley       # Detail viewing in a non-distributed mesh is broken because the DMLabelView() is collective, but the label is not shared
1680c4762a1bSJed Brown       suffix: 5_dist0
1681c4762a1bSJed Brown       nsize: 2
168230602db0SMatthew G. Knepley       args: -distribute 0 -interpolate {{none create}separate output} -dm_view
1683c4762a1bSJed Brown     test:
1684c4762a1bSJed Brown       suffix: 5_dist1
1685c4762a1bSJed Brown       nsize: 2
1686c4762a1bSJed Brown       args: -distribute 1 -interpolate {{none create after_distribute}separate output}
1687c4762a1bSJed Brown 
1688c4762a1bSJed Brown   testset:
1689c4762a1bSJed Brown     nsize: {{1 2 4}}
1690c4762a1bSJed Brown     args: -use_generator
1691c4762a1bSJed Brown     args: -dm_plex_check_all
1692c4762a1bSJed Brown     args: -distribute -interpolate none
1693c4762a1bSJed Brown     test:
1694c4762a1bSJed Brown       suffix: 6_tri
1695c4762a1bSJed Brown       requires: triangle
1696c0517cd5SMatthew G. Knepley       args: -faces {{2,2  1,3  7,4}} -cell_simplex 1 -dm_generator triangle
1697c4762a1bSJed Brown     test:
1698c4762a1bSJed Brown       suffix: 6_quad
1699c4762a1bSJed Brown       args: -faces {{2,2  1,3  7,4}} -cell_simplex 0
1700c4762a1bSJed Brown     test:
1701c4762a1bSJed Brown       suffix: 6_tet
1702c4762a1bSJed Brown       requires: ctetgen
1703c0517cd5SMatthew G. Knepley       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 1 -dm_generator ctetgen
1704c4762a1bSJed Brown     test:
1705c4762a1bSJed Brown       suffix: 6_hex
1706c4762a1bSJed Brown       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 0
1707c4762a1bSJed Brown   testset:
1708c4762a1bSJed Brown     nsize: {{1 2 4}}
1709c4762a1bSJed Brown     args: -use_generator
1710c4762a1bSJed Brown     args: -dm_plex_check_all
1711c4762a1bSJed Brown     args: -distribute -interpolate create
1712c4762a1bSJed Brown     test:
1713c4762a1bSJed Brown       suffix: 6_int_tri
1714c4762a1bSJed Brown       requires: triangle
1715c0517cd5SMatthew G. Knepley       args: -faces {{2,2  1,3  7,4}} -cell_simplex 1 -dm_generator triangle
1716c4762a1bSJed Brown     test:
1717c4762a1bSJed Brown       suffix: 6_int_quad
1718c4762a1bSJed Brown       args: -faces {{2,2  1,3  7,4}} -cell_simplex 0
1719c4762a1bSJed Brown     test:
1720c4762a1bSJed Brown       suffix: 6_int_tet
1721c4762a1bSJed Brown       requires: ctetgen
1722c0517cd5SMatthew G. Knepley       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 1 -dm_generator ctetgen
1723c4762a1bSJed Brown     test:
1724c4762a1bSJed Brown       suffix: 6_int_hex
1725c4762a1bSJed Brown       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 0
1726c4762a1bSJed Brown   testset:
1727c4762a1bSJed Brown     nsize: {{2 4}}
1728c4762a1bSJed Brown     args: -use_generator
1729c4762a1bSJed Brown     args: -dm_plex_check_all
1730c4762a1bSJed Brown     args: -distribute -interpolate after_distribute
1731c4762a1bSJed Brown     test:
1732c4762a1bSJed Brown       suffix: 6_parint_tri
1733c4762a1bSJed Brown       requires: triangle
1734c0517cd5SMatthew G. Knepley       args: -faces {{2,2  1,3  7,4}} -cell_simplex 1 -dm_generator triangle
1735c4762a1bSJed Brown     test:
1736c4762a1bSJed Brown       suffix: 6_parint_quad
1737c4762a1bSJed Brown       args: -faces {{2,2  1,3  7,4}} -cell_simplex 0
1738c4762a1bSJed Brown     test:
1739c4762a1bSJed Brown       suffix: 6_parint_tet
1740c4762a1bSJed Brown       requires: ctetgen
1741c0517cd5SMatthew G. Knepley       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 1 -dm_generator ctetgen
1742c4762a1bSJed Brown     test:
1743c4762a1bSJed Brown       suffix: 6_parint_hex
1744c4762a1bSJed Brown       args: -faces {{2,2,2  1,3,5  3,4,7}} -cell_simplex 0
1745c4762a1bSJed Brown 
1746c4762a1bSJed Brown   testset: # 7 EXODUS
1747c4762a1bSJed Brown     requires: exodusii
1748c4762a1bSJed Brown     args: -dm_plex_check_all
1749c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/blockcylinder-50.exo
1750c4762a1bSJed Brown     args: -distribute
1751c4762a1bSJed Brown     test: # seq load, simple partitioner
1752c4762a1bSJed Brown       suffix: 7_exo
1753c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1754c4762a1bSJed Brown       args: -interpolate none
1755c4762a1bSJed Brown     test: # seq load, seq interpolation, simple partitioner
1756c4762a1bSJed Brown       suffix: 7_exo_int_simple
1757c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1758c4762a1bSJed Brown       args: -interpolate create
1759c4762a1bSJed Brown     test: # seq load, seq interpolation, metis partitioner
1760c4762a1bSJed Brown       suffix: 7_exo_int_metis
1761c4762a1bSJed Brown       requires: parmetis
1762c4762a1bSJed Brown       nsize: {{2 4 5}}
1763c4762a1bSJed Brown       args: -interpolate create
1764c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1765c4762a1bSJed Brown     test: # seq load, simple partitioner, par interpolation
1766c4762a1bSJed Brown       suffix: 7_exo_simple_int
1767c4762a1bSJed Brown       nsize: {{2 4 5}}
1768c4762a1bSJed Brown       args: -interpolate after_distribute
1769c4762a1bSJed Brown     test: # seq load, metis partitioner, par interpolation
1770c4762a1bSJed Brown       suffix: 7_exo_metis_int
1771c4762a1bSJed Brown       requires: parmetis
1772c4762a1bSJed Brown       nsize: {{2 4 5}}
1773c4762a1bSJed Brown       args: -interpolate after_distribute
1774c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1775c4762a1bSJed Brown 
1776c4762a1bSJed Brown   testset: # 7 HDF5 SEQUANTIAL LOAD
1777c4762a1bSJed Brown     requires: hdf5 !complex
1778c4762a1bSJed Brown     args: -dm_plex_check_all
1779c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/blockcylinder-50.h5 -dm_plex_create_from_hdf5_xdmf
1780c4762a1bSJed Brown     args: -dm_plex_hdf5_force_sequential
1781c4762a1bSJed Brown     args: -distribute
1782c4762a1bSJed Brown     test: # seq load, simple partitioner
1783c4762a1bSJed Brown       suffix: 7_seq_hdf5_simple
1784c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1785c4762a1bSJed Brown       args: -interpolate none
1786c4762a1bSJed Brown     test: # seq load, seq interpolation, simple partitioner
1787c4762a1bSJed Brown       suffix: 7_seq_hdf5_int_simple
1788c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1789c4762a1bSJed Brown       args: -interpolate after_create
1790c4762a1bSJed Brown     test: # seq load, seq interpolation, metis partitioner
1791c4762a1bSJed Brown       nsize: {{2 4 5}}
1792c4762a1bSJed Brown       suffix: 7_seq_hdf5_int_metis
1793c4762a1bSJed Brown       requires: parmetis
1794c4762a1bSJed Brown       args: -interpolate after_create
1795c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1796c4762a1bSJed Brown     test: # seq load, simple partitioner, par interpolation
1797c4762a1bSJed Brown       suffix: 7_seq_hdf5_simple_int
1798c4762a1bSJed Brown       nsize: {{2 4 5}}
1799c4762a1bSJed Brown       args: -interpolate after_distribute
1800c4762a1bSJed Brown     test: # seq load, metis partitioner, par interpolation
1801c4762a1bSJed Brown       nsize: {{2 4 5}}
1802c4762a1bSJed Brown       suffix: 7_seq_hdf5_metis_int
1803c4762a1bSJed Brown       requires: parmetis
1804c4762a1bSJed Brown       args: -interpolate after_distribute
1805c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1806c4762a1bSJed Brown 
1807c4762a1bSJed Brown   testset: # 7 HDF5 PARALLEL LOAD
1808c4762a1bSJed Brown     requires: hdf5 !complex
1809c4762a1bSJed Brown     nsize: {{2 4 5}}
1810c4762a1bSJed Brown     args: -dm_plex_check_all
1811c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/blockcylinder-50.h5 -dm_plex_create_from_hdf5_xdmf
1812c4762a1bSJed Brown     test: # par load
1813c4762a1bSJed Brown       suffix: 7_par_hdf5
1814c4762a1bSJed Brown       args: -interpolate none
1815c4762a1bSJed Brown     test: # par load, par interpolation
1816c4762a1bSJed Brown       suffix: 7_par_hdf5_int
1817c4762a1bSJed Brown       args: -interpolate after_create
1818c4762a1bSJed Brown     test: # par load, parmetis repartitioner
1819c4762a1bSJed Brown       TODO: Parallel partitioning of uninterpolated meshes not supported
1820c4762a1bSJed Brown       suffix: 7_par_hdf5_parmetis
1821c4762a1bSJed Brown       requires: parmetis
1822c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1823c4762a1bSJed Brown       args: -interpolate none
1824c4762a1bSJed Brown     test: # par load, par interpolation, parmetis repartitioner
1825c4762a1bSJed Brown       suffix: 7_par_hdf5_int_parmetis
1826c4762a1bSJed Brown       requires: parmetis
1827c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1828c4762a1bSJed Brown       args: -interpolate after_create
1829c4762a1bSJed Brown     test: # par load, parmetis partitioner, par interpolation
1830c4762a1bSJed Brown       TODO: Parallel partitioning of uninterpolated meshes not supported
1831c4762a1bSJed Brown       suffix: 7_par_hdf5_parmetis_int
1832c4762a1bSJed Brown       requires: parmetis
1833c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1834c4762a1bSJed Brown       args: -interpolate after_distribute
1835c4762a1bSJed Brown 
1836c4762a1bSJed Brown     test:
1837c4762a1bSJed Brown       suffix: 7_hdf5_hierarch
1838c4762a1bSJed Brown       requires: hdf5 ptscotch !complex
1839c4762a1bSJed Brown       nsize: {{2 3 4}separate output}
1840c4762a1bSJed Brown       args: -distribute
1841c4762a1bSJed Brown       args: -interpolate after_create
1842c4762a1bSJed Brown       args: -petscpartitioner_type matpartitioning -petscpartitioner_view ::ascii_info
1843c4762a1bSJed Brown       args: -mat_partitioning_type hierarch -mat_partitioning_hierarchical_nfineparts 2
1844c4762a1bSJed Brown       args: -mat_partitioning_hierarchical_coarseparttype ptscotch -mat_partitioning_hierarchical_fineparttype ptscotch
1845c4762a1bSJed Brown 
1846c4762a1bSJed Brown   test:
1847c4762a1bSJed Brown     suffix: 8
1848c4762a1bSJed Brown     requires: hdf5 !complex
1849c4762a1bSJed Brown     nsize: 4
1850c4762a1bSJed Brown     args: -filename ${wPETSC_DIR}/share/petsc/datafiles/meshes/blockcylinder-50.h5 -dm_plex_create_from_hdf5_xdmf
1851c4762a1bSJed Brown     args: -distribute 0 -interpolate after_create
1852c4762a1bSJed Brown     args: -view_vertices_from_coords 0.,1.,0.,-0.5,1.,0.,0.583,-0.644,0.,-2.,-2.,-2. -view_vertices_from_coords_tol 1e-3
1853c4762a1bSJed Brown     args: -dm_plex_check_all
1854c4762a1bSJed Brown     args: -custom_view
1855c4762a1bSJed Brown 
1856c4762a1bSJed Brown   testset: # 9 HDF5 SEQUANTIAL LOAD
1857c4762a1bSJed Brown     requires: hdf5 !complex datafilespath
1858c4762a1bSJed Brown     args: -dm_plex_check_all
1859c4762a1bSJed Brown     args: -filename ${DATAFILESPATH}/meshes/cube-hexahedra-refined.h5 -dm_plex_create_from_hdf5_xdmf -dm_plex_hdf5_topology_path /cells -dm_plex_hdf5_geometry_path /coordinates
1860c4762a1bSJed Brown     args: -dm_plex_hdf5_force_sequential
1861c4762a1bSJed Brown     args: -distribute
1862c4762a1bSJed Brown     test: # seq load, simple partitioner
1863c4762a1bSJed Brown       suffix: 9_seq_hdf5_simple
1864c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1865c4762a1bSJed Brown       args: -interpolate none
1866c4762a1bSJed Brown     test: # seq load, seq interpolation, simple partitioner
1867c4762a1bSJed Brown       suffix: 9_seq_hdf5_int_simple
1868c4762a1bSJed Brown       nsize: {{1 2 4 5}}
1869c4762a1bSJed Brown       args: -interpolate after_create
1870c4762a1bSJed Brown     test: # seq load, seq interpolation, metis partitioner
1871c4762a1bSJed Brown       nsize: {{2 4 5}}
1872c4762a1bSJed Brown       suffix: 9_seq_hdf5_int_metis
1873c4762a1bSJed Brown       requires: parmetis
1874c4762a1bSJed Brown       args: -interpolate after_create
1875c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1876c4762a1bSJed Brown     test: # seq load, simple partitioner, par interpolation
1877c4762a1bSJed Brown       suffix: 9_seq_hdf5_simple_int
1878c4762a1bSJed Brown       nsize: {{2 4 5}}
1879c4762a1bSJed Brown       args: -interpolate after_distribute
1880c4762a1bSJed Brown     test: # seq load, simple partitioner, par interpolation
1881c4762a1bSJed Brown       # This is like 9_seq_hdf5_simple_int but testing error output of DMPlexCheckPointSFHeavy().
1882c4762a1bSJed Brown       # Once 9_seq_hdf5_simple_int gets fixed, this one gets broken.
1883c4762a1bSJed Brown       # We can then provide an intentionally broken mesh instead.
1884c4762a1bSJed Brown       TODO: This test is broken because PointSF is fixed.
1885c4762a1bSJed Brown       suffix: 9_seq_hdf5_simple_int_err
1886c4762a1bSJed Brown       nsize: 4
1887c4762a1bSJed Brown       args: -interpolate after_distribute
1888c4762a1bSJed Brown       filter: sed -e "/PETSC ERROR/,$$d"
1889c4762a1bSJed Brown     test: # seq load, metis partitioner, par interpolation
1890c4762a1bSJed Brown       nsize: {{2 4 5}}
1891c4762a1bSJed Brown       suffix: 9_seq_hdf5_metis_int
1892c4762a1bSJed Brown       requires: parmetis
1893c4762a1bSJed Brown       args: -interpolate after_distribute
1894c4762a1bSJed Brown       args: -petscpartitioner_type parmetis
1895c4762a1bSJed Brown 
1896c4762a1bSJed Brown   testset: # 9 HDF5 PARALLEL LOAD
1897c4762a1bSJed Brown     requires: hdf5 !complex datafilespath
1898c4762a1bSJed Brown     nsize: {{2 4 5}}
1899c4762a1bSJed Brown     args: -dm_plex_check_all
1900c4762a1bSJed Brown     args: -filename ${DATAFILESPATH}/meshes/cube-hexahedra-refined.h5 -dm_plex_create_from_hdf5_xdmf -dm_plex_hdf5_topology_path /cells -dm_plex_hdf5_geometry_path /coordinates
1901c4762a1bSJed Brown     test: # par load
1902c4762a1bSJed Brown       suffix: 9_par_hdf5
1903c4762a1bSJed Brown       args: -interpolate none
1904c4762a1bSJed Brown     test: # par load, par interpolation
1905c4762a1bSJed Brown       suffix: 9_par_hdf5_int
1906c4762a1bSJed Brown       args: -interpolate after_create
1907c4762a1bSJed Brown     test: # par load, parmetis repartitioner
1908c4762a1bSJed Brown       TODO: Parallel partitioning of uninterpolated meshes not supported
1909c4762a1bSJed Brown       suffix: 9_par_hdf5_parmetis
1910c4762a1bSJed Brown       requires: parmetis
1911c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1912c4762a1bSJed Brown       args: -interpolate none
1913c4762a1bSJed Brown     test: # par load, par interpolation, parmetis repartitioner
1914c4762a1bSJed Brown       suffix: 9_par_hdf5_int_parmetis
1915c4762a1bSJed Brown       requires: parmetis
1916c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1917c4762a1bSJed Brown       args: -interpolate after_create
1918c4762a1bSJed Brown     test: # par load, parmetis partitioner, par interpolation
1919c4762a1bSJed Brown       TODO: Parallel partitioning of uninterpolated meshes not supported
1920c4762a1bSJed Brown       suffix: 9_par_hdf5_parmetis_int
1921c4762a1bSJed Brown       requires: parmetis
1922c4762a1bSJed Brown       args: -distribute -petscpartitioner_type parmetis
1923c4762a1bSJed Brown       args: -interpolate after_distribute
1924c4762a1bSJed Brown 
1925c4762a1bSJed Brown   testset: # 10 HDF5 PARALLEL LOAD
1926c4762a1bSJed Brown     requires: hdf5 !complex datafilespath
1927c4762a1bSJed Brown     nsize: {{2 4 7}}
1928c4762a1bSJed Brown     args: -dm_plex_check_all
1929c4762a1bSJed Brown     args: -filename ${DATAFILESPATH}/meshes/cube-hexahedra-refined2.h5 -dm_plex_create_from_hdf5_xdmf -dm_plex_hdf5_topology_path /topo -dm_plex_hdf5_geometry_path /geom
1930c4762a1bSJed Brown     test: # par load, par interpolation
1931c4762a1bSJed Brown       suffix: 10_par_hdf5_int
1932c4762a1bSJed Brown       args: -interpolate after_create
1933c4762a1bSJed Brown TEST*/
1934