xref: /petsc/src/dm/impls/plex/plexfem.c (revision 48162695a465a6c624c8d78ab14e47e37b4ffd85)
1af0996ceSBarry Smith #include <petsc/private/dmpleximpl.h> /*I      "petscdmplex.h"   I*/
2da97024aSMatthew G. Knepley #include <petscsf.h>
3cb1e1211SMatthew G Knepley 
48e3a2eefSMatthew G. Knepley #include <petscblaslapack.h>
5e8f14785SLisandro Dalcin #include <petsc/private/hashsetij.h>
6af0996ceSBarry Smith #include <petsc/private/petscfeimpl.h>
7af0996ceSBarry Smith #include <petsc/private/petscfvimpl.h>
8a0845e3aSMatthew G. Knepley 
95f0b18bfSMatthew G. Knepley PetscBool  Clementcite       = PETSC_FALSE;
105f0b18bfSMatthew G. Knepley const char ClementCitation[] = "@article{clement1975approximation,\n"
115f0b18bfSMatthew G. Knepley                                "  title   = {Approximation by finite element functions using local regularization},\n"
125f0b18bfSMatthew G. Knepley                                "  author  = {Philippe Cl{\\'e}ment},\n"
135f0b18bfSMatthew G. Knepley                                "  journal = {Revue fran{\\c{c}}aise d'automatique, informatique, recherche op{\\'e}rationnelle. Analyse num{\\'e}rique},\n"
145f0b18bfSMatthew G. Knepley                                "  volume  = {9},\n"
155f0b18bfSMatthew G. Knepley                                "  number  = {R2},\n"
165f0b18bfSMatthew G. Knepley                                "  pages   = {77--84},\n"
175f0b18bfSMatthew G. Knepley                                "  year    = {1975}\n}\n";
185f0b18bfSMatthew G. Knepley 
19d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexConvertPlex(DM dm, DM *plex, PetscBool copy)
20d71ae5a4SJacob Faibussowitsch {
212f856554SMatthew G. Knepley   PetscBool isPlex;
222f856554SMatthew G. Knepley 
232f856554SMatthew G. Knepley   PetscFunctionBegin;
249566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex));
252f856554SMatthew G. Knepley   if (isPlex) {
262f856554SMatthew G. Knepley     *plex = dm;
279566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)dm));
282f856554SMatthew G. Knepley   } else {
299566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex));
302f856554SMatthew G. Knepley     if (!*plex) {
319566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm, DMPLEX, plex));
329566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex));
33cbf8eb3cSStefano Zampini     } else {
34cbf8eb3cSStefano Zampini       PetscCall(PetscObjectReference((PetscObject)*plex));
35cbf8eb3cSStefano Zampini     }
362f856554SMatthew G. Knepley     if (copy) {
372f856554SMatthew G. Knepley       DMSubDomainHookLink link;
389a2a23afSMatthew G. Knepley 
39bb4b53efSMatthew G. Knepley       PetscCall(DMCopyDS(dm, PETSC_DETERMINE, PETSC_DETERMINE, *plex));
409566063dSJacob Faibussowitsch       PetscCall(DMCopyAuxiliaryVec(dm, *plex));
419a2a23afSMatthew G. Knepley       /* Run the subdomain hook (this will copy the DMSNES/DMTS) */
422f856554SMatthew G. Knepley       for (link = dm->subdomainhook; link; link = link->next) {
439566063dSJacob Faibussowitsch         if (link->ddhook) PetscCall((*link->ddhook)(dm, *plex, link->ctx));
442f856554SMatthew G. Knepley       }
452f856554SMatthew G. Knepley     }
462f856554SMatthew G. Knepley   }
473ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
482f856554SMatthew G. Knepley }
492f856554SMatthew G. Knepley 
5049abdd8aSBarry Smith static PetscErrorCode PetscContainerCtxDestroy_PetscFEGeom(void **ctx)
51d71ae5a4SJacob Faibussowitsch {
5249abdd8aSBarry Smith   PetscFEGeom *geom = (PetscFEGeom *)*ctx;
539b6f715bSMatthew G. Knepley 
549b6f715bSMatthew G. Knepley   PetscFunctionBegin;
559566063dSJacob Faibussowitsch   PetscCall(PetscFEGeomDestroy(&geom));
563ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
579b6f715bSMatthew G. Knepley }
589b6f715bSMatthew G. Knepley 
59ac9d17c7SMatthew G. Knepley static PetscErrorCode DMPlexGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscFEGeomMode mode, PetscFEGeom **geom)
60d71ae5a4SJacob Faibussowitsch {
619b6f715bSMatthew G. Knepley   char           composeStr[33] = {0};
629b6f715bSMatthew G. Knepley   PetscObjectId  id;
639b6f715bSMatthew G. Knepley   PetscContainer container;
649b6f715bSMatthew G. Knepley 
659b6f715bSMatthew G. Knepley   PetscFunctionBegin;
669566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad, &id));
6763a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMPlexGetFEGeom_%" PetscInt64_FMT "\n", id));
689566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container));
699b6f715bSMatthew G. Knepley   if (container) {
709566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **)geom));
719b6f715bSMatthew G. Knepley   } else {
72ac9d17c7SMatthew G. Knepley     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, mode, geom));
739566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container));
749566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *)*geom));
7549abdd8aSBarry Smith     PetscCall(PetscContainerSetCtxDestroy(container, PetscContainerCtxDestroy_PetscFEGeom));
769566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container));
779566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
789b6f715bSMatthew G. Knepley   }
793ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
809b6f715bSMatthew G. Knepley }
819b6f715bSMatthew G. Knepley 
82ac9d17c7SMatthew G. Knepley static PetscErrorCode DMPlexRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscFEGeomMode mode, PetscFEGeom **geom)
83d71ae5a4SJacob Faibussowitsch {
849b6f715bSMatthew G. Knepley   PetscFunctionBegin;
859b6f715bSMatthew G. Knepley   *geom = NULL;
863ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
879b6f715bSMatthew G. Knepley }
889b6f715bSMatthew G. Knepley 
8946fa42a0SMatthew G. Knepley /*@
9046fa42a0SMatthew G. Knepley   DMPlexGetScale - Get the scale for the specified fundamental unit
9146fa42a0SMatthew G. Knepley 
9220f4b53cSBarry Smith   Not Collective
9346fa42a0SMatthew G. Knepley 
944165533cSJose E. Roman   Input Parameters:
95a1cb98faSBarry Smith + dm   - the `DM`
9646fa42a0SMatthew G. Knepley - unit - The SI unit
9746fa42a0SMatthew G. Knepley 
984165533cSJose E. Roman   Output Parameter:
9946fa42a0SMatthew G. Knepley . scale - The value used to scale all quantities with this unit
10046fa42a0SMatthew G. Knepley 
10146fa42a0SMatthew G. Knepley   Level: advanced
10246fa42a0SMatthew G. Knepley 
1031cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetScale()`, `PetscUnit`
10446fa42a0SMatthew G. Knepley @*/
105d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetScale(DM dm, PetscUnit unit, PetscReal *scale)
106d71ae5a4SJacob Faibussowitsch {
107cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
108cb1e1211SMatthew G Knepley 
109cb1e1211SMatthew G Knepley   PetscFunctionBegin;
110cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1114f572ea9SToby Isaac   PetscAssertPointer(scale, 3);
112cb1e1211SMatthew G Knepley   *scale = mesh->scale[unit];
1133ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
114cb1e1211SMatthew G Knepley }
115cb1e1211SMatthew G Knepley 
11646fa42a0SMatthew G. Knepley /*@
11746fa42a0SMatthew G. Knepley   DMPlexSetScale - Set the scale for the specified fundamental unit
11846fa42a0SMatthew G. Knepley 
11920f4b53cSBarry Smith   Not Collective
12046fa42a0SMatthew G. Knepley 
1214165533cSJose E. Roman   Input Parameters:
122a1cb98faSBarry Smith + dm    - the `DM`
12346fa42a0SMatthew G. Knepley . unit  - The SI unit
12446fa42a0SMatthew G. Knepley - scale - The value used to scale all quantities with this unit
12546fa42a0SMatthew G. Knepley 
12646fa42a0SMatthew G. Knepley   Level: advanced
12746fa42a0SMatthew G. Knepley 
1281cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetScale()`, `PetscUnit`
12946fa42a0SMatthew G. Knepley @*/
130d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexSetScale(DM dm, PetscUnit unit, PetscReal scale)
131d71ae5a4SJacob Faibussowitsch {
132cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
133cb1e1211SMatthew G Knepley 
134cb1e1211SMatthew G Knepley   PetscFunctionBegin;
135cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
136cb1e1211SMatthew G Knepley   mesh->scale[unit] = scale;
1373ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
138cb1e1211SMatthew G Knepley }
139cb1e1211SMatthew G Knepley 
140d2b2dc1eSMatthew G. Knepley PetscErrorCode DMPlexGetUseCeed_Plex(DM dm, PetscBool *useCeed)
141d2b2dc1eSMatthew G. Knepley {
142d2b2dc1eSMatthew G. Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
143d2b2dc1eSMatthew G. Knepley 
144d2b2dc1eSMatthew G. Knepley   PetscFunctionBegin;
145d2b2dc1eSMatthew G. Knepley   *useCeed = mesh->useCeed;
146d2b2dc1eSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
147d2b2dc1eSMatthew G. Knepley }
148d2b2dc1eSMatthew G. Knepley PetscErrorCode DMPlexSetUseCeed_Plex(DM dm, PetscBool useCeed)
149d2b2dc1eSMatthew G. Knepley {
150d2b2dc1eSMatthew G. Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
151d2b2dc1eSMatthew G. Knepley 
152d2b2dc1eSMatthew G. Knepley   PetscFunctionBegin;
153d2b2dc1eSMatthew G. Knepley   mesh->useCeed = useCeed;
154d2b2dc1eSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
155d2b2dc1eSMatthew G. Knepley }
156d2b2dc1eSMatthew G. Knepley 
157d2b2dc1eSMatthew G. Knepley /*@
158d2b2dc1eSMatthew G. Knepley   DMPlexGetUseCeed - Get flag for using the LibCEED backend
159d2b2dc1eSMatthew G. Knepley 
160d2b2dc1eSMatthew G. Knepley   Not collective
161d2b2dc1eSMatthew G. Knepley 
162d2b2dc1eSMatthew G. Knepley   Input Parameter:
163d2b2dc1eSMatthew G. Knepley . dm - The `DM`
164d2b2dc1eSMatthew G. Knepley 
165d2b2dc1eSMatthew G. Knepley   Output Parameter:
166d2b2dc1eSMatthew G. Knepley . useCeed - The flag
167d2b2dc1eSMatthew G. Knepley 
168d2b2dc1eSMatthew G. Knepley   Level: intermediate
169d2b2dc1eSMatthew G. Knepley 
170d2b2dc1eSMatthew G. Knepley .seealso: `DMPlexSetUseCeed()`
171d2b2dc1eSMatthew G. Knepley @*/
172d2b2dc1eSMatthew G. Knepley PetscErrorCode DMPlexGetUseCeed(DM dm, PetscBool *useCeed)
173d2b2dc1eSMatthew G. Knepley {
174d2b2dc1eSMatthew G. Knepley   PetscFunctionBegin;
175d2b2dc1eSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
176d2b2dc1eSMatthew G. Knepley   PetscAssertPointer(useCeed, 2);
177d2b2dc1eSMatthew G. Knepley   *useCeed = PETSC_FALSE;
178d2b2dc1eSMatthew G. Knepley   PetscTryMethod(dm, "DMPlexGetUseCeed_C", (DM, PetscBool *), (dm, useCeed));
179d2b2dc1eSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
180d2b2dc1eSMatthew G. Knepley }
181d2b2dc1eSMatthew G. Knepley 
182d2b2dc1eSMatthew G. Knepley /*@
183d2b2dc1eSMatthew G. Knepley   DMPlexSetUseCeed - Set flag for using the LibCEED backend
184d2b2dc1eSMatthew G. Knepley 
185d2b2dc1eSMatthew G. Knepley   Not collective
186d2b2dc1eSMatthew G. Knepley 
187d2b2dc1eSMatthew G. Knepley   Input Parameters:
188d2b2dc1eSMatthew G. Knepley + dm      - The `DM`
189d2b2dc1eSMatthew G. Knepley - useCeed - The flag
190d2b2dc1eSMatthew G. Knepley 
191d2b2dc1eSMatthew G. Knepley   Level: intermediate
192d2b2dc1eSMatthew G. Knepley 
193fe8e7dddSPierre Jolivet .seealso: `DMPlexGetUseCeed()`
194d2b2dc1eSMatthew G. Knepley @*/
195d2b2dc1eSMatthew G. Knepley PetscErrorCode DMPlexSetUseCeed(DM dm, PetscBool useCeed)
196d2b2dc1eSMatthew G. Knepley {
197d2b2dc1eSMatthew G. Knepley   PetscFunctionBegin;
198d2b2dc1eSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
199d2b2dc1eSMatthew G. Knepley   PetscValidLogicalCollectiveBool(dm, useCeed, 2);
200d2b2dc1eSMatthew G. Knepley   PetscUseMethod(dm, "DMPlexSetUseCeed_C", (DM, PetscBool), (dm, useCeed));
201d2b2dc1eSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
202d2b2dc1eSMatthew G. Knepley }
203d2b2dc1eSMatthew G. Knepley 
204e8e188d2SZach Atkins /*@
205e8e188d2SZach Atkins   DMPlexGetUseMatClosurePermutation - Get flag for using a closure permutation for matrix insertion
206e8e188d2SZach Atkins 
207e8e188d2SZach Atkins   Not collective
208e8e188d2SZach Atkins 
209e8e188d2SZach Atkins   Input Parameter:
210e8e188d2SZach Atkins . dm - The `DM`
211e8e188d2SZach Atkins 
212e8e188d2SZach Atkins   Output Parameter:
213e8e188d2SZach Atkins . useClPerm - The flag
214e8e188d2SZach Atkins 
215e8e188d2SZach Atkins   Level: intermediate
216e8e188d2SZach Atkins 
217e8e188d2SZach Atkins .seealso: `DMPlexSetUseMatClosurePermutation()`
218e8e188d2SZach Atkins @*/
219e8e188d2SZach Atkins PetscErrorCode DMPlexGetUseMatClosurePermutation(DM dm, PetscBool *useClPerm)
220e8e188d2SZach Atkins {
221e8e188d2SZach Atkins   DM_Plex *mesh = (DM_Plex *)dm->data;
222e8e188d2SZach Atkins 
223e8e188d2SZach Atkins   PetscFunctionBegin;
224e8e188d2SZach Atkins   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
225e8e188d2SZach Atkins   PetscAssertPointer(useClPerm, 2);
226e8e188d2SZach Atkins   *useClPerm = mesh->useMatClPerm;
227e8e188d2SZach Atkins   PetscFunctionReturn(PETSC_SUCCESS);
228e8e188d2SZach Atkins }
229e8e188d2SZach Atkins 
230e8e188d2SZach Atkins /*@
231e8e188d2SZach Atkins   DMPlexSetUseMatClosurePermutation - Set flag for using a closure permutation for matrix insertion
232e8e188d2SZach Atkins 
233e8e188d2SZach Atkins   Not collective
234e8e188d2SZach Atkins 
235e8e188d2SZach Atkins   Input Parameters:
236e8e188d2SZach Atkins + dm        - The `DM`
237e8e188d2SZach Atkins - useClPerm - The flag
238e8e188d2SZach Atkins 
239e8e188d2SZach Atkins   Level: intermediate
240e8e188d2SZach Atkins 
241e8e188d2SZach Atkins .seealso: `DMPlexGetUseMatClosurePermutation()`
242e8e188d2SZach Atkins @*/
243e8e188d2SZach Atkins PetscErrorCode DMPlexSetUseMatClosurePermutation(DM dm, PetscBool useClPerm)
244e8e188d2SZach Atkins {
245e8e188d2SZach Atkins   DM_Plex *mesh = (DM_Plex *)dm->data;
246e8e188d2SZach Atkins 
247e8e188d2SZach Atkins   PetscFunctionBegin;
248e8e188d2SZach Atkins   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
249e8e188d2SZach Atkins   PetscValidLogicalCollectiveBool(dm, useClPerm, 2);
250e8e188d2SZach Atkins   mesh->useMatClPerm = useClPerm;
251e8e188d2SZach Atkins   PetscFunctionReturn(PETSC_SUCCESS);
252e8e188d2SZach Atkins }
253e8e188d2SZach Atkins 
254d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexProjectRigidBody_Private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nc, PetscScalar *mode, void *ctx)
255d71ae5a4SJacob Faibussowitsch {
2569371c9d4SSatish Balay   const PetscInt eps[3][3][3] = {
2579371c9d4SSatish Balay     {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2589371c9d4SSatish Balay     {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2599371c9d4SSatish Balay     {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2609371c9d4SSatish Balay   };
261026175e5SToby Isaac   PetscInt *ctxInt = (PetscInt *)ctx;
262ad917190SMatthew G. Knepley   PetscInt  dim2   = ctxInt[0];
263026175e5SToby Isaac   PetscInt  d      = ctxInt[1];
264026175e5SToby Isaac   PetscInt  i, j, k = dim > 2 ? d - dim : d;
265026175e5SToby Isaac 
266ad917190SMatthew G. Knepley   PetscFunctionBegin;
26763a3b9bcSJacob Faibussowitsch   PetscCheck(dim == dim2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Input dimension %" PetscInt_FMT " does not match context dimension %" PetscInt_FMT, dim, dim2);
268026175e5SToby Isaac   for (i = 0; i < dim; i++) mode[i] = 0.;
269026175e5SToby Isaac   if (d < dim) {
27012adca46SMatthew G. Knepley     mode[d] = 1.; /* Translation along axis d */
271ad917190SMatthew G. Knepley   } else {
272026175e5SToby Isaac     for (i = 0; i < dim; i++) {
2739371c9d4SSatish Balay       for (j = 0; j < dim; j++) { mode[j] += eps[i][j][k] * X[i]; /* Rotation about axis d */ }
274026175e5SToby Isaac     }
275026175e5SToby Isaac   }
2763ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
277026175e5SToby Isaac }
278026175e5SToby Isaac 
279cc4e42d9SMartin Diehl /*@
28012adca46SMatthew G. Knepley   DMPlexCreateRigidBody - For the default global section, create rigid body modes by function space interpolation
281cb1e1211SMatthew G Knepley 
28220f4b53cSBarry Smith   Collective
283cb1e1211SMatthew G Knepley 
2844165533cSJose E. Roman   Input Parameters:
285a1cb98faSBarry Smith + dm    - the `DM`
28656cf3b9cSMatthew G. Knepley - field - The field number for the rigid body space, or 0 for the default
287cb1e1211SMatthew G Knepley 
2884165533cSJose E. Roman   Output Parameter:
289cb1e1211SMatthew G Knepley . sp - the null space
290cb1e1211SMatthew G Knepley 
291cb1e1211SMatthew G Knepley   Level: advanced
292cb1e1211SMatthew G Knepley 
293a1cb98faSBarry Smith   Note:
294a1cb98faSBarry Smith   This is necessary to provide a suitable coarse space for algebraic multigrid
295a1cb98faSBarry Smith 
2961cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `MatNullSpaceCreate()`, `PCGAMG`
297cb1e1211SMatthew G Knepley @*/
298d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexCreateRigidBody(DM dm, PetscInt field, MatNullSpace *sp)
299d71ae5a4SJacob Faibussowitsch {
30056cf3b9cSMatthew G. Knepley   PetscErrorCode (**func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *);
301cb1e1211SMatthew G Knepley   MPI_Comm     comm;
302026175e5SToby Isaac   Vec          mode[6];
303026175e5SToby Isaac   PetscSection section, globalSection;
30456cf3b9cSMatthew G. Knepley   PetscInt     dim, dimEmbed, Nf, n, m, mmin, d, i, j;
305db14aad5SMatthew G. Knepley   void       **ctxs;
306cb1e1211SMatthew G Knepley 
307cb1e1211SMatthew G Knepley   PetscFunctionBegin;
3089566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3099566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
3109566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
3119566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
31263a3b9bcSJacob Faibussowitsch   PetscCheck(!Nf || !(field < 0 || field >= Nf), comm, PETSC_ERR_ARG_OUTOFRANGE, "Field %" PetscInt_FMT " is not in [0, %" PetscInt_FMT ")", field, Nf);
31356cf3b9cSMatthew G. Knepley   if (dim == 1 && Nf < 2) {
3149566063dSJacob Faibussowitsch     PetscCall(MatNullSpaceCreate(comm, PETSC_TRUE, 0, NULL, sp));
3153ba16761SJacob Faibussowitsch     PetscFunctionReturn(PETSC_SUCCESS);
316cb1e1211SMatthew G Knepley   }
3179566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
3189566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
3199566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
320db14aad5SMatthew G. Knepley   PetscCall(PetscCalloc2(Nf, &func, Nf, &ctxs));
321b247467aSMatthew G. Knepley   m = (dim * (dim + 1)) / 2;
3229566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
3239566063dSJacob Faibussowitsch   PetscCall(VecSetType(mode[0], dm->vectype));
3249566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
3259566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
3269566063dSJacob Faibussowitsch   PetscCall(VecGetSize(mode[0], &n));
327b247467aSMatthew G. Knepley   mmin        = PetscMin(m, n);
32856cf3b9cSMatthew G. Knepley   func[field] = DMPlexProjectRigidBody_Private;
3299566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
330026175e5SToby Isaac   for (d = 0; d < m; d++) {
331330485fdSToby Isaac     PetscInt ctx[2];
332cb1e1211SMatthew G Knepley 
333db14aad5SMatthew G. Knepley     ctxs[field] = (void *)(&ctx[0]);
3349d8fbdeaSMatthew G. Knepley     ctx[0]      = dimEmbed;
335330485fdSToby Isaac     ctx[1]      = d;
336db14aad5SMatthew G. Knepley     PetscCall(DMProjectFunction(dm, 0.0, func, ctxs, INSERT_VALUES, mode[d]));
337cb1e1211SMatthew G Knepley   }
3383b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
339b50a2c0aSJacob Faibussowitsch   for (i = 0; i < mmin; ++i) {
340b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
341b50a2c0aSJacob Faibussowitsch 
3429566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
3439566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], mmin - i - 1, mode + i + 1, dots + i + 1));
344b50a2c0aSJacob Faibussowitsch     for (j = i + 1; j < mmin; ++j) {
345b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
3469566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
347b50a2c0aSJacob Faibussowitsch     }
348cb1e1211SMatthew G Knepley   }
3499566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, mmin, mode, sp));
3509566063dSJacob Faibussowitsch   for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i]));
351db14aad5SMatthew G. Knepley   PetscCall(PetscFree2(func, ctxs));
3523ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
353cb1e1211SMatthew G Knepley }
354cb1e1211SMatthew G Knepley 
355cc4e42d9SMartin Diehl /*@
35612adca46SMatthew G. Knepley   DMPlexCreateRigidBodies - For the default global section, create rigid body modes by function space interpolation
35712adca46SMatthew G. Knepley 
35820f4b53cSBarry Smith   Collective
35912adca46SMatthew G. Knepley 
3604165533cSJose E. Roman   Input Parameters:
361a1cb98faSBarry Smith + dm    - the `DM`
36212adca46SMatthew G. Knepley . nb    - The number of bodies
363a1cb98faSBarry Smith . label - The `DMLabel` marking each domain
36412adca46SMatthew G. Knepley . nids  - The number of ids per body
36512adca46SMatthew G. Knepley - ids   - An array of the label ids in sequence for each domain
36612adca46SMatthew G. Knepley 
3674165533cSJose E. Roman   Output Parameter:
36812adca46SMatthew G. Knepley . sp - the null space
36912adca46SMatthew G. Knepley 
37012adca46SMatthew G. Knepley   Level: advanced
37112adca46SMatthew G. Knepley 
372a1cb98faSBarry Smith   Note:
373a1cb98faSBarry Smith   This is necessary to provide a suitable coarse space for algebraic multigrid
374a1cb98faSBarry Smith 
3751cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `MatNullSpaceCreate()`
37612adca46SMatthew G. Knepley @*/
377d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexCreateRigidBodies(DM dm, PetscInt nb, DMLabel label, const PetscInt nids[], const PetscInt ids[], MatNullSpace *sp)
378d71ae5a4SJacob Faibussowitsch {
37912adca46SMatthew G. Knepley   MPI_Comm     comm;
38012adca46SMatthew G. Knepley   PetscSection section, globalSection;
38112adca46SMatthew G. Knepley   Vec         *mode;
38212adca46SMatthew G. Knepley   PetscScalar *dots;
38312adca46SMatthew G. Knepley   PetscInt     dim, dimEmbed, n, m, b, d, i, j, off;
38412adca46SMatthew G. Knepley 
38512adca46SMatthew G. Knepley   PetscFunctionBegin;
3869566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3879566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
3889566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
3899566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
3909566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
3919566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
39212adca46SMatthew G. Knepley   m = nb * (dim * (dim + 1)) / 2;
3939566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(m, &mode, m, &dots));
3949566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
3959566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
3969566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
3979566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
39812adca46SMatthew G. Knepley   for (b = 0, off = 0; b < nb; ++b) {
39912adca46SMatthew G. Knepley     for (d = 0; d < m / nb; ++d) {
40012adca46SMatthew G. Knepley       PetscInt ctx[2];
40112adca46SMatthew G. Knepley       PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private;
40212adca46SMatthew G. Knepley       void *voidctx                                                                                   = (void *)(&ctx[0]);
40312adca46SMatthew G. Knepley 
40412adca46SMatthew G. Knepley       ctx[0] = dimEmbed;
40512adca46SMatthew G. Knepley       ctx[1] = d;
4069566063dSJacob Faibussowitsch       PetscCall(DMProjectFunctionLabel(dm, 0.0, label, nids[b], &ids[off], 0, NULL, &func, &voidctx, INSERT_VALUES, mode[d]));
40712adca46SMatthew G. Knepley       off += nids[b];
40812adca46SMatthew G. Knepley     }
40912adca46SMatthew G. Knepley   }
4103b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
411606c1a1cSJacob Faibussowitsch   for (i = 0; i < m; ++i) {
412b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
4135a0e29b9SJacob Faibussowitsch 
4149566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
4159566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], m - i - 1, mode + i + 1, dots + i + 1));
4165a0e29b9SJacob Faibussowitsch     for (j = i + 1; j < m; ++j) {
417b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
4189566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
4195a0e29b9SJacob Faibussowitsch     }
42012adca46SMatthew G. Knepley   }
4219566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, m, mode, sp));
4229566063dSJacob Faibussowitsch   for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i]));
4239566063dSJacob Faibussowitsch   PetscCall(PetscFree2(mode, dots));
4243ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
42512adca46SMatthew G. Knepley }
42612adca46SMatthew G. Knepley 
427b29cfa1cSToby Isaac /*@
428b29cfa1cSToby Isaac   DMPlexSetMaxProjectionHeight - In DMPlexProjectXXXLocal() functions, the projected values of a basis function's dofs
429b29cfa1cSToby Isaac   are computed by associating the basis function with one of the mesh points in its transitively-closed support, and
430a4e35b19SJacob Faibussowitsch   evaluating the dual space basis of that point.
431b29cfa1cSToby Isaac 
432b29cfa1cSToby Isaac   Input Parameters:
433a1cb98faSBarry Smith + dm     - the `DMPLEX` object
434b29cfa1cSToby Isaac - height - the maximum projection height >= 0
435b29cfa1cSToby Isaac 
436b29cfa1cSToby Isaac   Level: advanced
437b29cfa1cSToby Isaac 
438a4e35b19SJacob Faibussowitsch   Notes:
439a4e35b19SJacob Faibussowitsch   A basis function is associated with the point in its transitively-closed support whose mesh
440a4e35b19SJacob Faibussowitsch   height is highest (w.r.t. DAG height), but not greater than the maximum projection height,
441a4e35b19SJacob Faibussowitsch   which is set with this function.  By default, the maximum projection height is zero, which
442a4e35b19SJacob Faibussowitsch   means that only mesh cells are used to project basis functions.  A height of one, for
443a4e35b19SJacob Faibussowitsch   example, evaluates a cell-interior basis functions using its cells dual space basis, but all
444a4e35b19SJacob Faibussowitsch   other basis functions with the dual space basis of a face.
445a4e35b19SJacob Faibussowitsch 
4461cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()`
447b29cfa1cSToby Isaac @*/
448d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexSetMaxProjectionHeight(DM dm, PetscInt height)
449d71ae5a4SJacob Faibussowitsch {
450b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *)dm->data;
451b29cfa1cSToby Isaac 
452b29cfa1cSToby Isaac   PetscFunctionBegin;
453b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
454b29cfa1cSToby Isaac   plex->maxProjectionHeight = height;
4553ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
456b29cfa1cSToby Isaac }
457b29cfa1cSToby Isaac 
458b29cfa1cSToby Isaac /*@
459b29cfa1cSToby Isaac   DMPlexGetMaxProjectionHeight - Get the maximum height (w.r.t. DAG) of mesh points used to evaluate dual bases in
460b29cfa1cSToby Isaac   DMPlexProjectXXXLocal() functions.
461b29cfa1cSToby Isaac 
4622fe279fdSBarry Smith   Input Parameter:
463a1cb98faSBarry Smith . dm - the `DMPLEX` object
464b29cfa1cSToby Isaac 
4652fe279fdSBarry Smith   Output Parameter:
466b29cfa1cSToby Isaac . height - the maximum projection height
467b29cfa1cSToby Isaac 
468b29cfa1cSToby Isaac   Level: intermediate
469b29cfa1cSToby Isaac 
4701cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()`
471b29cfa1cSToby Isaac @*/
472d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetMaxProjectionHeight(DM dm, PetscInt *height)
473d71ae5a4SJacob Faibussowitsch {
474b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *)dm->data;
475b29cfa1cSToby Isaac 
476b29cfa1cSToby Isaac   PetscFunctionBegin;
477b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
478b29cfa1cSToby Isaac   *height = plex->maxProjectionHeight;
4793ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
480b29cfa1cSToby Isaac }
481b29cfa1cSToby Isaac 
482ca3d3a14SMatthew G. Knepley typedef struct {
483ca3d3a14SMatthew G. Knepley   PetscReal    alpha; /* The first Euler angle, and in 2D the only one */
484ca3d3a14SMatthew G. Knepley   PetscReal    beta;  /* The second Euler angle */
485ca3d3a14SMatthew G. Knepley   PetscReal    gamma; /* The third Euler angle */
486ca3d3a14SMatthew G. Knepley   PetscInt     dim;   /* The dimension of R */
487ca3d3a14SMatthew G. Knepley   PetscScalar *R;     /* The rotation matrix, transforming a vector in the local basis to the global basis */
488ca3d3a14SMatthew G. Knepley   PetscScalar *RT;    /* The transposed rotation matrix, transforming a vector in the global basis to the local basis */
489ca3d3a14SMatthew G. Knepley } RotCtx;
490ca3d3a14SMatthew G. Knepley 
491ca3d3a14SMatthew G. Knepley /*
492ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
493ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
494ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
495ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
496ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
497ca3d3a14SMatthew G. Knepley */
498d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformSetUp_Rotation_Internal(DM dm, void *ctx)
499d71ae5a4SJacob Faibussowitsch {
500ca3d3a14SMatthew G. Knepley   RotCtx   *rc  = (RotCtx *)ctx;
501ca3d3a14SMatthew G. Knepley   PetscInt  dim = rc->dim;
502ca3d3a14SMatthew G. Knepley   PetscReal c1, s1, c2, s2, c3, s3;
503ca3d3a14SMatthew G. Knepley 
504ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5059566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(PetscSqr(dim), &rc->R, PetscSqr(dim), &rc->RT));
506ca3d3a14SMatthew G. Knepley   switch (dim) {
507ca3d3a14SMatthew G. Knepley   case 2:
5089371c9d4SSatish Balay     c1       = PetscCosReal(rc->alpha);
5099371c9d4SSatish Balay     s1       = PetscSinReal(rc->alpha);
5109371c9d4SSatish Balay     rc->R[0] = c1;
5119371c9d4SSatish Balay     rc->R[1] = s1;
5129371c9d4SSatish Balay     rc->R[2] = -s1;
5139371c9d4SSatish Balay     rc->R[3] = c1;
5149566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
515b458e8f1SJose E. Roman     DMPlex_Transpose2D_Internal(rc->RT);
516ca3d3a14SMatthew G. Knepley     break;
517ca3d3a14SMatthew G. Knepley   case 3:
5189371c9d4SSatish Balay     c1       = PetscCosReal(rc->alpha);
5199371c9d4SSatish Balay     s1       = PetscSinReal(rc->alpha);
5209371c9d4SSatish Balay     c2       = PetscCosReal(rc->beta);
5219371c9d4SSatish Balay     s2       = PetscSinReal(rc->beta);
5229371c9d4SSatish Balay     c3       = PetscCosReal(rc->gamma);
5239371c9d4SSatish Balay     s3       = PetscSinReal(rc->gamma);
5249371c9d4SSatish Balay     rc->R[0] = c1 * c3 - c2 * s1 * s3;
5259371c9d4SSatish Balay     rc->R[1] = c3 * s1 + c1 * c2 * s3;
5269371c9d4SSatish Balay     rc->R[2] = s2 * s3;
5279371c9d4SSatish Balay     rc->R[3] = -c1 * s3 - c2 * c3 * s1;
5289371c9d4SSatish Balay     rc->R[4] = c1 * c2 * c3 - s1 * s3;
5299371c9d4SSatish Balay     rc->R[5] = c3 * s2;
5309371c9d4SSatish Balay     rc->R[6] = s1 * s2;
5319371c9d4SSatish Balay     rc->R[7] = -c1 * s2;
5329371c9d4SSatish Balay     rc->R[8] = c2;
5339566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
534b458e8f1SJose E. Roman     DMPlex_Transpose3D_Internal(rc->RT);
535ca3d3a14SMatthew G. Knepley     break;
536d71ae5a4SJacob Faibussowitsch   default:
537d71ae5a4SJacob Faibussowitsch     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " not supported", dim);
538ca3d3a14SMatthew G. Knepley   }
5393ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
540ca3d3a14SMatthew G. Knepley }
541ca3d3a14SMatthew G. Knepley 
542d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformDestroy_Rotation_Internal(DM dm, void *ctx)
543d71ae5a4SJacob Faibussowitsch {
544ca3d3a14SMatthew G. Knepley   RotCtx *rc = (RotCtx *)ctx;
545ca3d3a14SMatthew G. Knepley 
546ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5479566063dSJacob Faibussowitsch   PetscCall(PetscFree2(rc->R, rc->RT));
5489566063dSJacob Faibussowitsch   PetscCall(PetscFree(rc));
5493ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
550ca3d3a14SMatthew G. Knepley }
551ca3d3a14SMatthew G. Knepley 
552d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformGetMatrix_Rotation_Internal(DM dm, const PetscReal x[], PetscBool l2g, const PetscScalar **A, void *ctx)
553d71ae5a4SJacob Faibussowitsch {
554ca3d3a14SMatthew G. Knepley   RotCtx *rc = (RotCtx *)ctx;
555ca3d3a14SMatthew G. Knepley 
556ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
5574f572ea9SToby Isaac   PetscAssertPointer(ctx, 5);
5589371c9d4SSatish Balay   if (l2g) {
5599371c9d4SSatish Balay     *A = rc->R;
5609371c9d4SSatish Balay   } else {
5619371c9d4SSatish Balay     *A = rc->RT;
5629371c9d4SSatish Balay   }
5633ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
564ca3d3a14SMatthew G. Knepley }
565ca3d3a14SMatthew G. Knepley 
566d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexBasisTransformApplyReal_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscReal *y, PetscReal *z, void *ctx)
567d71ae5a4SJacob Faibussowitsch {
568ec277c0fSMatthew G. Knepley   PetscFunctionBegin;
569ab6a9622SMatthew G. Knepley #if defined(PETSC_USE_COMPLEX)
570ab6a9622SMatthew G. Knepley   switch (dim) {
5719371c9d4SSatish Balay   case 2: {
57227104ee2SJacob Faibussowitsch     PetscScalar yt[2] = {y[0], y[1]}, zt[2] = {0.0, 0.0};
573ab6a9622SMatthew G. Knepley 
5749566063dSJacob Faibussowitsch     PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
5759371c9d4SSatish Balay     z[0] = PetscRealPart(zt[0]);
5769371c9d4SSatish Balay     z[1] = PetscRealPart(zt[1]);
5779371c9d4SSatish Balay   } break;
5789371c9d4SSatish Balay   case 3: {
57927104ee2SJacob Faibussowitsch     PetscScalar yt[3] = {y[0], y[1], y[2]}, zt[3] = {0.0, 0.0, 0.0};
580ab6a9622SMatthew G. Knepley 
5819566063dSJacob Faibussowitsch     PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
5829371c9d4SSatish Balay     z[0] = PetscRealPart(zt[0]);
5839371c9d4SSatish Balay     z[1] = PetscRealPart(zt[1]);
5849371c9d4SSatish Balay     z[2] = PetscRealPart(zt[2]);
5859371c9d4SSatish Balay   } break;
586ab6a9622SMatthew G. Knepley   }
587ab6a9622SMatthew G. Knepley #else
5889566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, y, z, ctx));
589ab6a9622SMatthew G. Knepley #endif
5903ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
591ab6a9622SMatthew G. Knepley }
592ab6a9622SMatthew G. Knepley 
593d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexBasisTransformApply_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscScalar *y, PetscScalar *z, void *ctx)
594d71ae5a4SJacob Faibussowitsch {
595ca3d3a14SMatthew G. Knepley   const PetscScalar *A;
596ca3d3a14SMatthew G. Knepley 
597ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
5989566063dSJacob Faibussowitsch   PetscCall((*dm->transformGetMatrix)(dm, x, l2g, &A, ctx));
599ca3d3a14SMatthew G. Knepley   switch (dim) {
600d71ae5a4SJacob Faibussowitsch   case 2:
601d71ae5a4SJacob Faibussowitsch     DMPlex_Mult2D_Internal(A, 1, y, z);
602d71ae5a4SJacob Faibussowitsch     break;
603d71ae5a4SJacob Faibussowitsch   case 3:
604d71ae5a4SJacob Faibussowitsch     DMPlex_Mult3D_Internal(A, 1, y, z);
605d71ae5a4SJacob Faibussowitsch     break;
606ca3d3a14SMatthew G. Knepley   }
6073ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
608ca3d3a14SMatthew G. Knepley }
609ca3d3a14SMatthew G. Knepley 
610d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformField_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscInt f, PetscBool l2g, PetscScalar *a)
611d71ae5a4SJacob Faibussowitsch {
612ca3d3a14SMatthew G. Knepley   PetscSection       ts;
613ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tva;
614ca3d3a14SMatthew G. Knepley   PetscInt           dof;
615ca3d3a14SMatthew G. Knepley 
616ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
6179566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
6189566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, p, f, &dof));
6199566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
6209566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, p, f, ta, &tva));
621ca3d3a14SMatthew G. Knepley   if (l2g) {
622ca3d3a14SMatthew G. Knepley     switch (dof) {
623d71ae5a4SJacob Faibussowitsch     case 4:
624d71ae5a4SJacob Faibussowitsch       DMPlex_Mult2D_Internal(tva, 1, a, a);
625d71ae5a4SJacob Faibussowitsch       break;
626d71ae5a4SJacob Faibussowitsch     case 9:
627d71ae5a4SJacob Faibussowitsch       DMPlex_Mult3D_Internal(tva, 1, a, a);
628d71ae5a4SJacob Faibussowitsch       break;
629ca3d3a14SMatthew G. Knepley     }
630ca3d3a14SMatthew G. Knepley   } else {
631ca3d3a14SMatthew G. Knepley     switch (dof) {
632d71ae5a4SJacob Faibussowitsch     case 4:
633d71ae5a4SJacob Faibussowitsch       DMPlex_MultTranspose2D_Internal(tva, 1, a, a);
634d71ae5a4SJacob Faibussowitsch       break;
635d71ae5a4SJacob Faibussowitsch     case 9:
636d71ae5a4SJacob Faibussowitsch       DMPlex_MultTranspose3D_Internal(tva, 1, a, a);
637d71ae5a4SJacob Faibussowitsch       break;
638ca3d3a14SMatthew G. Knepley     }
639ca3d3a14SMatthew G. Knepley   }
6409566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
6413ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
642ca3d3a14SMatthew G. Knepley }
643ca3d3a14SMatthew G. Knepley 
644d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformFieldTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt pf, PetscInt f, PetscInt pg, PetscInt g, PetscBool l2g, PetscInt lda, PetscScalar *a)
645d71ae5a4SJacob Faibussowitsch {
646ca3d3a14SMatthew G. Knepley   PetscSection       s, ts;
647ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tvaf, *tvag;
648ca3d3a14SMatthew G. Knepley   PetscInt           fdof, gdof, fpdof, gpdof;
649ca3d3a14SMatthew G. Knepley 
650ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
6519566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
6529566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
6539566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pf, f, &fpdof));
6549566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pg, g, &gpdof));
6559566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pf, f, &fdof));
6569566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pg, g, &gdof));
6579566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
6589566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pf, f, ta, &tvaf));
6599566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pg, g, ta, &tvag));
660ca3d3a14SMatthew G. Knepley   if (l2g) {
661ca3d3a14SMatthew G. Knepley     switch (fdof) {
662d71ae5a4SJacob Faibussowitsch     case 4:
663d71ae5a4SJacob Faibussowitsch       DMPlex_MatMult2D_Internal(tvaf, gpdof, lda, a, a);
664d71ae5a4SJacob Faibussowitsch       break;
665d71ae5a4SJacob Faibussowitsch     case 9:
666d71ae5a4SJacob Faibussowitsch       DMPlex_MatMult3D_Internal(tvaf, gpdof, lda, a, a);
667d71ae5a4SJacob Faibussowitsch       break;
668ca3d3a14SMatthew G. Knepley     }
669ca3d3a14SMatthew G. Knepley     switch (gdof) {
670d71ae5a4SJacob Faibussowitsch     case 4:
671d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultTransposeLeft2D_Internal(tvag, fpdof, lda, a, a);
672d71ae5a4SJacob Faibussowitsch       break;
673d71ae5a4SJacob Faibussowitsch     case 9:
674d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultTransposeLeft3D_Internal(tvag, fpdof, lda, a, a);
675d71ae5a4SJacob Faibussowitsch       break;
676ca3d3a14SMatthew G. Knepley     }
677ca3d3a14SMatthew G. Knepley   } else {
678ca3d3a14SMatthew G. Knepley     switch (fdof) {
679d71ae5a4SJacob Faibussowitsch     case 4:
680d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultTranspose2D_Internal(tvaf, gpdof, lda, a, a);
681d71ae5a4SJacob Faibussowitsch       break;
682d71ae5a4SJacob Faibussowitsch     case 9:
683d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultTranspose3D_Internal(tvaf, gpdof, lda, a, a);
684d71ae5a4SJacob Faibussowitsch       break;
685ca3d3a14SMatthew G. Knepley     }
686ca3d3a14SMatthew G. Knepley     switch (gdof) {
687d71ae5a4SJacob Faibussowitsch     case 4:
688d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultLeft2D_Internal(tvag, fpdof, lda, a, a);
689d71ae5a4SJacob Faibussowitsch       break;
690d71ae5a4SJacob Faibussowitsch     case 9:
691d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultLeft3D_Internal(tvag, fpdof, lda, a, a);
692d71ae5a4SJacob Faibussowitsch       break;
693ca3d3a14SMatthew G. Knepley     }
694ca3d3a14SMatthew G. Knepley   }
6959566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
6963ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
697ca3d3a14SMatthew G. Knepley }
698ca3d3a14SMatthew G. Knepley 
699d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexBasisTransformPoint_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool fieldActive[], PetscBool l2g, PetscScalar *a)
700d71ae5a4SJacob Faibussowitsch {
701ca3d3a14SMatthew G. Knepley   PetscSection    s;
702ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
703ca3d3a14SMatthew G. Knepley   IS              clPoints;
704ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
705ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
706ca3d3a14SMatthew G. Knepley   PetscInt        Nf, f, Np, cp, dof, d = 0;
707ca3d3a14SMatthew G. Knepley 
708ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
7099566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
7109566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
71107218a29SMatthew G. Knepley   PetscCall(DMPlexGetCompressedClosure(dm, s, p, 0, &Np, &points, &clSection, &clPoints, &clp));
712ca3d3a14SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
713ca3d3a14SMatthew G. Knepley     for (cp = 0; cp < Np * 2; cp += 2) {
7149566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cp], f, &dof));
715ca3d3a14SMatthew G. Knepley       if (!dof) continue;
7169566063dSJacob Faibussowitsch       if (fieldActive[f]) PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, points[cp], f, l2g, &a[d]));
717ca3d3a14SMatthew G. Knepley       d += dof;
718ca3d3a14SMatthew G. Knepley     }
719ca3d3a14SMatthew G. Knepley   }
7209566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
7213ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
722ca3d3a14SMatthew G. Knepley }
723ca3d3a14SMatthew G. Knepley 
724d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexBasisTransformPointTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool l2g, PetscInt lda, PetscScalar *a)
725d71ae5a4SJacob Faibussowitsch {
726ca3d3a14SMatthew G. Knepley   PetscSection    s;
727ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
728ca3d3a14SMatthew G. Knepley   IS              clPoints;
729ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
730ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
7318bdb3c71SMatthew G. Knepley   PetscInt        Nf, f, g, Np, cpf, cpg, fdof, gdof, r, c = 0;
732ca3d3a14SMatthew G. Knepley 
733ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
7349566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
7359566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
73607218a29SMatthew G. Knepley   PetscCall(DMPlexGetCompressedClosure(dm, s, p, 0, &Np, &points, &clSection, &clPoints, &clp));
737ca3d3a14SMatthew G. Knepley   for (f = 0, r = 0; f < Nf; ++f) {
738ca3d3a14SMatthew G. Knepley     for (cpf = 0; cpf < Np * 2; cpf += 2) {
7399566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cpf], f, &fdof));
740ca3d3a14SMatthew G. Knepley       for (g = 0, c = 0; g < Nf; ++g) {
741ca3d3a14SMatthew G. Knepley         for (cpg = 0; cpg < Np * 2; cpg += 2) {
7429566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldDof(s, points[cpg], g, &gdof));
7439566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformFieldTensor_Internal(dm, tdm, tv, points[cpf], f, points[cpg], g, l2g, lda, &a[r * lda + c]));
744ca3d3a14SMatthew G. Knepley           c += gdof;
745ca3d3a14SMatthew G. Knepley         }
746ca3d3a14SMatthew G. Knepley       }
74763a3b9bcSJacob Faibussowitsch       PetscCheck(c == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of columns %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
748ca3d3a14SMatthew G. Knepley       r += fdof;
749ca3d3a14SMatthew G. Knepley     }
750ca3d3a14SMatthew G. Knepley   }
75163a3b9bcSJacob Faibussowitsch   PetscCheck(r == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of rows %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
7529566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
7533ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
754ca3d3a14SMatthew G. Knepley }
755ca3d3a14SMatthew G. Knepley 
756d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransform_Internal(DM dm, Vec lv, PetscBool l2g)
757d71ae5a4SJacob Faibussowitsch {
758ca3d3a14SMatthew G. Knepley   DM                 tdm;
759ca3d3a14SMatthew G. Knepley   Vec                tv;
760ca3d3a14SMatthew G. Knepley   PetscSection       ts, s;
761ca3d3a14SMatthew G. Knepley   const PetscScalar *ta;
762ca3d3a14SMatthew G. Knepley   PetscScalar       *a, *va;
763ca3d3a14SMatthew G. Knepley   PetscInt           pStart, pEnd, p, Nf, f;
764ca3d3a14SMatthew G. Knepley 
765ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
7669566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
7679566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
7689566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
7699566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
7709566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetChart(s, &pStart, &pEnd));
7719566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
7729566063dSJacob Faibussowitsch   PetscCall(VecGetArray(lv, &a));
7739566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
774ca3d3a14SMatthew G. Knepley   for (p = pStart; p < pEnd; ++p) {
775ca3d3a14SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
7769566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalFieldRef(dm, p, f, a, &va));
7779566063dSJacob Faibussowitsch       PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, p, f, l2g, va));
778ca3d3a14SMatthew G. Knepley     }
779ca3d3a14SMatthew G. Knepley   }
7809566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(lv, &a));
7819566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
7823ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
783ca3d3a14SMatthew G. Knepley }
784ca3d3a14SMatthew G. Knepley 
785ca3d3a14SMatthew G. Knepley /*@
786ca3d3a14SMatthew G. Knepley   DMPlexGlobalToLocalBasis - Transform the values in the given local vector from the global basis to the local basis
787ca3d3a14SMatthew G. Knepley 
788ca3d3a14SMatthew G. Knepley   Input Parameters:
789a1cb98faSBarry Smith + dm - The `DM`
790ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the global basis
791ca3d3a14SMatthew G. Knepley 
7922fe279fdSBarry Smith   Output Parameter:
793ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the local basis
794ca3d3a14SMatthew G. Knepley 
795ca3d3a14SMatthew G. Knepley   Level: developer
796ca3d3a14SMatthew G. Knepley 
797a1cb98faSBarry Smith   Note:
798a1cb98faSBarry Smith   This method is only intended to be called inside `DMGlobalToLocal()`. It is unlikely that a user will have a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
799a1cb98faSBarry Smith 
8001cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLocalToGlobalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()`
801ca3d3a14SMatthew G. Knepley @*/
802d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGlobalToLocalBasis(DM dm, Vec lv)
803d71ae5a4SJacob Faibussowitsch {
804ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
805ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
806ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
8079566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_FALSE));
8083ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
809ca3d3a14SMatthew G. Knepley }
810ca3d3a14SMatthew G. Knepley 
811ca3d3a14SMatthew G. Knepley /*@
812ca3d3a14SMatthew G. Knepley   DMPlexLocalToGlobalBasis - Transform the values in the given local vector from the local basis to the global basis
813ca3d3a14SMatthew G. Knepley 
814ca3d3a14SMatthew G. Knepley   Input Parameters:
815a1cb98faSBarry Smith + dm - The `DM`
816ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the local basis
817ca3d3a14SMatthew G. Knepley 
8182fe279fdSBarry Smith   Output Parameter:
819ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the global basis
820ca3d3a14SMatthew G. Knepley 
821ca3d3a14SMatthew G. Knepley   Level: developer
822ca3d3a14SMatthew G. Knepley 
823a1cb98faSBarry Smith   Note:
824a1cb98faSBarry Smith   This method is only intended to be called inside `DMGlobalToLocal()`. It is unlikely that a user would want a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
825a1cb98faSBarry Smith 
8261cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGlobalToLocalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()`
827ca3d3a14SMatthew G. Knepley @*/
828d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexLocalToGlobalBasis(DM dm, Vec lv)
829d71ae5a4SJacob Faibussowitsch {
830ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
831ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
832ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
8339566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_TRUE));
8343ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
835ca3d3a14SMatthew G. Knepley }
836ca3d3a14SMatthew G. Knepley 
837ca3d3a14SMatthew G. Knepley /*@
838ca3d3a14SMatthew G. Knepley   DMPlexCreateBasisRotation - Create an internal transformation from the global basis, used to specify boundary conditions
839ca3d3a14SMatthew G. Knepley   and global solutions, to a local basis, appropriate for discretization integrals and assembly.
840ca3d3a14SMatthew G. Knepley 
841ca3d3a14SMatthew G. Knepley   Input Parameters:
842a1cb98faSBarry Smith + dm    - The `DM`
843ca3d3a14SMatthew G. Knepley . alpha - The first Euler angle, and in 2D the only one
844ca3d3a14SMatthew G. Knepley . beta  - The second Euler angle
845f0fc11ceSJed Brown - gamma - The third Euler angle
846ca3d3a14SMatthew G. Knepley 
847ca3d3a14SMatthew G. Knepley   Level: developer
848ca3d3a14SMatthew G. Knepley 
849a1cb98faSBarry Smith   Note:
850a1cb98faSBarry Smith   Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
851a1cb98faSBarry Smith   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows
852a1cb98faSBarry Smith .vb
853a1cb98faSBarry Smith    The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
854a1cb98faSBarry Smith    The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
855a1cb98faSBarry Smith    The XYZ system rotates a third time about the z axis by gamma.
856a1cb98faSBarry Smith .ve
857a1cb98faSBarry Smith 
8581cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGlobalToLocalBasis()`, `DMPlexLocalToGlobalBasis()`
859ca3d3a14SMatthew G. Knepley @*/
860d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexCreateBasisRotation(DM dm, PetscReal alpha, PetscReal beta, PetscReal gamma)
861d71ae5a4SJacob Faibussowitsch {
862ca3d3a14SMatthew G. Knepley   RotCtx  *rc;
863ca3d3a14SMatthew G. Knepley   PetscInt cdim;
864ca3d3a14SMatthew G. Knepley 
865362febeeSStefano Zampini   PetscFunctionBegin;
8669566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
8679566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(1, &rc));
868ca3d3a14SMatthew G. Knepley   dm->transformCtx       = rc;
869ca3d3a14SMatthew G. Knepley   dm->transformSetUp     = DMPlexBasisTransformSetUp_Rotation_Internal;
870ca3d3a14SMatthew G. Knepley   dm->transformDestroy   = DMPlexBasisTransformDestroy_Rotation_Internal;
871ca3d3a14SMatthew G. Knepley   dm->transformGetMatrix = DMPlexBasisTransformGetMatrix_Rotation_Internal;
872ca3d3a14SMatthew G. Knepley   rc->dim                = cdim;
873ca3d3a14SMatthew G. Knepley   rc->alpha              = alpha;
874ca3d3a14SMatthew G. Knepley   rc->beta               = beta;
875ca3d3a14SMatthew G. Knepley   rc->gamma              = gamma;
8769566063dSJacob Faibussowitsch   PetscCall((*dm->transformSetUp)(dm, dm->transformCtx));
8779566063dSJacob Faibussowitsch   PetscCall(DMConstructBasisTransform_Internal(dm));
8783ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
879ca3d3a14SMatthew G. Knepley }
880ca3d3a14SMatthew G. Knepley 
881b278463cSMatthew G. Knepley /*@C
882ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssential - Insert boundary values into a local vector using a function of the coordinates
883b278463cSMatthew G. Knepley 
884b278463cSMatthew G. Knepley   Input Parameters:
885a1cb98faSBarry Smith + dm     - The `DM`, with a `PetscDS` that matches the problem being constrained
886b278463cSMatthew G. Knepley . time   - The time
887b278463cSMatthew G. Knepley . field  - The field to constrain
8881c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
88920f4b53cSBarry Smith . comps  - An array of constrained component numbers, or `NULL` for all components
890a1cb98faSBarry Smith . label  - The `DMLabel` defining constrained points
891a1cb98faSBarry Smith . numids - The number of `DMLabel` ids for constrained points
892b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
893b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
894b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
895b278463cSMatthew G. Knepley 
896b278463cSMatthew G. Knepley   Output Parameter:
897b278463cSMatthew G. Knepley . locX - A local vector to receives the boundary values
898b278463cSMatthew G. Knepley 
899b278463cSMatthew G. Knepley   Level: developer
900b278463cSMatthew G. Knepley 
9011cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLabel`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()`
902b278463cSMatthew G. Knepley @*/
903d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValuesEssential(DM dm, PetscReal time, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void *ctx, Vec locX)
904d71ae5a4SJacob Faibussowitsch {
9050163fd50SMatthew G. Knepley   PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx);
90655f2e967SMatthew G. Knepley   void   **ctxs;
907d7ddef95SMatthew G. Knepley   PetscInt numFields;
908d7ddef95SMatthew G. Knepley 
909d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
9109566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
9119566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs));
912d7ddef95SMatthew G. Knepley   funcs[field] = func;
913d7ddef95SMatthew G. Knepley   ctxs[field]  = ctx;
9149566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_BC_VALUES, locX));
9159566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs, ctxs));
9163ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
917d7ddef95SMatthew G. Knepley }
918d7ddef95SMatthew G. Knepley 
919b278463cSMatthew G. Knepley /*@C
920ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialField - Insert boundary values into a local vector using a function of the coordinates and field data
921b278463cSMatthew G. Knepley 
922b278463cSMatthew G. Knepley   Input Parameters:
923a1cb98faSBarry Smith + dm     - The `DM`, with a `PetscDS` that matches the problem being constrained
924b278463cSMatthew G. Knepley . time   - The time
925b278463cSMatthew G. Knepley . locU   - A local vector with the input solution values
926b278463cSMatthew G. Knepley . field  - The field to constrain
9271c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
92820f4b53cSBarry Smith . comps  - An array of constrained component numbers, or `NULL` for all components
929a1cb98faSBarry Smith . label  - The `DMLabel` defining constrained points
930a1cb98faSBarry Smith . numids - The number of `DMLabel` ids for constrained points
931b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
932b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
933b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
934b278463cSMatthew G. Knepley 
935b278463cSMatthew G. Knepley   Output Parameter:
936b278463cSMatthew G. Knepley . locX - A local vector to receives the boundary values
937b278463cSMatthew G. Knepley 
938b278463cSMatthew G. Knepley   Level: developer
939b278463cSMatthew G. Knepley 
9401cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()`
941b278463cSMatthew G. Knepley @*/
942d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValuesEssentialField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX)
943d71ae5a4SJacob Faibussowitsch {
9449371c9d4SSatish Balay   void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
945c60e475cSMatthew G. Knepley   void   **ctxs;
946c60e475cSMatthew G. Knepley   PetscInt numFields;
947c60e475cSMatthew G. Knepley 
948c60e475cSMatthew G. Knepley   PetscFunctionBegin;
9499566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
9509566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs));
951c60e475cSMatthew G. Knepley   funcs[field] = func;
952c60e475cSMatthew G. Knepley   ctxs[field]  = ctx;
9539566063dSJacob Faibussowitsch   PetscCall(DMProjectFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
9549566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs, ctxs));
9553ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
956c60e475cSMatthew G. Knepley }
957c60e475cSMatthew G. Knepley 
958b278463cSMatthew G. Knepley /*@C
959d5b43468SJose E. Roman   DMPlexInsertBoundaryValuesEssentialBdField - Insert boundary values into a local vector using a function of the coordinates and boundary field data
960ece3a9fcSMatthew G. Knepley 
96120f4b53cSBarry Smith   Collective
962ece3a9fcSMatthew G. Knepley 
963ece3a9fcSMatthew G. Knepley   Input Parameters:
964a1cb98faSBarry Smith + dm     - The `DM`, with a `PetscDS` that matches the problem being constrained
965ece3a9fcSMatthew G. Knepley . time   - The time
966ece3a9fcSMatthew G. Knepley . locU   - A local vector with the input solution values
967ece3a9fcSMatthew G. Knepley . field  - The field to constrain
968ece3a9fcSMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
96920f4b53cSBarry Smith . comps  - An array of constrained component numbers, or `NULL` for all components
970a1cb98faSBarry Smith . label  - The `DMLabel` defining constrained points
971a1cb98faSBarry Smith . numids - The number of `DMLabel` ids for constrained points
972ece3a9fcSMatthew G. Knepley . ids    - An array of ids for constrained points
97320f4b53cSBarry Smith . func   - A pointwise function giving boundary values, the calling sequence is given in `DMProjectBdFieldLabelLocal()`
97420f4b53cSBarry Smith - ctx    - An optional user context for `func`
975ece3a9fcSMatthew G. Knepley 
976ece3a9fcSMatthew G. Knepley   Output Parameter:
977ece3a9fcSMatthew G. Knepley . locX - A local vector to receive the boundary values
978ece3a9fcSMatthew G. Knepley 
979ece3a9fcSMatthew G. Knepley   Level: developer
980ece3a9fcSMatthew G. Knepley 
9811cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectBdFieldLabelLocal()`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()`
982ece3a9fcSMatthew G. Knepley @*/
983d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValuesEssentialBdField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX)
984d71ae5a4SJacob Faibussowitsch {
9859371c9d4SSatish Balay   void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
986ece3a9fcSMatthew G. Knepley   void   **ctxs;
987ece3a9fcSMatthew G. Knepley   PetscInt numFields;
988ece3a9fcSMatthew G. Knepley 
989ece3a9fcSMatthew G. Knepley   PetscFunctionBegin;
9909566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
9919566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs));
992ece3a9fcSMatthew G. Knepley   funcs[field] = func;
993ece3a9fcSMatthew G. Knepley   ctxs[field]  = ctx;
9949566063dSJacob Faibussowitsch   PetscCall(DMProjectBdFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
9959566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs, ctxs));
9963ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
997ece3a9fcSMatthew G. Knepley }
998ece3a9fcSMatthew G. Knepley 
999ece3a9fcSMatthew G. Knepley /*@C
1000b278463cSMatthew G. Knepley   DMPlexInsertBoundaryValuesRiemann - Insert boundary values into a local vector
1001b278463cSMatthew G. Knepley 
1002b278463cSMatthew G. Knepley   Input Parameters:
1003a1cb98faSBarry Smith + dm           - The `DM`, with a `PetscDS` that matches the problem being constrained
1004b278463cSMatthew G. Knepley . time         - The time
1005b278463cSMatthew G. Knepley . faceGeometry - A vector with the FVM face geometry information
1006b278463cSMatthew G. Knepley . cellGeometry - A vector with the FVM cell geometry information
1007b278463cSMatthew G. Knepley . Grad         - A vector with the FVM cell gradient information
1008b278463cSMatthew G. Knepley . field        - The field to constrain
10091c531cf8SMatthew G. Knepley . Nc           - The number of constrained field components, or 0 for all components
101020f4b53cSBarry Smith . comps        - An array of constrained component numbers, or `NULL` for all components
1011a1cb98faSBarry Smith . label        - The `DMLabel` defining constrained points
1012a1cb98faSBarry Smith . numids       - The number of `DMLabel` ids for constrained points
1013b278463cSMatthew G. Knepley . ids          - An array of ids for constrained points
1014b278463cSMatthew G. Knepley . func         - A pointwise function giving boundary values
1015b278463cSMatthew G. Knepley - ctx          - An optional user context for bcFunc
1016b278463cSMatthew G. Knepley 
1017b278463cSMatthew G. Knepley   Output Parameter:
1018b278463cSMatthew G. Knepley . locX - A local vector to receives the boundary values
1019b278463cSMatthew G. Knepley 
1020b278463cSMatthew G. Knepley   Level: developer
1021b278463cSMatthew G. Knepley 
1022a1cb98faSBarry Smith   Note:
1023a1cb98faSBarry Smith   This implementation currently ignores the numcomps/comps argument from `DMAddBoundary()`
1024a1cb98faSBarry Smith 
10251cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()`
1026b278463cSMatthew G. Knepley @*/
1027d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValuesRiemann(DM dm, PetscReal time, Vec faceGeometry, Vec cellGeometry, Vec Grad, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *), void *ctx, Vec locX)
1028d71ae5a4SJacob Faibussowitsch {
102961f58d28SMatthew G. Knepley   PetscDS            prob;
1030da97024aSMatthew G. Knepley   PetscSF            sf;
1031d7ddef95SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad;
103220369375SToby Isaac   const PetscScalar *facegeom, *cellgeom = NULL, *grad;
1033da97024aSMatthew G. Knepley   const PetscInt    *leaves;
1034d7ddef95SMatthew G. Knepley   PetscScalar       *x, *fx;
1035520b3818SMatthew G. Knepley   PetscInt           dim, nleaves, loc, fStart, fEnd, pdim, i;
10363ba16761SJacob Faibussowitsch   PetscErrorCode     ierru = PETSC_SUCCESS;
1037d7ddef95SMatthew G. Knepley 
1038d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
10399566063dSJacob Faibussowitsch   PetscCall(DMGetPointSF(dm, &sf));
10409566063dSJacob Faibussowitsch   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
1041da97024aSMatthew G. Knepley   nleaves = PetscMax(0, nleaves);
10429566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
10439566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
10449566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
10459566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
10469566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
104720369375SToby Isaac   if (cellGeometry) {
10489566063dSJacob Faibussowitsch     PetscCall(VecGetDM(cellGeometry, &dmCell));
10499566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
105020369375SToby Isaac   }
1051d7ddef95SMatthew G. Knepley   if (Grad) {
1052c0a6632aSMatthew G. Knepley     PetscFV fv;
1053c0a6632aSMatthew G. Knepley 
10549566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fv));
10559566063dSJacob Faibussowitsch     PetscCall(VecGetDM(Grad, &dmGrad));
10569566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(Grad, &grad));
10579566063dSJacob Faibussowitsch     PetscCall(PetscFVGetNumComponents(fv, &pdim));
10589566063dSJacob Faibussowitsch     PetscCall(DMGetWorkArray(dm, pdim, MPIU_SCALAR, &fx));
1059d7ddef95SMatthew G. Knepley   }
10609566063dSJacob Faibussowitsch   PetscCall(VecGetArray(locX, &x));
1061d7ddef95SMatthew G. Knepley   for (i = 0; i < numids; ++i) {
1062d7ddef95SMatthew G. Knepley     IS              faceIS;
1063d7ddef95SMatthew G. Knepley     const PetscInt *faces;
1064d7ddef95SMatthew G. Knepley     PetscInt        numFaces, f;
1065d7ddef95SMatthew G. Knepley 
10669566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, ids[i], &faceIS));
1067d7ddef95SMatthew G. Knepley     if (!faceIS) continue; /* No points with that id on this process */
10689566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(faceIS, &numFaces));
10699566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(faceIS, &faces));
1070d7ddef95SMatthew G. Knepley     for (f = 0; f < numFaces; ++f) {
1071d7ddef95SMatthew G. Knepley       const PetscInt   face = faces[f], *cells;
1072640bce14SSatish Balay       PetscFVFaceGeom *fg;
1073d7ddef95SMatthew G. Knepley 
1074d7ddef95SMatthew G. Knepley       if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */
10759566063dSJacob Faibussowitsch       PetscCall(PetscFindInt(face, nleaves, (PetscInt *)leaves, &loc));
1076da97024aSMatthew G. Knepley       if (loc >= 0) continue;
10779566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
10789566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, face, &cells));
1079d7ddef95SMatthew G. Knepley       if (Grad) {
1080640bce14SSatish Balay         PetscFVCellGeom *cg;
1081640bce14SSatish Balay         PetscScalar     *cx, *cgrad;
1082d7ddef95SMatthew G. Knepley         PetscScalar     *xG;
1083d7ddef95SMatthew G. Knepley         PetscReal        dx[3];
1084d7ddef95SMatthew G. Knepley         PetscInt         d;
1085d7ddef95SMatthew G. Knepley 
10869566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg));
10879566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &cx));
10889566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad));
10899566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
1090d7ddef95SMatthew G. Knepley         DMPlex_WaxpyD_Internal(dim, -1, cg->centroid, fg->centroid, dx);
1091d7ddef95SMatthew G. Knepley         for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DMPlex_DotD_Internal(dim, &cgrad[d * dim], dx);
10929566063dSJacob Faibussowitsch         PetscCall((*func)(time, fg->centroid, fg->normal, fx, xG, ctx));
1093d7ddef95SMatthew G. Knepley       } else {
1094640bce14SSatish Balay         PetscScalar *xI;
1095d7ddef95SMatthew G. Knepley         PetscScalar *xG;
1096d7ddef95SMatthew G. Knepley 
10979566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &xI));
10989566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
1099e735a8a9SMatthew G. Knepley         ierru = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx);
1100e735a8a9SMatthew G. Knepley         if (ierru) {
11019566063dSJacob Faibussowitsch           PetscCall(ISRestoreIndices(faceIS, &faces));
11029566063dSJacob Faibussowitsch           PetscCall(ISDestroy(&faceIS));
1103e735a8a9SMatthew G. Knepley           goto cleanup;
1104e735a8a9SMatthew G. Knepley         }
1105d7ddef95SMatthew G. Knepley       }
1106d7ddef95SMatthew G. Knepley     }
11079566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(faceIS, &faces));
11089566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&faceIS));
1109d7ddef95SMatthew G. Knepley   }
1110e735a8a9SMatthew G. Knepley cleanup:
11119566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(locX, &x));
1112d7ddef95SMatthew G. Knepley   if (Grad) {
11139566063dSJacob Faibussowitsch     PetscCall(DMRestoreWorkArray(dm, pdim, MPIU_SCALAR, &fx));
11149566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(Grad, &grad));
1115d7ddef95SMatthew G. Knepley   }
11169566063dSJacob Faibussowitsch   if (cellGeometry) PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
11179566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
11189566063dSJacob Faibussowitsch   PetscCall(ierru);
11193ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1120d7ddef95SMatthew G. Knepley }
1121d7ddef95SMatthew G. Knepley 
1122d71ae5a4SJacob Faibussowitsch static PetscErrorCode zero(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nc, PetscScalar *u, void *ctx)
1123d71ae5a4SJacob Faibussowitsch {
11240c364540SMatthew G. Knepley   PetscInt c;
11250c364540SMatthew G. Knepley   for (c = 0; c < Nc; ++c) u[c] = 0.0;
11263ba16761SJacob Faibussowitsch   return PETSC_SUCCESS;
11270c364540SMatthew G. Knepley }
11280c364540SMatthew G. Knepley 
1129d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1130d71ae5a4SJacob Faibussowitsch {
11310c364540SMatthew G. Knepley   PetscObject isZero;
1132e5e52638SMatthew G. Knepley   PetscDS     prob;
1133d7ddef95SMatthew G. Knepley   PetscInt    numBd, b;
113455f2e967SMatthew G. Knepley 
113555f2e967SMatthew G. Knepley   PetscFunctionBegin;
11369566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
11379566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
11389566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero));
1139450c7a9fSMatthew G. Knepley   PetscCall(PetscDSUpdateBoundaryLabels(prob, dm));
114055f2e967SMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
114145480ffeSMatthew G. Knepley     PetscWeakForm           wf;
1142f971fd6bSMatthew G. Knepley     DMBoundaryConditionType type;
114345480ffeSMatthew G. Knepley     const char             *name;
1144d7ddef95SMatthew G. Knepley     DMLabel                 label;
11451c531cf8SMatthew G. Knepley     PetscInt                field, Nc;
11461c531cf8SMatthew G. Knepley     const PetscInt         *comps;
1147d7ddef95SMatthew G. Knepley     PetscObject             obj;
1148d7ddef95SMatthew G. Knepley     PetscClassId            id;
114945480ffeSMatthew G. Knepley     void (*bvfunc)(void);
1150d7ddef95SMatthew G. Knepley     PetscInt        numids;
1151d7ddef95SMatthew G. Knepley     const PetscInt *ids;
115255f2e967SMatthew G. Knepley     void           *ctx;
115355f2e967SMatthew G. Knepley 
11549566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, &bvfunc, NULL, &ctx));
1155f971fd6bSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
11569566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
11579566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1158d7ddef95SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1159c60e475cSMatthew G. Knepley       switch (type) {
1160c60e475cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
11619371c9d4SSatish Balay       case DM_BC_ESSENTIAL: {
11628434afd1SBarry Smith         PetscSimplePointFn *func = (PetscSimplePointFn *)bvfunc;
116345480ffeSMatthew G. Knepley 
116445480ffeSMatthew G. Knepley         if (isZero) func = zero;
11659566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
11669566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func, ctx, locX));
11679566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
11689371c9d4SSatish Balay       } break;
11699371c9d4SSatish Balay       case DM_BC_ESSENTIAL_FIELD: {
11702192575eSBarry Smith         PetscPointFn *func = (PetscPointFn *)bvfunc;
117145480ffeSMatthew G. Knepley 
11729566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
11739566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func, ctx, locX));
11749566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
11759371c9d4SSatish Balay       } break;
1176d71ae5a4SJacob Faibussowitsch       default:
1177d71ae5a4SJacob Faibussowitsch         break;
1178c60e475cSMatthew G. Knepley       }
1179d7ddef95SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
118045480ffeSMatthew G. Knepley       {
118145480ffeSMatthew G. Knepley         PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *) = (PetscErrorCode (*)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *))bvfunc;
118245480ffeSMatthew G. Knepley 
118343ea7facSMatthew G. Knepley         if (!faceGeomFVM) continue;
11849566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesRiemann(dm, time, faceGeomFVM, cellGeomFVM, gradFVM, field, Nc, comps, label, numids, ids, func, ctx, locX));
118545480ffeSMatthew G. Knepley       }
118663a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
118755f2e967SMatthew G. Knepley   }
11883ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
118955f2e967SMatthew G. Knepley }
119055f2e967SMatthew G. Knepley 
1191d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1192d71ae5a4SJacob Faibussowitsch {
119356cf3b9cSMatthew G. Knepley   PetscObject isZero;
119456cf3b9cSMatthew G. Knepley   PetscDS     prob;
119556cf3b9cSMatthew G. Knepley   PetscInt    numBd, b;
119656cf3b9cSMatthew G. Knepley 
119756cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
11983ba16761SJacob Faibussowitsch   if (!locX) PetscFunctionReturn(PETSC_SUCCESS);
11999566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
12009566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
12019566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero));
120256cf3b9cSMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
120345480ffeSMatthew G. Knepley     PetscWeakForm           wf;
120456cf3b9cSMatthew G. Knepley     DMBoundaryConditionType type;
120545480ffeSMatthew G. Knepley     const char             *name;
120656cf3b9cSMatthew G. Knepley     DMLabel                 label;
120756cf3b9cSMatthew G. Knepley     PetscInt                field, Nc;
120856cf3b9cSMatthew G. Knepley     const PetscInt         *comps;
120956cf3b9cSMatthew G. Knepley     PetscObject             obj;
121056cf3b9cSMatthew G. Knepley     PetscClassId            id;
121156cf3b9cSMatthew G. Knepley     PetscInt                numids;
121256cf3b9cSMatthew G. Knepley     const PetscInt         *ids;
121345480ffeSMatthew G. Knepley     void (*bvfunc)(void);
121456cf3b9cSMatthew G. Knepley     void *ctx;
121556cf3b9cSMatthew G. Knepley 
12169566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, NULL, &bvfunc, &ctx));
121756cf3b9cSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
12189566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
12199566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
122056cf3b9cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
122156cf3b9cSMatthew G. Knepley       switch (type) {
122256cf3b9cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
12239371c9d4SSatish Balay       case DM_BC_ESSENTIAL: {
12248434afd1SBarry Smith         PetscSimplePointFn *func_t = (PetscSimplePointFn *)bvfunc;
122545480ffeSMatthew G. Knepley 
122645480ffeSMatthew G. Knepley         if (isZero) func_t = zero;
12279566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
12289566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
12299566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
12309371c9d4SSatish Balay       } break;
12319371c9d4SSatish Balay       case DM_BC_ESSENTIAL_FIELD: {
12322192575eSBarry Smith         PetscPointFn *func_t = (PetscPointFn *)bvfunc;
123345480ffeSMatthew G. Knepley 
12349566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
12359566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
12369566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
12379371c9d4SSatish Balay       } break;
1238d71ae5a4SJacob Faibussowitsch       default:
1239d71ae5a4SJacob Faibussowitsch         break;
124056cf3b9cSMatthew G. Knepley       }
124163a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
124256cf3b9cSMatthew G. Knepley   }
12433ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
124456cf3b9cSMatthew G. Knepley }
124556cf3b9cSMatthew G. Knepley 
124601468941SMatthew G. Knepley PetscErrorCode DMPlexInsertBounds_Plex(DM dm, PetscBool lower, PetscReal time, Vec locB)
124701468941SMatthew G. Knepley {
124801468941SMatthew G. Knepley   PetscDS  ds;
124901468941SMatthew G. Knepley   PetscInt numBd;
125001468941SMatthew G. Knepley 
125101468941SMatthew G. Knepley   PetscFunctionBegin;
125201468941SMatthew G. Knepley   PetscCall(DMGetDS(dm, &ds));
125301468941SMatthew G. Knepley   PetscCall(PetscDSGetNumBoundary(ds, &numBd));
125401468941SMatthew G. Knepley   PetscCall(PetscDSUpdateBoundaryLabels(ds, dm));
125501468941SMatthew G. Knepley   for (PetscInt b = 0; b < numBd; ++b) {
125601468941SMatthew G. Knepley     PetscWeakForm           wf;
125701468941SMatthew G. Knepley     DMBoundaryConditionType type;
125801468941SMatthew G. Knepley     const char             *name;
125901468941SMatthew G. Knepley     DMLabel                 label;
126001468941SMatthew G. Knepley     PetscInt                numids;
126101468941SMatthew G. Knepley     const PetscInt         *ids;
126201468941SMatthew G. Knepley     PetscInt                field, Nc;
126301468941SMatthew G. Knepley     const PetscInt         *comps;
126401468941SMatthew G. Knepley     void (*bvfunc)(void);
126501468941SMatthew G. Knepley     void *ctx;
126601468941SMatthew G. Knepley 
126701468941SMatthew G. Knepley     PetscCall(PetscDSGetBoundary(ds, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, &bvfunc, NULL, &ctx));
126801468941SMatthew G. Knepley     if (lower && type != DM_BC_LOWER_BOUND) continue;
126901468941SMatthew G. Knepley     if (!lower && type != DM_BC_UPPER_BOUND) continue;
127001468941SMatthew G. Knepley     PetscCall(DMPlexLabelAddCells(dm, label));
127101468941SMatthew G. Knepley     {
127201468941SMatthew G. Knepley       PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx);
127301468941SMatthew G. Knepley       void   **ctxs;
127401468941SMatthew G. Knepley       PetscInt Nf;
127501468941SMatthew G. Knepley 
127601468941SMatthew G. Knepley       PetscCall(DMGetNumFields(dm, &Nf));
127701468941SMatthew G. Knepley       PetscCall(PetscCalloc2(Nf, &funcs, Nf, &ctxs));
127801468941SMatthew G. Knepley       funcs[field] = (PetscSimplePointFn *)bvfunc;
127901468941SMatthew G. Knepley       ctxs[field]  = ctx;
128001468941SMatthew G. Knepley       PetscCall(DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_ALL_VALUES, locB));
128101468941SMatthew G. Knepley       PetscCall(PetscFree2(funcs, ctxs));
128201468941SMatthew G. Knepley     }
128301468941SMatthew G. Knepley     PetscCall(DMPlexLabelClearCells(dm, label));
128401468941SMatthew G. Knepley   }
128501468941SMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
128601468941SMatthew G. Knepley }
128701468941SMatthew G. Knepley 
1288f1d73a7aSMatthew G. Knepley /*@
1289f1d73a7aSMatthew G. Knepley   DMPlexInsertBoundaryValues - Puts coefficients which represent boundary values into the local solution vector
1290f1d73a7aSMatthew G. Knepley 
1291ed808b8fSJed Brown   Not Collective
1292ed808b8fSJed Brown 
1293f1d73a7aSMatthew G. Knepley   Input Parameters:
1294a1cb98faSBarry Smith + dm              - The `DM`
1295f1d73a7aSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
1296f1d73a7aSMatthew G. Knepley . time            - The time
1297f1d73a7aSMatthew G. Knepley . faceGeomFVM     - Face geometry data for FV discretizations
1298f1d73a7aSMatthew G. Knepley . cellGeomFVM     - Cell geometry data for FV discretizations
1299f1d73a7aSMatthew G. Knepley - gradFVM         - Gradient reconstruction data for FV discretizations
1300f1d73a7aSMatthew G. Knepley 
13012fe279fdSBarry Smith   Output Parameter:
1302f1d73a7aSMatthew G. Knepley . locX - Solution updated with boundary values
1303f1d73a7aSMatthew G. Knepley 
1304ed808b8fSJed Brown   Level: intermediate
1305f1d73a7aSMatthew G. Knepley 
13061cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunctionLabelLocal()`, `DMAddBoundary()`
1307f1d73a7aSMatthew G. Knepley @*/
1308d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValues(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1309d71ae5a4SJacob Faibussowitsch {
1310f1d73a7aSMatthew G. Knepley   PetscFunctionBegin;
1311f1d73a7aSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1312064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
1313ad540459SPierre Jolivet   if (faceGeomFVM) PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);
1314ad540459SPierre Jolivet   if (cellGeomFVM) PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);
1315ad540459SPierre Jolivet   if (gradFVM) PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);
1316cac4c232SBarry Smith   PetscTryMethod(dm, "DMPlexInsertBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX, time, faceGeomFVM, cellGeomFVM, gradFVM));
13173ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1318f1d73a7aSMatthew G. Knepley }
1319f1d73a7aSMatthew G. Knepley 
132056cf3b9cSMatthew G. Knepley /*@
1321a5b23f4aSJose E. Roman   DMPlexInsertTimeDerivativeBoundaryValues - Puts coefficients which represent boundary values of the time derivative into the local solution vector
132256cf3b9cSMatthew G. Knepley 
132356cf3b9cSMatthew G. Knepley   Input Parameters:
1324a1cb98faSBarry Smith + dm              - The `DM`
132556cf3b9cSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
132656cf3b9cSMatthew G. Knepley . time            - The time
132756cf3b9cSMatthew G. Knepley . faceGeomFVM     - Face geometry data for FV discretizations
132856cf3b9cSMatthew G. Knepley . cellGeomFVM     - Cell geometry data for FV discretizations
132956cf3b9cSMatthew G. Knepley - gradFVM         - Gradient reconstruction data for FV discretizations
133056cf3b9cSMatthew G. Knepley 
13312fe279fdSBarry Smith   Output Parameter:
133256cf3b9cSMatthew G. Knepley . locX_t - Solution updated with boundary values
133356cf3b9cSMatthew G. Knepley 
133456cf3b9cSMatthew G. Knepley   Level: developer
133556cf3b9cSMatthew G. Knepley 
13361cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunctionLabelLocal()`
133756cf3b9cSMatthew G. Knepley @*/
1338d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues(DM dm, PetscBool insertEssential, Vec locX_t, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1339d71ae5a4SJacob Faibussowitsch {
134056cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
134156cf3b9cSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1342ad540459SPierre Jolivet   if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 3);
1343ad540459SPierre Jolivet   if (faceGeomFVM) PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);
1344ad540459SPierre Jolivet   if (cellGeomFVM) PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);
1345ad540459SPierre Jolivet   if (gradFVM) PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);
13466c51210dSStefano Zampini   PetscTryMethod(dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX_t, time, faceGeomFVM, cellGeomFVM, gradFVM));
13473ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
134856cf3b9cSMatthew G. Knepley }
134956cf3b9cSMatthew G. Knepley 
135001468941SMatthew G. Knepley /*@
135101468941SMatthew G. Knepley   DMPlexInsertBounds - Puts coefficients which represent solution bounds into the local bounds vector
135201468941SMatthew G. Knepley 
135301468941SMatthew G. Knepley   Not Collective
135401468941SMatthew G. Knepley 
135501468941SMatthew G. Knepley   Input Parameters:
135601468941SMatthew G. Knepley + dm    - The `DM`
135701468941SMatthew G. Knepley . lower - If `PETSC_TRUE` use `DM_BC_LOWER_BOUND` conditions, otherwise use `DM_BC_UPPER_BOUND`
135801468941SMatthew G. Knepley - time  - The time
135901468941SMatthew G. Knepley 
136001468941SMatthew G. Knepley   Output Parameter:
136101468941SMatthew G. Knepley . locB - Bounds vector updated with new bounds
136201468941SMatthew G. Knepley 
136301468941SMatthew G. Knepley   Level: intermediate
136401468941SMatthew G. Knepley 
136501468941SMatthew G. Knepley .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunctionLabelLocal()`, `PetscDSAddBoundary()`
136601468941SMatthew G. Knepley @*/
136701468941SMatthew G. Knepley PetscErrorCode DMPlexInsertBounds(DM dm, PetscBool lower, PetscReal time, Vec locB)
136801468941SMatthew G. Knepley {
136901468941SMatthew G. Knepley   PetscFunctionBegin;
137001468941SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
137101468941SMatthew G. Knepley   PetscValidHeaderSpecific(locB, VEC_CLASSID, 4);
137201468941SMatthew G. Knepley   PetscTryMethod(dm, "DMPlexInsertBounds_C", (DM, PetscBool, PetscReal, Vec), (dm, lower, time, locB));
137301468941SMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
137401468941SMatthew G. Knepley }
137501468941SMatthew G. Knepley 
13765962854dSMatthew G. Knepley // Handle non-essential (e.g. outflow) boundary values
13775962854dSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesFVM(DM dm, PetscFV fv, Vec locX, PetscReal time, Vec *locGradient)
13785962854dSMatthew G. Knepley {
13795962854dSMatthew G. Knepley   DM  dmGrad;
13805962854dSMatthew G. Knepley   Vec cellGeometryFVM, faceGeometryFVM, locGrad = NULL;
13815962854dSMatthew G. Knepley 
13825962854dSMatthew G. Knepley   PetscFunctionBegin;
13835962854dSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
13845962854dSMatthew G. Knepley   PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);
13855962854dSMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
13865962854dSMatthew G. Knepley   if (locGradient) {
13875962854dSMatthew G. Knepley     PetscAssertPointer(locGradient, 5);
13885962854dSMatthew G. Knepley     *locGradient = NULL;
13895962854dSMatthew G. Knepley   }
13905962854dSMatthew G. Knepley   PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL));
13915962854dSMatthew G. Knepley   /* Reconstruct and limit cell gradients */
13925962854dSMatthew G. Knepley   PetscCall(DMPlexGetGradientDM(dm, fv, &dmGrad));
13935962854dSMatthew G. Knepley   if (dmGrad) {
13945962854dSMatthew G. Knepley     Vec      grad;
13955962854dSMatthew G. Knepley     PetscInt fStart, fEnd;
13965962854dSMatthew G. Knepley 
13975962854dSMatthew G. Knepley     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
13985962854dSMatthew G. Knepley     PetscCall(DMGetGlobalVector(dmGrad, &grad));
13995962854dSMatthew G. Knepley     PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
14005962854dSMatthew G. Knepley     /* Communicate gradient values */
14015962854dSMatthew G. Knepley     PetscCall(DMGetLocalVector(dmGrad, &locGrad));
14025962854dSMatthew G. Knepley     PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
14035962854dSMatthew G. Knepley     PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
14045962854dSMatthew G. Knepley     PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
14055962854dSMatthew G. Knepley   }
14065962854dSMatthew G. Knepley   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, time, faceGeometryFVM, cellGeometryFVM, locGrad));
14075962854dSMatthew G. Knepley   if (locGradient) *locGradient = locGrad;
14085962854dSMatthew G. Knepley   else if (locGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
14095962854dSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
14105962854dSMatthew G. Knepley }
14115962854dSMatthew G. Knepley 
1412d71ae5a4SJacob Faibussowitsch PetscErrorCode DMComputeL2Diff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
1413d71ae5a4SJacob Faibussowitsch {
1414574a98acSMatthew G. Knepley   Vec localX;
1415574a98acSMatthew G. Knepley 
1416574a98acSMatthew G. Knepley   PetscFunctionBegin;
14179566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
14189566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, localX, time, NULL, NULL, NULL));
14199566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
14209566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
14219566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeL2DiffLocal(dm, time, funcs, ctxs, localX, diff));
14229566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
14233ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1424574a98acSMatthew G. Knepley }
1425574a98acSMatthew G. Knepley 
1426574a98acSMatthew G. Knepley /*@C
142760225df5SJacob Faibussowitsch   DMPlexComputeL2DiffLocal - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h.
1428574a98acSMatthew G. Knepley 
142920f4b53cSBarry Smith   Collective
1430c0f8e1fdSMatthew G. Knepley 
1431574a98acSMatthew G. Knepley   Input Parameters:
1432a1cb98faSBarry Smith + dm     - The `DM`
1433574a98acSMatthew G. Knepley . time   - The time
1434574a98acSMatthew G. Knepley . funcs  - The functions to evaluate for each field component
143520f4b53cSBarry Smith . ctxs   - Optional array of contexts to pass to each function, or `NULL`.
1436574a98acSMatthew G. Knepley - localX - The coefficient vector u_h, a local vector
1437574a98acSMatthew G. Knepley 
1438574a98acSMatthew G. Knepley   Output Parameter:
1439574a98acSMatthew G. Knepley . diff - The diff ||u - u_h||_2
1440574a98acSMatthew G. Knepley 
1441574a98acSMatthew G. Knepley   Level: developer
1442574a98acSMatthew G. Knepley 
14431cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
1444574a98acSMatthew G. Knepley @*/
1445d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeL2DiffLocal(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec localX, PetscReal *diff)
1446d71ae5a4SJacob Faibussowitsch {
14470f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex *)dm->data)->printL2;
1448ca3d3a14SMatthew G. Knepley   DM               tdm;
1449ca3d3a14SMatthew G. Knepley   Vec              tv;
1450cb1e1211SMatthew G Knepley   PetscSection     section;
1451c5bbbd5bSMatthew G. Knepley   PetscQuadrature  quad;
14524bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
145315496722SMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
14544bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
1455cb1e1211SMatthew G Knepley   PetscReal        localDiff = 0.0;
14567318780aSToby Isaac   const PetscReal *quadWeights;
1457412e9a14SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cellHeight, cStart, cEnd, c, field, fieldOffset;
1458ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1459cb1e1211SMatthew G Knepley 
1460cb1e1211SMatthew G Knepley   PetscFunctionBegin;
14619566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
14629566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
14632a4e142eSMatthew G. Knepley   fegeom.dimEmbed = coordDim;
14649566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
14659566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
14669566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
14679566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
14689566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1469eae3dc7dSJacob Faibussowitsch   PetscCheck(numFields, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
1470cb1e1211SMatthew G Knepley   for (field = 0; field < numFields; ++field) {
147115496722SMatthew G. Knepley     PetscObject  obj;
147215496722SMatthew G. Knepley     PetscClassId id;
1473c5bbbd5bSMatthew G. Knepley     PetscInt     Nc;
1474c5bbbd5bSMatthew G. Knepley 
14759566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
14769566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
147715496722SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
147815496722SMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
147915496722SMatthew G. Knepley 
14809566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
14819566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
148215496722SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
148315496722SMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
148415496722SMatthew G. Knepley 
14859566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
14869566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
148763a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1488c5bbbd5bSMatthew G. Knepley     numComponents += Nc;
1489cb1e1211SMatthew G Knepley   }
14909566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
149163a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
1492c9cb6370SYANG Zongze   PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * (Nq + 1), &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ));
14939566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
14949566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
1495cb1e1211SMatthew G Knepley   for (c = cStart; c < cEnd; ++c) {
1496a1e44745SMatthew G. Knepley     PetscScalar *x        = NULL;
1497cb1e1211SMatthew G Knepley     PetscReal    elemDiff = 0.0;
14989c3cf19fSMatthew G. Knepley     PetscInt     qc       = 0;
1499cb1e1211SMatthew G Knepley 
15009566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
1501*48162695SZach Atkins     PetscCall(DMPlexVecGetOrientedClosure(dm, NULL, PETSC_FALSE, localX, c, 0, NULL, &x));
1502cb1e1211SMatthew G Knepley 
150315496722SMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
150415496722SMatthew G. Knepley       PetscObject  obj;
150515496722SMatthew G. Knepley       PetscClassId id;
1506c110b1eeSGeoffrey Irving       void *const  ctx = ctxs ? ctxs[field] : NULL;
150715496722SMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1508cb1e1211SMatthew G Knepley 
15099566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
15109566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
15119371c9d4SSatish Balay       if (id == PETSCFE_CLASSID) {
15129371c9d4SSatish Balay         PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
15139371c9d4SSatish Balay         PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
15149371c9d4SSatish Balay       } else if (id == PETSCFV_CLASSID) {
15159371c9d4SSatish Balay         PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
15169371c9d4SSatish Balay         Nb = 1;
15179371c9d4SSatish Balay       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1518cb1e1211SMatthew G Knepley       if (debug) {
1519cb1e1211SMatthew G Knepley         char title[1024];
152063a3b9bcSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
15219566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1522cb1e1211SMatthew G Knepley       }
15237318780aSToby Isaac       for (q = 0; q < Nq; ++q) {
15242a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1525d0609cedSBarry Smith         PetscErrorCode ierr;
15262a4e142eSMatthew G. Knepley 
15272a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
15282a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q * coordDim * coordDim];
15292a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
15302a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
153163a3b9bcSJacob Faibussowitsch         PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", point %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1532d3a7d86cSMatthew G. Knepley         if (transform) {
1533d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * Nq];
15349566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx));
1535d3a7d86cSMatthew G. Knepley         } else {
1536d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * q];
1537d3a7d86cSMatthew G. Knepley         }
15389566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal, Nc));
1539ca3d3a14SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, Nc, funcVal, ctx);
1540e735a8a9SMatthew G. Knepley         if (ierr) {
15419566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
15429566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
15439566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
1544e735a8a9SMatthew G. Knepley         }
15459566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
15469566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant));
15479566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant));
15482df84da0SMatthew G. Knepley         else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
154915496722SMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1550beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
15519371c9d4SSatish Balay           if (debug)
1552835f2295SStefano Zampini             PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g (%g, %g)\n", c, field, fc, (double)(coordDim > 0 ? coords[coordDim * q] : 0), (double)(coordDim > 1 ? coords[coordDim * q + 1] : 0), (double)(coordDim > 2 ? coords[coordDim * q + 2] : 0),
15539371c9d4SSatish Balay                                   (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]), (double)PetscRealPart(interpolant[fc]), (double)PetscRealPart(funcVal[fc])));
15544bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
1555cb1e1211SMatthew G Knepley         }
1556cb1e1211SMatthew G Knepley       }
15579c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
1558beaa55a6SMatthew G. Knepley       qc += Nc;
1559cb1e1211SMatthew G Knepley     }
15609566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
15619566063dSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
1562cb1e1211SMatthew G Knepley     localDiff += elemDiff;
1563cb1e1211SMatthew G Knepley   }
15649566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
1565462c564dSBarry Smith   PetscCallMPI(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
1566cb1e1211SMatthew G Knepley   *diff = PetscSqrtReal(*diff);
15673ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1568cb1e1211SMatthew G Knepley }
1569cb1e1211SMatthew G Knepley 
1570d71ae5a4SJacob Faibussowitsch PetscErrorCode DMComputeL2GradientDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, const PetscReal n[], PetscReal *diff)
1571d71ae5a4SJacob Faibussowitsch {
15720f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex *)dm->data)->printL2;
1573ca3d3a14SMatthew G. Knepley   DM               tdm;
1574cb1e1211SMatthew G Knepley   PetscSection     section;
157540e14135SMatthew G. Knepley   PetscQuadrature  quad;
1576ca3d3a14SMatthew G. Knepley   Vec              localX, tv;
15779c3cf19fSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
15782a4e142eSMatthew G. Knepley   const PetscReal *quadWeights;
15794bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
15804bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
158140e14135SMatthew G. Knepley   PetscReal        localDiff = 0.0;
1582485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, qNc = 0, Nq = 0, numFields, numComponents = 0, cStart, cEnd, c, field, fieldOffset;
1583ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1584cb1e1211SMatthew G Knepley 
1585cb1e1211SMatthew G Knepley   PetscFunctionBegin;
15869566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
15879566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
15884bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
15899566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
15909566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
15919566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
15929566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
15939566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
15949566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
15959566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
15969566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1597652b88e8SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
15980f2d7e86SMatthew G. Knepley     PetscFE  fe;
159940e14135SMatthew G. Knepley     PetscInt Nc;
1600652b88e8SMatthew G. Knepley 
16019566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe));
16029566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
16039566063dSJacob Faibussowitsch     PetscCall(PetscFEGetNumComponents(fe, &Nc));
160440e14135SMatthew G. Knepley     numComponents += Nc;
1605652b88e8SMatthew G. Knepley   }
16069566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
160763a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
16089566063dSJacob Faibussowitsch   /* PetscCall(DMProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX)); */
1609c9cb6370SYANG Zongze   PetscCall(PetscMalloc6(numComponents, &funcVal, coordDim * (Nq + 1), &coords, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ, numComponents * coordDim, &interpolant, Nq, &fegeom.detJ));
16109566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
161140e14135SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
161240e14135SMatthew G. Knepley     PetscScalar *x        = NULL;
161340e14135SMatthew G. Knepley     PetscReal    elemDiff = 0.0;
16149c3cf19fSMatthew G. Knepley     PetscInt     qc       = 0;
1615652b88e8SMatthew G. Knepley 
16169566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
1617*48162695SZach Atkins     PetscCall(DMPlexVecGetOrientedClosure(dm, NULL, PETSC_FALSE, localX, c, 0, NULL, &x));
161840e14135SMatthew G. Knepley 
16199c3cf19fSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
16200f2d7e86SMatthew G. Knepley       PetscFE     fe;
162151259fa3SMatthew G. Knepley       void *const ctx = ctxs ? ctxs[field] : NULL;
16229c3cf19fSMatthew G. Knepley       PetscInt    Nb, Nc, q, fc;
162340e14135SMatthew G. Knepley 
16249566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe));
16259566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
16269566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
162740e14135SMatthew G. Knepley       if (debug) {
162840e14135SMatthew G. Knepley         char title[1024];
16299566063dSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
16309566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1631652b88e8SMatthew G. Knepley       }
16329c3cf19fSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
16332a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1634d0609cedSBarry Smith         PetscErrorCode ierr;
16352a4e142eSMatthew G. Knepley 
16362a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
16372a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q * coordDim * coordDim];
16382a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
16392a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
16402df84da0SMatthew G. Knepley         PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1641d3a7d86cSMatthew G. Knepley         if (transform) {
1642d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * Nq];
16439566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx));
1644d3a7d86cSMatthew G. Knepley         } else {
1645d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * q];
1646d3a7d86cSMatthew G. Knepley         }
16479566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal, Nc));
16484bee2e38SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, n, Nc, funcVal, ctx);
1649e735a8a9SMatthew G. Knepley         if (ierr) {
16509566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
16519566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
16529566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ));
1653e735a8a9SMatthew G. Knepley         }
16549566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
16559566063dSJacob Faibussowitsch         PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[fieldOffset], &qgeom, q, interpolant));
16564bee2e38SMatthew G. Knepley         /* Overwrite with the dot product if the normal is given */
16574bee2e38SMatthew G. Knepley         if (n) {
16584bee2e38SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
16594bee2e38SMatthew G. Knepley             PetscScalar sum = 0.0;
16604bee2e38SMatthew G. Knepley             PetscInt    d;
16614bee2e38SMatthew G. Knepley             for (d = 0; d < dim; ++d) sum += interpolant[fc * dim + d] * n[d];
16624bee2e38SMatthew G. Knepley             interpolant[fc] = sum;
16634bee2e38SMatthew G. Knepley           }
16644bee2e38SMatthew G. Knepley         }
16659c3cf19fSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1666beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
166763a3b9bcSJacob Faibussowitsch           if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " fieldDer %" PetscInt_FMT ",%" PetscInt_FMT " diff %g\n", c, field, fc, (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q])));
16684bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
166940e14135SMatthew G. Knepley         }
167040e14135SMatthew G. Knepley       }
16719c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
16729c3cf19fSMatthew G. Knepley       qc += Nc;
167340e14135SMatthew G. Knepley     }
16749566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
167563a3b9bcSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
167640e14135SMatthew G. Knepley     localDiff += elemDiff;
167740e14135SMatthew G. Knepley   }
16789566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ));
16799566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
1680462c564dSBarry Smith   PetscCallMPI(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
168140e14135SMatthew G. Knepley   *diff = PetscSqrtReal(*diff);
16823ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1683cb1e1211SMatthew G Knepley }
1684cb1e1211SMatthew G Knepley 
1685d71ae5a4SJacob Faibussowitsch PetscErrorCode DMComputeL2FieldDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
1686d71ae5a4SJacob Faibussowitsch {
16870f09c10fSMatthew G. Knepley   const PetscInt debug = ((DM_Plex *)dm->data)->printL2;
1688ca3d3a14SMatthew G. Knepley   DM             tdm;
1689083401c6SMatthew G. Knepley   DMLabel        depthLabel;
169073d901b8SMatthew G. Knepley   PetscSection   section;
1691ca3d3a14SMatthew G. Knepley   Vec            localX, tv;
169273d901b8SMatthew G. Knepley   PetscReal     *localDiff;
1693083401c6SMatthew G. Knepley   PetscInt       dim, depth, dE, Nf, f, Nds, s;
1694ca3d3a14SMatthew G. Knepley   PetscBool      transform;
169573d901b8SMatthew G. Knepley 
169673d901b8SMatthew G. Knepley   PetscFunctionBegin;
16979566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
16989566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dE));
16999566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
17009566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
17019566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
17029566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
17039566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
17049566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
17059566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
17069566063dSJacob Faibussowitsch   PetscCall(DMLabelGetNumValues(depthLabel, &depth));
1707083401c6SMatthew G. Knepley 
17089566063dSJacob Faibussowitsch   PetscCall(VecSet(localX, 0.0));
17099566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
17109566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
17119566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
17129566063dSJacob Faibussowitsch   PetscCall(DMGetNumDS(dm, &Nds));
17139566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(Nf, &localDiff));
1714083401c6SMatthew G. Knepley   for (s = 0; s < Nds; ++s) {
1715083401c6SMatthew G. Knepley     PetscDS          ds;
1716083401c6SMatthew G. Knepley     DMLabel          label;
1717083401c6SMatthew G. Knepley     IS               fieldIS, pointIS;
1718083401c6SMatthew G. Knepley     const PetscInt  *fields, *points = NULL;
1719083401c6SMatthew G. Knepley     PetscQuadrature  quad;
1720083401c6SMatthew G. Knepley     const PetscReal *quadPoints, *quadWeights;
1721083401c6SMatthew G. Knepley     PetscFEGeom      fegeom;
1722083401c6SMatthew G. Knepley     PetscReal       *coords, *gcoords;
1723083401c6SMatthew G. Knepley     PetscScalar     *funcVal, *interpolant;
17245fedec97SMatthew G. Knepley     PetscBool        isCohesive;
1725083401c6SMatthew G. Knepley     PetscInt         qNc, Nq, totNc, cStart = 0, cEnd, c, dsNf;
172673d901b8SMatthew G. Knepley 
172707218a29SMatthew G. Knepley     PetscCall(DMGetRegionNumDS(dm, s, &label, &fieldIS, &ds, NULL));
17289566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(fieldIS, &fields));
17299566063dSJacob Faibussowitsch     PetscCall(PetscDSIsCohesive(ds, &isCohesive));
17309566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(ds, &dsNf));
17319566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalComponents(ds, &totNc));
17329566063dSJacob Faibussowitsch     PetscCall(PetscDSGetQuadrature(ds, &quad));
17339566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
173463a3b9bcSJacob Faibussowitsch     PetscCheck(!(qNc != 1) || !(qNc != totNc), PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, totNc);
17359566063dSJacob Faibussowitsch     PetscCall(PetscCalloc6(totNc, &funcVal, totNc, &interpolant, dE * (Nq + 1), &coords, Nq, &fegeom.detJ, dE * dE * Nq, &fegeom.J, dE * dE * Nq, &fegeom.invJ));
1736083401c6SMatthew G. Knepley     if (!label) {
17379566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1738083401c6SMatthew G. Knepley     } else {
17399566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(label, 1, &pointIS));
17409566063dSJacob Faibussowitsch       PetscCall(ISGetLocalSize(pointIS, &cEnd));
17419566063dSJacob Faibussowitsch       PetscCall(ISGetIndices(pointIS, &points));
1742083401c6SMatthew G. Knepley     }
174373d901b8SMatthew G. Knepley     for (c = cStart; c < cEnd; ++c) {
1744083401c6SMatthew G. Knepley       const PetscInt  cell = points ? points[c] : c;
174573d901b8SMatthew G. Knepley       PetscScalar    *x    = NULL;
17465fedec97SMatthew G. Knepley       const PetscInt *cone;
17475fedec97SMatthew G. Knepley       PetscInt        qc = 0, fOff = 0, dep;
174873d901b8SMatthew G. Knepley 
17499566063dSJacob Faibussowitsch       PetscCall(DMLabelGetValue(depthLabel, cell, &dep));
1750083401c6SMatthew G. Knepley       if (dep != depth - 1) continue;
17515fedec97SMatthew G. Knepley       if (isCohesive) {
17529566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cell, &cone));
17539566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cone[0], quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
175496959cd1SMatthew G. Knepley       } else {
17559566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
175696959cd1SMatthew G. Knepley       }
1757*48162695SZach Atkins       PetscCall(DMPlexVecGetOrientedClosure(dm, NULL, PETSC_FALSE, localX, cell, 0, NULL, &x));
17585fedec97SMatthew G. Knepley       for (f = 0; f < dsNf; ++f) {
175915496722SMatthew G. Knepley         PetscObject  obj;
176015496722SMatthew G. Knepley         PetscClassId id;
1761083401c6SMatthew G. Knepley         void *const  ctx = ctxs ? ctxs[fields[f]] : NULL;
176215496722SMatthew G. Knepley         PetscInt     Nb, Nc, q, fc;
176315496722SMatthew G. Knepley         PetscReal    elemDiff = 0.0;
17645fedec97SMatthew G. Knepley         PetscBool    cohesive;
176515496722SMatthew G. Knepley 
17669566063dSJacob Faibussowitsch         PetscCall(PetscDSGetCohesive(ds, f, &cohesive));
17679566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
17689566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
17699371c9d4SSatish Balay         if (id == PETSCFE_CLASSID) {
17709371c9d4SSatish Balay           PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
17719371c9d4SSatish Balay           PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
17729371c9d4SSatish Balay         } else if (id == PETSCFV_CLASSID) {
17739371c9d4SSatish Balay           PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
17749371c9d4SSatish Balay           Nb = 1;
17759371c9d4SSatish Balay         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
1776c82db6dbSMatthew G. Knepley         if (isCohesive && !cohesive) {
1777c82db6dbSMatthew G. Knepley           fOff += Nb * 2;
1778c82db6dbSMatthew G. Knepley           qc += Nc;
1779c82db6dbSMatthew G. Knepley           continue;
1780c82db6dbSMatthew G. Knepley         }
178173d901b8SMatthew G. Knepley         if (debug) {
178273d901b8SMatthew G. Knepley           char title[1024];
178363a3b9bcSJacob Faibussowitsch           PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, fields[f]));
17849566063dSJacob Faibussowitsch           PetscCall(DMPrintCellVector(cell, title, Nb, &x[fOff]));
178573d901b8SMatthew G. Knepley         }
17867318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
17872a4e142eSMatthew G. Knepley           PetscFEGeom    qgeom;
1788d0609cedSBarry Smith           PetscErrorCode ierr;
17892a4e142eSMatthew G. Knepley 
17902a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
1791083401c6SMatthew G. Knepley           qgeom.J        = &fegeom.J[q * dE * dE];
1792083401c6SMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * dE * dE];
17932a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
179463a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for cell %" PetscInt_FMT ", quadrature point %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
1795d3a7d86cSMatthew G. Knepley           if (transform) {
1796083401c6SMatthew G. Knepley             gcoords = &coords[dE * Nq];
17979566063dSJacob Faibussowitsch             PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[dE * q], PETSC_TRUE, dE, &coords[dE * q], gcoords, dm->transformCtx));
1798d3a7d86cSMatthew G. Knepley           } else {
1799083401c6SMatthew G. Knepley             gcoords = &coords[dE * q];
1800d3a7d86cSMatthew G. Knepley           }
18012df84da0SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) funcVal[fc] = 0.;
1802083401c6SMatthew G. Knepley           ierr = (*funcs[fields[f]])(dE, time, gcoords, Nc, funcVal, ctx);
1803e735a8a9SMatthew G. Knepley           if (ierr) {
18049566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
18059566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
18069566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
1807e735a8a9SMatthew G. Knepley           }
18089566063dSJacob Faibussowitsch           if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[dE * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
180996959cd1SMatthew G. Knepley           /* Call once for each face, except for lagrange field */
18109566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fOff], &qgeom, q, interpolant));
18119566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fOff], q, interpolant));
181263a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
181315496722SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1814beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
18159371c9d4SSatish Balay             if (debug)
1816835f2295SStefano Zampini               PetscCall(PetscPrintf(PETSC_COMM_SELF, "    cell %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g\n", cell, fields[f], fc, (double)(dE > 0 ? coords[dE * q] : 0), (double)(dE > 1 ? coords[dE * q + 1] : 0), (double)(dE > 2 ? coords[dE * q + 2] : 0),
18179371c9d4SSatish Balay                                     (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q])));
18184bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
181973d901b8SMatthew G. Knepley           }
182073d901b8SMatthew G. Knepley         }
1821083401c6SMatthew G. Knepley         fOff += Nb;
18229c3cf19fSMatthew G. Knepley         qc += Nc;
1823083401c6SMatthew G. Knepley         localDiff[fields[f]] += elemDiff;
182463a3b9bcSJacob Faibussowitsch         if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  cell %" PetscInt_FMT " field %" PetscInt_FMT " cum diff %g\n", cell, fields[f], (double)localDiff[fields[f]]));
182573d901b8SMatthew G. Knepley       }
18269566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
1827083401c6SMatthew G. Knepley     }
1828083401c6SMatthew G. Knepley     if (label) {
18299566063dSJacob Faibussowitsch       PetscCall(ISRestoreIndices(pointIS, &points));
18309566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
1831083401c6SMatthew G. Knepley     }
18329566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(fieldIS, &fields));
18339566063dSJacob Faibussowitsch     PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
183473d901b8SMatthew G. Knepley   }
18359566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
1836e91c04dfSPierre Jolivet   PetscCallMPI(MPIU_Allreduce(localDiff, diff, Nf, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
18379566063dSJacob Faibussowitsch   PetscCall(PetscFree(localDiff));
1838083401c6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) diff[f] = PetscSqrtReal(diff[f]);
18393ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
184073d901b8SMatthew G. Knepley }
184173d901b8SMatthew G. Knepley 
1842e729f68cSMatthew G. Knepley /*@C
1843e729f68cSMatthew G. Knepley   DMPlexComputeL2DiffVec - This function computes the cellwise L_2 difference between a function u and an FEM interpolant solution u_h, and stores it in a Vec.
1844e729f68cSMatthew G. Knepley 
184520f4b53cSBarry Smith   Collective
1846c0f8e1fdSMatthew G. Knepley 
1847e729f68cSMatthew G. Knepley   Input Parameters:
1848a1cb98faSBarry Smith + dm    - The `DM`
18490163fd50SMatthew G. Knepley . time  - The time
185020f4b53cSBarry Smith . funcs - The functions to evaluate for each field component: `NULL` means that component does not contribute to error calculation
185120f4b53cSBarry Smith . ctxs  - Optional array of contexts to pass to each function, or `NULL`.
1852e729f68cSMatthew G. Knepley - X     - The coefficient vector u_h
1853e729f68cSMatthew G. Knepley 
1854e729f68cSMatthew G. Knepley   Output Parameter:
185520f4b53cSBarry Smith . D - A `Vec` which holds the difference ||u - u_h||_2 for each cell
1856e729f68cSMatthew G. Knepley 
1857e729f68cSMatthew G. Knepley   Level: developer
1858e729f68cSMatthew G. Knepley 
18591cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
1860e729f68cSMatthew G. Knepley @*/
1861d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeL2DiffVec(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, Vec D)
1862d71ae5a4SJacob Faibussowitsch {
1863e729f68cSMatthew G. Knepley   PetscSection     section;
1864e729f68cSMatthew G. Knepley   PetscQuadrature  quad;
1865e729f68cSMatthew G. Knepley   Vec              localX;
18664bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
1867e729f68cSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
18684bee2e38SMatthew G. Knepley   PetscReal       *coords;
1869e729f68cSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1870485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, c, field, fieldOffset;
1871e729f68cSMatthew G. Knepley 
1872e729f68cSMatthew G. Knepley   PetscFunctionBegin;
18739566063dSJacob Faibussowitsch   PetscCall(VecSet(D, 0.0));
18749566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
18759566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
18769566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
18779566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
18789566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
18799566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
18809566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
18819566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
1882e729f68cSMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
1883e729f68cSMatthew G. Knepley     PetscObject  obj;
1884e729f68cSMatthew G. Knepley     PetscClassId id;
1885e729f68cSMatthew G. Knepley     PetscInt     Nc;
1886e729f68cSMatthew G. Knepley 
18879566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
18889566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1889e729f68cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1890e729f68cSMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
1891e729f68cSMatthew G. Knepley 
18929566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
18939566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
1894e729f68cSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
1895e729f68cSMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
1896e729f68cSMatthew G. Knepley 
18979566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
18989566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
189963a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1900e729f68cSMatthew G. Knepley     numComponents += Nc;
1901e729f68cSMatthew G. Knepley   }
19029566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
190363a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
19049566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ));
19059566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1906e729f68cSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
1907e729f68cSMatthew G. Knepley     PetscScalar *x        = NULL;
19086f288a59SMatthew G. Knepley     PetscScalar  elemDiff = 0.0;
19099c3cf19fSMatthew G. Knepley     PetscInt     qc       = 0;
1910e729f68cSMatthew G. Knepley 
19119566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
1912*48162695SZach Atkins     PetscCall(DMPlexVecGetOrientedClosure(dm, NULL, PETSC_FALSE, localX, c, 0, NULL, &x));
1913e729f68cSMatthew G. Knepley 
1914e729f68cSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
1915e729f68cSMatthew G. Knepley       PetscObject  obj;
1916e729f68cSMatthew G. Knepley       PetscClassId id;
1917e729f68cSMatthew G. Knepley       void *const  ctx = ctxs ? ctxs[field] : NULL;
1918e729f68cSMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1919e729f68cSMatthew G. Knepley 
19209566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
19219566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
19229371c9d4SSatish Balay       if (id == PETSCFE_CLASSID) {
19239371c9d4SSatish Balay         PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
19249371c9d4SSatish Balay         PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
19259371c9d4SSatish Balay       } else if (id == PETSCFV_CLASSID) {
19269371c9d4SSatish Balay         PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
19279371c9d4SSatish Balay         Nb = 1;
19289371c9d4SSatish Balay       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
192923f34ed2SToby Isaac       if (funcs[field]) {
19307318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
19312a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
19322a4e142eSMatthew G. Knepley 
19332a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
19342a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q * coordDim * coordDim];
19352a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
19362a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
193763a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
19389566063dSJacob Faibussowitsch           PetscCall((*funcs[field])(coordDim, time, &coords[q * coordDim], Nc, funcVal, ctx));
1939c3e24edfSBarry Smith #if defined(needs_fix_with_return_code_argument)
1940e735a8a9SMatthew G. Knepley           if (ierr) {
19419566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
19429566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
19439566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
1944e735a8a9SMatthew G. Knepley           }
1945c3e24edfSBarry Smith #endif
19469566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant));
19479566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant));
194863a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1949e729f68cSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1950beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
19514bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
1952e729f68cSMatthew G. Knepley           }
1953e729f68cSMatthew G. Knepley         }
195423f34ed2SToby Isaac       }
1955beaa55a6SMatthew G. Knepley       fieldOffset += Nb;
19569c3cf19fSMatthew G. Knepley       qc += Nc;
1957e729f68cSMatthew G. Knepley     }
19589566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
19599566063dSJacob Faibussowitsch     PetscCall(VecSetValue(D, c - cStart, elemDiff, INSERT_VALUES));
1960e729f68cSMatthew G. Knepley   }
19619566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
19629566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
19639566063dSJacob Faibussowitsch   PetscCall(VecSqrtAbs(D));
19643ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1965e729f68cSMatthew G. Knepley }
1966e729f68cSMatthew G. Knepley 
19675f0b18bfSMatthew G. Knepley /*@
19682353bfffSMatthew G. Knepley   DMPlexComputeL2FluxDiffVecLocal - This function computes the integral of the difference between the gradient of field `f`in `u` and field `mf` in `mu`
19692353bfffSMatthew G. Knepley 
19702353bfffSMatthew G. Knepley   Collective
19712353bfffSMatthew G. Knepley 
19722353bfffSMatthew G. Knepley   Input Parameters:
19732353bfffSMatthew G. Knepley + lu  - The local `Vec` containing the primal solution
19742353bfffSMatthew G. Knepley . f   - The field number for the potential
19752353bfffSMatthew G. Knepley . lmu - The local `Vec` containing the mixed solution
19762353bfffSMatthew G. Knepley - mf  - The field number for the flux
19772353bfffSMatthew G. Knepley 
19782353bfffSMatthew G. Knepley   Output Parameter:
19792353bfffSMatthew G. Knepley . eFlux - A global `Vec` which holds $||\nabla u_f - \mu_{mf}||$
19802353bfffSMatthew G. Knepley 
19812353bfffSMatthew G. Knepley   Level: advanced
19822353bfffSMatthew G. Knepley 
19832353bfffSMatthew G. Knepley   Notes:
19842353bfffSMatthew G. Knepley   We assume that the `DM` for each solution has the same topology, geometry, and quadrature.
19852353bfffSMatthew G. Knepley 
19862353bfffSMatthew G. Knepley   This is usually used to get an error estimate for the primal solution, using the flux from a mixed solution.
19872353bfffSMatthew G. Knepley 
19882353bfffSMatthew G. Knepley .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeL2FluxDiffVec()`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
19892353bfffSMatthew G. Knepley @*/
19902353bfffSMatthew G. Knepley PetscErrorCode DMPlexComputeL2FluxDiffVecLocal(Vec lu, PetscInt f, Vec lmu, PetscInt mf, Vec eFlux)
19912353bfffSMatthew G. Knepley {
19922353bfffSMatthew G. Knepley   DM               dm, mdm, edm;
19932353bfffSMatthew G. Knepley   PetscFE          fe, mfe;
19942353bfffSMatthew G. Knepley   PetscFEGeom      fegeom;
19952353bfffSMatthew G. Knepley   PetscQuadrature  quad;
19962353bfffSMatthew G. Knepley   const PetscReal *quadWeights;
19972353bfffSMatthew G. Knepley   PetscReal       *coords;
19982353bfffSMatthew G. Knepley   PetscScalar     *interpolant, *minterpolant, *earray;
19992353bfffSMatthew G. Knepley   PetscInt         cdim, mcdim, cStart, cEnd, Nc, mNc, qNc, Nq;
20002353bfffSMatthew G. Knepley   MPI_Comm         comm;
20012353bfffSMatthew G. Knepley 
20022353bfffSMatthew G. Knepley   PetscFunctionBegin;
20032353bfffSMatthew G. Knepley   PetscCall(VecGetDM(lu, &dm));
20042353bfffSMatthew G. Knepley   PetscCall(VecGetDM(lmu, &mdm));
20052353bfffSMatthew G. Knepley   PetscCall(VecGetDM(eFlux, &edm));
20062353bfffSMatthew G. Knepley   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
20072353bfffSMatthew G. Knepley   PetscCall(VecSet(eFlux, 0.0));
20082353bfffSMatthew G. Knepley 
20092353bfffSMatthew G. Knepley   // Check if the both problems are on the same mesh
20102353bfffSMatthew G. Knepley   PetscCall(DMGetCoordinateDim(dm, &cdim));
20112353bfffSMatthew G. Knepley   PetscCall(DMGetCoordinateDim(mdm, &mcdim));
20122353bfffSMatthew G. Knepley   PetscCheck(cdim == mcdim, comm, PETSC_ERR_ARG_SIZ, "primal coordinate Dim %" PetscInt_FMT " != %" PetscInt_FMT " mixed coordinate Dim", cdim, mcdim);
20132353bfffSMatthew G. Knepley   fegeom.dimEmbed = cdim;
20142353bfffSMatthew G. Knepley 
20152353bfffSMatthew G. Knepley   PetscCall(DMGetField(dm, f, NULL, (PetscObject *)&fe));
20162353bfffSMatthew G. Knepley   PetscCall(DMGetField(mdm, mf, NULL, (PetscObject *)&mfe));
20172353bfffSMatthew G. Knepley   PetscCall(PetscFEGetNumComponents(fe, &Nc));
20182353bfffSMatthew G. Knepley   PetscCall(PetscFEGetNumComponents(mfe, &mNc));
20192353bfffSMatthew G. Knepley   PetscCall(PetscFEGetQuadrature(fe, &quad));
20202353bfffSMatthew G. Knepley   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
20212353bfffSMatthew G. Knepley   PetscCheck(qNc == 1 || qNc == mNc, comm, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, mNc);
20222353bfffSMatthew G. Knepley 
20232353bfffSMatthew G. Knepley   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
20242353bfffSMatthew G. Knepley   PetscCall(VecGetArrayWrite(eFlux, &earray));
20252353bfffSMatthew G. Knepley   PetscCall(PetscMalloc6(Nc * cdim, &interpolant, mNc * cdim, &minterpolant, cdim * (Nq + 1), &coords, cdim * cdim * Nq, &fegeom.J, cdim * cdim * Nq, &fegeom.invJ, Nq, &fegeom.detJ));
20262353bfffSMatthew G. Knepley   for (PetscInt c = cStart; c < cEnd; ++c) {
20272353bfffSMatthew G. Knepley     PetscScalar *x            = NULL;
20282353bfffSMatthew G. Knepley     PetscScalar *mx           = NULL;
20292353bfffSMatthew G. Knepley     PetscScalar *eval         = NULL;
20302353bfffSMatthew G. Knepley     PetscReal    fluxElemDiff = 0.0;
20312353bfffSMatthew G. Knepley 
20322353bfffSMatthew G. Knepley     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
20332353bfffSMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(dm, NULL, lu, c, NULL, &x));
20342353bfffSMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(mdm, NULL, lmu, c, NULL, &mx));
20352353bfffSMatthew G. Knepley 
20362353bfffSMatthew G. Knepley     for (PetscInt q = 0; q < Nq; ++q) {
20372353bfffSMatthew G. Knepley       PetscFEGeom qgeom;
20382353bfffSMatthew G. Knepley 
20392353bfffSMatthew G. Knepley       qgeom.dimEmbed = fegeom.dimEmbed;
20402353bfffSMatthew G. Knepley       qgeom.J        = &fegeom.J[q * cdim * cdim];
20412353bfffSMatthew G. Knepley       qgeom.invJ     = &fegeom.invJ[q * cdim * cdim];
20422353bfffSMatthew G. Knepley       qgeom.detJ     = &fegeom.detJ[q];
20432353bfffSMatthew G. Knepley 
20442353bfffSMatthew G. Knepley       PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
20452353bfffSMatthew G. Knepley 
20462353bfffSMatthew G. Knepley       PetscCall(PetscFEInterpolate_Static(mfe, &mx[0], &qgeom, q, minterpolant));
20472353bfffSMatthew G. Knepley       PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[0], &qgeom, q, interpolant));
20482353bfffSMatthew G. Knepley 
20492353bfffSMatthew G. Knepley       /* Now take the elementwise difference and store that in a vector. */
20502353bfffSMatthew G. Knepley       for (PetscInt fc = 0; fc < mNc; ++fc) {
20512353bfffSMatthew G. Knepley         const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : fc)];
20522353bfffSMatthew G. Knepley         fluxElemDiff += PetscSqr(PetscRealPart(interpolant[fc] - minterpolant[fc])) * wt * fegeom.detJ[q];
20532353bfffSMatthew G. Knepley       }
20542353bfffSMatthew G. Knepley     }
20552353bfffSMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(dm, NULL, lu, c, NULL, &x));
20562353bfffSMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(mdm, NULL, lmu, c, NULL, &mx));
20572353bfffSMatthew G. Knepley     PetscCall(DMPlexPointGlobalRef(edm, c, earray, (void *)&eval));
20582353bfffSMatthew G. Knepley     if (eval) eval[0] = fluxElemDiff;
20592353bfffSMatthew G. Knepley   }
20602353bfffSMatthew G. Knepley   PetscCall(PetscFree6(interpolant, minterpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
20612353bfffSMatthew G. Knepley   PetscCall(VecRestoreArrayWrite(eFlux, &earray));
20622353bfffSMatthew G. Knepley 
20632353bfffSMatthew G. Knepley   PetscCall(VecAssemblyBegin(eFlux));
20642353bfffSMatthew G. Knepley   PetscCall(VecAssemblyEnd(eFlux));
20652353bfffSMatthew G. Knepley   PetscCall(VecSqrtAbs(eFlux));
20662353bfffSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
20672353bfffSMatthew G. Knepley }
20682353bfffSMatthew G. Knepley 
20692353bfffSMatthew G. Knepley /*@
20702353bfffSMatthew G. Knepley   DMPlexComputeL2FluxDiffVec - This function computes the integral of the difference between the gradient of field `f`in `u` and field `mf` in `mu`
20712353bfffSMatthew G. Knepley 
20722353bfffSMatthew G. Knepley   Collective
20732353bfffSMatthew G. Knepley 
20742353bfffSMatthew G. Knepley   Input Parameters:
20752353bfffSMatthew G. Knepley + u  - The global `Vec` containing the primal solution
20762353bfffSMatthew G. Knepley . f  - The field number for the potential
20772353bfffSMatthew G. Knepley . mu - The global `Vec` containing the mixed solution
20782353bfffSMatthew G. Knepley - mf - The field number for the flux
20792353bfffSMatthew G. Knepley 
20802353bfffSMatthew G. Knepley   Output Parameter:
20812353bfffSMatthew G. Knepley . eFlux - A global `Vec` which holds $||\nabla u_f - \mu_{mf}||$
20822353bfffSMatthew G. Knepley 
20832353bfffSMatthew G. Knepley   Level: advanced
20842353bfffSMatthew G. Knepley 
20852353bfffSMatthew G. Knepley   Notes:
20862353bfffSMatthew G. Knepley   We assume that the `DM` for each solution has the same topology, geometry, and quadrature.
20872353bfffSMatthew G. Knepley 
20882353bfffSMatthew G. Knepley   This is usually used to get an error estimate for the primal solution, using the flux from a mixed solution.
20892353bfffSMatthew G. Knepley 
20902353bfffSMatthew G. Knepley .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeL2FluxDiffVecLocal()`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
20912353bfffSMatthew G. Knepley @*/
20922353bfffSMatthew G. Knepley PetscErrorCode DMPlexComputeL2FluxDiffVec(Vec u, PetscInt f, Vec mu, PetscInt mf, Vec eFlux)
20932353bfffSMatthew G. Knepley {
20942353bfffSMatthew G. Knepley   DM  dm, mdm;
20952353bfffSMatthew G. Knepley   Vec lu, lmu;
20962353bfffSMatthew G. Knepley 
20972353bfffSMatthew G. Knepley   PetscFunctionBegin;
20982353bfffSMatthew G. Knepley   PetscCall(VecGetDM(u, &dm));
20992353bfffSMatthew G. Knepley   PetscCall(DMGetLocalVector(dm, &lu));
21002353bfffSMatthew G. Knepley   PetscCall(DMGlobalToLocal(dm, u, INSERT_VALUES, lu));
21012353bfffSMatthew G. Knepley   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, lu, 0.0, NULL, NULL, NULL));
21022353bfffSMatthew G. Knepley 
21032353bfffSMatthew G. Knepley   PetscCall(VecGetDM(mu, &mdm));
21042353bfffSMatthew G. Knepley   PetscCall(DMGetLocalVector(mdm, &lmu));
21052353bfffSMatthew G. Knepley   PetscCall(DMGlobalToLocal(mdm, mu, INSERT_VALUES, lmu));
21062353bfffSMatthew G. Knepley   PetscCall(DMPlexInsertBoundaryValues(mdm, PETSC_TRUE, lmu, 0.0, NULL, NULL, NULL));
21072353bfffSMatthew G. Knepley 
21082353bfffSMatthew G. Knepley   PetscCall(DMPlexComputeL2FluxDiffVecLocal(lu, f, lmu, mf, eFlux));
21092353bfffSMatthew G. Knepley 
21102353bfffSMatthew G. Knepley   PetscCall(DMRestoreLocalVector(dm, &lu));
21112353bfffSMatthew G. Knepley   PetscCall(DMRestoreLocalVector(mdm, &lmu));
21122353bfffSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
21132353bfffSMatthew G. Knepley }
21142353bfffSMatthew G. Knepley 
21152353bfffSMatthew G. Knepley /*@
211620f4b53cSBarry Smith   DMPlexComputeClementInterpolant - This function computes the L2 projection of the cellwise values of a function u onto P1
21175f0b18bfSMatthew G. Knepley 
211820f4b53cSBarry Smith   Collective
21195f0b18bfSMatthew G. Knepley 
21205f0b18bfSMatthew G. Knepley   Input Parameters:
2121a1cb98faSBarry Smith + dm   - The `DM`
21225f0b18bfSMatthew G. Knepley - locX - The coefficient vector u_h
21235f0b18bfSMatthew G. Knepley 
21245f0b18bfSMatthew G. Knepley   Output Parameter:
2125a1cb98faSBarry Smith . locC - A `Vec` which holds the Clement interpolant of the function
21265f0b18bfSMatthew G. Knepley 
21275f0b18bfSMatthew G. Knepley   Level: developer
21285f0b18bfSMatthew G. Knepley 
2129a1cb98faSBarry Smith   Note:
2130a1cb98faSBarry Smith   $ u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| $ where $ |T_i| $ is the cell volume
2131a1cb98faSBarry Smith 
21321cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
21335f0b18bfSMatthew G. Knepley @*/
2134d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeClementInterpolant(DM dm, Vec locX, Vec locC)
2135d71ae5a4SJacob Faibussowitsch {
21365f0b18bfSMatthew G. Knepley   PetscInt         debug = ((DM_Plex *)dm->data)->printFEM;
21375f0b18bfSMatthew G. Knepley   DM               dmc;
21385f0b18bfSMatthew G. Knepley   PetscQuadrature  quad;
21395f0b18bfSMatthew G. Knepley   PetscScalar     *interpolant, *valsum;
21405f0b18bfSMatthew G. Knepley   PetscFEGeom      fegeom;
21415f0b18bfSMatthew G. Knepley   PetscReal       *coords;
21425f0b18bfSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
21435f0b18bfSMatthew G. Knepley   PetscInt         dim, cdim, Nf, f, Nc = 0, Nq, qNc, cStart, cEnd, vStart, vEnd, v;
21445f0b18bfSMatthew G. Knepley 
21455f0b18bfSMatthew G. Knepley   PetscFunctionBegin;
21469566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
21479566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmc));
21489566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
21499566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
21509566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
21515f0b18bfSMatthew G. Knepley   fegeom.dimEmbed = cdim;
21529566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
21535f0b18bfSMatthew G. Knepley   PetscCheck(Nf > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
21545f0b18bfSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
21555f0b18bfSMatthew G. Knepley     PetscObject  obj;
21565f0b18bfSMatthew G. Knepley     PetscClassId id;
21575f0b18bfSMatthew G. Knepley     PetscInt     fNc;
21585f0b18bfSMatthew G. Knepley 
21599566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, f, NULL, &obj));
21609566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
21615f0b18bfSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
21625f0b18bfSMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
21635f0b18bfSMatthew G. Knepley 
21649566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
21659566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &fNc));
21665f0b18bfSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
21675f0b18bfSMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
21685f0b18bfSMatthew G. Knepley 
21699566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
21709566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &fNc));
217163a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
21725f0b18bfSMatthew G. Knepley     Nc += fNc;
21735f0b18bfSMatthew G. Knepley   }
21749566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
217563a3b9bcSJacob Faibussowitsch   PetscCheck(qNc == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " > 1", qNc);
21769566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(Nc * 2, &valsum, Nc, &interpolant, cdim * Nq, &coords, Nq, &fegeom.detJ, cdim * cdim * Nq, &fegeom.J, cdim * cdim * Nq, &fegeom.invJ));
21779566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
21789566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
21795f0b18bfSMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
21805f0b18bfSMatthew G. Knepley     PetscScalar volsum = 0.0;
21815f0b18bfSMatthew G. Knepley     PetscInt   *star   = NULL;
21825f0b18bfSMatthew G. Knepley     PetscInt    starSize, st, fc;
21835f0b18bfSMatthew G. Knepley 
21849566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(valsum, Nc));
21859566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
21865f0b18bfSMatthew G. Knepley     for (st = 0; st < starSize * 2; st += 2) {
21875f0b18bfSMatthew G. Knepley       const PetscInt cell = star[st];
21885f0b18bfSMatthew G. Knepley       PetscScalar   *val  = &valsum[Nc];
21895f0b18bfSMatthew G. Knepley       PetscScalar   *x    = NULL;
21905f0b18bfSMatthew G. Knepley       PetscReal      vol  = 0.0;
21915f0b18bfSMatthew G. Knepley       PetscInt       foff = 0;
21925f0b18bfSMatthew G. Knepley 
21935f0b18bfSMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
21949566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
21959566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
21965f0b18bfSMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
21975f0b18bfSMatthew G. Knepley         PetscObject  obj;
21985f0b18bfSMatthew G. Knepley         PetscClassId id;
21995f0b18bfSMatthew G. Knepley         PetscInt     Nb, fNc, q;
22005f0b18bfSMatthew G. Knepley 
22019566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(val, Nc));
22029566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, f, NULL, &obj));
22039566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
22049371c9d4SSatish Balay         if (id == PETSCFE_CLASSID) {
22059371c9d4SSatish Balay           PetscCall(PetscFEGetNumComponents((PetscFE)obj, &fNc));
22069371c9d4SSatish Balay           PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
22079371c9d4SSatish Balay         } else if (id == PETSCFV_CLASSID) {
22089371c9d4SSatish Balay           PetscCall(PetscFVGetNumComponents((PetscFV)obj, &fNc));
22099371c9d4SSatish Balay           Nb = 1;
22109371c9d4SSatish Balay         } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
22115f0b18bfSMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
22125f0b18bfSMatthew G. Knepley           const PetscReal wt = quadWeights[q] * fegeom.detJ[q];
22135f0b18bfSMatthew G. Knepley           PetscFEGeom     qgeom;
22145f0b18bfSMatthew G. Knepley 
22155f0b18bfSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
22165f0b18bfSMatthew G. Knepley           qgeom.J        = &fegeom.J[q * cdim * cdim];
22175f0b18bfSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * cdim * cdim];
22185f0b18bfSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
221963a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
22209566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[foff], &qgeom, q, interpolant));
222163a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
22225f0b18bfSMatthew G. Knepley           for (fc = 0; fc < fNc; ++fc) val[foff + fc] += interpolant[fc] * wt;
22235f0b18bfSMatthew G. Knepley           vol += wt;
22245f0b18bfSMatthew G. Knepley         }
22255f0b18bfSMatthew G. Knepley         foff += Nb;
22265f0b18bfSMatthew G. Knepley       }
22279566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
22285f0b18bfSMatthew G. Knepley       for (fc = 0; fc < Nc; ++fc) valsum[fc] += val[fc];
22295f0b18bfSMatthew G. Knepley       volsum += vol;
22305f0b18bfSMatthew G. Knepley       if (debug) {
22319566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " value: [", v, cell));
22325f0b18bfSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
22339566063dSJacob Faibussowitsch           if (fc) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
22349566063dSJacob Faibussowitsch           PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(val[fc])));
22355f0b18bfSMatthew G. Knepley         }
22369566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
22375f0b18bfSMatthew G. Knepley       }
22385f0b18bfSMatthew G. Knepley     }
22395f0b18bfSMatthew G. Knepley     for (fc = 0; fc < Nc; ++fc) valsum[fc] /= volsum;
22409566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
22419566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmc, NULL, locC, v, valsum, INSERT_VALUES));
22425f0b18bfSMatthew G. Knepley   }
22439566063dSJacob Faibussowitsch   PetscCall(PetscFree6(valsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
22443ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
22455f0b18bfSMatthew G. Knepley }
22465f0b18bfSMatthew G. Knepley 
22475f0b18bfSMatthew G. Knepley /*@
224820f4b53cSBarry Smith   DMPlexComputeGradientClementInterpolant - This function computes the L2 projection of the cellwise gradient of a function u onto P1
22491555c271SMatthew G. Knepley 
225020f4b53cSBarry Smith   Collective
2251c0f8e1fdSMatthew G. Knepley 
22521555c271SMatthew G. Knepley   Input Parameters:
2253a1cb98faSBarry Smith + dm   - The `DM`
22545f0b18bfSMatthew G. Knepley - locX - The coefficient vector u_h
22551555c271SMatthew G. Knepley 
22561555c271SMatthew G. Knepley   Output Parameter:
2257a1cb98faSBarry Smith . locC - A `Vec` which holds the Clement interpolant of the gradient
22581555c271SMatthew G. Knepley 
22591555c271SMatthew G. Knepley   Level: developer
22601555c271SMatthew G. Knepley 
2261a1cb98faSBarry Smith   Note:
2262a1cb98faSBarry Smith   $\nabla u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| \nabla u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| $ where $ |T_i| $ is the cell volume
2263a1cb98faSBarry Smith 
22641cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
22651555c271SMatthew G. Knepley @*/
2266d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeGradientClementInterpolant(DM dm, Vec locX, Vec locC)
2267d71ae5a4SJacob Faibussowitsch {
2268db1066baSMatthew G. Knepley   DM_Plex         *mesh  = (DM_Plex *)dm->data;
2269db1066baSMatthew G. Knepley   PetscInt         debug = mesh->printFEM;
22701555c271SMatthew G. Knepley   DM               dmC;
22711555c271SMatthew G. Knepley   PetscQuadrature  quad;
22721555c271SMatthew G. Knepley   PetscScalar     *interpolant, *gradsum;
22734bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
22744bee2e38SMatthew G. Knepley   PetscReal       *coords;
22751555c271SMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
2276485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, vStart, vEnd, v, field, fieldOffset;
22771555c271SMatthew G. Knepley 
22781555c271SMatthew G. Knepley   PetscFunctionBegin;
22799566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
22809566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmC));
22819566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
22829566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
22839566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
22844bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
22859566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
22865f80ce2aSJacob Faibussowitsch   PetscCheck(numFields, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
22871555c271SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
22881555c271SMatthew G. Knepley     PetscObject  obj;
22891555c271SMatthew G. Knepley     PetscClassId id;
22901555c271SMatthew G. Knepley     PetscInt     Nc;
22911555c271SMatthew G. Knepley 
22929566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
22939566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
22941555c271SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
22951555c271SMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
22961555c271SMatthew G. Knepley 
22979566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
22989566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
22991555c271SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
23001555c271SMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
23011555c271SMatthew G. Knepley 
23029566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
23039566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
230463a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
23051555c271SMatthew G. Knepley     numComponents += Nc;
23061555c271SMatthew G. Knepley   }
23079566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
230863a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
23099566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(coordDim * numComponents * 2, &gradsum, coordDim * numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ));
23109566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
23119566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
23121555c271SMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
23131555c271SMatthew G. Knepley     PetscScalar volsum = 0.0;
23141555c271SMatthew G. Knepley     PetscInt   *star   = NULL;
23151555c271SMatthew G. Knepley     PetscInt    starSize, st, d, fc;
23161555c271SMatthew G. Knepley 
23179566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(gradsum, coordDim * numComponents));
23189566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
23191555c271SMatthew G. Knepley     for (st = 0; st < starSize * 2; st += 2) {
23201555c271SMatthew G. Knepley       const PetscInt cell = star[st];
23211555c271SMatthew G. Knepley       PetscScalar   *grad = &gradsum[coordDim * numComponents];
23221555c271SMatthew G. Knepley       PetscScalar   *x    = NULL;
23231555c271SMatthew G. Knepley       PetscReal      vol  = 0.0;
23241555c271SMatthew G. Knepley 
23251555c271SMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
23269566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
23279566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
23281555c271SMatthew G. Knepley       for (field = 0, fieldOffset = 0; field < numFields; ++field) {
23291555c271SMatthew G. Knepley         PetscObject  obj;
23301555c271SMatthew G. Knepley         PetscClassId id;
23311555c271SMatthew G. Knepley         PetscInt     Nb, Nc, q, qc = 0;
23321555c271SMatthew G. Knepley 
23339566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(grad, coordDim * numComponents));
23349566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, field, NULL, &obj));
23359566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
23369371c9d4SSatish Balay         if (id == PETSCFE_CLASSID) {
23379371c9d4SSatish Balay           PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
23389371c9d4SSatish Balay           PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
23399371c9d4SSatish Balay         } else if (id == PETSCFV_CLASSID) {
23409371c9d4SSatish Balay           PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
23419371c9d4SSatish Balay           Nb = 1;
23429371c9d4SSatish Balay         } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
23431555c271SMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
23442a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
23452a4e142eSMatthew G. Knepley 
23462a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
23472a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q * coordDim * coordDim];
23482a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
23492a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
235063a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
23519566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolateGradient_Static((PetscFE)obj, 1, &x[fieldOffset], &qgeom, q, interpolant));
235263a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
23531555c271SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
235453d2db2dSJoe Wallwork             const PetscReal wt = quadWeights[q * qNc + qc];
23551555c271SMatthew G. Knepley 
23564bee2e38SMatthew G. Knepley             for (d = 0; d < coordDim; ++d) grad[fc * coordDim + d] += interpolant[fc * dim + d] * wt * fegeom.detJ[q];
23571555c271SMatthew G. Knepley           }
23584bee2e38SMatthew G. Knepley           vol += quadWeights[q * qNc] * fegeom.detJ[q];
23591555c271SMatthew G. Knepley         }
23601555c271SMatthew G. Knepley         fieldOffset += Nb;
23611555c271SMatthew G. Knepley         qc += Nc;
23621555c271SMatthew G. Knepley       }
23639566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
2364f8527842SMatthew G. Knepley       for (fc = 0; fc < numComponents; ++fc) {
2365ad540459SPierre Jolivet         for (d = 0; d < coordDim; ++d) gradsum[fc * coordDim + d] += grad[fc * coordDim + d];
2366f8527842SMatthew G. Knepley       }
2367f8527842SMatthew G. Knepley       volsum += vol;
2368db1066baSMatthew G. Knepley       if (debug) {
23699566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " gradient: [", v, cell));
23701555c271SMatthew G. Knepley         for (fc = 0; fc < numComponents; ++fc) {
23711555c271SMatthew G. Knepley           for (d = 0; d < coordDim; ++d) {
23729566063dSJacob Faibussowitsch             if (fc || d > 0) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
23739566063dSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(grad[fc * coordDim + d])));
23741555c271SMatthew G. Knepley           }
23751555c271SMatthew G. Knepley         }
23769566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
2377db1066baSMatthew G. Knepley       }
23781555c271SMatthew G. Knepley     }
23791555c271SMatthew G. Knepley     for (fc = 0; fc < numComponents; ++fc) {
23801555c271SMatthew G. Knepley       for (d = 0; d < coordDim; ++d) gradsum[fc * coordDim + d] /= volsum;
23811555c271SMatthew G. Knepley     }
23829566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
23839566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmC, NULL, locC, v, gradsum, INSERT_VALUES));
23841555c271SMatthew G. Knepley   }
23859566063dSJacob Faibussowitsch   PetscCall(PetscFree6(gradsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
23863ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
23871555c271SMatthew G. Knepley }
23881555c271SMatthew G. Knepley 
23896493148fSStefano Zampini PetscErrorCode DMPlexComputeIntegral_Internal(DM dm, Vec locX, PetscInt cStart, PetscInt cEnd, PetscScalar *cintegral, void *user)
2390d71ae5a4SJacob Faibussowitsch {
2391cbf8eb3cSStefano Zampini   DM           dmAux = NULL, plexA = NULL;
239261aaff12SToby Isaac   PetscDS      prob, probAux       = NULL;
239373d901b8SMatthew G. Knepley   PetscSection section, sectionAux;
23946493148fSStefano Zampini   Vec          locA;
2395c330f8ffSToby Isaac   PetscInt     dim, numCells = cEnd - cStart, c, f;
2396c330f8ffSToby Isaac   PetscBool    useFVM = PETSC_FALSE;
2397338f77d5SMatthew G. Knepley   /* DS */
2398338f77d5SMatthew G. Knepley   PetscInt           Nf, totDim, *uOff, *uOff_x, numConstants;
2399338f77d5SMatthew G. Knepley   PetscInt           NfAux, totDimAux, *aOff;
24008e3a54c0SPierre Jolivet   PetscScalar       *u, *a = NULL;
2401338f77d5SMatthew G. Knepley   const PetscScalar *constants;
2402338f77d5SMatthew G. Knepley   /* Geometry */
2403c330f8ffSToby Isaac   PetscFEGeom       *cgeomFEM;
2404338f77d5SMatthew G. Knepley   DM                 dmGrad;
2405c330f8ffSToby Isaac   PetscQuadrature    affineQuad      = NULL;
2406338f77d5SMatthew G. Knepley   Vec                cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL;
2407b5a3613cSMatthew G. Knepley   PetscFVCellGeom   *cgeomFVM;
2408338f77d5SMatthew G. Knepley   const PetscScalar *lgrad;
2409b7260050SToby Isaac   PetscInt           maxDegree;
2410c330f8ffSToby Isaac   DMField            coordField;
2411c330f8ffSToby Isaac   IS                 cellIS;
241273d901b8SMatthew G. Knepley 
241373d901b8SMatthew G. Knepley   PetscFunctionBegin;
24149566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
24159566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
24169566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
24179566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
2418338f77d5SMatthew G. Knepley   /* Determine which discretizations we have */
2419b5a3613cSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
2420b5a3613cSMatthew G. Knepley     PetscObject  obj;
2421b5a3613cSMatthew G. Knepley     PetscClassId id;
2422b5a3613cSMatthew G. Knepley 
24239566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
24249566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
2425338f77d5SMatthew G. Knepley     if (id == PETSCFV_CLASSID) useFVM = PETSC_TRUE;
2426338f77d5SMatthew G. Knepley   }
2427338f77d5SMatthew G. Knepley   /* Read DS information */
24289566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
24299566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
24309566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
24319566063dSJacob Faibussowitsch   PetscCall(ISCreateStride(PETSC_COMM_SELF, numCells, cStart, 1, &cellIS));
24329566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
2433338f77d5SMatthew G. Knepley   /* Read Auxiliary DS information */
24349566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
24359a2a23afSMatthew G. Knepley   if (locA) {
24369566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
2437cbf8eb3cSStefano Zampini     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
24389566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
24399566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
24409566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
24419566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
24429566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
2443338f77d5SMatthew G. Knepley   }
2444338f77d5SMatthew G. Knepley   /* Allocate data  arrays */
24459566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(numCells * totDim, &u));
24469566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
2447338f77d5SMatthew G. Knepley   /* Read out geometry */
24489566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
24499566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
2450b7260050SToby Isaac   if (maxDegree <= 1) {
24519566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad));
2452ac9d17c7SMatthew G. Knepley     if (affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, affineQuad, PETSC_FEGEOM_BASIC, &cgeomFEM));
2453b5a3613cSMatthew G. Knepley   }
2454b5a3613cSMatthew G. Knepley   if (useFVM) {
2455338f77d5SMatthew G. Knepley     PetscFV   fv = NULL;
2456b5a3613cSMatthew G. Knepley     Vec       grad;
2457b5a3613cSMatthew G. Knepley     PetscInt  fStart, fEnd;
2458b5a3613cSMatthew G. Knepley     PetscBool compGrad;
2459b5a3613cSMatthew G. Knepley 
2460338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
2461338f77d5SMatthew G. Knepley       PetscObject  obj;
2462338f77d5SMatthew G. Knepley       PetscClassId id;
2463338f77d5SMatthew G. Knepley 
24649566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
24659566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
24669371c9d4SSatish Balay       if (id == PETSCFV_CLASSID) {
24679371c9d4SSatish Balay         fv = (PetscFV)obj;
24689371c9d4SSatish Balay         break;
24699371c9d4SSatish Balay       }
2470338f77d5SMatthew G. Knepley     }
24719566063dSJacob Faibussowitsch     PetscCall(PetscFVGetComputeGradients(fv, &compGrad));
24729566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, PETSC_TRUE));
24739566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGeometryFVM(dm, &cellGeometryFVM, &faceGeometryFVM));
24749566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGradientFVM(dm, fv, faceGeometryFVM, cellGeometryFVM, &dmGrad));
24759566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, compGrad));
24769566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM));
2477b5a3613cSMatthew G. Knepley     /* Reconstruct and limit cell gradients */
24789566063dSJacob Faibussowitsch     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
24799566063dSJacob Faibussowitsch     PetscCall(DMGetGlobalVector(dmGrad, &grad));
24809566063dSJacob Faibussowitsch     PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
2481b5a3613cSMatthew G. Knepley     /* Communicate gradient values */
24829566063dSJacob Faibussowitsch     PetscCall(DMGetLocalVector(dmGrad, &locGrad));
24839566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
24849566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
24859566063dSJacob Faibussowitsch     PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
2486b5a3613cSMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
24879566063dSJacob Faibussowitsch     PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, 0.0, faceGeometryFVM, cellGeometryFVM, locGrad));
24889566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
2489b5a3613cSMatthew G. Knepley   }
2490338f77d5SMatthew G. Knepley   /* Read out data from inputs */
249173d901b8SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
249273d901b8SMatthew G. Knepley     PetscScalar *x = NULL;
249373d901b8SMatthew G. Knepley     PetscInt     i;
249473d901b8SMatthew G. Knepley 
24959566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, section, locX, c, NULL, &x));
24960f2d7e86SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i];
24979566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, section, locX, c, NULL, &x));
249873d901b8SMatthew G. Knepley     if (dmAux) {
2499cbf8eb3cSStefano Zampini       PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, c, NULL, &x));
25000f2d7e86SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i];
2501cbf8eb3cSStefano Zampini       PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, c, NULL, &x));
250273d901b8SMatthew G. Knepley     }
250373d901b8SMatthew G. Knepley   }
2504338f77d5SMatthew G. Knepley   /* Do integration for each field */
250573d901b8SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
2506c1f031eeSMatthew G. Knepley     PetscObject  obj;
2507c1f031eeSMatthew G. Knepley     PetscClassId id;
2508c1f031eeSMatthew G. Knepley     PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
250973d901b8SMatthew G. Knepley 
25109566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
25119566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
2512c1f031eeSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
2513c1f031eeSMatthew G. Knepley       PetscFE         fe = (PetscFE)obj;
2514c1f031eeSMatthew G. Knepley       PetscQuadrature q;
2515c330f8ffSToby Isaac       PetscFEGeom    *chunkGeom = NULL;
2516c1f031eeSMatthew G. Knepley       PetscInt        Nq, Nb;
2517c1f031eeSMatthew G. Knepley 
25189566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
25199566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &q));
25209566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
25219566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
2522c1f031eeSMatthew G. Knepley       blockSize = Nb * Nq;
252373d901b8SMatthew G. Knepley       batchSize = numBlocks * blockSize;
25249566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
252573d901b8SMatthew G. Knepley       numChunks = numCells / (numBatches * batchSize);
252673d901b8SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
252773d901b8SMatthew G. Knepley       Nr        = numCells % (numBatches * batchSize);
252873d901b8SMatthew G. Knepley       offset    = numCells - Nr;
2529ac9d17c7SMatthew G. Knepley       if (!affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, q, PETSC_FEGEOM_BASIC, &cgeomFEM));
25309566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
25319566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrate(prob, f, Ne, chunkGeom, u, probAux, a, cintegral));
25329566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &chunkGeom));
25338e3a54c0SPierre Jolivet       PetscCall(PetscFEIntegrate(prob, f, Nr, chunkGeom, &u[offset * totDim], probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), &cintegral[offset * Nf]));
25349566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &chunkGeom));
253548a46eb9SPierre Jolivet       if (!affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM));
2536c1f031eeSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
2537c1f031eeSMatthew G. Knepley       PetscInt      foff;
25382192575eSBarry Smith       PetscPointFn *obj_func;
2539c1f031eeSMatthew G. Knepley 
25409566063dSJacob Faibussowitsch       PetscCall(PetscDSGetObjective(prob, f, &obj_func));
25419566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, f, &foff));
2542c1f031eeSMatthew G. Knepley       if (obj_func) {
2543c1f031eeSMatthew G. Knepley         for (c = 0; c < numCells; ++c) {
2544b5a3613cSMatthew G. Knepley           PetscScalar *u_x;
2545d627b919SMatthew G. Knepley           PetscScalar  lint = 0.;
2546b5a3613cSMatthew G. Knepley 
25479566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, c, lgrad, &u_x));
254855bffe1bSPierre Jolivet           obj_func(dim, Nf, NfAux, uOff, uOff_x, &u[totDim * c + foff], NULL, u_x, aOff, NULL, PetscSafePointerPlusOffset(a, totDimAux * c), NULL, NULL, 0.0, cgeomFVM[c].centroid, numConstants, constants, &lint);
2549338f77d5SMatthew G. Knepley           cintegral[c * Nf + f] += PetscRealPart(lint) * cgeomFVM[c].volume;
255073d901b8SMatthew G. Knepley         }
2551c1f031eeSMatthew G. Knepley       }
255263a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
2553c1f031eeSMatthew G. Knepley   }
2554338f77d5SMatthew G. Knepley   /* Cleanup data arrays */
2555b5a3613cSMatthew G. Knepley   if (useFVM) {
25569566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
25579566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM));
25589566063dSJacob Faibussowitsch     PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
25599566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&faceGeometryFVM));
25609566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&cellGeometryFVM));
25619566063dSJacob Faibussowitsch     PetscCall(DMDestroy(&dmGrad));
2562b5a3613cSMatthew G. Knepley   }
25639566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscFree(a));
2564cbf8eb3cSStefano Zampini   PetscCall(DMDestroy(&plexA));
25659566063dSJacob Faibussowitsch   PetscCall(PetscFree(u));
2566338f77d5SMatthew G. Knepley   /* Cleanup */
256748a46eb9SPierre Jolivet   if (affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM));
25689566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&affineQuad));
25699566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cellIS));
25703ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
2571338f77d5SMatthew G. Knepley }
2572338f77d5SMatthew G. Knepley 
2573338f77d5SMatthew G. Knepley /*@
2574338f77d5SMatthew G. Knepley   DMPlexComputeIntegralFEM - Form the integral over the domain from the global input X using pointwise functions specified by the user
2575338f77d5SMatthew G. Knepley 
2576338f77d5SMatthew G. Knepley   Input Parameters:
2577338f77d5SMatthew G. Knepley + dm   - The mesh
2578338f77d5SMatthew G. Knepley . X    - Global input vector
2579338f77d5SMatthew G. Knepley - user - The user context
2580338f77d5SMatthew G. Knepley 
2581338f77d5SMatthew G. Knepley   Output Parameter:
2582338f77d5SMatthew G. Knepley . integral - Integral for each field
2583338f77d5SMatthew G. Knepley 
2584338f77d5SMatthew G. Knepley   Level: developer
2585338f77d5SMatthew G. Knepley 
25861cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSNESComputeResidualFEM()`
2587338f77d5SMatthew G. Knepley @*/
2588d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeIntegralFEM(DM dm, Vec X, PetscScalar *integral, void *user)
2589d71ae5a4SJacob Faibussowitsch {
2590cbf8eb3cSStefano Zampini   PetscInt     printFEM;
2591b8feb594SMatthew G. Knepley   PetscScalar *cintegral, *lintegral;
2592412e9a14SMatthew G. Knepley   PetscInt     Nf, f, cellHeight, cStart, cEnd, cell;
25936493148fSStefano Zampini   Vec          locX;
2594338f77d5SMatthew G. Knepley 
2595338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2596338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2597338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
25984f572ea9SToby Isaac   PetscAssertPointer(integral, 3);
25999566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0));
2600cbf8eb3cSStefano Zampini   PetscCall(DMPlexConvertPlex(dm, &dm, PETSC_TRUE));
26019566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
26029566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
26039566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2604338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
26059566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &lintegral, (cEnd - cStart) * Nf, &cintegral));
26066493148fSStefano Zampini   /* Get local solution with boundary values */
26076493148fSStefano Zampini   PetscCall(DMGetLocalVector(dm, &locX));
26086493148fSStefano Zampini   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
26096493148fSStefano Zampini   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
26106493148fSStefano Zampini   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
26116493148fSStefano Zampini   PetscCall(DMPlexComputeIntegral_Internal(dm, locX, cStart, cEnd, cintegral, user));
26126493148fSStefano Zampini   PetscCall(DMRestoreLocalVector(dm, &locX));
2613cbf8eb3cSStefano Zampini   printFEM = ((DM_Plex *)dm->data)->printFEM;
2614338f77d5SMatthew G. Knepley   /* Sum up values */
2615338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2616338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2617338f77d5SMatthew G. Knepley 
2618cbf8eb3cSStefano Zampini     if (printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf]));
2619b8feb594SMatthew G. Knepley     for (f = 0; f < Nf; ++f) lintegral[f] += cintegral[c * Nf + f];
2620338f77d5SMatthew G. Knepley   }
2621e91c04dfSPierre Jolivet   PetscCallMPI(MPIU_Allreduce(lintegral, integral, Nf, MPIU_SCALAR, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
2622cbf8eb3cSStefano Zampini   if (printFEM) {
26239566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "Integral:"));
26249566063dSJacob Faibussowitsch     for (f = 0; f < Nf; ++f) PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), " %g", (double)PetscRealPart(integral[f])));
26259566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "\n"));
262673d901b8SMatthew G. Knepley   }
26279566063dSJacob Faibussowitsch   PetscCall(PetscFree2(lintegral, cintegral));
26289566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0));
2629cbf8eb3cSStefano Zampini   PetscCall(DMDestroy(&dm));
26303ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
2631338f77d5SMatthew G. Knepley }
2632338f77d5SMatthew G. Knepley 
2633338f77d5SMatthew G. Knepley /*@
2634338f77d5SMatthew G. Knepley   DMPlexComputeCellwiseIntegralFEM - Form the vector of cellwise integrals F from the global input X using pointwise functions specified by the user
2635338f77d5SMatthew G. Knepley 
2636338f77d5SMatthew G. Knepley   Input Parameters:
2637338f77d5SMatthew G. Knepley + dm   - The mesh
2638338f77d5SMatthew G. Knepley . X    - Global input vector
2639338f77d5SMatthew G. Knepley - user - The user context
2640338f77d5SMatthew G. Knepley 
2641338f77d5SMatthew G. Knepley   Output Parameter:
264260225df5SJacob Faibussowitsch . F - Cellwise integrals for each field
2643338f77d5SMatthew G. Knepley 
2644338f77d5SMatthew G. Knepley   Level: developer
2645338f77d5SMatthew G. Knepley 
26461cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSNESComputeResidualFEM()`
2647338f77d5SMatthew G. Knepley @*/
2648d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeCellwiseIntegralFEM(DM dm, Vec X, Vec F, void *user)
2649d71ae5a4SJacob Faibussowitsch {
2650cbf8eb3cSStefano Zampini   PetscInt     printFEM;
2651338f77d5SMatthew G. Knepley   DM           dmF;
26529c8ab049SStefano Zampini   PetscSection sectionF = NULL;
2653338f77d5SMatthew G. Knepley   PetscScalar *cintegral, *af;
26549c8ab049SStefano Zampini   PetscInt     Nf, f, cellHeight, cStart, cEnd, cell, n;
26556493148fSStefano Zampini   Vec          locX;
2656338f77d5SMatthew G. Knepley 
2657338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2658338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2659338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2660338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(F, VEC_CLASSID, 3);
26619566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0));
2662cbf8eb3cSStefano Zampini   PetscCall(DMPlexConvertPlex(dm, &dm, PETSC_TRUE));
26639566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
26649566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
26659566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2666338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
26679566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1((cEnd - cStart) * Nf, &cintegral));
26686493148fSStefano Zampini   /* Get local solution with boundary values */
26696493148fSStefano Zampini   PetscCall(DMGetLocalVector(dm, &locX));
26706493148fSStefano Zampini   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
26716493148fSStefano Zampini   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
26726493148fSStefano Zampini   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
26736493148fSStefano Zampini   PetscCall(DMPlexComputeIntegral_Internal(dm, locX, cStart, cEnd, cintegral, user));
26746493148fSStefano Zampini   PetscCall(DMRestoreLocalVector(dm, &locX));
2675338f77d5SMatthew G. Knepley   /* Put values in F */
26769566063dSJacob Faibussowitsch   PetscCall(VecGetArray(F, &af));
26779c8ab049SStefano Zampini   PetscCall(VecGetDM(F, &dmF));
26789c8ab049SStefano Zampini   if (dmF) PetscCall(DMGetLocalSection(dmF, &sectionF));
26799c8ab049SStefano Zampini   PetscCall(VecGetLocalSize(F, &n));
26809c8ab049SStefano Zampini   PetscCheck(n >= (cEnd - cStart) * Nf, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Vector size %" PetscInt_FMT " < %" PetscInt_FMT, n, (cEnd - cStart) * Nf);
2681cbf8eb3cSStefano Zampini   printFEM = ((DM_Plex *)dm->data)->printFEM;
2682338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2683338f77d5SMatthew G. Knepley     const PetscInt c   = cell - cStart;
26849c8ab049SStefano Zampini     PetscInt       dof = Nf, off = c * Nf;
2685338f77d5SMatthew G. Knepley 
2686cbf8eb3cSStefano Zampini     if (printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf]));
26879c8ab049SStefano Zampini     if (sectionF) {
26889566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetDof(sectionF, cell, &dof));
26899566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetOffset(sectionF, cell, &off));
26909c8ab049SStefano Zampini     }
269163a3b9bcSJacob Faibussowitsch     PetscCheck(dof == Nf, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of cell dofs %" PetscInt_FMT " != %" PetscInt_FMT, dof, Nf);
2692338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) af[off + f] = cintegral[c * Nf + f];
2693338f77d5SMatthew G. Knepley   }
26949566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(F, &af));
26959566063dSJacob Faibussowitsch   PetscCall(PetscFree(cintegral));
26969566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0));
2697cbf8eb3cSStefano Zampini   PetscCall(DMDestroy(&dm));
26983ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
269973d901b8SMatthew G. Knepley }
270073d901b8SMatthew G. Knepley 
270179ab67a3SMatthew G. Knepley static PetscErrorCode DMPlexComputeBdIntegral_Internal(DM dm, Vec locX, IS pointIS, void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *fintegral, void *user)
2702d71ae5a4SJacob Faibussowitsch {
27039b6f715bSMatthew G. Knepley   DM                 plex = NULL, plexA = NULL;
2704a6e0b375SMatthew G. Knepley   DMEnclosureType    encAux;
27059b6f715bSMatthew G. Knepley   PetscDS            prob, probAux       = NULL;
27069b6f715bSMatthew G. Knepley   PetscSection       section, sectionAux = NULL;
27079b6f715bSMatthew G. Knepley   Vec                locA = NULL;
27089b6f715bSMatthew G. Knepley   DMField            coordField;
27099b6f715bSMatthew G. Knepley   PetscInt           Nf, totDim, *uOff, *uOff_x;
27109b6f715bSMatthew G. Knepley   PetscInt           NfAux = 0, totDimAux = 0, *aOff = NULL;
27119b6f715bSMatthew G. Knepley   PetscScalar       *u, *a = NULL;
271264c72086SMatthew G. Knepley   const PetscScalar *constants;
27139b6f715bSMatthew G. Knepley   PetscInt           numConstants, f;
271464c72086SMatthew G. Knepley 
271564c72086SMatthew G. Knepley   PetscFunctionBegin;
27169566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
27179566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
27189566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
27199566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
27209566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
272164c72086SMatthew G. Knepley   /* Determine which discretizations we have */
27229b6f715bSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
272364c72086SMatthew G. Knepley     PetscObject  obj;
272464c72086SMatthew G. Knepley     PetscClassId id;
272564c72086SMatthew G. Knepley 
27269566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
27279566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
272863a3b9bcSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not supported for FVM (field %" PetscInt_FMT ")", f);
272964c72086SMatthew G. Knepley   }
273064c72086SMatthew G. Knepley   /* Read DS information */
27319566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
27329566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
27339566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
27349566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
273564c72086SMatthew G. Knepley   /* Read Auxiliary DS information */
27369566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
27379b6f715bSMatthew G. Knepley   if (locA) {
27389b6f715bSMatthew G. Knepley     DM dmAux;
27399b6f715bSMatthew G. Knepley 
27409566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
27419566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
27429566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
27439566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
27449566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
27459566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
27469566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
27479566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
274864c72086SMatthew G. Knepley   }
27499b6f715bSMatthew G. Knepley   /* Integrate over points */
27509b6f715bSMatthew G. Knepley   {
27519b6f715bSMatthew G. Knepley     PetscFEGeom    *fgeom, *chunkGeom = NULL;
2752b7260050SToby Isaac     PetscInt        maxDegree;
27539b6f715bSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
27549b6f715bSMatthew G. Knepley     const PetscInt *points;
27559b6f715bSMatthew G. Knepley     PetscInt        numFaces, face, Nq, field;
27569b6f715bSMatthew G. Knepley     PetscInt        numChunks, chunkSize, chunk, Nr, offset;
275764c72086SMatthew G. Knepley 
27589566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
27599566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
276032603206SJames Wright     PetscCall(PetscCalloc2(numFaces * totDim, &u, (locA ? (size_t)numFaces * totDimAux : 0), &a));
27619566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
276279ab67a3SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
276379ab67a3SMatthew G. Knepley       const PetscInt point = points[face], *support;
276479ab67a3SMatthew G. Knepley       PetscScalar   *x     = NULL;
276579ab67a3SMatthew G. Knepley 
276679ab67a3SMatthew G. Knepley       PetscCall(DMPlexGetSupport(dm, point, &support));
276779ab67a3SMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
276879ab67a3SMatthew G. Knepley       for (PetscInt i = 0; i < totDim; ++i) u[face * totDim + i] = x[i];
276979ab67a3SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
277079ab67a3SMatthew G. Knepley       if (locA) {
277179ab67a3SMatthew G. Knepley         PetscInt subp;
277279ab67a3SMatthew G. Knepley         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
277379ab67a3SMatthew G. Knepley         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
277479ab67a3SMatthew G. Knepley         for (PetscInt i = 0; i < totDimAux; ++i) a[f * totDimAux + i] = x[i];
277579ab67a3SMatthew G. Knepley         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
277679ab67a3SMatthew G. Knepley       }
277779ab67a3SMatthew G. Knepley     }
277864c72086SMatthew G. Knepley     for (field = 0; field < Nf; ++field) {
277964c72086SMatthew G. Knepley       PetscFE fe;
278064c72086SMatthew G. Knepley 
27819566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fe));
27829566063dSJacob Faibussowitsch       if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
27839b6f715bSMatthew G. Knepley       if (!qGeom) {
27849566063dSJacob Faibussowitsch         PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
27859566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)qGeom));
27869b6f715bSMatthew G. Knepley       }
27879566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
2788ac9d17c7SMatthew G. Knepley       PetscCall(DMPlexGetFEGeom(coordField, pointIS, qGeom, PETSC_FEGEOM_BOUNDARY, &fgeom));
27899b6f715bSMatthew G. Knepley       /* Get blocking */
27909b6f715bSMatthew G. Knepley       {
27919b6f715bSMatthew G. Knepley         PetscQuadrature q;
27929b6f715bSMatthew G. Knepley         PetscInt        numBatches, batchSize, numBlocks, blockSize;
27939b6f715bSMatthew G. Knepley         PetscInt        Nq, Nb;
27949b6f715bSMatthew G. Knepley 
27959566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
27969566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &q));
27979566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
27989566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
279964c72086SMatthew G. Knepley         blockSize = Nb * Nq;
280064c72086SMatthew G. Knepley         batchSize = numBlocks * blockSize;
28019b6f715bSMatthew G. Knepley         chunkSize = numBatches * batchSize;
28029566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
28039b6f715bSMatthew G. Knepley         numChunks = numFaces / chunkSize;
28049b6f715bSMatthew G. Knepley         Nr        = numFaces % chunkSize;
280564c72086SMatthew G. Knepley         offset    = numFaces - Nr;
280664c72086SMatthew G. Knepley       }
28079b6f715bSMatthew G. Knepley       /* Do integration for each field */
28089b6f715bSMatthew G. Knepley       for (chunk = 0; chunk < numChunks; ++chunk) {
28099566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(fgeom, chunk * chunkSize, (chunk + 1) * chunkSize, &chunkGeom));
281079ab67a3SMatthew G. Knepley         PetscCall(PetscFEIntegrateBd(prob, field, funcs[field], chunkSize, chunkGeom, &u[chunk * chunkSize * totDim], probAux, PetscSafePointerPlusOffset(a, chunk * chunkSize * totDimAux), &fintegral[chunk * chunkSize * Nf]));
28119566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
281264c72086SMatthew G. Knepley       }
28139566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
281479ab67a3SMatthew G. Knepley       PetscCall(PetscFEIntegrateBd(prob, field, funcs[field], Nr, chunkGeom, &u[offset * totDim], probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), &fintegral[offset * Nf]));
28159566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
281664c72086SMatthew G. Knepley       /* Cleanup data arrays */
2817ac9d17c7SMatthew G. Knepley       PetscCall(DMPlexRestoreFEGeom(coordField, pointIS, qGeom, PETSC_FEGEOM_BOUNDARY, &fgeom));
28189566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&qGeom));
281979ab67a3SMatthew G. Knepley     }
28209566063dSJacob Faibussowitsch     PetscCall(PetscFree2(u, a));
28219566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
282264c72086SMatthew G. Knepley   }
28239566063dSJacob Faibussowitsch   if (plex) PetscCall(DMDestroy(&plex));
28249566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
28253ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
28269b6f715bSMatthew G. Knepley }
28279b6f715bSMatthew G. Knepley 
282860225df5SJacob Faibussowitsch /*@C
28299b6f715bSMatthew G. Knepley   DMPlexComputeBdIntegral - Form the integral over the specified boundary from the global input X using pointwise functions specified by the user
28309b6f715bSMatthew G. Knepley 
28319b6f715bSMatthew G. Knepley   Input Parameters:
28329b6f715bSMatthew G. Knepley + dm      - The mesh
28339b6f715bSMatthew G. Knepley . X       - Global input vector
2834a1cb98faSBarry Smith . label   - The boundary `DMLabel`
2835a1cb98faSBarry Smith . numVals - The number of label values to use, or `PETSC_DETERMINE` for all values
2836a1cb98faSBarry Smith . vals    - The label values to use, or NULL for all values
283779ab67a3SMatthew G. Knepley . funcs   - The functions to integrate along the boundary for each field
28389b6f715bSMatthew G. Knepley - user    - The user context
28399b6f715bSMatthew G. Knepley 
28409b6f715bSMatthew G. Knepley   Output Parameter:
28419b6f715bSMatthew G. Knepley . integral - Integral for each field
28429b6f715bSMatthew G. Knepley 
28439b6f715bSMatthew G. Knepley   Level: developer
28449b6f715bSMatthew G. Knepley 
28451cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeIntegralFEM()`, `DMPlexComputeBdResidualFEM()`
28469b6f715bSMatthew G. Knepley @*/
284779ab67a3SMatthew G. Knepley PetscErrorCode DMPlexComputeBdIntegral(DM dm, Vec X, DMLabel label, PetscInt numVals, const PetscInt vals[], void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *integral, void *user)
2848d71ae5a4SJacob Faibussowitsch {
28499b6f715bSMatthew G. Knepley   Vec          locX;
28509b6f715bSMatthew G. Knepley   PetscSection section;
28519b6f715bSMatthew G. Knepley   DMLabel      depthLabel;
28529b6f715bSMatthew G. Knepley   IS           facetIS;
28539b6f715bSMatthew G. Knepley   PetscInt     dim, Nf, f, v;
28549b6f715bSMatthew G. Knepley 
28559b6f715bSMatthew G. Knepley   PetscFunctionBegin;
28569b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
28579b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2858292bffcbSToby Isaac   PetscValidHeaderSpecific(label, DMLABEL_CLASSID, 3);
28594f572ea9SToby Isaac   if (vals) PetscAssertPointer(vals, 5);
28604f572ea9SToby Isaac   PetscAssertPointer(integral, 7);
28619566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0));
28629566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
28639566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
28649566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
28659566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
28669566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
28679b6f715bSMatthew G. Knepley   /* Get local solution with boundary values */
28689566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &locX));
28699566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
28709566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
28719566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
28729b6f715bSMatthew G. Knepley   /* Loop over label values */
28739566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(integral, Nf));
28749b6f715bSMatthew G. Knepley   for (v = 0; v < numVals; ++v) {
28759b6f715bSMatthew G. Knepley     IS           pointIS;
28769b6f715bSMatthew G. Knepley     PetscInt     numFaces, face;
28779b6f715bSMatthew G. Knepley     PetscScalar *fintegral;
28789b6f715bSMatthew G. Knepley 
28799566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, vals[v], &pointIS));
28809b6f715bSMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
28819b6f715bSMatthew G. Knepley     {
28829b6f715bSMatthew G. Knepley       IS isectIS;
28839b6f715bSMatthew G. Knepley 
28849b6f715bSMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
28859566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
28869566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
28879b6f715bSMatthew G. Knepley       pointIS = isectIS;
28889b6f715bSMatthew G. Knepley     }
28899566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
28909566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(numFaces * Nf, &fintegral));
289179ab67a3SMatthew G. Knepley     PetscCall(DMPlexComputeBdIntegral_Internal(dm, locX, pointIS, funcs, fintegral, user));
28929b6f715bSMatthew G. Knepley     /* Sum point contributions into integral */
28939371c9d4SSatish Balay     for (f = 0; f < Nf; ++f)
28949371c9d4SSatish Balay       for (face = 0; face < numFaces; ++face) integral[f] += fintegral[face * Nf + f];
28959566063dSJacob Faibussowitsch     PetscCall(PetscFree(fintegral));
28969566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
28979b6f715bSMatthew G. Knepley   }
28989566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &locX));
28999566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
29009566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0));
29013ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
290264c72086SMatthew G. Knepley }
290364c72086SMatthew G. Knepley 
2904d69c5d34SMatthew G. Knepley /*@
29050318f8a0SStefano Zampini   DMPlexComputeInterpolatorNested - Form the local portion of the interpolation matrix from the coarse `DM` to a uniformly refined `DM`.
2906d69c5d34SMatthew G. Knepley 
2907d69c5d34SMatthew G. Knepley   Input Parameters:
2908cf51de39SMatthew G. Knepley + dmc       - The coarse mesh
2909cf51de39SMatthew G. Knepley . dmf       - The fine mesh
2910cf51de39SMatthew G. Knepley . isRefined - Flag indicating regular refinement, rather than the same topology
2911d69c5d34SMatthew G. Knepley - user      - The user context
2912d69c5d34SMatthew G. Knepley 
2913d69c5d34SMatthew G. Knepley   Output Parameter:
2914934789fcSMatthew G. Knepley . In - The interpolation matrix
2915d69c5d34SMatthew G. Knepley 
2916d69c5d34SMatthew G. Knepley   Level: developer
2917d69c5d34SMatthew G. Knepley 
2918ae3cf2c1SStefano Zampini .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorGeneral()`
2919d69c5d34SMatthew G. Knepley @*/
2920d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeInterpolatorNested(DM dmc, DM dmf, PetscBool isRefined, Mat In, void *user)
2921d71ae5a4SJacob Faibussowitsch {
2922d69c5d34SMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dmc->data;
2923d69c5d34SMatthew G. Knepley   const char  *name = "Interpolator";
2924d69c5d34SMatthew G. Knepley   PetscFE     *feRef;
292597c42addSMatthew G. Knepley   PetscFV     *fvRef;
2926d69c5d34SMatthew G. Knepley   PetscSection fsection, fglobalSection;
2927d69c5d34SMatthew G. Knepley   PetscSection csection, cglobalSection;
2928d69c5d34SMatthew G. Knepley   PetscScalar *elemMat;
2929485ad865SMatthew G. Knepley   PetscInt     dim, Nf, f, fieldI, fieldJ, offsetI, offsetJ, cStart, cEnd, c;
29302ea9c922SToby Isaac   PetscInt     cTotDim = 0, rTotDim = 0;
2931d69c5d34SMatthew G. Knepley 
2932d69c5d34SMatthew G. Knepley   PetscFunctionBegin;
29339566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
29349566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
29359566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
29369566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
29379566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
29389566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
29399566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
29409566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
29419566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &feRef, Nf, &fvRef));
2942d69c5d34SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
29432ea9c922SToby Isaac     PetscObject  obj, objc;
29442ea9c922SToby Isaac     PetscClassId id, idc;
29452ea9c922SToby Isaac     PetscInt     rNb = 0, Nc = 0, cNb = 0;
2946d69c5d34SMatthew G. Knepley 
29479566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmf, f, NULL, &obj));
29489566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
294997c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
295097c42addSMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
295197c42addSMatthew G. Knepley 
2952cf51de39SMatthew G. Knepley       if (isRefined) {
29539566063dSJacob Faibussowitsch         PetscCall(PetscFERefine(fe, &feRef[f]));
2954cf51de39SMatthew G. Knepley       } else {
29559566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)fe));
2956cf51de39SMatthew G. Knepley         feRef[f] = fe;
2957cf51de39SMatthew G. Knepley       }
29589566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &rNb));
29599566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
296097c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
296197c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV)obj;
296297c42addSMatthew G. Knepley       PetscDualSpace Q;
296397c42addSMatthew G. Knepley 
2964cf51de39SMatthew G. Knepley       if (isRefined) {
29659566063dSJacob Faibussowitsch         PetscCall(PetscFVRefine(fv, &fvRef[f]));
2966cf51de39SMatthew G. Knepley       } else {
29679566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)fv));
2968cf51de39SMatthew G. Knepley         fvRef[f] = fv;
2969cf51de39SMatthew G. Knepley       }
29709566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
29719566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &rNb));
29729566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
29739566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
297497c42addSMatthew G. Knepley     }
29759566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmc, f, NULL, &objc));
29769566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(objc, &idc));
29772ea9c922SToby Isaac     if (idc == PETSCFE_CLASSID) {
29782ea9c922SToby Isaac       PetscFE fe = (PetscFE)objc;
29792ea9c922SToby Isaac 
29809566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &cNb));
29812ea9c922SToby Isaac     } else if (id == PETSCFV_CLASSID) {
29822ea9c922SToby Isaac       PetscFV        fv = (PetscFV)obj;
29832ea9c922SToby Isaac       PetscDualSpace Q;
29842ea9c922SToby Isaac 
29859566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
29869566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &cNb));
2987d69c5d34SMatthew G. Knepley     }
29882ea9c922SToby Isaac     rTotDim += rNb;
29892ea9c922SToby Isaac     cTotDim += cNb;
29902ea9c922SToby Isaac   }
29919566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(rTotDim * cTotDim, &elemMat));
29929566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, rTotDim * cTotDim));
2993d69c5d34SMatthew G. Knepley   for (fieldI = 0, offsetI = 0; fieldI < Nf; ++fieldI) {
2994d69c5d34SMatthew G. Knepley     PetscDualSpace   Qref;
2995d69c5d34SMatthew G. Knepley     PetscQuadrature  f;
2996d69c5d34SMatthew G. Knepley     const PetscReal *qpoints, *qweights;
2997d69c5d34SMatthew G. Knepley     PetscReal       *points;
2998d69c5d34SMatthew G. Knepley     PetscInt         npoints = 0, Nc, Np, fpdim, i, k, p, d;
2999d69c5d34SMatthew G. Knepley 
3000d69c5d34SMatthew G. Knepley     /* Compose points from all dual basis functionals */
300197c42addSMatthew G. Knepley     if (feRef[fieldI]) {
30029566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[fieldI], &Qref));
30039566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[fieldI], &Nc));
300497c42addSMatthew G. Knepley     } else {
30059566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[fieldI], &Qref));
30069566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[fieldI], &Nc));
300797c42addSMatthew G. Knepley     }
30089566063dSJacob Faibussowitsch     PetscCall(PetscDualSpaceGetDimension(Qref, &fpdim));
3009d69c5d34SMatthew G. Knepley     for (i = 0; i < fpdim; ++i) {
30109566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
30119566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, NULL, NULL));
3012d69c5d34SMatthew G. Knepley       npoints += Np;
3013d69c5d34SMatthew G. Knepley     }
30149566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(npoints * dim, &points));
3015d69c5d34SMatthew G. Knepley     for (i = 0, k = 0; i < fpdim; ++i) {
30169566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
30179566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL));
30189371c9d4SSatish Balay       for (p = 0; p < Np; ++p, ++k)
30199371c9d4SSatish Balay         for (d = 0; d < dim; ++d) points[k * dim + d] = qpoints[p * dim + d];
3020d69c5d34SMatthew G. Knepley     }
3021d69c5d34SMatthew G. Knepley 
3022d69c5d34SMatthew G. Knepley     for (fieldJ = 0, offsetJ = 0; fieldJ < Nf; ++fieldJ) {
302397c42addSMatthew G. Knepley       PetscObject  obj;
302497c42addSMatthew G. Knepley       PetscClassId id;
30259c3cf19fSMatthew G. Knepley       PetscInt     NcJ = 0, cpdim = 0, j, qNc;
3026d69c5d34SMatthew G. Knepley 
30279566063dSJacob Faibussowitsch       PetscCall(DMGetField(dmc, fieldJ, NULL, &obj));
30289566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
302997c42addSMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
303097c42addSMatthew G. Knepley         PetscFE         fe = (PetscFE)obj;
3031ef0bb6c7SMatthew G. Knepley         PetscTabulation T  = NULL;
3032d69c5d34SMatthew G. Knepley 
3033d69c5d34SMatthew G. Knepley         /* Evaluate basis at points */
30349566063dSJacob Faibussowitsch         PetscCall(PetscFEGetNumComponents(fe, &NcJ));
30359566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &cpdim));
3036ffe73a53SMatthew G. Knepley         /* For now, fields only interpolate themselves */
3037ffe73a53SMatthew G. Knepley         if (fieldI == fieldJ) {
303863a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
30399566063dSJacob Faibussowitsch           PetscCall(PetscFECreateTabulation(fe, 1, npoints, points, 0, &T));
3040d69c5d34SMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
30419566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
30429566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
304363a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
3044d69c5d34SMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
304536a6d9c0SMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
3046d172c84bSMatthew G. Knepley                 /*
3047d172c84bSMatthew G. Knepley                    cTotDim:            Total columns in element interpolation matrix, sum of number of dual basis functionals in each field
3048d172c84bSMatthew G. Knepley                    offsetI, offsetJ:   Offsets into the larger element interpolation matrix for different fields
3049d172c84bSMatthew G. Knepley                    fpdim, i, cpdim, j: Dofs for fine and coarse grids, correspond to dual space basis functionals
3050d172c84bSMatthew G. Knepley                    qNC, Nc, Ncj, c:    Number of components in this field
3051d172c84bSMatthew G. Knepley                    Np, p:              Number of quad points in the fine grid functional i
3052d172c84bSMatthew G. Knepley                    k:                  i*Np + p, overall point number for the interpolation
3053d172c84bSMatthew G. Knepley                 */
3054ef0bb6c7SMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += T->T[0][k * cpdim * NcJ + j * Nc + c] * qweights[p * qNc + c];
305536a6d9c0SMatthew G. Knepley               }
3056d69c5d34SMatthew G. Knepley             }
3057d69c5d34SMatthew G. Knepley           }
30589566063dSJacob Faibussowitsch           PetscCall(PetscTabulationDestroy(&T));
3059ffe73a53SMatthew G. Knepley         }
306097c42addSMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
306197c42addSMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
306297c42addSMatthew G. Knepley 
306397c42addSMatthew G. Knepley         /* Evaluate constant function at points */
30649566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &NcJ));
306597c42addSMatthew G. Knepley         cpdim = 1;
306697c42addSMatthew G. Knepley         /* For now, fields only interpolate themselves */
306797c42addSMatthew G. Knepley         if (fieldI == fieldJ) {
306863a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
306997c42addSMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
30709566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
30719566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
307263a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
307397c42addSMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
307497c42addSMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
3075458eb97cSMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += 1.0 * qweights[p * qNc + c];
307697c42addSMatthew G. Knepley               }
307797c42addSMatthew G. Knepley             }
307897c42addSMatthew G. Knepley           }
307997c42addSMatthew G. Knepley         }
308097c42addSMatthew G. Knepley       }
3081d172c84bSMatthew G. Knepley       offsetJ += cpdim;
3082d69c5d34SMatthew G. Knepley     }
3083d172c84bSMatthew G. Knepley     offsetI += fpdim;
30849566063dSJacob Faibussowitsch     PetscCall(PetscFree(points));
3085d69c5d34SMatthew G. Knepley   }
30869566063dSJacob Faibussowitsch   if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(0, name, rTotDim, cTotDim, elemMat));
30877f5b169aSMatthew G. Knepley   /* Preallocate matrix */
30887f5b169aSMatthew G. Knepley   {
3089c094ef40SMatthew G. Knepley     Mat          preallocator;
3090c094ef40SMatthew G. Knepley     PetscScalar *vals;
3091c094ef40SMatthew G. Knepley     PetscInt    *cellCIndices, *cellFIndices;
3092c094ef40SMatthew G. Knepley     PetscInt     locRows, locCols, cell;
30937f5b169aSMatthew G. Knepley 
30949566063dSJacob Faibussowitsch     PetscCall(MatGetLocalSize(In, &locRows, &locCols));
30959566063dSJacob Faibussowitsch     PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &preallocator));
30969566063dSJacob Faibussowitsch     PetscCall(MatSetType(preallocator, MATPREALLOCATOR));
30979566063dSJacob Faibussowitsch     PetscCall(MatSetSizes(preallocator, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE));
30989566063dSJacob Faibussowitsch     PetscCall(MatSetUp(preallocator));
30999566063dSJacob Faibussowitsch     PetscCall(PetscCalloc3(rTotDim * cTotDim, &vals, cTotDim, &cellCIndices, rTotDim, &cellFIndices));
3100b4ec2976SMatthew G. Knepley     if (locRows || locCols) {
31017f5b169aSMatthew G. Knepley       for (cell = cStart; cell < cEnd; ++cell) {
3102cf51de39SMatthew G. Knepley         if (isRefined) {
31039566063dSJacob Faibussowitsch           PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, cell, cellCIndices, cellFIndices));
31049566063dSJacob Faibussowitsch           PetscCall(MatSetValues(preallocator, rTotDim, cellFIndices, cTotDim, cellCIndices, vals, INSERT_VALUES));
3105cf51de39SMatthew G. Knepley         } else {
3106e8e188d2SZach Atkins           PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, PETSC_FALSE, dmc, csection, cglobalSection, PETSC_FALSE, preallocator, cell, vals, INSERT_VALUES));
3107cf51de39SMatthew G. Knepley         }
31087f5b169aSMatthew G. Knepley       }
3109b4ec2976SMatthew G. Knepley     }
31109566063dSJacob Faibussowitsch     PetscCall(PetscFree3(vals, cellCIndices, cellFIndices));
31119566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(preallocator, MAT_FINAL_ASSEMBLY));
31129566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(preallocator, MAT_FINAL_ASSEMBLY));
31139566063dSJacob Faibussowitsch     PetscCall(MatPreallocatorPreallocate(preallocator, PETSC_TRUE, In));
31149566063dSJacob Faibussowitsch     PetscCall(MatDestroy(&preallocator));
31157f5b169aSMatthew G. Knepley   }
31167f5b169aSMatthew G. Knepley   /* Fill matrix */
31179566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(In));
3118d69c5d34SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
3119cf51de39SMatthew G. Knepley     if (isRefined) {
31209566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosureRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES));
3121cf51de39SMatthew G. Knepley     } else {
3122e8e188d2SZach Atkins       PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, PETSC_FALSE, dmc, csection, cglobalSection, PETSC_FALSE, In, c, elemMat, INSERT_VALUES));
3123cf51de39SMatthew G. Knepley     }
3124d69c5d34SMatthew G. Knepley   }
31259566063dSJacob Faibussowitsch   for (f = 0; f < Nf; ++f) PetscCall(PetscFEDestroy(&feRef[f]));
31269566063dSJacob Faibussowitsch   PetscCall(PetscFree2(feRef, fvRef));
31279566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
31289566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
31299566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
31309318fe57SMatthew G. Knepley   if (mesh->printFEM > 1) {
31319566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)In), "%s:\n", name));
31322ce66baaSPierre Jolivet     PetscCall(MatFilter(In, 1.0e-10, PETSC_FALSE, PETSC_FALSE));
31339566063dSJacob Faibussowitsch     PetscCall(MatView(In, NULL));
3134d69c5d34SMatthew G. Knepley   }
31359566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
31363ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
3137d69c5d34SMatthew G. Knepley }
31386c73c22cSMatthew G. Knepley 
3139d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeMassMatrixNested(DM dmc, DM dmf, Mat mass, void *user)
3140d71ae5a4SJacob Faibussowitsch {
3141bd041c0cSMatthew G. Knepley   SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_SUP, "Laziness");
3142bd041c0cSMatthew G. Knepley }
3143bd041c0cSMatthew G. Knepley 
314468132eb9SMatthew G. Knepley /*@
31450318f8a0SStefano Zampini   DMPlexComputeInterpolatorGeneral - Form the local portion of the interpolation matrix from the coarse `DM` to a non-nested fine `DM`.
314668132eb9SMatthew G. Knepley 
314768132eb9SMatthew G. Knepley   Input Parameters:
314868132eb9SMatthew G. Knepley + dmf  - The fine mesh
314968132eb9SMatthew G. Knepley . dmc  - The coarse mesh
315068132eb9SMatthew G. Knepley - user - The user context
315168132eb9SMatthew G. Knepley 
315268132eb9SMatthew G. Knepley   Output Parameter:
315368132eb9SMatthew G. Knepley . In - The interpolation matrix
315468132eb9SMatthew G. Knepley 
315568132eb9SMatthew G. Knepley   Level: developer
315668132eb9SMatthew G. Knepley 
3157ae3cf2c1SStefano Zampini .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorNested()`
315868132eb9SMatthew G. Knepley @*/
3159d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeInterpolatorGeneral(DM dmc, DM dmf, Mat In, void *user)
3160d71ae5a4SJacob Faibussowitsch {
316164e98e1dSMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dmf->data;
316264e98e1dSMatthew G. Knepley   const char  *name = "Interpolator";
31634ef9d792SMatthew G. Knepley   PetscDS      prob;
3164a0806964SMatthew G. Knepley   Mat          interp;
3165a0806964SMatthew G. Knepley   PetscSection fsection, globalFSection;
3166a0806964SMatthew G. Knepley   PetscSection csection, globalCSection;
3167a0806964SMatthew G. Knepley   PetscInt     locRows, locCols;
31684ef9d792SMatthew G. Knepley   PetscReal   *x, *v0, *J, *invJ, detJ;
31694ef9d792SMatthew G. Knepley   PetscReal   *v0c, *Jc, *invJc, detJc;
31704ef9d792SMatthew G. Knepley   PetscScalar *elemMat;
3171a0806964SMatthew G. Knepley   PetscInt     dim, Nf, field, totDim, cStart, cEnd, cell, ccell, s;
31724ef9d792SMatthew G. Knepley 
31734ef9d792SMatthew G. Knepley   PetscFunctionBegin;
31749566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
31759566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
31769566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
31779566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
31789566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
31799566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ));
31809566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc));
31819566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
31829566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
31839566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
31849566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
3185a0806964SMatthew G. Knepley   PetscCall(DMPlexGetSimplexOrBoxCells(dmf, 0, &cStart, &cEnd));
31869566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
31879566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
31884ef9d792SMatthew G. Knepley 
3189a0806964SMatthew G. Knepley   PetscCall(MatGetLocalSize(In, &locRows, &locCols));
3190a0806964SMatthew G. Knepley   PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &interp));
3191a0806964SMatthew G. Knepley   PetscCall(MatSetType(interp, MATPREALLOCATOR));
3192a0806964SMatthew G. Knepley   PetscCall(MatSetSizes(interp, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE));
3193a0806964SMatthew G. Knepley   PetscCall(MatSetUp(interp));
3194a0806964SMatthew G. Knepley   for (s = 0; s < 2; ++s) {
31954ef9d792SMatthew G. Knepley     for (field = 0; field < Nf; ++field) {
31964ef9d792SMatthew G. Knepley       PetscObject      obj;
31974ef9d792SMatthew G. Knepley       PetscClassId     id;
3198c0d7054bSMatthew G. Knepley       PetscDualSpace   Q = NULL;
3199ef0bb6c7SMatthew G. Knepley       PetscTabulation  T = NULL;
32004ef9d792SMatthew G. Knepley       PetscQuadrature  f;
32014ef9d792SMatthew G. Knepley       const PetscReal *qpoints, *qweights;
3202d0f6233fSMatthew G. Knepley       PetscInt         Nc, qNc, Np, fpdim, off, i, d;
32034ef9d792SMatthew G. Knepley 
3204d0f6233fSMatthew G. Knepley       PetscCall(PetscDSGetFieldOffset(prob, field, &off));
32059566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, &obj));
32069566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
32074ef9d792SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
32084ef9d792SMatthew G. Knepley         PetscFE fe = (PetscFE)obj;
32094ef9d792SMatthew G. Knepley 
32109566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDualSpace(fe, &Q));
32119566063dSJacob Faibussowitsch         PetscCall(PetscFEGetNumComponents(fe, &Nc));
3212a0806964SMatthew G. Knepley         if (s) PetscCall(PetscFECreateTabulation(fe, 1, 1, x, 0, &T));
32134ef9d792SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
32144ef9d792SMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
32154ef9d792SMatthew G. Knepley 
32169566063dSJacob Faibussowitsch         PetscCall(PetscFVGetDualSpace(fv, &Q));
32174ef9d792SMatthew G. Knepley         Nc = 1;
321863a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
32199566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &fpdim));
32204ef9d792SMatthew G. Knepley       /* For each fine grid cell */
32214ef9d792SMatthew G. Knepley       for (cell = cStart; cell < cEnd; ++cell) {
32224ef9d792SMatthew G. Knepley         PetscInt *findices, *cindices;
32234ef9d792SMatthew G. Knepley         PetscInt  numFIndices, numCIndices;
32244ef9d792SMatthew G. Knepley 
32259566063dSJacob Faibussowitsch         PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
32269566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3227d0f6233fSMatthew G. Knepley         PetscCheck(numFIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %" PetscInt_FMT " != %" PetscInt_FMT " dual basis vecs", numFIndices, totDim);
32284ef9d792SMatthew G. Knepley         for (i = 0; i < fpdim; ++i) {
32294ef9d792SMatthew G. Knepley           Vec                pointVec;
32304ef9d792SMatthew G. Knepley           PetscScalar       *pV;
323112111d7cSToby Isaac           PetscSF            coarseCellSF = NULL;
32323a93e3b7SToby Isaac           const PetscSFNode *coarseCells;
3233d0f6233fSMatthew G. Knepley           PetscInt           numCoarseCells, cpdim, row = findices[i + off], q, c, j;
32344ef9d792SMatthew G. Knepley 
32354ef9d792SMatthew G. Knepley           /* Get points from the dual basis functional quadrature */
32369566063dSJacob Faibussowitsch           PetscCall(PetscDualSpaceGetFunctional(Q, i, &f));
32379566063dSJacob Faibussowitsch           PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, &qpoints, &qweights));
323863a3b9bcSJacob Faibussowitsch           PetscCheck(qNc == Nc, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, Nc);
32399566063dSJacob Faibussowitsch           PetscCall(VecCreateSeq(PETSC_COMM_SELF, Np * dim, &pointVec));
32409566063dSJacob Faibussowitsch           PetscCall(VecSetBlockSize(pointVec, dim));
32419566063dSJacob Faibussowitsch           PetscCall(VecGetArray(pointVec, &pV));
32424ef9d792SMatthew G. Knepley           for (q = 0; q < Np; ++q) {
3243c330f8ffSToby Isaac             const PetscReal xi0[3] = {-1., -1., -1.};
3244c330f8ffSToby Isaac 
32454ef9d792SMatthew G. Knepley             /* Transform point to real space */
3246c330f8ffSToby Isaac             CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x);
32474ef9d792SMatthew G. Knepley             for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d];
32484ef9d792SMatthew G. Knepley           }
32499566063dSJacob Faibussowitsch           PetscCall(VecRestoreArray(pointVec, &pV));
32504ef9d792SMatthew G. Knepley           /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
32511555c271SMatthew G. Knepley           /* OPT: Read this out from preallocation information */
32529566063dSJacob Faibussowitsch           PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
32534ef9d792SMatthew G. Knepley           /* Update preallocation info */
32549566063dSJacob Faibussowitsch           PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
32555f80ce2aSJacob Faibussowitsch           PetscCheck(numCoarseCells == Np, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located");
32569566063dSJacob Faibussowitsch           PetscCall(VecGetArray(pointVec, &pV));
32574ef9d792SMatthew G. Knepley           for (ccell = 0; ccell < numCoarseCells; ++ccell) {
3258826eb36dSMatthew G. Knepley             PetscReal       pVReal[3];
3259c330f8ffSToby Isaac             const PetscReal xi0[3] = {-1., -1., -1.};
3260826eb36dSMatthew G. Knepley 
32619566063dSJacob Faibussowitsch             PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3262a0806964SMatthew G. Knepley             if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetDimension((PetscFE)obj, &cpdim));
3263a0806964SMatthew G. Knepley             else cpdim = 1;
3264a0806964SMatthew G. Knepley 
3265a0806964SMatthew G. Knepley             if (s) {
32664ef9d792SMatthew G. Knepley               /* Transform points from real space to coarse reference space */
32679566063dSJacob Faibussowitsch               PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
3268e2d86523SMatthew G. Knepley               for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]);
3269c330f8ffSToby Isaac               CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
32704ef9d792SMatthew G. Knepley 
32714ef9d792SMatthew G. Knepley               if (id == PETSCFE_CLASSID) {
32724ef9d792SMatthew G. Knepley                 /* Evaluate coarse basis on contained point */
3273a0806964SMatthew G. Knepley                 PetscCall(PetscFEComputeTabulation((PetscFE)obj, 1, x, 0, T));
32749566063dSJacob Faibussowitsch                 PetscCall(PetscArrayzero(elemMat, cpdim));
32754ef9d792SMatthew G. Knepley                 /* Get elemMat entries by multiplying by weight */
32764ef9d792SMatthew G. Knepley                 for (j = 0; j < cpdim; ++j) {
3277ef0bb6c7SMatthew G. Knepley                   for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * qweights[ccell * qNc + c];
32784ef9d792SMatthew G. Knepley                 }
32794ef9d792SMatthew G. Knepley               } else {
32804ef9d792SMatthew G. Knepley                 for (j = 0; j < cpdim; ++j) {
32819c3cf19fSMatthew G. Knepley                   for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * qweights[ccell * qNc + c];
32824ef9d792SMatthew G. Knepley                 }
32834ef9d792SMatthew G. Knepley               }
32849566063dSJacob Faibussowitsch               if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
3285a0806964SMatthew G. Knepley             }
3286a0806964SMatthew G. Knepley             /* Update interpolator */
3287d0f6233fSMatthew G. Knepley             PetscCheck(numCIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, totDim);
3288a0806964SMatthew G. Knepley             PetscCall(MatSetValues(interp, 1, &row, cpdim, &cindices[off], elemMat, INSERT_VALUES));
32899566063dSJacob Faibussowitsch             PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
32904ef9d792SMatthew G. Knepley           }
32919566063dSJacob Faibussowitsch           PetscCall(VecRestoreArray(pointVec, &pV));
32929566063dSJacob Faibussowitsch           PetscCall(PetscSFDestroy(&coarseCellSF));
32939566063dSJacob Faibussowitsch           PetscCall(VecDestroy(&pointVec));
32944ef9d792SMatthew G. Knepley         }
32959566063dSJacob Faibussowitsch         PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
32964ef9d792SMatthew G. Knepley       }
3297a0806964SMatthew G. Knepley       if (s && id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
3298a0806964SMatthew G. Knepley     }
3299a0806964SMatthew G. Knepley     if (!s) {
3300a0806964SMatthew G. Knepley       PetscCall(MatAssemblyBegin(interp, MAT_FINAL_ASSEMBLY));
3301a0806964SMatthew G. Knepley       PetscCall(MatAssemblyEnd(interp, MAT_FINAL_ASSEMBLY));
3302a0806964SMatthew G. Knepley       PetscCall(MatPreallocatorPreallocate(interp, PETSC_TRUE, In));
3303a0806964SMatthew G. Knepley       PetscCall(MatDestroy(&interp));
3304a0806964SMatthew G. Knepley       interp = In;
3305a0806964SMatthew G. Knepley     }
33064ef9d792SMatthew G. Knepley   }
33079566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0, J, invJ));
33089566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c, Jc, invJc));
33099566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
33109566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
33119566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
33129566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
33133ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
33144ef9d792SMatthew G. Knepley }
33154ef9d792SMatthew G. Knepley 
331646fa42a0SMatthew G. Knepley /*@
33170318f8a0SStefano Zampini   DMPlexComputeMassMatrixGeneral - Form the local portion of the mass matrix from the coarse `DM` to a non-nested fine `DM`.
3318bd041c0cSMatthew G. Knepley 
3319bd041c0cSMatthew G. Knepley   Input Parameters:
3320bd041c0cSMatthew G. Knepley + dmf  - The fine mesh
3321bd041c0cSMatthew G. Knepley . dmc  - The coarse mesh
3322bd041c0cSMatthew G. Knepley - user - The user context
3323bd041c0cSMatthew G. Knepley 
3324bd041c0cSMatthew G. Knepley   Output Parameter:
3325bd041c0cSMatthew G. Knepley . mass - The mass matrix
3326bd041c0cSMatthew G. Knepley 
3327bd041c0cSMatthew G. Knepley   Level: developer
3328bd041c0cSMatthew G. Knepley 
3329ae3cf2c1SStefano Zampini .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeMassMatrixNested()`, `DMPlexComputeInterpolatorNested()`, `DMPlexComputeInterpolatorGeneral()`
3330bd041c0cSMatthew G. Knepley @*/
3331d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeMassMatrixGeneral(DM dmc, DM dmf, Mat mass, void *user)
3332d71ae5a4SJacob Faibussowitsch {
3333bd041c0cSMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dmf->data;
3334bd041c0cSMatthew G. Knepley   const char  *name = "Mass Matrix";
3335bd041c0cSMatthew G. Knepley   PetscDS      prob;
3336bd041c0cSMatthew G. Knepley   PetscSection fsection, csection, globalFSection, globalCSection;
3337e8f14785SLisandro Dalcin   PetscHSetIJ  ht;
3338bd041c0cSMatthew G. Knepley   PetscLayout  rLayout;
3339bd041c0cSMatthew G. Knepley   PetscInt    *dnz, *onz;
3340bd041c0cSMatthew G. Knepley   PetscInt     locRows, rStart, rEnd;
3341bd041c0cSMatthew G. Knepley   PetscReal   *x, *v0, *J, *invJ, detJ;
3342bd041c0cSMatthew G. Knepley   PetscReal   *v0c, *Jc, *invJc, detJc;
3343bd041c0cSMatthew G. Knepley   PetscScalar *elemMat;
3344bd041c0cSMatthew G. Knepley   PetscInt     dim, Nf, field, totDim, cStart, cEnd, cell, ccell;
3345bd041c0cSMatthew G. Knepley 
3346bd041c0cSMatthew G. Knepley   PetscFunctionBegin;
33479566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
33489566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
33499566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
33509566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
33519566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ));
33529566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc));
33539566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
33549566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
33559566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
33569566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
33579566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd));
33589566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
33599566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
3360bd041c0cSMatthew G. Knepley 
33619566063dSJacob Faibussowitsch   PetscCall(MatGetLocalSize(mass, &locRows, NULL));
33629566063dSJacob Faibussowitsch   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)mass), &rLayout));
33639566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetLocalSize(rLayout, locRows));
33649566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetBlockSize(rLayout, 1));
33659566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetUp(rLayout));
33669566063dSJacob Faibussowitsch   PetscCall(PetscLayoutGetRange(rLayout, &rStart, &rEnd));
33679566063dSJacob Faibussowitsch   PetscCall(PetscLayoutDestroy(&rLayout));
33689566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(locRows, &dnz, locRows, &onz));
33699566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJCreate(&ht));
3370bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
3371bd041c0cSMatthew G. Knepley     PetscObject      obj;
3372bd041c0cSMatthew G. Knepley     PetscClassId     id;
3373bd041c0cSMatthew G. Knepley     PetscQuadrature  quad;
3374bd041c0cSMatthew G. Knepley     const PetscReal *qpoints;
3375bd041c0cSMatthew G. Knepley     PetscInt         Nq, Nc, i, d;
3376bd041c0cSMatthew G. Knepley 
33779566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
33789566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
33799566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad));
33809566063dSJacob Faibussowitsch     else PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad));
33819566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, NULL));
3382bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
3383bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3384bd041c0cSMatthew G. Knepley       Vec                pointVec;
3385bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
3386bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
3387bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
3388bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, q, c;
3389bd041c0cSMatthew G. Knepley       PetscInt          *findices, *cindices;
3390bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
3391bd041c0cSMatthew G. Knepley 
33929566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
33939566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3394bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
33959566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec));
33969566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
33979566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3398bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
3399c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3400c330f8ffSToby Isaac 
3401bd041c0cSMatthew G. Knepley         /* Transform point to real space */
3402c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x);
3403bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d];
3404bd041c0cSMatthew G. Knepley       }
34059566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
3406bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
34079566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
34089566063dSJacob Faibussowitsch       PetscCall(PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view"));
3409bd041c0cSMatthew G. Knepley       /* Update preallocation info */
34109566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
34115f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located");
3412bd041c0cSMatthew G. Knepley       {
3413e8f14785SLisandro Dalcin         PetscHashIJKey key;
3414e8f14785SLisandro Dalcin         PetscBool      missing;
3415bd041c0cSMatthew G. Knepley 
3416bd041c0cSMatthew G. Knepley         for (i = 0; i < numFIndices; ++i) {
3417e8f14785SLisandro Dalcin           key.i = findices[i];
3418e8f14785SLisandro Dalcin           if (key.i >= 0) {
3419bd041c0cSMatthew G. Knepley             /* Get indices for coarse elements */
3420bd041c0cSMatthew G. Knepley             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
34219566063dSJacob Faibussowitsch               PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3422bd041c0cSMatthew G. Knepley               for (c = 0; c < numCIndices; ++c) {
3423e8f14785SLisandro Dalcin                 key.j = cindices[c];
3424e8f14785SLisandro Dalcin                 if (key.j < 0) continue;
34259566063dSJacob Faibussowitsch                 PetscCall(PetscHSetIJQueryAdd(ht, key, &missing));
3426bd041c0cSMatthew G. Knepley                 if (missing) {
3427e8f14785SLisandro Dalcin                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i - rStart];
3428e8f14785SLisandro Dalcin                   else ++onz[key.i - rStart];
3429bd041c0cSMatthew G. Knepley                 }
3430bd041c0cSMatthew G. Knepley               }
34319566063dSJacob Faibussowitsch               PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3432bd041c0cSMatthew G. Knepley             }
3433bd041c0cSMatthew G. Knepley           }
3434bd041c0cSMatthew G. Knepley         }
3435bd041c0cSMatthew G. Knepley       }
34369566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
34379566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
34389566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
3439bd041c0cSMatthew G. Knepley     }
3440bd041c0cSMatthew G. Knepley   }
34419566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJDestroy(&ht));
34429566063dSJacob Faibussowitsch   PetscCall(MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL));
34439566063dSJacob Faibussowitsch   PetscCall(MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));
34449566063dSJacob Faibussowitsch   PetscCall(PetscFree2(dnz, onz));
3445bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
3446bd041c0cSMatthew G. Knepley     PetscObject      obj;
3447bd041c0cSMatthew G. Knepley     PetscClassId     id;
3448ef0bb6c7SMatthew G. Knepley     PetscTabulation  T, Tfine;
3449bd041c0cSMatthew G. Knepley     PetscQuadrature  quad;
3450bd041c0cSMatthew G. Knepley     const PetscReal *qpoints, *qweights;
3451bd041c0cSMatthew G. Knepley     PetscInt         Nq, Nc, i, d;
3452bd041c0cSMatthew G. Knepley 
34539566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
34549566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
3455ef0bb6c7SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
34569566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad));
34579566063dSJacob Faibussowitsch       PetscCall(PetscFEGetCellTabulation((PetscFE)obj, 1, &Tfine));
34589566063dSJacob Faibussowitsch       PetscCall(PetscFECreateTabulation((PetscFE)obj, 1, 1, x, 0, &T));
3459ef0bb6c7SMatthew G. Knepley     } else {
34609566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad));
3461ef0bb6c7SMatthew G. Knepley     }
34629566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, &qweights));
3463bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
3464bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3465bd041c0cSMatthew G. Knepley       Vec                pointVec;
3466bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
3467bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
3468bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
3469bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, cpdim, q, c, j;
3470bd041c0cSMatthew G. Knepley       PetscInt          *findices, *cindices;
3471bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
3472bd041c0cSMatthew G. Knepley 
34739566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
34749566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3475bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
34769566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec));
34779566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
34789566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3479bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
3480c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3481c330f8ffSToby Isaac 
3482bd041c0cSMatthew G. Knepley         /* Transform point to real space */
3483c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x);
3484bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d];
3485bd041c0cSMatthew G. Knepley       }
34869566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
3487bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
34889566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
3489bd041c0cSMatthew G. Knepley       /* Update matrix */
34909566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
34915f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located");
34929566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3493bd041c0cSMatthew G. Knepley       for (ccell = 0; ccell < numCoarseCells; ++ccell) {
3494bd041c0cSMatthew G. Knepley         PetscReal       pVReal[3];
3495c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3496c330f8ffSToby Isaac 
34979566063dSJacob Faibussowitsch         PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3498bd041c0cSMatthew G. Knepley         /* Transform points from real space to coarse reference space */
34999566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
3500bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]);
3501c330f8ffSToby Isaac         CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
3502bd041c0cSMatthew G. Knepley 
3503bd041c0cSMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
3504bd041c0cSMatthew G. Knepley           PetscFE fe = (PetscFE)obj;
3505bd041c0cSMatthew G. Knepley 
3506bd041c0cSMatthew G. Knepley           /* Evaluate coarse basis on contained point */
35079566063dSJacob Faibussowitsch           PetscCall(PetscFEGetDimension(fe, &cpdim));
35089566063dSJacob Faibussowitsch           PetscCall(PetscFEComputeTabulation(fe, 1, x, 0, T));
3509bd041c0cSMatthew G. Knepley           /* Get elemMat entries by multiplying by weight */
3510bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
35119566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3512bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3513ef0bb6c7SMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * Tfine->T[0][(ccell * numFIndices + i) * Nc + c] * qweights[ccell * Nc + c] * detJ;
3514bd041c0cSMatthew G. Knepley             }
3515bd041c0cSMatthew G. Knepley             /* Update interpolator */
35169566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
351763a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
35189566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3519bd041c0cSMatthew G. Knepley           }
3520bd041c0cSMatthew G. Knepley         } else {
3521bd041c0cSMatthew G. Knepley           cpdim = 1;
3522bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
35239566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3524bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3525bd041c0cSMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * 1.0 * qweights[ccell * Nc + c] * detJ;
3526bd041c0cSMatthew G. Knepley             }
3527bd041c0cSMatthew G. Knepley             /* Update interpolator */
35289566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
352963a3b9bcSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "Nq: %" PetscInt_FMT " %" PetscInt_FMT " Nf: %" PetscInt_FMT " %" PetscInt_FMT " Nc: %" PetscInt_FMT " %" PetscInt_FMT "\n", ccell, Nq, i, numFIndices, j, numCIndices));
353063a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
35319566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3532bd041c0cSMatthew G. Knepley           }
3533bd041c0cSMatthew G. Knepley         }
35349566063dSJacob Faibussowitsch         PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3535bd041c0cSMatthew G. Knepley       }
35369566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
35379566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
35389566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
35399566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
3540bd041c0cSMatthew G. Knepley     }
35419566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
3542bd041c0cSMatthew G. Knepley   }
35439566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0, J, invJ));
35449566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c, Jc, invJc));
35459566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
35469566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY));
35479566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY));
35483ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
3549bd041c0cSMatthew G. Knepley }
3550bd041c0cSMatthew G. Knepley 
3551bd041c0cSMatthew G. Knepley /*@
355246fa42a0SMatthew G. Knepley   DMPlexComputeInjectorFEM - Compute a mapping from coarse unknowns to fine unknowns
355346fa42a0SMatthew G. Knepley 
355446fa42a0SMatthew G. Knepley   Input Parameters:
355546fa42a0SMatthew G. Knepley + dmc  - The coarse mesh
355660225df5SJacob Faibussowitsch . dmf  - The fine mesh
355746fa42a0SMatthew G. Knepley - user - The user context
355846fa42a0SMatthew G. Knepley 
355946fa42a0SMatthew G. Knepley   Output Parameter:
356046fa42a0SMatthew G. Knepley . sc - The mapping
356146fa42a0SMatthew G. Knepley 
356246fa42a0SMatthew G. Knepley   Level: developer
356346fa42a0SMatthew G. Knepley 
3564ae3cf2c1SStefano Zampini .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorNested()`
356546fa42a0SMatthew G. Knepley @*/
3566d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeInjectorFEM(DM dmc, DM dmf, VecScatter *sc, void *user)
3567d71ae5a4SJacob Faibussowitsch {
3568e9d4ef1bSMatthew G. Knepley   PetscDS      prob;
35697c927364SMatthew G. Knepley   PetscFE     *feRef;
357097c42addSMatthew G. Knepley   PetscFV     *fvRef;
35717c927364SMatthew G. Knepley   Vec          fv, cv;
35727c927364SMatthew G. Knepley   IS           fis, cis;
35737c927364SMatthew G. Knepley   PetscSection fsection, fglobalSection, csection, cglobalSection;
35747c927364SMatthew G. Knepley   PetscInt    *cmap, *cellCIndices, *cellFIndices, *cindices, *findices;
3575485ad865SMatthew G. Knepley   PetscInt     cTotDim, fTotDim = 0, Nf, f, field, cStart, cEnd, c, dim, d, startC, endC, offsetC, offsetF, m;
35766f3d3cbcSMatthew G. Knepley   PetscBool   *needAvg;
35777c927364SMatthew G. Knepley 
35787c927364SMatthew G. Knepley   PetscFunctionBegin;
35799566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InjectorFEM, dmc, dmf, 0, 0));
35809566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
35819566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
35829566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
35839566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
35849566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
35859566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
35869566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
35879566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
35889566063dSJacob Faibussowitsch   PetscCall(PetscCalloc3(Nf, &feRef, Nf, &fvRef, Nf, &needAvg));
35897c927364SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
359097c42addSMatthew G. Knepley     PetscObject  obj;
359197c42addSMatthew G. Knepley     PetscClassId id;
3592aa7890ccSMatthew G. Knepley     PetscInt     fNb = 0, Nc = 0;
35937c927364SMatthew G. Knepley 
35949566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
35959566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
359697c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
359797c42addSMatthew G. Knepley       PetscFE    fe = (PetscFE)obj;
35986f3d3cbcSMatthew G. Knepley       PetscSpace sp;
35999b2fc754SMatthew G. Knepley       PetscInt   maxDegree;
360097c42addSMatthew G. Knepley 
36019566063dSJacob Faibussowitsch       PetscCall(PetscFERefine(fe, &feRef[f]));
36029566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &fNb));
36039566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
36049566063dSJacob Faibussowitsch       PetscCall(PetscFEGetBasisSpace(fe, &sp));
36059566063dSJacob Faibussowitsch       PetscCall(PetscSpaceGetDegree(sp, NULL, &maxDegree));
36069b2fc754SMatthew G. Knepley       if (!maxDegree) needAvg[f] = PETSC_TRUE;
360797c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
360897c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV)obj;
360997c42addSMatthew G. Knepley       PetscDualSpace Q;
361097c42addSMatthew G. Knepley 
36119566063dSJacob Faibussowitsch       PetscCall(PetscFVRefine(fv, &fvRef[f]));
36129566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
36139566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &fNb));
36149566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
36156f3d3cbcSMatthew G. Knepley       needAvg[f] = PETSC_TRUE;
361697c42addSMatthew G. Knepley     }
3617d172c84bSMatthew G. Knepley     fTotDim += fNb;
36187c927364SMatthew G. Knepley   }
36199566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &cTotDim));
36209566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(cTotDim, &cmap));
36217c927364SMatthew G. Knepley   for (field = 0, offsetC = 0, offsetF = 0; field < Nf; ++field) {
36227c927364SMatthew G. Knepley     PetscFE        feC;
362397c42addSMatthew G. Knepley     PetscFV        fvC;
36247c927364SMatthew G. Knepley     PetscDualSpace QF, QC;
3625d172c84bSMatthew G. Knepley     PetscInt       order = -1, NcF, NcC, fpdim, cpdim;
36267c927364SMatthew G. Knepley 
362797c42addSMatthew G. Knepley     if (feRef[field]) {
36289566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&feC));
36299566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feC, &NcC));
36309566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[field], &NcF));
36319566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[field], &QF));
36329566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetOrder(QF, &order));
36339566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
36349566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feC, &QC));
36359566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
363697c42addSMatthew G. Knepley     } else {
36379566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fvC));
36389566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvC, &NcC));
36399566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[field], &NcF));
36409566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[field], &QF));
36419566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
36429566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvC, &QC));
36439566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
364497c42addSMatthew G. Knepley     }
364563a3b9bcSJacob Faibussowitsch     PetscCheck(NcF == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, NcF, NcC);
36467c927364SMatthew G. Knepley     for (c = 0; c < cpdim; ++c) {
36477c927364SMatthew G. Knepley       PetscQuadrature  cfunc;
3648d172c84bSMatthew G. Knepley       const PetscReal *cqpoints, *cqweights;
3649d172c84bSMatthew G. Knepley       PetscInt         NqcC, NpC;
365097c42addSMatthew G. Knepley       PetscBool        found = PETSC_FALSE;
36517c927364SMatthew G. Knepley 
36529566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(QC, c, &cfunc));
36539566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(cfunc, NULL, &NqcC, &NpC, &cqpoints, &cqweights));
365463a3b9bcSJacob Faibussowitsch       PetscCheck(NqcC == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcC, NcC);
36551dca8a05SBarry Smith       PetscCheck(NpC == 1 || !feRef[field], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Do not know how to do injection for moments");
36567c927364SMatthew G. Knepley       for (f = 0; f < fpdim; ++f) {
36577c927364SMatthew G. Knepley         PetscQuadrature  ffunc;
3658d172c84bSMatthew G. Knepley         const PetscReal *fqpoints, *fqweights;
36597c927364SMatthew G. Knepley         PetscReal        sum = 0.0;
3660d172c84bSMatthew G. Knepley         PetscInt         NqcF, NpF;
36617c927364SMatthew G. Knepley 
36629566063dSJacob Faibussowitsch         PetscCall(PetscDualSpaceGetFunctional(QF, f, &ffunc));
36639566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(ffunc, NULL, &NqcF, &NpF, &fqpoints, &fqweights));
366463a3b9bcSJacob Faibussowitsch         PetscCheck(NqcF == NcF, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcF, NcF);
36657c927364SMatthew G. Knepley         if (NpC != NpF) continue;
36667c927364SMatthew G. Knepley         for (d = 0; d < dim; ++d) sum += PetscAbsReal(cqpoints[d] - fqpoints[d]);
36677c927364SMatthew G. Knepley         if (sum > 1.0e-9) continue;
3668d172c84bSMatthew G. Knepley         for (d = 0; d < NcC; ++d) sum += PetscAbsReal(cqweights[d] * fqweights[d]);
3669d172c84bSMatthew G. Knepley         if (sum < 1.0e-9) continue;
3670d172c84bSMatthew G. Knepley         cmap[offsetC + c] = offsetF + f;
367197c42addSMatthew G. Knepley         found             = PETSC_TRUE;
36727c927364SMatthew G. Knepley         break;
36737c927364SMatthew G. Knepley       }
367497c42addSMatthew G. Knepley       if (!found) {
367597c42addSMatthew G. Knepley         /* TODO We really want the average here, but some asshole put VecScatter in the interface */
3676d172c84bSMatthew G. Knepley         if (fvRef[field] || (feRef[field] && order == 0)) {
3677d172c84bSMatthew G. Knepley           cmap[offsetC + c] = offsetF + 0;
367897c42addSMatthew G. Knepley         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate matching functional for injection");
367997c42addSMatthew G. Knepley       }
36807c927364SMatthew G. Knepley     }
3681d172c84bSMatthew G. Knepley     offsetC += cpdim;
3682d172c84bSMatthew G. Knepley     offsetF += fpdim;
36837c927364SMatthew G. Knepley   }
36849371c9d4SSatish Balay   for (f = 0; f < Nf; ++f) {
36859371c9d4SSatish Balay     PetscCall(PetscFEDestroy(&feRef[f]));
36869371c9d4SSatish Balay     PetscCall(PetscFVDestroy(&fvRef[f]));
36879371c9d4SSatish Balay   }
36889566063dSJacob Faibussowitsch   PetscCall(PetscFree3(feRef, fvRef, needAvg));
36897c927364SMatthew G. Knepley 
36909566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmf, &fv));
36919566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmc, &cv));
36929566063dSJacob Faibussowitsch   PetscCall(VecGetOwnershipRange(cv, &startC, &endC));
36939566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(cglobalSection, &m));
36949566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(cTotDim, &cellCIndices, fTotDim, &cellFIndices));
36959566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m, &cindices));
36969566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m, &findices));
36977c927364SMatthew G. Knepley   for (d = 0; d < m; ++d) cindices[d] = findices[d] = -1;
36987c927364SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
36999566063dSJacob Faibussowitsch     PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, c, cellCIndices, cellFIndices));
37007c927364SMatthew G. Knepley     for (d = 0; d < cTotDim; ++d) {
37010bd915a7SMatthew G. Knepley       if ((cellCIndices[d] < startC) || (cellCIndices[d] >= endC)) continue;
3702d2457c26SMatthew G. Knepley       PetscCheck(!(findices[cellCIndices[d] - startC] >= 0) || !(findices[cellCIndices[d] - startC] != cellFIndices[cmap[d]]), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " Coarse dof %" PetscInt_FMT " maps to both %" PetscInt_FMT " and %" PetscInt_FMT, c, cindices[cellCIndices[d] - startC], findices[cellCIndices[d] - startC], cellFIndices[cmap[d]]);
37037c927364SMatthew G. Knepley       cindices[cellCIndices[d] - startC] = cellCIndices[d];
37047c927364SMatthew G. Knepley       findices[cellCIndices[d] - startC] = cellFIndices[cmap[d]];
37057c927364SMatthew G. Knepley     }
37067c927364SMatthew G. Knepley   }
37079566063dSJacob Faibussowitsch   PetscCall(PetscFree(cmap));
37089566063dSJacob Faibussowitsch   PetscCall(PetscFree2(cellCIndices, cellFIndices));
37097c927364SMatthew G. Knepley 
37109566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis));
37119566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis));
37129566063dSJacob Faibussowitsch   PetscCall(VecScatterCreate(cv, cis, fv, fis, sc));
37139566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cis));
37149566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&fis));
37159566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmf, &fv));
37169566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmc, &cv));
37179566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InjectorFEM, dmc, dmf, 0, 0));
37183ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
3719cb1e1211SMatthew G Knepley }
3720a1cf66bbSMatthew G. Knepley 
37212f856554SMatthew G. Knepley /*@C
37222f856554SMatthew G. Knepley   DMPlexGetCellFields - Retrieve the field values values for a chunk of cells
37232f856554SMatthew G. Knepley 
37242f856554SMatthew G. Knepley   Input Parameters:
3725a1cb98faSBarry Smith + dm     - The `DM`
37262f856554SMatthew G. Knepley . cellIS - The cells to include
37272f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
3728ce78bad3SBarry Smith . locX_t - A local vector with solution field time derivatives, or `NULL`
3729ce78bad3SBarry Smith - locA   - A local vector with auxiliary fields, or `NULL`
37302f856554SMatthew G. Knepley 
37312f856554SMatthew G. Knepley   Output Parameters:
37322f856554SMatthew G. Knepley + u   - The field coefficients
37332f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
37342f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
37352f856554SMatthew G. Knepley 
37362f856554SMatthew G. Knepley   Level: developer
37372f856554SMatthew G. Knepley 
37381cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()`
37392f856554SMatthew G. Knepley @*/
3740ce78bad3SBarry Smith PetscErrorCode DMPlexGetCellFields(DM dm, IS cellIS, Vec locX, PeOp Vec locX_t, PeOp Vec locA, PetscScalar *u[], PetscScalar *u_t[], PetscScalar *a[])
3741d71ae5a4SJacob Faibussowitsch {
37422f856554SMatthew G. Knepley   DM              plex, plexA = NULL;
3743a6e0b375SMatthew G. Knepley   DMEnclosureType encAux;
37442f856554SMatthew G. Knepley   PetscSection    section, sectionAux;
37452f856554SMatthew G. Knepley   PetscDS         prob;
37462f856554SMatthew G. Knepley   const PetscInt *cells;
37472f856554SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, totDim, totDimAux, c;
37482f856554SMatthew G. Knepley 
37492f856554SMatthew G. Knepley   PetscFunctionBegin;
37502f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3751064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
3752ad540459SPierre Jolivet   if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4);
3753ad540459SPierre Jolivet   if (locA) PetscValidHeaderSpecific(locA, VEC_CLASSID, 5);
37544f572ea9SToby Isaac   PetscAssertPointer(u, 6);
37554f572ea9SToby Isaac   PetscAssertPointer(u_t, 7);
37564f572ea9SToby Isaac   PetscAssertPointer(a, 8);
37579566063dSJacob Faibussowitsch   PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE));
37589566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
37599566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
376007218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL));
37619566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
37622f856554SMatthew G. Knepley   if (locA) {
37632f856554SMatthew G. Knepley     DM      dmAux;
37642f856554SMatthew G. Knepley     PetscDS probAux;
37652f856554SMatthew G. Knepley 
37669566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
37679566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
37689566063dSJacob Faibussowitsch     PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE));
37699566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
37709566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
37719566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
37722f856554SMatthew G. Knepley   }
37732f856554SMatthew G. Knepley   numCells = cEnd - cStart;
37749566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u));
37759371c9d4SSatish Balay   if (locX_t) PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u_t));
3776ad540459SPierre Jolivet   else *u_t = NULL;
37779371c9d4SSatish Balay   if (locA) PetscCall(DMGetWorkArray(dm, numCells * totDimAux, MPIU_SCALAR, a));
3778ad540459SPierre Jolivet   else *a = NULL;
37792f856554SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
37802f856554SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
37812f856554SMatthew G. Knepley     const PetscInt cind = c - cStart;
37822f856554SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL, *ul = *u, *ul_t = *u_t, *al = *a;
37832f856554SMatthew G. Knepley     PetscInt       i;
37842f856554SMatthew G. Knepley 
37859566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x));
37862f856554SMatthew G. Knepley     for (i = 0; i < totDim; ++i) ul[cind * totDim + i] = x[i];
37879566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x));
37882f856554SMatthew G. Knepley     if (locX_t) {
37899566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t));
37902f856554SMatthew G. Knepley       for (i = 0; i < totDim; ++i) ul_t[cind * totDim + i] = x_t[i];
37919566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t));
37922f856554SMatthew G. Knepley     }
37932f856554SMatthew G. Knepley     if (locA) {
37942f856554SMatthew G. Knepley       PetscInt subcell;
37959566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell));
37969566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, NULL, &x));
37972f856554SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) al[cind * totDimAux + i] = x[i];
37989566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, NULL, &x));
37992f856554SMatthew G. Knepley     }
38002f856554SMatthew G. Knepley   }
38019566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
38029566063dSJacob Faibussowitsch   if (locA) PetscCall(DMDestroy(&plexA));
38039566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
38043ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
38052f856554SMatthew G. Knepley }
38062f856554SMatthew G. Knepley 
38072f856554SMatthew G. Knepley /*@C
38082f856554SMatthew G. Knepley   DMPlexRestoreCellFields - Restore the field values values for a chunk of cells
38092f856554SMatthew G. Knepley 
38102f856554SMatthew G. Knepley   Input Parameters:
3811a1cb98faSBarry Smith + dm     - The `DM`
38122f856554SMatthew G. Knepley . cellIS - The cells to include
38132f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
3814ce78bad3SBarry Smith . locX_t - A local vector with solution field time derivatives, or `NULL`
3815ce78bad3SBarry Smith - locA   - A local vector with auxiliary fields, or `NULL`
38162f856554SMatthew G. Knepley 
38172f856554SMatthew G. Knepley   Output Parameters:
38182f856554SMatthew G. Knepley + u   - The field coefficients
38192f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
38202f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
38212f856554SMatthew G. Knepley 
38222f856554SMatthew G. Knepley   Level: developer
38232f856554SMatthew G. Knepley 
38241cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()`
38252f856554SMatthew G. Knepley @*/
3826ce78bad3SBarry Smith PetscErrorCode DMPlexRestoreCellFields(DM dm, IS cellIS, Vec locX, PeOp Vec locX_t, PeOp Vec locA, PetscScalar *u[], PetscScalar *u_t[], PetscScalar *a[])
3827d71ae5a4SJacob Faibussowitsch {
38282f856554SMatthew G. Knepley   PetscFunctionBegin;
38299566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u));
38309566063dSJacob Faibussowitsch   if (locX_t) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u_t));
38319566063dSJacob Faibussowitsch   if (locA) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, a));
38323ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
38332f856554SMatthew G. Knepley }
38342f856554SMatthew G. Knepley 
3835a4e35b19SJacob Faibussowitsch static PetscErrorCode DMPlexGetHybridCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
3836d71ae5a4SJacob Faibussowitsch {
383707218a29SMatthew G. Knepley   DM              plex, plexA = NULL;
383807218a29SMatthew G. Knepley   DMEnclosureType encAux;
383907218a29SMatthew G. Knepley   PetscSection    section, sectionAux;
384007218a29SMatthew G. Knepley   PetscDS         ds, dsIn;
38416528b96dSMatthew G. Knepley   const PetscInt *cells;
384207218a29SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, c, totDim, totDimAux, Nf, f;
38436528b96dSMatthew G. Knepley 
38446528b96dSMatthew G. Knepley   PetscFunctionBegin;
384507218a29SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
384607218a29SMatthew G. Knepley   PetscValidHeaderSpecific(cellIS, IS_CLASSID, 2);
384707218a29SMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
384807218a29SMatthew G. Knepley   if (locX_t) { PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4); }
384907218a29SMatthew G. Knepley   if (locA) { PetscValidHeaderSpecific(locA, VEC_CLASSID, 5); }
38504f572ea9SToby Isaac   PetscAssertPointer(u, 6);
38514f572ea9SToby Isaac   PetscAssertPointer(u_t, 7);
38524f572ea9SToby Isaac   PetscAssertPointer(a, 8);
385307218a29SMatthew G. Knepley   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
385407218a29SMatthew G. Knepley   numCells = cEnd - cStart;
385507218a29SMatthew G. Knepley   PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE));
385607218a29SMatthew G. Knepley   PetscCall(DMGetLocalSection(dm, &section));
385707218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn));
385807218a29SMatthew G. Knepley   PetscCall(PetscDSGetNumFields(dsIn, &Nf));
385907218a29SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(dsIn, &totDim));
386007218a29SMatthew G. Knepley   if (locA) {
386107218a29SMatthew G. Knepley     DM      dmAux;
386207218a29SMatthew G. Knepley     PetscDS probAux;
386307218a29SMatthew G. Knepley 
386407218a29SMatthew G. Knepley     PetscCall(VecGetDM(locA, &dmAux));
386507218a29SMatthew G. Knepley     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
386607218a29SMatthew G. Knepley     PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE));
386707218a29SMatthew G. Knepley     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
386807218a29SMatthew G. Knepley     PetscCall(DMGetDS(dmAux, &probAux));
386907218a29SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
387007218a29SMatthew G. Knepley   }
387107218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u));
387207218a29SMatthew G. Knepley   if (locX_t) PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u_t));
387307218a29SMatthew G. Knepley   else {
387407218a29SMatthew G. Knepley     *u_t = NULL;
387507218a29SMatthew G. Knepley   }
387607218a29SMatthew G. Knepley   if (locA) PetscCall(DMGetWorkArray(dm, numCells * totDimAux, MPIU_SCALAR, a));
387707218a29SMatthew G. Knepley   else {
387807218a29SMatthew G. Knepley     *a = NULL;
387907218a29SMatthew G. Knepley   }
388007218a29SMatthew G. Knepley   // Loop over cohesive cells
388107218a29SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
388207218a29SMatthew G. Knepley     const PetscInt  cell = cells ? cells[c] : c;
388307218a29SMatthew G. Knepley     const PetscInt  cind = c - cStart;
388474a0c561SMatthew G. Knepley     PetscScalar    *xf = NULL, *xc = NULL, *x = NULL, *xf_t = NULL, *xc_t = NULL;
38858e3a54c0SPierre Jolivet     PetscScalar    *ul = &(*u)[cind * totDim], *ul_t = PetscSafePointerPlusOffset(*u_t, cind * totDim);
388607218a29SMatthew G. Knepley     const PetscInt *cone, *ornt;
388707218a29SMatthew G. Knepley     PetscInt        Nx = 0, Nxf, s;
388807218a29SMatthew G. Knepley 
388907218a29SMatthew G. Knepley     PetscCall(DMPlexGetCone(dm, cell, &cone));
389007218a29SMatthew G. Knepley     PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
389107218a29SMatthew G. Knepley     // Put in cohesive unknowns
389207218a29SMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, &Nxf, &xf));
389374a0c561SMatthew G. Knepley     if (locX_t) PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &xf_t));
389407218a29SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
389507218a29SMatthew G. Knepley       PetscInt  fdofIn, foff, foffIn;
389607218a29SMatthew G. Knepley       PetscBool cohesive;
389707218a29SMatthew G. Knepley 
389807218a29SMatthew G. Knepley       PetscCall(PetscDSGetCohesive(dsIn, f, &cohesive));
389907218a29SMatthew G. Knepley       if (!cohesive) continue;
390007218a29SMatthew G. Knepley       PetscCall(PetscDSGetFieldSize(dsIn, f, &fdofIn));
390107218a29SMatthew G. Knepley       PetscCall(PetscDSGetFieldOffsetCohesive(ds, f, &foff));
390207218a29SMatthew G. Knepley       PetscCall(PetscDSGetFieldOffsetCohesive(dsIn, f, &foffIn));
390307218a29SMatthew G. Knepley       for (PetscInt i = 0; i < fdofIn; ++i) ul[foffIn + i] = xf[foff + i];
390474a0c561SMatthew G. Knepley       if (locX_t)
390574a0c561SMatthew G. Knepley         for (PetscInt i = 0; i < fdofIn; ++i) ul_t[foffIn + i] = xf_t[foff + i];
390607218a29SMatthew G. Knepley       Nx += fdofIn;
390707218a29SMatthew G. Knepley     }
390807218a29SMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, &Nxf, &xf));
390974a0c561SMatthew G. Knepley     if (locX_t) PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &xf_t));
391007218a29SMatthew G. Knepley     // Loop over sides of surface
391107218a29SMatthew G. Knepley     for (s = 0; s < 2; ++s) {
391207218a29SMatthew G. Knepley       const PetscInt *support;
391307218a29SMatthew G. Knepley       const PetscInt  face = cone[s];
3914989fa639SBrad Aagaard       PetscDS         dsC;
391507218a29SMatthew G. Knepley       PetscInt        ssize, ncell, Nxc;
391607218a29SMatthew G. Knepley 
391707218a29SMatthew G. Knepley       // I don't think I need the face to have 0 orientation in the hybrid cell
391807218a29SMatthew G. Knepley       //PetscCheck(!ornt[s], PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", face, cell, ornt[s]);
391907218a29SMatthew G. Knepley       PetscCall(DMPlexGetSupport(dm, face, &support));
392007218a29SMatthew G. Knepley       PetscCall(DMPlexGetSupportSize(dm, face, &ssize));
392107218a29SMatthew G. Knepley       if (support[0] == cell) ncell = support[1];
392207218a29SMatthew G. Knepley       else if (support[1] == cell) ncell = support[0];
392307218a29SMatthew G. Knepley       else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", face, cell);
392407218a29SMatthew G. Knepley       // Get closure of both face and cell, stick in cell for normal fields and face for cohesive fields
3925989fa639SBrad Aagaard       PetscCall(DMGetCellDS(dm, ncell, &dsC, NULL));
392607218a29SMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(plex, section, locX, ncell, &Nxc, &xc));
392774a0c561SMatthew G. Knepley       if (locX_t) PetscCall(DMPlexVecGetClosure(plex, section, locX_t, ncell, NULL, &xc_t));
392807218a29SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
3929989fa639SBrad Aagaard         PetscInt  fdofIn, foffIn, foff;
393007218a29SMatthew G. Knepley         PetscBool cohesive;
393107218a29SMatthew G. Knepley 
393207218a29SMatthew G. Knepley         PetscCall(PetscDSGetCohesive(dsIn, f, &cohesive));
393307218a29SMatthew G. Knepley         if (cohesive) continue;
393407218a29SMatthew G. Knepley         PetscCall(PetscDSGetFieldSize(dsIn, f, &fdofIn));
3935989fa639SBrad Aagaard         PetscCall(PetscDSGetFieldOffset(dsC, f, &foff));
393607218a29SMatthew G. Knepley         PetscCall(PetscDSGetFieldOffsetCohesive(dsIn, f, &foffIn));
3937989fa639SBrad Aagaard         for (PetscInt i = 0; i < fdofIn; ++i) ul[foffIn + s * fdofIn + i] = xc[foff + i];
393874a0c561SMatthew G. Knepley         if (locX_t)
3939989fa639SBrad Aagaard           for (PetscInt i = 0; i < fdofIn; ++i) ul_t[foffIn + s * fdofIn + i] = xc_t[foff + i];
394007218a29SMatthew G. Knepley         Nx += fdofIn;
394107218a29SMatthew G. Knepley       }
394207218a29SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, ncell, &Nxc, &xc));
394374a0c561SMatthew G. Knepley       if (locX_t) PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, ncell, NULL, &xc_t));
394407218a29SMatthew G. Knepley     }
394507218a29SMatthew G. Knepley     PetscCheck(Nx == totDim, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for cell %" PetscInt_FMT " does not match DS size %" PetscInt_FMT, Nx, cell, totDim);
394607218a29SMatthew G. Knepley 
394707218a29SMatthew G. Knepley     if (locA) {
394807218a29SMatthew G. Knepley       PetscScalar *al = &(*a)[cind * totDimAux];
394907218a29SMatthew G. Knepley       PetscInt     subcell;
395007218a29SMatthew G. Knepley 
395107218a29SMatthew G. Knepley       PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell));
395207218a29SMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, &Nx, &x));
395307218a29SMatthew G. Knepley       PetscCheck(Nx == totDimAux, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for subcell %" PetscInt_FMT "does not match DS size %" PetscInt_FMT, Nx, subcell, totDimAux);
395407218a29SMatthew G. Knepley       for (PetscInt i = 0; i < totDimAux; ++i) al[i] = x[i];
395507218a29SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, &Nx, &x));
395607218a29SMatthew G. Knepley     }
395707218a29SMatthew G. Knepley   }
395807218a29SMatthew G. Knepley   PetscCall(DMDestroy(&plex));
395907218a29SMatthew G. Knepley   PetscCall(DMDestroy(&plexA));
396007218a29SMatthew G. Knepley   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
396107218a29SMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
396207218a29SMatthew G. Knepley }
396307218a29SMatthew G. Knepley 
396407218a29SMatthew G. Knepley /*
39653e2b0218SMatthew G. Knepley   DMPlexGetHybridFields - Get the field values for the negative side (s = 0) and positive side (s = 1) of the interface
396607218a29SMatthew G. Knepley 
396707218a29SMatthew G. Knepley   Input Parameters:
396807218a29SMatthew G. Knepley + dm      - The full domain DM
396907218a29SMatthew G. Knepley . dmX     - An array of DM for the field, say an auxiliary DM, indexed by s
397007218a29SMatthew G. Knepley . dsX     - An array of PetscDS for the field, indexed by s
397107218a29SMatthew G. Knepley . cellIS  - The interface cells for which we want values
397207218a29SMatthew G. Knepley . locX    - An array of local vectors with the field values, indexed by s
397307218a29SMatthew G. Knepley - useCell - Flag to have values come from neighboring cell rather than endcap face
397407218a29SMatthew G. Knepley 
397507218a29SMatthew G. Knepley   Output Parameter:
397607218a29SMatthew G. Knepley . x       - An array of field values, indexed by s
397707218a29SMatthew G. Knepley 
397807218a29SMatthew G. Knepley   Note:
397976fbde31SPierre Jolivet   The arrays in `x` will be allocated using `DMGetWorkArray()`, and must be returned using `DMPlexRestoreHybridFields()`.
398007218a29SMatthew G. Knepley 
398107218a29SMatthew G. Knepley   Level: advanced
398207218a29SMatthew G. Knepley 
398376fbde31SPierre Jolivet .seealso: `DMPlexRestoreHybridFields()`, `DMGetWorkArray()`
398407218a29SMatthew G. Knepley */
398507218a29SMatthew G. Knepley static PetscErrorCode DMPlexGetHybridFields(DM dm, DM dmX[], PetscDS dsX[], IS cellIS, Vec locX[], PetscBool useCell, PetscScalar *x[])
398607218a29SMatthew G. Knepley {
398707218a29SMatthew G. Knepley   DM              plexX[2];
398807218a29SMatthew G. Knepley   DMEnclosureType encX[2];
398907218a29SMatthew G. Knepley   PetscSection    sectionX[2];
399007218a29SMatthew G. Knepley   const PetscInt *cells;
399107218a29SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, c, s, totDimX[2];
399207218a29SMatthew G. Knepley 
399307218a29SMatthew G. Knepley   PetscFunctionBegin;
39944f572ea9SToby Isaac   PetscAssertPointer(locX, 5);
399507218a29SMatthew G. Knepley   if (!locX[0] || !locX[1]) PetscFunctionReturn(PETSC_SUCCESS);
39964f572ea9SToby Isaac   PetscAssertPointer(dmX, 2);
39974f572ea9SToby Isaac   PetscAssertPointer(dsX, 3);
399807218a29SMatthew G. Knepley   PetscValidHeaderSpecific(cellIS, IS_CLASSID, 4);
39994f572ea9SToby Isaac   PetscAssertPointer(x, 7);
40009566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
40016528b96dSMatthew G. Knepley   numCells = cEnd - cStart;
4002148442b3SMatthew G. Knepley   for (s = 0; s < 2; ++s) {
400307218a29SMatthew G. Knepley     PetscValidHeaderSpecific(dmX[s], DM_CLASSID, 2);
400407218a29SMatthew G. Knepley     PetscValidHeaderSpecific(dsX[s], PETSCDS_CLASSID, 3);
400507218a29SMatthew G. Knepley     PetscValidHeaderSpecific(locX[s], VEC_CLASSID, 5);
400607218a29SMatthew G. Knepley     PetscCall(DMPlexConvertPlex(dmX[s], &plexX[s], PETSC_FALSE));
400707218a29SMatthew G. Knepley     PetscCall(DMGetEnclosureRelation(dmX[s], dm, &encX[s]));
400807218a29SMatthew G. Knepley     PetscCall(DMGetLocalSection(dmX[s], &sectionX[s]));
400907218a29SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(dsX[s], &totDimX[s]));
401007218a29SMatthew G. Knepley     PetscCall(DMGetWorkArray(dmX[s], numCells * totDimX[s], MPIU_SCALAR, &x[s]));
401104c51a94SMatthew G. Knepley   }
4012148442b3SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
40136528b96dSMatthew G. Knepley     const PetscInt  cell = cells ? cells[c] : c;
40146528b96dSMatthew G. Knepley     const PetscInt  cind = c - cStart;
40156528b96dSMatthew G. Knepley     const PetscInt *cone, *ornt;
40166528b96dSMatthew G. Knepley 
40179566063dSJacob Faibussowitsch     PetscCall(DMPlexGetCone(dm, cell, &cone));
40189566063dSJacob Faibussowitsch     PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
401907218a29SMatthew G. Knepley     //PetscCheck(!ornt[0], PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", cone[0], cell, ornt[0]);
4020148442b3SMatthew G. Knepley     for (s = 0; s < 2; ++s) {
402107218a29SMatthew G. Knepley       const PetscInt tdX     = totDimX[s];
402207218a29SMatthew G. Knepley       PetscScalar   *closure = NULL, *xl = &x[s][cind * tdX];
402307218a29SMatthew G. Knepley       PetscInt       face = cone[s], point = face, subpoint, Nx, i;
402407218a29SMatthew G. Knepley 
402507218a29SMatthew G. Knepley       if (useCell) {
40265fedec97SMatthew G. Knepley         const PetscInt *support;
402707218a29SMatthew G. Knepley         PetscInt        ssize;
40286528b96dSMatthew G. Knepley 
402907218a29SMatthew G. Knepley         PetscCall(DMPlexGetSupport(dm, face, &support));
403007218a29SMatthew G. Knepley         PetscCall(DMPlexGetSupportSize(dm, face, &ssize));
403107218a29SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", face, cell, ssize);
403207218a29SMatthew G. Knepley         if (support[0] == cell) point = support[1];
403307218a29SMatthew G. Knepley         else if (support[1] == cell) point = support[0];
403407218a29SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", face, cell);
403507218a29SMatthew G. Knepley       }
403607218a29SMatthew G. Knepley       PetscCall(DMGetEnclosurePoint(plexX[s], dm, encX[s], point, &subpoint));
4037*48162695SZach Atkins       PetscCall(DMPlexVecGetOrientedClosure(plexX[s], sectionX[s], PETSC_FALSE, locX[s], subpoint, ornt[s], &Nx, &closure));
403807218a29SMatthew G. Knepley       PetscCheck(Nx == tdX, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for subpoint %" PetscInt_FMT " does not match DS size %" PetscInt_FMT, Nx, subpoint, tdX);
403907218a29SMatthew G. Knepley       for (i = 0; i < Nx; ++i) xl[i] = closure[i];
404007218a29SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plexX[s], sectionX[s], locX[s], subpoint, &Nx, &closure));
40416528b96dSMatthew G. Knepley     }
40426528b96dSMatthew G. Knepley   }
404307218a29SMatthew G. Knepley   for (s = 0; s < 2; ++s) PetscCall(DMDestroy(&plexX[s]));
40449566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
40453ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
40466528b96dSMatthew G. Knepley }
40476528b96dSMatthew G. Knepley 
404807218a29SMatthew G. Knepley static PetscErrorCode DMPlexRestoreHybridFields(DM dm, DM dmX[], PetscDS dsX[], IS cellIS, Vec locX[], PetscBool useCell, PetscScalar *x[])
4049d71ae5a4SJacob Faibussowitsch {
40506528b96dSMatthew G. Knepley   PetscFunctionBegin;
405107218a29SMatthew G. Knepley   if (!locX[0] || !locX[1]) PetscFunctionReturn(PETSC_SUCCESS);
405207218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dmX[0], 0, MPIU_SCALAR, &x[0]));
405307218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dmX[1], 0, MPIU_SCALAR, &x[1]));
40543ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
40556528b96dSMatthew G. Knepley }
40566528b96dSMatthew G. Knepley 
40572f856554SMatthew G. Knepley /*@C
40582f856554SMatthew G. Knepley   DMPlexGetFaceFields - Retrieve the field values values for a chunk of faces
40592f856554SMatthew G. Knepley 
40602f856554SMatthew G. Knepley   Input Parameters:
4061a1cb98faSBarry Smith + dm           - The `DM`
40622f856554SMatthew G. Knepley . fStart       - The first face to include
40632f856554SMatthew G. Knepley . fEnd         - The first face to exclude
40642f856554SMatthew G. Knepley . locX         - A local vector with the solution fields
4065ce78bad3SBarry Smith . locX_t       - A local vector with solution field time derivatives, or `NULL`
40662f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
40672f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
4068ce78bad3SBarry Smith - locGrad      - A local vector with field gradients, or `NULL`
40692f856554SMatthew G. Knepley 
40702f856554SMatthew G. Knepley   Output Parameters:
40712f856554SMatthew G. Knepley + Nface - The number of faces with field values
40722f856554SMatthew G. Knepley . uL    - The field values at the left side of the face
40732f856554SMatthew G. Knepley - uR    - The field values at the right side of the face
40742f856554SMatthew G. Knepley 
40752f856554SMatthew G. Knepley   Level: developer
40762f856554SMatthew G. Knepley 
40771cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellFields()`
40782f856554SMatthew G. Knepley @*/
4079ce78bad3SBarry Smith PetscErrorCode DMPlexGetFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, PeOp Vec locX_t, Vec faceGeometry, Vec cellGeometry, PeOp Vec locGrad, PetscInt *Nface, PetscScalar *uL[], PetscScalar *uR[])
4080d71ae5a4SJacob Faibussowitsch {
40812f856554SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad = NULL;
40822f856554SMatthew G. Knepley   PetscSection       section;
40832f856554SMatthew G. Knepley   PetscDS            prob;
40842f856554SMatthew G. Knepley   DMLabel            ghostLabel;
40852f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom, *x, *lgrad;
40862f856554SMatthew G. Knepley   PetscBool         *isFE;
40872f856554SMatthew G. Knepley   PetscInt           dim, Nf, f, Nc, numFaces = fEnd - fStart, iface, face;
40882f856554SMatthew G. Knepley 
40892f856554SMatthew G. Knepley   PetscFunctionBegin;
40902f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
40912f856554SMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 4);
4092ad540459SPierre Jolivet   if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 5);
40932f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 6);
40942f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 7);
4095ad540459SPierre Jolivet   if (locGrad) PetscValidHeaderSpecific(locGrad, VEC_CLASSID, 8);
40964f572ea9SToby Isaac   PetscAssertPointer(uL, 10);
40974f572ea9SToby Isaac   PetscAssertPointer(uR, 11);
40989566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
40999566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
41009566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
41019566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
41029566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalComponents(prob, &Nc));
41039566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(Nf, &isFE));
41042f856554SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
41052f856554SMatthew G. Knepley     PetscObject  obj;
41062f856554SMatthew G. Knepley     PetscClassId id;
41072f856554SMatthew G. Knepley 
41089566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
41099566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
41109371c9d4SSatish Balay     if (id == PETSCFE_CLASSID) {
41119371c9d4SSatish Balay       isFE[f] = PETSC_TRUE;
41129371c9d4SSatish Balay     } else if (id == PETSCFV_CLASSID) {
41139371c9d4SSatish Balay       isFE[f] = PETSC_FALSE;
41149371c9d4SSatish Balay     } else {
41159371c9d4SSatish Balay       isFE[f] = PETSC_FALSE;
41169371c9d4SSatish Balay     }
41172f856554SMatthew G. Knepley   }
41189566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
41199566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(locX, &x));
41209566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
41219566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
41229566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
41239566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
41242f856554SMatthew G. Knepley   if (locGrad) {
41259566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locGrad, &dmGrad));
41269566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
41272f856554SMatthew G. Knepley   }
41289566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uL));
41299566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uR));
41302f856554SMatthew G. Knepley   /* Right now just eat the extra work for FE (could make a cell loop) */
41312f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
41322f856554SMatthew G. Knepley     const PetscInt  *cells;
41332f856554SMatthew G. Knepley     PetscFVFaceGeom *fg;
41342f856554SMatthew G. Knepley     PetscFVCellGeom *cgL, *cgR;
41352f856554SMatthew G. Knepley     PetscScalar     *xL, *xR, *gL, *gR;
41362f856554SMatthew G. Knepley     PetscScalar     *uLl = *uL, *uRl = *uR;
41372f856554SMatthew G. Knepley     PetscInt         ghost, nsupp, nchild;
41382f856554SMatthew G. Knepley 
41399566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
41409566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
41419566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
41422f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
41439566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
41449566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
41459566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
41469566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
41472f856554SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
41482f856554SMatthew G. Knepley       PetscInt off;
41492f856554SMatthew G. Knepley 
41509566063dSJacob Faibussowitsch       PetscCall(PetscDSGetComponentOffset(prob, f, &off));
41512f856554SMatthew G. Knepley       if (isFE[f]) {
41522f856554SMatthew G. Knepley         const PetscInt *cone;
41532f856554SMatthew G. Knepley         PetscInt        comp, coneSizeL, coneSizeR, faceLocL, faceLocR, ldof, rdof, d;
41542f856554SMatthew G. Knepley 
41552f856554SMatthew G. Knepley         xL = xR = NULL;
41569566063dSJacob Faibussowitsch         PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
4157835f2295SStefano Zampini         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[0], &ldof, &xL));
4158835f2295SStefano Zampini         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[1], &rdof, &xR));
41599566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[0], &cone));
41609566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[0], &coneSizeL));
41619371c9d4SSatish Balay         for (faceLocL = 0; faceLocL < coneSizeL; ++faceLocL)
41629371c9d4SSatish Balay           if (cone[faceLocL] == face) break;
41639566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[1], &cone));
41649566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[1], &coneSizeR));
41659371c9d4SSatish Balay         for (faceLocR = 0; faceLocR < coneSizeR; ++faceLocR)
41669371c9d4SSatish Balay           if (cone[faceLocR] == face) break;
41671dca8a05SBarry Smith         PetscCheck(faceLocL != coneSizeL || faceLocR != coneSizeR, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %" PetscInt_FMT " in cone of cell %" PetscInt_FMT " or cell %" PetscInt_FMT, face, cells[0], cells[1]);
41682f856554SMatthew G. Knepley         /* Check that FEM field has values in the right cell (sometimes its an FV ghost cell) */
41692f856554SMatthew G. Knepley         /* TODO: this is a hack that might not be right for nonconforming */
41702f856554SMatthew G. Knepley         if (faceLocL < coneSizeL) {
41719566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocL, xL, &uLl[iface * Nc + off]));
41729566063dSJacob Faibussowitsch           if (rdof == ldof && faceLocR < coneSizeR) PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off]));
41732f856554SMatthew G. Knepley           else {
41749371c9d4SSatish Balay             for (d = 0; d < comp; ++d) uRl[iface * Nc + off + d] = uLl[iface * Nc + off + d];
41759371c9d4SSatish Balay           }
41769371c9d4SSatish Balay         } else {
41779566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off]));
41789566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
41792f856554SMatthew G. Knepley           for (d = 0; d < comp; ++d) uLl[iface * Nc + off + d] = uRl[iface * Nc + off + d];
41802f856554SMatthew G. Knepley         }
4181835f2295SStefano Zampini         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[0], &ldof, &xL));
4182835f2295SStefano Zampini         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[1], &rdof, &xR));
41832f856554SMatthew G. Knepley       } else {
41842f856554SMatthew G. Knepley         PetscFV  fv;
41852f856554SMatthew G. Knepley         PetscInt numComp, c;
41862f856554SMatthew G. Knepley 
41879566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, (PetscObject *)&fv));
41889566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &numComp));
41899566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[0], f, x, &xL));
41909566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[1], f, x, &xR));
41912f856554SMatthew G. Knepley         if (dmGrad) {
41922f856554SMatthew G. Knepley           PetscReal dxL[3], dxR[3];
41932f856554SMatthew G. Knepley 
41949566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], lgrad, &gL));
41959566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[1], lgrad, &gR));
41962f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgL->centroid, fg->centroid, dxL);
41972f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgR->centroid, fg->centroid, dxR);
41982f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
41992f856554SMatthew G. Knepley             uLl[iface * Nc + off + c] = xL[c] + DMPlex_DotD_Internal(dim, &gL[c * dim], dxL);
42002f856554SMatthew G. Knepley             uRl[iface * Nc + off + c] = xR[c] + DMPlex_DotD_Internal(dim, &gR[c * dim], dxR);
42012f856554SMatthew G. Knepley           }
42022f856554SMatthew G. Knepley         } else {
42032f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
42042f856554SMatthew G. Knepley             uLl[iface * Nc + off + c] = xL[c];
42052f856554SMatthew G. Knepley             uRl[iface * Nc + off + c] = xR[c];
42062f856554SMatthew G. Knepley           }
42072f856554SMatthew G. Knepley         }
42082f856554SMatthew G. Knepley       }
42092f856554SMatthew G. Knepley     }
42102f856554SMatthew G. Knepley     ++iface;
42112f856554SMatthew G. Knepley   }
42122f856554SMatthew G. Knepley   *Nface = iface;
42139566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(locX, &x));
42149566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
42159566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
421648a46eb9SPierre Jolivet   if (locGrad) PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
42179566063dSJacob Faibussowitsch   PetscCall(PetscFree(isFE));
42183ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
42192f856554SMatthew G. Knepley }
42202f856554SMatthew G. Knepley 
42212f856554SMatthew G. Knepley /*@C
42222f856554SMatthew G. Knepley   DMPlexRestoreFaceFields - Restore the field values values for a chunk of faces
42232f856554SMatthew G. Knepley 
42242f856554SMatthew G. Knepley   Input Parameters:
4225a1cb98faSBarry Smith + dm           - The `DM`
42262f856554SMatthew G. Knepley . fStart       - The first face to include
42272f856554SMatthew G. Knepley . fEnd         - The first face to exclude
42282f856554SMatthew G. Knepley . locX         - A local vector with the solution fields
4229ce78bad3SBarry Smith . locX_t       - A local vector with solution field time derivatives, or `NULL`
42302f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
42312f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
4232ce78bad3SBarry Smith - locGrad      - A local vector with field gradients, or `NULL`
42332f856554SMatthew G. Knepley 
42342f856554SMatthew G. Knepley   Output Parameters:
42352f856554SMatthew G. Knepley + Nface - The number of faces with field values
42362f856554SMatthew G. Knepley . uL    - The field values at the left side of the face
42372f856554SMatthew G. Knepley - uR    - The field values at the right side of the face
42382f856554SMatthew G. Knepley 
42392f856554SMatthew G. Knepley   Level: developer
42402f856554SMatthew G. Knepley 
42411cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()`
42422f856554SMatthew G. Knepley @*/
4243ce78bad3SBarry Smith PetscErrorCode DMPlexRestoreFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, PeOp Vec locX_t, Vec faceGeometry, Vec cellGeometry, PeOp Vec locGrad, PetscInt *Nface, PetscScalar *uL[], PetscScalar *uR[])
4244d71ae5a4SJacob Faibussowitsch {
42452f856554SMatthew G. Knepley   PetscFunctionBegin;
42469566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uL));
42479566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uR));
42483ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
42492f856554SMatthew G. Knepley }
42502f856554SMatthew G. Knepley 
42512f856554SMatthew G. Knepley /*@C
42522f856554SMatthew G. Knepley   DMPlexGetFaceGeometry - Retrieve the geometric values for a chunk of faces
42532f856554SMatthew G. Knepley 
42542f856554SMatthew G. Knepley   Input Parameters:
4255a1cb98faSBarry Smith + dm           - The `DM`
42562f856554SMatthew G. Knepley . fStart       - The first face to include
42572f856554SMatthew G. Knepley . fEnd         - The first face to exclude
42582f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
42592f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
42602f856554SMatthew G. Knepley 
42612f856554SMatthew G. Knepley   Output Parameters:
42622f856554SMatthew G. Knepley + Nface - The number of faces with field values
4263ce78bad3SBarry Smith . fgeom - The face centroid and normals
4264ce78bad3SBarry Smith - vol   - The cell volumes
42652f856554SMatthew G. Knepley 
42662f856554SMatthew G. Knepley   Level: developer
42672f856554SMatthew G. Knepley 
42681cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellFields()`
42692f856554SMatthew G. Knepley @*/
4270ce78bad3SBarry Smith PetscErrorCode DMPlexGetFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom *fgeom[], PetscReal *vol[])
4271d71ae5a4SJacob Faibussowitsch {
42722f856554SMatthew G. Knepley   DM                 dmFace, dmCell;
42732f856554SMatthew G. Knepley   DMLabel            ghostLabel;
42742f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom;
42752f856554SMatthew G. Knepley   PetscInt           dim, numFaces = fEnd - fStart, iface, face;
42762f856554SMatthew G. Knepley 
42772f856554SMatthew G. Knepley   PetscFunctionBegin;
42782f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42792f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 4);
42802f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 5);
42814f572ea9SToby Isaac   PetscAssertPointer(fgeom, 7);
42824f572ea9SToby Isaac   PetscAssertPointer(vol, 8);
42839566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
42849566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
42859566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
42869566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
42879566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
42889566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
42899566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(numFaces, fgeom));
42909566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces * 2, MPIU_SCALAR, vol));
42912f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
42922f856554SMatthew G. Knepley     const PetscInt  *cells;
42932f856554SMatthew G. Knepley     PetscFVFaceGeom *fg;
42942f856554SMatthew G. Knepley     PetscFVCellGeom *cgL, *cgR;
42952f856554SMatthew G. Knepley     PetscFVFaceGeom *fgeoml = *fgeom;
42962f856554SMatthew G. Knepley     PetscReal       *voll   = *vol;
42972f856554SMatthew G. Knepley     PetscInt         ghost, d, nchild, nsupp;
42982f856554SMatthew G. Knepley 
42999566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
43009566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
43019566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
43022f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
43039566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
43049566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
43059566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
43069566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
43072f856554SMatthew G. Knepley     for (d = 0; d < dim; ++d) {
43082f856554SMatthew G. Knepley       fgeoml[iface].centroid[d] = fg->centroid[d];
43092f856554SMatthew G. Knepley       fgeoml[iface].normal[d]   = fg->normal[d];
43102f856554SMatthew G. Knepley     }
43112f856554SMatthew G. Knepley     voll[iface * 2 + 0] = cgL->volume;
43122f856554SMatthew G. Knepley     voll[iface * 2 + 1] = cgR->volume;
43132f856554SMatthew G. Knepley     ++iface;
43142f856554SMatthew G. Knepley   }
43152f856554SMatthew G. Knepley   *Nface = iface;
43169566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
43179566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
43183ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
43192f856554SMatthew G. Knepley }
43202f856554SMatthew G. Knepley 
43212f856554SMatthew G. Knepley /*@C
43222f856554SMatthew G. Knepley   DMPlexRestoreFaceGeometry - Restore the field values values for a chunk of faces
43232f856554SMatthew G. Knepley 
43242f856554SMatthew G. Knepley   Input Parameters:
4325a1cb98faSBarry Smith + dm           - The `DM`
43262f856554SMatthew G. Knepley . fStart       - The first face to include
43272f856554SMatthew G. Knepley . fEnd         - The first face to exclude
43282f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
43292f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
43302f856554SMatthew G. Knepley 
43312f856554SMatthew G. Knepley   Output Parameters:
43322f856554SMatthew G. Knepley + Nface - The number of faces with field values
4333ce78bad3SBarry Smith . fgeom - The face centroid and normals
4334ce78bad3SBarry Smith - vol   - The cell volumes
43352f856554SMatthew G. Knepley 
43362f856554SMatthew G. Knepley   Level: developer
43372f856554SMatthew G. Knepley 
43381cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()`
43392f856554SMatthew G. Knepley @*/
4340ce78bad3SBarry Smith PetscErrorCode DMPlexRestoreFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom *fgeom[], PetscReal *vol[])
4341d71ae5a4SJacob Faibussowitsch {
43422f856554SMatthew G. Knepley   PetscFunctionBegin;
43439566063dSJacob Faibussowitsch   PetscCall(PetscFree(*fgeom));
43449566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_REAL, vol));
43453ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
43462f856554SMatthew G. Knepley }
43472f856554SMatthew G. Knepley 
4348ac9d17c7SMatthew G. Knepley PetscErrorCode DMSNESGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscFEGeomMode mode, PetscFEGeom **geom)
4349d71ae5a4SJacob Faibussowitsch {
4350a1cf66bbSMatthew G. Knepley   char           composeStr[33] = {0};
4351a1cf66bbSMatthew G. Knepley   PetscObjectId  id;
4352a1cf66bbSMatthew G. Knepley   PetscContainer container;
4353a1cf66bbSMatthew G. Knepley 
4354a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
43559566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad, &id));
435663a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMSNESGetFEGeom_%" PetscInt64_FMT "\n", id));
43579566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container));
4358a1cf66bbSMatthew G. Knepley   if (container) {
43599566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **)geom));
4360a1cf66bbSMatthew G. Knepley   } else {
4361ac9d17c7SMatthew G. Knepley     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, mode, geom));
43629566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container));
43639566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *)*geom));
436449abdd8aSBarry Smith     PetscCall(PetscContainerSetCtxDestroy(container, PetscContainerCtxDestroy_PetscFEGeom));
43659566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container));
43669566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
4367a1cf66bbSMatthew G. Knepley   }
43683ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
4369a1cf66bbSMatthew G. Knepley }
4370a1cf66bbSMatthew G. Knepley 
4371d71ae5a4SJacob Faibussowitsch PetscErrorCode DMSNESRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
4372d71ae5a4SJacob Faibussowitsch {
4373a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
4374a1cf66bbSMatthew G. Knepley   *geom = NULL;
43753ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
4376a1cf66bbSMatthew G. Knepley }
4377a1cf66bbSMatthew G. Knepley 
4378d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeResidual_Patch_Internal(DM dm, PetscSection section, IS cellIS, PetscReal t, Vec locX, Vec locX_t, Vec locF, void *user)
4379d71ae5a4SJacob Faibussowitsch {
438092d50984SMatthew G. Knepley   DM_Plex        *mesh       = (DM_Plex *)dm->data;
438192d50984SMatthew G. Knepley   const char     *name       = "Residual";
438292d50984SMatthew G. Knepley   DM              dmAux      = NULL;
438392d50984SMatthew G. Knepley   DMLabel         ghostLabel = NULL;
438492d50984SMatthew G. Knepley   PetscDS         prob       = NULL;
438592d50984SMatthew G. Knepley   PetscDS         probAux    = NULL;
438692d50984SMatthew G. Knepley   PetscBool       useFEM     = PETSC_FALSE;
438792d50984SMatthew G. Knepley   PetscBool       isImplicit = (locX_t || t == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
438892d50984SMatthew G. Knepley   DMField         coordField = NULL;
4389c0006e53SPatrick Farrell   Vec             locA;
4390c0006e53SPatrick Farrell   PetscScalar    *u = NULL, *u_t, *a, *uL = NULL, *uR = NULL;
439192d50984SMatthew G. Knepley   IS              chunkIS;
439292d50984SMatthew G. Knepley   const PetscInt *cells;
439392d50984SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
4394364207b6SKarl Rupp   PetscInt        Nf, f, totDim, totDimAux, numChunks, cellChunkSize, chunk, fStart, fEnd;
43951690c2aeSBarry Smith   PetscInt        maxDegree = PETSC_INT_MAX;
439606ad1575SMatthew G. Knepley   PetscFormKey    key;
439792d50984SMatthew G. Knepley   PetscQuadrature affineQuad = NULL, *quads = NULL;
439892d50984SMatthew G. Knepley   PetscFEGeom    *affineGeom = NULL, **geoms = NULL;
439992d50984SMatthew G. Knepley 
440092d50984SMatthew G. Knepley   PetscFunctionBegin;
44019566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0));
440292d50984SMatthew G. Knepley   /* FEM+FVM */
440392d50984SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
44049566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
44059566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
44069566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
44079566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
44089566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
440992d50984SMatthew G. Knepley   if (locA) {
44109566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
44119566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
44129566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
441392d50984SMatthew G. Knepley   }
441492d50984SMatthew G. Knepley   /* 2: Get geometric data */
441592d50984SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
441692d50984SMatthew G. Knepley     PetscObject  obj;
441792d50984SMatthew G. Knepley     PetscClassId id;
441892d50984SMatthew G. Knepley     PetscBool    fimp;
441992d50984SMatthew G. Knepley 
44209566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(prob, f, &fimp));
442192d50984SMatthew G. Knepley     if (isImplicit != fimp) continue;
44229566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
44239566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
4424ad540459SPierre Jolivet     if (id == PETSCFE_CLASSID) useFEM = PETSC_TRUE;
44255f80ce2aSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Use of FVM with PCPATCH not yet implemented");
442692d50984SMatthew G. Knepley   }
442792d50984SMatthew G. Knepley   if (useFEM) {
44289566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
44299566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
443092d50984SMatthew G. Knepley     if (maxDegree <= 1) {
44319566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad));
4432ac9d17c7SMatthew G. Knepley       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FEGEOM_BASIC, &affineGeom));
443392d50984SMatthew G. Knepley     } else {
44349566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
443592d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
443692d50984SMatthew G. Knepley         PetscObject  obj;
443792d50984SMatthew G. Knepley         PetscClassId id;
443892d50984SMatthew G. Knepley         PetscBool    fimp;
443992d50984SMatthew G. Knepley 
44409566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(prob, f, &fimp));
444192d50984SMatthew G. Knepley         if (isImplicit != fimp) continue;
44429566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
44439566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
444492d50984SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
444592d50984SMatthew G. Knepley           PetscFE fe = (PetscFE)obj;
444692d50984SMatthew G. Knepley 
44479566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
44489566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
4449ac9d17c7SMatthew G. Knepley           PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FEGEOM_BASIC, &geoms[f]));
445092d50984SMatthew G. Knepley         }
445192d50984SMatthew G. Knepley       }
445292d50984SMatthew G. Knepley     }
445392d50984SMatthew G. Knepley   }
445492d50984SMatthew G. Knepley   /* Loop over chunks */
44559566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
44569566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
44579566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
445892d50984SMatthew G. Knepley   numCells      = cEnd - cStart;
445992d50984SMatthew G. Knepley   numChunks     = 1;
446092d50984SMatthew G. Knepley   cellChunkSize = numCells / numChunks;
446192d50984SMatthew G. Knepley   numChunks     = PetscMin(1, numCells);
44626528b96dSMatthew G. Knepley   key.label     = NULL;
44636528b96dSMatthew G. Knepley   key.value     = 0;
446406ad1575SMatthew G. Knepley   key.part      = 0;
446592d50984SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
4466c0006e53SPatrick Farrell     PetscScalar     *elemVec, *fluxL = NULL, *fluxR = NULL;
4467c0006e53SPatrick Farrell     PetscReal       *vol   = NULL;
4468c0006e53SPatrick Farrell     PetscFVFaceGeom *fgeom = NULL;
446992d50984SMatthew G. Knepley     PetscInt         cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
4470c0006e53SPatrick Farrell     PetscInt         numFaces = 0;
447192d50984SMatthew G. Knepley 
447292d50984SMatthew G. Knepley     /* Extract field coefficients */
447392d50984SMatthew G. Knepley     if (useFEM) {
44749566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
44759566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
44769566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
44779566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells * totDim));
447892d50984SMatthew G. Knepley     }
447992d50984SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
448092d50984SMatthew G. Knepley     /* Loop over fields */
448192d50984SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
448292d50984SMatthew G. Knepley       PetscObject  obj;
448392d50984SMatthew G. Knepley       PetscClassId id;
448492d50984SMatthew G. Knepley       PetscBool    fimp;
448592d50984SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
448692d50984SMatthew G. Knepley 
44876528b96dSMatthew G. Knepley       key.field = f;
44889566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(prob, f, &fimp));
448992d50984SMatthew G. Knepley       if (isImplicit != fimp) continue;
44909566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
44919566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
449292d50984SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
449392d50984SMatthew G. Knepley         PetscFE         fe        = (PetscFE)obj;
449492d50984SMatthew G. Knepley         PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[f];
449592d50984SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
449692d50984SMatthew G. Knepley         PetscQuadrature quad      = affineQuad ? affineQuad : quads[f];
449792d50984SMatthew G. Knepley         PetscInt        Nq, Nb;
449892d50984SMatthew G. Knepley 
44999566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
45009566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
45019566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
450292d50984SMatthew G. Knepley         blockSize = Nb;
450392d50984SMatthew G. Knepley         batchSize = numBlocks * blockSize;
45049566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
450592d50984SMatthew G. Knepley         numChunks = numCells / (numBatches * batchSize);
450692d50984SMatthew G. Knepley         Ne        = numChunks * numBatches * batchSize;
450792d50984SMatthew G. Knepley         Nr        = numCells % (numBatches * batchSize);
450892d50984SMatthew G. Knepley         offset    = numCells - Nr;
450992d50984SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
451092d50984SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
45119566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom));
45129566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(prob, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
45139566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom));
45148e3a54c0SPierre Jolivet         PetscCall(PetscFEIntegrateResidual(prob, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, &a[offset * totDimAux], t, &elemVec[offset * totDim]));
45159566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom));
451692d50984SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
451792d50984SMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
451892d50984SMatthew G. Knepley 
451992d50984SMatthew G. Knepley         Ne = numFaces;
452092d50984SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
452192d50984SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
45229566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
452363a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
452492d50984SMatthew G. Knepley     }
452592d50984SMatthew G. Knepley     /* Loop over domain */
452692d50984SMatthew G. Knepley     if (useFEM) {
452792d50984SMatthew G. Knepley       /* Add elemVec to locX */
452892d50984SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
452992d50984SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
453092d50984SMatthew G. Knepley         const PetscInt cind = c - cStart;
453192d50984SMatthew G. Knepley 
45329566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim]));
453392d50984SMatthew G. Knepley         if (ghostLabel) {
453492d50984SMatthew G. Knepley           PetscInt ghostVal;
453592d50984SMatthew G. Knepley 
45369566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
453792d50984SMatthew G. Knepley           if (ghostVal > 0) continue;
453892d50984SMatthew G. Knepley         }
45399566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES));
454092d50984SMatthew G. Knepley       }
454192d50984SMatthew G. Knepley     }
454292d50984SMatthew G. Knepley     /* Handle time derivative */
454392d50984SMatthew G. Knepley     if (locX_t) {
454492d50984SMatthew G. Knepley       PetscScalar *x_t, *fa;
454592d50984SMatthew G. Knepley 
45469566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
45479566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
454892d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
454992d50984SMatthew G. Knepley         PetscFV      fv;
455092d50984SMatthew G. Knepley         PetscObject  obj;
455192d50984SMatthew G. Knepley         PetscClassId id;
455292d50984SMatthew G. Knepley         PetscInt     pdim, d;
455392d50984SMatthew G. Knepley 
45549566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
45559566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
455692d50984SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
455792d50984SMatthew G. Knepley         fv = (PetscFV)obj;
45589566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
455992d50984SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
456092d50984SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
456192d50984SMatthew G. Knepley           PetscScalar   *u_t, *r;
456292d50984SMatthew G. Knepley 
456392d50984SMatthew G. Knepley           if (ghostLabel) {
456492d50984SMatthew G. Knepley             PetscInt ghostVal;
456592d50984SMatthew G. Knepley 
45669566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
456792d50984SMatthew G. Knepley             if (ghostVal > 0) continue;
456892d50984SMatthew G. Knepley           }
45699566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
45709566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
457192d50984SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
457292d50984SMatthew G. Knepley         }
457392d50984SMatthew G. Knepley       }
45749566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
45759566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
457692d50984SMatthew G. Knepley     }
457792d50984SMatthew G. Knepley     if (useFEM) {
45789566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
45799566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
458092d50984SMatthew G. Knepley     }
458192d50984SMatthew G. Knepley   }
45829566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
45839566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
4584754e4fbaSMatthew G. Knepley   /* TODO Could include boundary residual here (see DMPlexComputeResidualByKey) */
458592d50984SMatthew G. Knepley   if (useFEM) {
458692d50984SMatthew G. Knepley     if (maxDegree <= 1) {
45879566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
45889566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
458992d50984SMatthew G. Knepley     } else {
459092d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
45919566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
45929566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
459392d50984SMatthew G. Knepley       }
45949566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads, geoms));
459592d50984SMatthew G. Knepley     }
459692d50984SMatthew G. Knepley   }
45979566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0));
45983ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
459992d50984SMatthew G. Knepley }
460092d50984SMatthew G. Knepley 
4601a1cf66bbSMatthew G. Knepley /*
4602a1cf66bbSMatthew G. Knepley   We always assemble JacP, and if the matrix is different from Jac and two different sets of point functions are provided, we also assemble Jac
4603a1cf66bbSMatthew G. Knepley 
4604a1cf66bbSMatthew G. Knepley   X   - The local solution vector
4605a5b23f4aSJose E. Roman   X_t - The local solution time derivative vector, or NULL
4606a1cf66bbSMatthew G. Knepley */
4607d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeJacobian_Patch_Internal(DM dm, PetscSection section, PetscSection globalSection, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *ctx)
4608d71ae5a4SJacob Faibussowitsch {
4609a1cf66bbSMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *)dm->data;
4610a1cf66bbSMatthew G. Knepley   const char     *name = "Jacobian", *nameP = "JacobianPre";
4611a1cf66bbSMatthew G. Knepley   DM              dmAux = NULL;
4612a1cf66bbSMatthew G. Knepley   PetscDS         prob, probAux = NULL;
4613a1cf66bbSMatthew G. Knepley   PetscSection    sectionAux = NULL;
4614a1cf66bbSMatthew G. Knepley   Vec             A;
4615a1cf66bbSMatthew G. Knepley   DMField         coordField;
4616a1cf66bbSMatthew G. Knepley   PetscFEGeom    *cgeomFEM;
4617a1cf66bbSMatthew G. Knepley   PetscQuadrature qGeom = NULL;
4618a1cf66bbSMatthew G. Knepley   Mat             J = Jac, JP = JacP;
4619a1cf66bbSMatthew G. Knepley   PetscScalar    *work, *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL, *elemMatD = NULL;
4620e432b41dSStefano Zampini   PetscBool       hasJac, hasPrec, hasDyn, assembleJac, *isFE, hasFV = PETSC_FALSE;
4621a1cf66bbSMatthew G. Knepley   const PetscInt *cells;
462206ad1575SMatthew G. Knepley   PetscFormKey    key;
46239b2fc754SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ, maxDegree, numCells, cStart, cEnd, numChunks, chunkSize, chunk, totDim, totDimAux = 0, sz, wsz, off = 0, offCell = 0;
4624a1cf66bbSMatthew G. Knepley 
4625a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
46269566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
46279566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
46289566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
46299566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
46309566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &A));
46319a2a23afSMatthew G. Knepley   if (A) {
46329566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
46339566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
46349566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
4635a1cf66bbSMatthew G. Knepley   }
4636a1cf66bbSMatthew G. Knepley   /* Get flags */
46379566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
46389566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, Nf, MPIU_BOOL, &isFE));
4639a1cf66bbSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
4640a1cf66bbSMatthew G. Knepley     PetscObject  disc;
4641a1cf66bbSMatthew G. Knepley     PetscClassId id;
46429566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &disc));
46439566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(disc, &id));
46449371c9d4SSatish Balay     if (id == PETSCFE_CLASSID) {
46459371c9d4SSatish Balay       isFE[fieldI] = PETSC_TRUE;
46469371c9d4SSatish Balay     } else if (id == PETSCFV_CLASSID) {
46479371c9d4SSatish Balay       hasFV        = PETSC_TRUE;
46489371c9d4SSatish Balay       isFE[fieldI] = PETSC_FALSE;
46499371c9d4SSatish Balay     }
4650a1cf66bbSMatthew G. Knepley   }
46519566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
46529566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
46539566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
4654a1cf66bbSMatthew G. Knepley   assembleJac = hasJac && hasPrec && (Jac != JacP) ? PETSC_TRUE : PETSC_FALSE;
4655a1cf66bbSMatthew G. Knepley   hasDyn      = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
46569566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE)); /* No allocated space for FV stuff, so ignore the zero entries */
46579566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
46589566063dSJacob Faibussowitsch   if (probAux) PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
4659a1cf66bbSMatthew G. Knepley   /* Compute batch sizes */
4660a1cf66bbSMatthew G. Knepley   if (isFE[0]) {
4661a1cf66bbSMatthew G. Knepley     PetscFE         fe;
4662a1cf66bbSMatthew G. Knepley     PetscQuadrature q;
4663a1cf66bbSMatthew G. Knepley     PetscInt        numQuadPoints, numBatches, batchSize, numBlocks, blockSize, Nb;
4664a1cf66bbSMatthew G. Knepley 
46659566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe));
46669566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &q));
46679566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(q, NULL, NULL, &numQuadPoints, NULL, NULL));
46689566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
46699566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
4670a1cf66bbSMatthew G. Knepley     blockSize = Nb * numQuadPoints;
4671a1cf66bbSMatthew G. Knepley     batchSize = numBlocks * blockSize;
4672a1cf66bbSMatthew G. Knepley     chunkSize = numBatches * batchSize;
4673a1cf66bbSMatthew G. Knepley     numChunks = numCells / chunkSize + numCells % chunkSize;
46749566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
4675a1cf66bbSMatthew G. Knepley   } else {
4676a1cf66bbSMatthew G. Knepley     chunkSize = numCells;
4677a1cf66bbSMatthew G. Knepley     numChunks = 1;
4678a1cf66bbSMatthew G. Knepley   }
4679a1cf66bbSMatthew G. Knepley   /* Get work space */
4680a1cf66bbSMatthew G. Knepley   wsz = (((X ? 1 : 0) + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize;
46819566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, wsz, MPIU_SCALAR, &work));
46829566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(work, wsz));
4683a1cf66bbSMatthew G. Knepley   off      = 0;
4684a1cf66bbSMatthew G. Knepley   u        = X ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL;
4685a1cf66bbSMatthew G. Knepley   u_t      = X_t ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL;
4686a1cf66bbSMatthew G. Knepley   a        = dmAux ? (sz = chunkSize * totDimAux, off += sz, work + off - sz) : NULL;
4687a1cf66bbSMatthew G. Knepley   elemMat  = hasJac ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL;
4688a1cf66bbSMatthew G. Knepley   elemMatP = hasPrec ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL;
4689a1cf66bbSMatthew G. Knepley   elemMatD = hasDyn ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL;
469063a3b9bcSJacob Faibussowitsch   PetscCheck(off == wsz, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error is workspace size %" PetscInt_FMT " should be %" PetscInt_FMT, off, wsz);
4691a1cf66bbSMatthew G. Knepley   /* Setup geometry */
46929566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
46939566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
46949566063dSJacob Faibussowitsch   if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
4695a1cf66bbSMatthew G. Knepley   if (!qGeom) {
4696a1cf66bbSMatthew G. Knepley     PetscFE fe;
4697a1cf66bbSMatthew G. Knepley 
46989566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe));
46999566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &qGeom));
47009566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)qGeom));
4701a1cf66bbSMatthew G. Knepley   }
4702ac9d17c7SMatthew G. Knepley   PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FEGEOM_BASIC, &cgeomFEM));
4703a1cf66bbSMatthew G. Knepley   /* Compute volume integrals */
47049566063dSJacob Faibussowitsch   if (assembleJac) PetscCall(MatZeroEntries(J));
47059566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(JP));
47066528b96dSMatthew G. Knepley   key.label = NULL;
47076528b96dSMatthew G. Knepley   key.value = 0;
470806ad1575SMatthew G. Knepley   key.part  = 0;
4709a1cf66bbSMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk, offCell += chunkSize) {
4710a1cf66bbSMatthew G. Knepley     const PetscInt Ncell = PetscMin(chunkSize, numCells - offCell);
4711a1cf66bbSMatthew G. Knepley     PetscInt       c;
4712a1cf66bbSMatthew G. Knepley 
4713a1cf66bbSMatthew G. Knepley     /* Extract values */
4714a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4715a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c + offCell] : c + offCell;
4716a1cf66bbSMatthew G. Knepley       PetscScalar   *x = NULL, *x_t = NULL;
4717a1cf66bbSMatthew G. Knepley       PetscInt       i;
4718a1cf66bbSMatthew G. Knepley 
4719a1cf66bbSMatthew G. Knepley       if (X) {
47209566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x));
4721a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i];
47229566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x));
4723a1cf66bbSMatthew G. Knepley       }
4724a1cf66bbSMatthew G. Knepley       if (X_t) {
47259566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t));
4726a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[c * totDim + i] = x_t[i];
47279566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t));
4728a1cf66bbSMatthew G. Knepley       }
4729a1cf66bbSMatthew G. Knepley       if (dmAux) {
47309566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, A, cell, NULL, &x));
4731a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i];
47329566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, A, cell, NULL, &x));
4733a1cf66bbSMatthew G. Knepley       }
4734a1cf66bbSMatthew G. Knepley     }
4735a1cf66bbSMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
4736a1cf66bbSMatthew G. Knepley       PetscFE fe;
47379566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
4738a1cf66bbSMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
47396528b96dSMatthew G. Knepley         key.field = fieldI * Nf + fieldJ;
47404561e6c9SMatthew G. Knepley         if (hasJac) PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMat));
47414561e6c9SMatthew G. Knepley         if (hasPrec) PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN_PRE, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatP));
47424561e6c9SMatthew G. Knepley         if (hasDyn) PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN_DYN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatD));
4743a1cf66bbSMatthew G. Knepley       }
4744a1cf66bbSMatthew G. Knepley       /* For finite volume, add the identity */
4745a1cf66bbSMatthew G. Knepley       if (!isFE[fieldI]) {
4746a1cf66bbSMatthew G. Knepley         PetscFV  fv;
4747a1cf66bbSMatthew G. Knepley         PetscInt eOffset = 0, Nc, fc, foff;
4748a1cf66bbSMatthew G. Knepley 
47499566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(prob, fieldI, &foff));
47509566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv));
47519566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &Nc));
4752a1cf66bbSMatthew G. Knepley         for (c = 0; c < chunkSize; ++c, eOffset += totDim * totDim) {
4753a1cf66bbSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
4754a1cf66bbSMatthew G. Knepley             const PetscInt i = foff + fc;
4755ad540459SPierre Jolivet             if (hasJac) elemMat[eOffset + i * totDim + i] = 1.0;
4756ad540459SPierre Jolivet             if (hasPrec) elemMatP[eOffset + i * totDim + i] = 1.0;
4757a1cf66bbSMatthew G. Knepley           }
4758a1cf66bbSMatthew G. Knepley         }
4759a1cf66bbSMatthew G. Knepley       }
4760a1cf66bbSMatthew G. Knepley     }
4761a1cf66bbSMatthew G. Knepley     /*   Add contribution from X_t */
47629371c9d4SSatish Balay     if (hasDyn) {
47639371c9d4SSatish Balay       for (c = 0; c < chunkSize * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
47649371c9d4SSatish Balay     }
4765a1cf66bbSMatthew G. Knepley     /* Insert values into matrix */
4766a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4767a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c + offCell] : c + offCell;
4768a1cf66bbSMatthew G. Knepley       if (mesh->printFEM > 1) {
47699566063dSJacob Faibussowitsch         if (hasJac) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[(c - cStart) * totDim * totDim]));
47709566063dSJacob Faibussowitsch         if (hasPrec) PetscCall(DMPrintCellMatrix(cell, nameP, totDim, totDim, &elemMatP[(c - cStart) * totDim * totDim]));
4771a1cf66bbSMatthew G. Knepley       }
4772e8e188d2SZach Atkins       if (assembleJac) PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, Jac, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES));
4773e8e188d2SZach Atkins       PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, JP, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES));
4774a1cf66bbSMatthew G. Knepley     }
4775a1cf66bbSMatthew G. Knepley   }
4776a1cf66bbSMatthew G. Knepley   /* Cleanup */
47779566063dSJacob Faibussowitsch   PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
47789566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&qGeom));
47799566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
47809566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, Nf, MPIU_BOOL, &isFE));
47819566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, ((1 + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize, MPIU_SCALAR, &work));
4782a1cf66bbSMatthew G. Knepley   /* Compute boundary integrals */
47839566063dSJacob Faibussowitsch   /* PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, ctx)); */
4784a1cf66bbSMatthew G. Knepley   /* Assemble matrix */
47859371c9d4SSatish Balay   if (assembleJac) {
47869371c9d4SSatish Balay     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
47879371c9d4SSatish Balay     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
47889371c9d4SSatish Balay   }
47899371c9d4SSatish Balay   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
47909371c9d4SSatish Balay   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
47919566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
47923ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
4793a1cf66bbSMatthew G. Knepley }
47943e9753d6SMatthew G. Knepley 
47954ee01570SBarry Smith /* FEM Assembly Function */
47963e9753d6SMatthew G. Knepley 
4797d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMConvertPlex_Internal(DM dm, DM *plex, PetscBool copy)
4798d71ae5a4SJacob Faibussowitsch {
47993e9753d6SMatthew G. Knepley   PetscBool isPlex;
48003e9753d6SMatthew G. Knepley 
48013e9753d6SMatthew G. Knepley   PetscFunctionBegin;
48029566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex));
48033e9753d6SMatthew G. Knepley   if (isPlex) {
48043e9753d6SMatthew G. Knepley     *plex = dm;
48059566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)dm));
48063e9753d6SMatthew G. Knepley   } else {
48079566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex));
48083e9753d6SMatthew G. Knepley     if (!*plex) {
48099566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm, DMPLEX, plex));
48109566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex));
48113e9753d6SMatthew G. Knepley     } else {
48129566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)*plex));
48133e9753d6SMatthew G. Knepley     }
4814cbf8eb3cSStefano Zampini     if (copy) PetscCall(DMCopyAuxiliaryVec(dm, *plex));
48153e9753d6SMatthew G. Knepley   }
48163ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
48173e9753d6SMatthew G. Knepley }
48183e9753d6SMatthew G. Knepley 
48193e9753d6SMatthew G. Knepley /*@
48203e9753d6SMatthew G. Knepley   DMPlexGetGeometryFVM - Return precomputed geometric data
48213e9753d6SMatthew G. Knepley 
482220f4b53cSBarry Smith   Collective
48233e9753d6SMatthew G. Knepley 
48243e9753d6SMatthew G. Knepley   Input Parameter:
4825a1cb98faSBarry Smith . dm - The `DM`
48263e9753d6SMatthew G. Knepley 
48273e9753d6SMatthew G. Knepley   Output Parameters:
48283e9753d6SMatthew G. Knepley + facegeom  - The values precomputed from face geometry
48293e9753d6SMatthew G. Knepley . cellgeom  - The values precomputed from cell geometry
4830ce78bad3SBarry Smith - minRadius - The minimum radius over the mesh of an inscribed sphere in a cell, or `NULL` if not needed
48313e9753d6SMatthew G. Knepley 
48323e9753d6SMatthew G. Knepley   Level: developer
48333e9753d6SMatthew G. Knepley 
48341cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMTSSetRHSFunctionLocal()`
48353e9753d6SMatthew G. Knepley @*/
4836ce78bad3SBarry Smith PetscErrorCode DMPlexGetGeometryFVM(DM dm, Vec *facegeom, Vec *cellgeom, PeOp PetscReal *minRadius)
4837d71ae5a4SJacob Faibussowitsch {
48383e9753d6SMatthew G. Knepley   DM plex;
48393e9753d6SMatthew G. Knepley 
48403e9753d6SMatthew G. Knepley   PetscFunctionBegin;
48413e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
48429566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE));
48439566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, NULL, cellgeom, facegeom, NULL));
48449566063dSJacob Faibussowitsch   if (minRadius) PetscCall(DMPlexGetMinRadius(plex, minRadius));
48459566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
48463ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
48473e9753d6SMatthew G. Knepley }
48483e9753d6SMatthew G. Knepley 
48493e9753d6SMatthew G. Knepley /*@
48503e9753d6SMatthew G. Knepley   DMPlexGetGradientDM - Return gradient data layout
48513e9753d6SMatthew G. Knepley 
485220f4b53cSBarry Smith   Collective
48533e9753d6SMatthew G. Knepley 
48543e9753d6SMatthew G. Knepley   Input Parameters:
4855a1cb98faSBarry Smith + dm - The `DM`
485620f4b53cSBarry Smith - fv - The `PetscFV`
48573e9753d6SMatthew G. Knepley 
48583e9753d6SMatthew G. Knepley   Output Parameter:
48593e9753d6SMatthew G. Knepley . dmGrad - The layout for gradient values
48603e9753d6SMatthew G. Knepley 
48613e9753d6SMatthew G. Knepley   Level: developer
48623e9753d6SMatthew G. Knepley 
48631cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetGeometryFVM()`
48643e9753d6SMatthew G. Knepley @*/
4865d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetGradientDM(DM dm, PetscFV fv, DM *dmGrad)
4866d71ae5a4SJacob Faibussowitsch {
48673e9753d6SMatthew G. Knepley   DM        plex;
48683e9753d6SMatthew G. Knepley   PetscBool computeGradients;
48693e9753d6SMatthew G. Knepley 
48703e9753d6SMatthew G. Knepley   PetscFunctionBegin;
48713e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
48723e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);
48734f572ea9SToby Isaac   PetscAssertPointer(dmGrad, 3);
48749566063dSJacob Faibussowitsch   PetscCall(PetscFVGetComputeGradients(fv, &computeGradients));
48759371c9d4SSatish Balay   if (!computeGradients) {
48769371c9d4SSatish Balay     *dmGrad = NULL;
48773ba16761SJacob Faibussowitsch     PetscFunctionReturn(PETSC_SUCCESS);
48789371c9d4SSatish Balay   }
48799566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE));
48809566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, fv, NULL, NULL, dmGrad));
48819566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
48823ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
48833e9753d6SMatthew G. Knepley }
48843e9753d6SMatthew G. Knepley 
4885997bf629SMatthew G. Knepley /*@
4886997bf629SMatthew G. Knepley   DMPlexComputeBdResidualSingleByKey - Compute the local boundary residual for terms matching the input key
4887997bf629SMatthew G. Knepley 
4888997bf629SMatthew G. Knepley   Not collective
4889997bf629SMatthew G. Knepley 
4890997bf629SMatthew G. Knepley   Input Parameters:
4891997bf629SMatthew G. Knepley + dm         - The output `DM`
4892997bf629SMatthew G. Knepley . wf         - The `PetscWeakForm` holding forms on this boundary
4893997bf629SMatthew G. Knepley . key        - The `PetscFormKey` indicating what should be integrated
4894997bf629SMatthew G. Knepley . facetIS    - The `IS` giving a set of faces to integrate over
4895997bf629SMatthew G. Knepley . locX       - The local solution
4896997bf629SMatthew G. Knepley . locX_t     - The time derivative of the local solution, or `NULL` for time-independent problems
4897997bf629SMatthew G. Knepley . t          - The time
4898997bf629SMatthew G. Knepley - coordField - The `DMField` object with coordinates for these faces
4899997bf629SMatthew G. Knepley 
4900997bf629SMatthew G. Knepley   Output Parameter:
4901997bf629SMatthew G. Knepley . locF - The local residual
4902997bf629SMatthew G. Knepley 
4903997bf629SMatthew G. Knepley   Level: developer
4904997bf629SMatthew G. Knepley 
4905997bf629SMatthew G. Knepley .seealso: `DMPlexComputeBdResidualSingle()`, `DMPlexComputeJacobianByKey()`, `DMPlexComputeResidualHybridByKey()`, `DMPlexComputeJacobianHybridByKey()`, `PetscFormKey`
4906997bf629SMatthew G. Knepley @*/
4907997bf629SMatthew G. Knepley PetscErrorCode DMPlexComputeBdResidualSingleByKey(DM dm, PetscWeakForm wf, PetscFormKey key, IS facetIS, Vec locX, Vec locX_t, PetscReal t, DMField coordField, Vec locF)
4908d71ae5a4SJacob Faibussowitsch {
49093e9753d6SMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *)dm->data;
49103e9753d6SMatthew G. Knepley   DM              plex = NULL, plexA = NULL;
4911673019faSMatthew G. Knepley   const char     *name = "BdResidual";
49123e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
49133e9753d6SMatthew G. Knepley   PetscDS         prob, probAux       = NULL;
49143e9753d6SMatthew G. Knepley   PetscSection    section, sectionAux = NULL;
49153e9753d6SMatthew G. Knepley   Vec             locA = NULL;
49163e9753d6SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t = NULL, *a = NULL, *elemVec = NULL;
49173e9753d6SMatthew G. Knepley   PetscInt        totDim, totDimAux = 0;
49183e9753d6SMatthew G. Knepley 
49193e9753d6SMatthew G. Knepley   PetscFunctionBegin;
49209566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
49219566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
49229566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
49239566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
49249566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
49253e9753d6SMatthew G. Knepley   if (locA) {
49263e9753d6SMatthew G. Knepley     DM dmAux;
49273e9753d6SMatthew G. Knepley 
49289566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
49299566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
49309566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
49319566063dSJacob Faibussowitsch     PetscCall(DMGetDS(plexA, &probAux));
49329566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
49339566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
49343e9753d6SMatthew G. Knepley   }
49350c290148SMatthew G. Knepley   {
49363e9753d6SMatthew G. Knepley     PetscFEGeom    *fgeom;
49373e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
49383e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
49393e9753d6SMatthew G. Knepley     IS              pointIS;
49403e9753d6SMatthew G. Knepley     const PetscInt *points;
49413e9753d6SMatthew G. Knepley     PetscInt        numFaces, face, Nq;
49423e9753d6SMatthew G. Knepley 
49439566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(key.label, key.value, &pointIS));
49440c290148SMatthew G. Knepley     if (!pointIS) goto end; /* No points with that id on this process */
49453e9753d6SMatthew G. Knepley     {
49463e9753d6SMatthew G. Knepley       IS isectIS;
49473e9753d6SMatthew G. Knepley 
49483e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
49499566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
49509566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
49513e9753d6SMatthew G. Knepley       pointIS = isectIS;
49523e9753d6SMatthew G. Knepley     }
49539566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
49549566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
495532603206SJames Wright     PetscCall(PetscMalloc4(numFaces * totDim, &u, (locX_t ? (size_t)numFaces * totDim : 0), &u_t, numFaces * totDim, &elemVec, (locA ? (size_t)numFaces * totDimAux : 0), &a));
49569566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
495748a46eb9SPierre Jolivet     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
49583e9753d6SMatthew G. Knepley     if (!qGeom) {
49593e9753d6SMatthew G. Knepley       PetscFE fe;
49603e9753d6SMatthew G. Knepley 
49619566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe));
49629566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
49639566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
49643e9753d6SMatthew G. Knepley     }
49659566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
4966ac9d17c7SMatthew G. Knepley     PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_FEGEOM_BOUNDARY, &fgeom));
49673e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
4968f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
49693e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
4970f15274beSMatthew Knepley       PetscInt       i;
49713e9753d6SMatthew G. Knepley 
49729566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
49739566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
49743e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i];
49759566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
49763e9753d6SMatthew G. Knepley       if (locX_t) {
49779566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
49783e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i];
49799566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
49803e9753d6SMatthew G. Knepley       }
49813e9753d6SMatthew G. Knepley       if (locA) {
49823e9753d6SMatthew G. Knepley         PetscInt subp;
49833e9753d6SMatthew G. Knepley 
49849566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
49859566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
49863e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i];
49879566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
49883e9753d6SMatthew G. Knepley       }
49893e9753d6SMatthew G. Knepley     }
49909566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(elemVec, numFaces * totDim));
49913e9753d6SMatthew G. Knepley     {
49923e9753d6SMatthew G. Knepley       PetscFE      fe;
49933e9753d6SMatthew G. Knepley       PetscInt     Nb;
49943e9753d6SMatthew G. Knepley       PetscFEGeom *chunkGeom = NULL;
49953e9753d6SMatthew G. Knepley       /* Conforming batches */
49963e9753d6SMatthew G. Knepley       PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
49973e9753d6SMatthew G. Knepley       /* Remainder */
49983e9753d6SMatthew G. Knepley       PetscInt Nr, offset;
49993e9753d6SMatthew G. Knepley 
50009566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe));
50019566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
50029566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
50033e9753d6SMatthew G. Knepley       /* TODO: documentation is unclear about what is going on with these numbers: how should Nb / Nq factor in ? */
50043e9753d6SMatthew G. Knepley       blockSize = Nb;
50053e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
50069566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
50073e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches * batchSize);
50083e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
50093e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches * batchSize);
50103e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
50119566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom));
50129566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
50139566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
50149566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
50158e3a54c0SPierre Jolivet       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, &elemVec[offset * totDim]));
50169566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
50173e9753d6SMatthew G. Knepley     }
50183e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
50193e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
50203e9753d6SMatthew G. Knepley 
5021673019faSMatthew G. Knepley       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(point, name, totDim, &elemVec[face * totDim]));
50229566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
50239566063dSJacob Faibussowitsch       PetscCall(DMPlexVecSetClosure(plex, NULL, locF, support[0], &elemVec[face * totDim], ADD_ALL_VALUES));
50243e9753d6SMatthew G. Knepley     }
50259566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
50269566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
50279566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
50289566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
50299566063dSJacob Faibussowitsch     PetscCall(PetscFree4(u, u_t, elemVec, a));
50303e9753d6SMatthew G. Knepley   }
50310c290148SMatthew G. Knepley end:
5032673019faSMatthew G. Knepley   if (mesh->printFEM) {
5033673019faSMatthew G. Knepley     PetscSection s;
5034673019faSMatthew G. Knepley     Vec          locFbc;
5035673019faSMatthew G. Knepley     PetscInt     pStart, pEnd, maxDof;
5036673019faSMatthew G. Knepley     PetscScalar *zeroes;
5037673019faSMatthew G. Knepley 
5038673019faSMatthew G. Knepley     PetscCall(DMGetLocalSection(dm, &s));
5039673019faSMatthew G. Knepley     PetscCall(VecDuplicate(locF, &locFbc));
5040673019faSMatthew G. Knepley     PetscCall(VecCopy(locF, locFbc));
5041673019faSMatthew G. Knepley     PetscCall(PetscSectionGetChart(s, &pStart, &pEnd));
5042673019faSMatthew G. Knepley     PetscCall(PetscSectionGetMaxDof(s, &maxDof));
5043673019faSMatthew G. Knepley     PetscCall(PetscCalloc1(maxDof, &zeroes));
5044673019faSMatthew G. Knepley     for (PetscInt p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, s, p, zeroes, INSERT_BC_VALUES));
5045673019faSMatthew G. Knepley     PetscCall(PetscFree(zeroes));
5046673019faSMatthew G. Knepley     PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc));
5047673019faSMatthew G. Knepley     PetscCall(VecDestroy(&locFbc));
5048673019faSMatthew G. Knepley   }
50499566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
50509566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexA));
50513ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
50523e9753d6SMatthew G. Knepley }
50533e9753d6SMatthew G. Knepley 
5054997bf629SMatthew G. Knepley /*@
5055997bf629SMatthew G. Knepley   DMPlexComputeBdResidualSingle - Compute the local boundary residual
5056997bf629SMatthew G. Knepley 
5057997bf629SMatthew G. Knepley   Not collective
5058997bf629SMatthew G. Knepley 
5059997bf629SMatthew G. Knepley   Input Parameters:
5060997bf629SMatthew G. Knepley + dm     - The output `DM`
5061997bf629SMatthew G. Knepley . wf     - The `PetscWeakForm` holding forms on this boundary
5062997bf629SMatthew G. Knepley . key    - The `PetscFormKey` indicating what should be integrated
5063997bf629SMatthew G. Knepley . locX   - The local solution
5064997bf629SMatthew G. Knepley . locX_t - The time derivative of the local solution, or `NULL` for time-independent problems
5065997bf629SMatthew G. Knepley - t      - The time
5066997bf629SMatthew G. Knepley 
5067997bf629SMatthew G. Knepley   Output Parameter:
5068997bf629SMatthew G. Knepley . locF - The local residual
5069997bf629SMatthew G. Knepley 
5070997bf629SMatthew G. Knepley   Level: developer
5071997bf629SMatthew G. Knepley 
5072997bf629SMatthew G. Knepley .seealso: `DMPlexComputeBdResidualSingleByKey()`, `DMPlexComputeJacobianByKey()`, `DMPlexComputeResidualHybridByKey()`, `DMPlexComputeJacobianHybridByKey()`, `PetscFormKey`
5073997bf629SMatthew G. Knepley @*/
5074997bf629SMatthew G. Knepley PetscErrorCode DMPlexComputeBdResidualSingle(DM dm, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, PetscReal t, Vec locF)
5075d71ae5a4SJacob Faibussowitsch {
50763e9753d6SMatthew G. Knepley   DMField  coordField;
50773e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
50783e9753d6SMatthew G. Knepley   IS       facetIS;
50793e9753d6SMatthew G. Knepley   PetscInt dim;
50803e9753d6SMatthew G. Knepley 
50813e9753d6SMatthew G. Knepley   PetscFunctionBegin;
50829566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
50839566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
50849566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
50859566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
5086997bf629SMatthew G. Knepley   PetscCall(DMPlexComputeBdResidualSingleByKey(dm, wf, key, facetIS, locX, locX_t, t, coordField, locF));
50879566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
50883ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
50893e9753d6SMatthew G. Knepley }
50903e9753d6SMatthew G. Knepley 
5091a4e35b19SJacob Faibussowitsch static PetscErrorCode DMPlexComputeBdResidual_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
5092d71ae5a4SJacob Faibussowitsch {
50933e9753d6SMatthew G. Knepley   PetscDS  prob;
50943e9753d6SMatthew G. Knepley   PetscInt numBd, bd;
50953e9753d6SMatthew G. Knepley   DMField  coordField = NULL;
50963e9753d6SMatthew G. Knepley   IS       facetIS    = NULL;
50973e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
50983e9753d6SMatthew G. Knepley   PetscInt dim;
50993e9753d6SMatthew G. Knepley 
51003e9753d6SMatthew G. Knepley   PetscFunctionBegin;
51019566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
51029566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
51039566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
51049566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
51059566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
51063e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
510745480ffeSMatthew G. Knepley     PetscWeakForm           wf;
51083e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
51093e9753d6SMatthew G. Knepley     DMLabel                 label;
51103e9753d6SMatthew G. Knepley     const PetscInt         *values;
51110c290148SMatthew G. Knepley     PetscInt                field, numValues, v;
51123e9753d6SMatthew G. Knepley     PetscObject             obj;
51133e9753d6SMatthew G. Knepley     PetscClassId            id;
51140c290148SMatthew G. Knepley     PetscFormKey            key;
51153e9753d6SMatthew G. Knepley 
51169566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &field, NULL, NULL, NULL, NULL, NULL));
51173d3e5d66SMatthew G. Knepley     if (type & DM_BC_ESSENTIAL) continue;
51189566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
51199566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
51203d3e5d66SMatthew G. Knepley     if (id != PETSCFE_CLASSID) continue;
51213e9753d6SMatthew G. Knepley     if (!facetIS) {
51223e9753d6SMatthew G. Knepley       DMLabel  depthLabel;
51233e9753d6SMatthew G. Knepley       PetscInt dim;
51243e9753d6SMatthew G. Knepley 
51259566063dSJacob Faibussowitsch       PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
51269566063dSJacob Faibussowitsch       PetscCall(DMGetDimension(dm, &dim));
51279566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
51283e9753d6SMatthew G. Knepley     }
51299566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
51300c290148SMatthew G. Knepley     for (v = 0; v < numValues; ++v) {
51310c290148SMatthew G. Knepley       key.label = label;
51320c290148SMatthew G. Knepley       key.value = values[v];
51330c290148SMatthew G. Knepley       key.field = field;
51340c290148SMatthew G. Knepley       key.part  = 0;
5135997bf629SMatthew G. Knepley       PetscCall(DMPlexComputeBdResidualSingleByKey(dm, wf, key, facetIS, locX, locX_t, t, coordField, locF));
51360c290148SMatthew G. Knepley     }
51373e9753d6SMatthew G. Knepley   }
51389566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
51393ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
51403e9753d6SMatthew G. Knepley }
51413e9753d6SMatthew G. Knepley 
5142754e4fbaSMatthew G. Knepley /*@
5143754e4fbaSMatthew G. Knepley   DMPlexComputeResidualByKey - Compute the local residual for terms matching the input key
5144754e4fbaSMatthew G. Knepley 
5145754e4fbaSMatthew G. Knepley   Collective
5146754e4fbaSMatthew G. Knepley 
5147754e4fbaSMatthew G. Knepley   Input Parameters:
5148754e4fbaSMatthew G. Knepley + dm     - The output `DM`
5149754e4fbaSMatthew G. Knepley . key    - The `PetscFormKey` indicating what should be integrated
5150754e4fbaSMatthew G. Knepley . cellIS - The `IS` giving a set of cells to integrate over
5151754e4fbaSMatthew G. Knepley . time   - The time, or `PETSC_MIN_REAL` to include implicit terms in a time-independent problems
5152754e4fbaSMatthew G. Knepley . locX   - The local solution
5153754e4fbaSMatthew G. Knepley . locX_t - The time derivative of the local solution, or `NULL` for time-independent problems
5154754e4fbaSMatthew G. Knepley . t      - The time
5155754e4fbaSMatthew G. Knepley - user   - An optional user context, passed to the pointwise functions
5156754e4fbaSMatthew G. Knepley 
5157754e4fbaSMatthew G. Knepley   Output Parameter:
5158754e4fbaSMatthew G. Knepley . locF - The local residual
5159754e4fbaSMatthew G. Knepley 
5160754e4fbaSMatthew G. Knepley   Level: developer
5161754e4fbaSMatthew G. Knepley 
5162754e4fbaSMatthew G. Knepley .seealso: `DMPlexComputeJacobianByKey()`, `DMPlexComputeResidualHybridByKey()`, `DMPlexComputeJacobianHybridByKey()`, `PetscFormKey`
5163754e4fbaSMatthew G. Knepley @*/
5164754e4fbaSMatthew G. Knepley PetscErrorCode DMPlexComputeResidualByKey(DM dm, PetscFormKey key, IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
5165d71ae5a4SJacob Faibussowitsch {
51663e9753d6SMatthew G. Knepley   DM_Plex        *mesh       = (DM_Plex *)dm->data;
51673e9753d6SMatthew G. Knepley   const char     *name       = "Residual";
51683e9753d6SMatthew G. Knepley   DM              dmAux      = NULL;
51693e9753d6SMatthew G. Knepley   DM              dmGrad     = NULL;
51703e9753d6SMatthew G. Knepley   DMLabel         ghostLabel = NULL;
51716528b96dSMatthew G. Knepley   PetscDS         ds         = NULL;
51726528b96dSMatthew G. Knepley   PetscDS         dsAux      = NULL;
51733e9753d6SMatthew G. Knepley   PetscSection    section    = NULL;
51743e9753d6SMatthew G. Knepley   PetscBool       useFEM     = PETSC_FALSE;
51753e9753d6SMatthew G. Knepley   PetscBool       useFVM     = PETSC_FALSE;
51763e9753d6SMatthew G. Knepley   PetscBool       isImplicit = (locX_t || time == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
51773e9753d6SMatthew G. Knepley   PetscFV         fvm        = NULL;
51783e9753d6SMatthew G. Knepley   DMField         coordField = NULL;
51795962854dSMatthew G. Knepley   Vec             locA, cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL;
51803e9753d6SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t, *a, *uL, *uR;
51813e9753d6SMatthew G. Knepley   IS              chunkIS;
51823e9753d6SMatthew G. Knepley   const PetscInt *cells;
51833e9753d6SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
51843e9753d6SMatthew G. Knepley   PetscInt        Nf, f, totDim, totDimAux, numChunks, cellChunkSize, faceChunkSize, chunk, fStart, fEnd;
51851690c2aeSBarry Smith   PetscInt        maxDegree  = PETSC_INT_MAX;
51863e9753d6SMatthew G. Knepley   PetscQuadrature affineQuad = NULL, *quads = NULL;
51873e9753d6SMatthew G. Knepley   PetscFEGeom    *affineGeom = NULL, **geoms = NULL;
51883e9753d6SMatthew G. Knepley 
51893e9753d6SMatthew G. Knepley   PetscFunctionBegin;
51909566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0));
51913cc88e6aSStefano Zampini   if (!cellIS) goto end;
51923cc88e6aSStefano Zampini   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
51933cc88e6aSStefano Zampini   if (cStart >= cEnd) goto end;
51943e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
51953e9753d6SMatthew G. Knepley   /* TODO The FVM geometry is over-manipulated. Make the precalc functions return exactly what we need */
51963e9753d6SMatthew G. Knepley   /* FEM+FVM */
51979566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
51983e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
51999566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
52009566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
520107218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, NULL));
52029566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
52039566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
52049566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
52053e9753d6SMatthew G. Knepley   if (locA) {
52063e9753d6SMatthew G. Knepley     PetscInt subcell;
52079566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
52081059d808SMatthew G. Knepley     PetscCall(DMGetEnclosurePoint(dmAux, dm, DM_ENC_UNKNOWN, cells ? cells[cStart] : cStart, &subcell));
520907218a29SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux, subcell, &dsAux, NULL));
52109566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux));
52113e9753d6SMatthew G. Knepley   }
52123e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
52133e9753d6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
52143e9753d6SMatthew G. Knepley     PetscObject  obj;
52153e9753d6SMatthew G. Knepley     PetscClassId id;
52163e9753d6SMatthew G. Knepley     PetscBool    fimp;
52173e9753d6SMatthew G. Knepley 
52189566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(ds, f, &fimp));
52193e9753d6SMatthew G. Knepley     if (isImplicit != fimp) continue;
52209566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(ds, f, &obj));
52219566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
5222ad540459SPierre Jolivet     if (id == PETSCFE_CLASSID) useFEM = PETSC_TRUE;
52239371c9d4SSatish Balay     if (id == PETSCFV_CLASSID) {
52249371c9d4SSatish Balay       useFVM = PETSC_TRUE;
52259371c9d4SSatish Balay       fvm    = (PetscFV)obj;
52269371c9d4SSatish Balay     }
52273e9753d6SMatthew G. Knepley   }
52283e9753d6SMatthew G. Knepley   if (useFEM) {
52299566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
52309566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
52313e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
52329566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad));
5233ac9d17c7SMatthew G. Knepley       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FEGEOM_BASIC, &affineGeom));
52343e9753d6SMatthew G. Knepley     } else {
52359566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
52363e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
52373e9753d6SMatthew G. Knepley         PetscObject  obj;
52383e9753d6SMatthew G. Knepley         PetscClassId id;
52393e9753d6SMatthew G. Knepley         PetscBool    fimp;
52403e9753d6SMatthew G. Knepley 
52419566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(ds, f, &fimp));
52423e9753d6SMatthew G. Knepley         if (isImplicit != fimp) continue;
52439566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
52449566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
52453e9753d6SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
52463e9753d6SMatthew G. Knepley           PetscFE fe = (PetscFE)obj;
52473e9753d6SMatthew G. Knepley 
52489566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
52499566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
5250ac9d17c7SMatthew G. Knepley           PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FEGEOM_BASIC, &geoms[f]));
52513e9753d6SMatthew G. Knepley         }
52523e9753d6SMatthew G. Knepley       }
52533e9753d6SMatthew G. Knepley     }
52543e9753d6SMatthew G. Knepley   }
52555962854dSMatthew G. Knepley   // Handle non-essential (e.g. outflow) boundary values
52563e9753d6SMatthew G. Knepley   if (useFVM) {
52575962854dSMatthew G. Knepley     PetscCall(DMPlexInsertBoundaryValuesFVM(dm, fvm, locX, time, &locGrad));
52589566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL));
52599566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGradientDM(dm, fvm, &dmGrad));
52603e9753d6SMatthew G. Knepley   }
52613e9753d6SMatthew G. Knepley   /* Loop over chunks */
52629566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
52633e9753d6SMatthew G. Knepley   numCells      = cEnd - cStart;
52643e9753d6SMatthew G. Knepley   numChunks     = 1;
52653e9753d6SMatthew G. Knepley   cellChunkSize = numCells / numChunks;
52663e9753d6SMatthew G. Knepley   faceChunkSize = (fEnd - fStart) / numChunks;
52673e9753d6SMatthew G. Knepley   numChunks     = PetscMin(1, numCells);
52683e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
52693e9753d6SMatthew G. Knepley     PetscScalar     *elemVec, *fluxL, *fluxR;
52703e9753d6SMatthew G. Knepley     PetscReal       *vol;
52713e9753d6SMatthew G. Knepley     PetscFVFaceGeom *fgeom;
52723e9753d6SMatthew G. Knepley     PetscInt         cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
52733e9753d6SMatthew G. Knepley     PetscInt         fS = fStart + chunk * faceChunkSize, fE = PetscMin(fS + faceChunkSize, fEnd), numFaces = 0, face;
52743e9753d6SMatthew G. Knepley 
52753e9753d6SMatthew G. Knepley     /* Extract field coefficients */
52763e9753d6SMatthew G. Knepley     if (useFEM) {
52779566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
52789566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
52799566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
52809566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells * totDim));
52813e9753d6SMatthew G. Knepley     }
52823e9753d6SMatthew G. Knepley     if (useFVM) {
52839566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
52849566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
52859566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL));
52869566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR));
52879566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxL, numFaces * totDim));
52889566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxR, numFaces * totDim));
52893e9753d6SMatthew G. Knepley     }
52903e9753d6SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
52913e9753d6SMatthew G. Knepley     /* Loop over fields */
52923e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
52933e9753d6SMatthew G. Knepley       PetscObject  obj;
52943e9753d6SMatthew G. Knepley       PetscClassId id;
52953e9753d6SMatthew G. Knepley       PetscBool    fimp;
52963e9753d6SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
52973e9753d6SMatthew G. Knepley 
52986528b96dSMatthew G. Knepley       key.field = f;
52999566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(ds, f, &fimp));
53003e9753d6SMatthew G. Knepley       if (isImplicit != fimp) continue;
53019566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, &obj));
53029566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
53033e9753d6SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
53043e9753d6SMatthew G. Knepley         PetscFE         fe        = (PetscFE)obj;
53053e9753d6SMatthew G. Knepley         PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[f];
53063e9753d6SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
53073e9753d6SMatthew G. Knepley         PetscQuadrature quad      = affineQuad ? affineQuad : quads[f];
53083e9753d6SMatthew G. Knepley         PetscInt        Nq, Nb;
53093e9753d6SMatthew G. Knepley 
53109566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
53119566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
53129566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
53133e9753d6SMatthew G. Knepley         blockSize = Nb;
53143e9753d6SMatthew G. Knepley         batchSize = numBlocks * blockSize;
53159566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
53163e9753d6SMatthew G. Knepley         numChunks = numCells / (numBatches * batchSize);
53173e9753d6SMatthew G. Knepley         Ne        = numChunks * numBatches * batchSize;
53183e9753d6SMatthew G. Knepley         Nr        = numCells % (numBatches * batchSize);
53193e9753d6SMatthew G. Knepley         offset    = numCells - Nr;
53203e9753d6SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
53213e9753d6SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
53229566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom));
53239566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(ds, key, Ne, chunkGeom, u, u_t, dsAux, a, t, elemVec));
53249566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom));
53258e3a54c0SPierre Jolivet         PetscCall(PetscFEIntegrateResidual(ds, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, &elemVec[offset * totDim]));
53269566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom));
53273e9753d6SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
53283e9753d6SMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
53293e9753d6SMatthew G. Knepley 
53303e9753d6SMatthew G. Knepley         Ne = numFaces;
53313e9753d6SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
53323e9753d6SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
53339566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, ds, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
533463a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
53353e9753d6SMatthew G. Knepley     }
53363e9753d6SMatthew G. Knepley     /* Loop over domain */
53373e9753d6SMatthew G. Knepley     if (useFEM) {
53383e9753d6SMatthew G. Knepley       /* Add elemVec to locX */
53393e9753d6SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
53403e9753d6SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
53413e9753d6SMatthew G. Knepley         const PetscInt cind = c - cStart;
53423e9753d6SMatthew G. Knepley 
53439566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim]));
53443e9753d6SMatthew G. Knepley         if (ghostLabel) {
53453e9753d6SMatthew G. Knepley           PetscInt ghostVal;
53463e9753d6SMatthew G. Knepley 
53479566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
53483e9753d6SMatthew G. Knepley           if (ghostVal > 0) continue;
53493e9753d6SMatthew G. Knepley         }
53509566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES));
53513e9753d6SMatthew G. Knepley       }
53523e9753d6SMatthew G. Knepley     }
53533e9753d6SMatthew G. Knepley     if (useFVM) {
53543e9753d6SMatthew G. Knepley       PetscScalar *fa;
53553e9753d6SMatthew G. Knepley       PetscInt     iface;
53563e9753d6SMatthew G. Knepley 
53579566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
53583e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
53593e9753d6SMatthew G. Knepley         PetscFV      fv;
53603e9753d6SMatthew G. Knepley         PetscObject  obj;
53613e9753d6SMatthew G. Knepley         PetscClassId id;
53625962854dSMatthew G. Knepley         PetscInt     cdim, foff, pdim;
53633e9753d6SMatthew G. Knepley 
53645962854dSMatthew G. Knepley         PetscCall(DMGetCoordinateDim(dm, &cdim));
53659566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
53669566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(ds, f, &foff));
53679566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
53683e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
53693e9753d6SMatthew G. Knepley         fv = (PetscFV)obj;
53709566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
53713e9753d6SMatthew G. Knepley         /* Accumulate fluxes to cells */
53723e9753d6SMatthew G. Knepley         for (face = fS, iface = 0; face < fE; ++face) {
53733e9753d6SMatthew G. Knepley           const PetscInt *scells;
53743e9753d6SMatthew G. Knepley           PetscScalar    *fL = NULL, *fR = NULL;
53753e9753d6SMatthew G. Knepley           PetscInt        ghost, d, nsupp, nchild;
53763e9753d6SMatthew G. Knepley 
53779566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
53789566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
53799566063dSJacob Faibussowitsch           PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
53803e9753d6SMatthew G. Knepley           if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
53819566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupport(dm, face, &scells));
53829566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, scells[0], &ghost));
53839566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[0], f, fa, &fL));
53849566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, scells[1], &ghost));
53859566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[1], f, fa, &fR));
53865962854dSMatthew G. Knepley           if (mesh->printFVM > 1) {
53875962854dSMatthew G. Knepley             PetscCall(DMPrintCellVectorReal(face, "Residual: normal", cdim, fgeom[iface].normal));
53885962854dSMatthew G. Knepley             PetscCall(DMPrintCellVector(face, "Residual: left state", pdim, &uL[iface * totDim + foff]));
53895962854dSMatthew G. Knepley             PetscCall(DMPrintCellVector(face, "Residual: right state", pdim, &uR[iface * totDim + foff]));
53905962854dSMatthew G. Knepley             PetscCall(DMPrintCellVector(face, "Residual: left flux", pdim, &fluxL[iface * totDim + foff]));
53915962854dSMatthew G. Knepley             PetscCall(DMPrintCellVector(face, "Residual: right flux", pdim, &fluxR[iface * totDim + foff]));
53925962854dSMatthew G. Knepley           }
53933e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) {
53943e9753d6SMatthew G. Knepley             if (fL) fL[d] -= fluxL[iface * totDim + foff + d];
53953e9753d6SMatthew G. Knepley             if (fR) fR[d] += fluxR[iface * totDim + foff + d];
53963e9753d6SMatthew G. Knepley           }
53973e9753d6SMatthew G. Knepley           ++iface;
53983e9753d6SMatthew G. Knepley         }
53993e9753d6SMatthew G. Knepley       }
54009566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
54013e9753d6SMatthew G. Knepley     }
54023e9753d6SMatthew G. Knepley     /* Handle time derivative */
54033e9753d6SMatthew G. Knepley     if (locX_t) {
54043e9753d6SMatthew G. Knepley       PetscScalar *x_t, *fa;
54053e9753d6SMatthew G. Knepley 
54069566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
54079566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
54083e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
54093e9753d6SMatthew G. Knepley         PetscFV      fv;
54103e9753d6SMatthew G. Knepley         PetscObject  obj;
54113e9753d6SMatthew G. Knepley         PetscClassId id;
54123e9753d6SMatthew G. Knepley         PetscInt     pdim, d;
54133e9753d6SMatthew G. Knepley 
54149566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
54159566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
54163e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
54173e9753d6SMatthew G. Knepley         fv = (PetscFV)obj;
54189566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
54193e9753d6SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
54203e9753d6SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
54213e9753d6SMatthew G. Knepley           PetscScalar   *u_t, *r;
54223e9753d6SMatthew G. Knepley 
54233e9753d6SMatthew G. Knepley           if (ghostLabel) {
54243e9753d6SMatthew G. Knepley             PetscInt ghostVal;
54253e9753d6SMatthew G. Knepley 
54269566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
54273e9753d6SMatthew G. Knepley             if (ghostVal > 0) continue;
54283e9753d6SMatthew G. Knepley           }
54299566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
54309566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
54313e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
54323e9753d6SMatthew G. Knepley         }
54333e9753d6SMatthew G. Knepley       }
54349566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
54359566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
54363e9753d6SMatthew G. Knepley     }
54373e9753d6SMatthew G. Knepley     if (useFEM) {
54389566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
54399566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
54403e9753d6SMatthew G. Knepley     }
54413e9753d6SMatthew G. Knepley     if (useFVM) {
54429566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
54439566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
54449566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL));
54459566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR));
54469566063dSJacob Faibussowitsch       if (dmGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
54473e9753d6SMatthew G. Knepley     }
54483e9753d6SMatthew G. Knepley   }
54499566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
54509566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
54513e9753d6SMatthew G. Knepley 
54523e9753d6SMatthew G. Knepley   if (useFEM) {
54539566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdResidual_Internal(dm, locX, locX_t, t, locF, user));
54543e9753d6SMatthew G. Knepley 
54553e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
54569566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
54579566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
54583e9753d6SMatthew G. Knepley     } else {
54593e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
54609566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
54619566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
54623e9753d6SMatthew G. Knepley       }
54639566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads, geoms));
54643e9753d6SMatthew G. Knepley     }
54653e9753d6SMatthew G. Knepley   }
54663e9753d6SMatthew G. Knepley 
54673e9753d6SMatthew G. Knepley   /* FEM */
54683e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
54693e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
54703e9753d6SMatthew G. Knepley   /* 3: Handle boundary values */
54713e9753d6SMatthew G. Knepley   /* 4: Loop over domain */
54723e9753d6SMatthew G. Knepley   /*   Extract coefficients */
54733e9753d6SMatthew G. Knepley   /* Loop over fields */
54743e9753d6SMatthew G. Knepley   /*   Set tiling for FE*/
54753e9753d6SMatthew G. Knepley   /*   Integrate FE residual to get elemVec */
54763e9753d6SMatthew G. Knepley   /*     Loop over subdomain */
54773e9753d6SMatthew G. Knepley   /*       Loop over quad points */
54783e9753d6SMatthew G. Knepley   /*         Transform coords to real space */
54793e9753d6SMatthew G. Knepley   /*         Evaluate field and aux fields at point */
54803e9753d6SMatthew G. Knepley   /*         Evaluate residual at point */
54813e9753d6SMatthew G. Knepley   /*         Transform residual to real space */
54823e9753d6SMatthew G. Knepley   /*       Add residual to elemVec */
54833e9753d6SMatthew G. Knepley   /* Loop over domain */
54843e9753d6SMatthew G. Knepley   /*   Add elemVec to locX */
54853e9753d6SMatthew G. Knepley 
54863e9753d6SMatthew G. Knepley   /* FVM */
54873e9753d6SMatthew G. Knepley   /* Get geometric data */
54883e9753d6SMatthew G. Knepley   /* If using gradients */
54893e9753d6SMatthew G. Knepley   /*   Compute gradient data */
54903e9753d6SMatthew G. Knepley   /*   Loop over domain faces */
54913e9753d6SMatthew G. Knepley   /*     Count computational faces */
54923e9753d6SMatthew G. Knepley   /*     Reconstruct cell gradient */
54933e9753d6SMatthew G. Knepley   /*   Loop over domain cells */
54943e9753d6SMatthew G. Knepley   /*     Limit cell gradients */
54953e9753d6SMatthew G. Knepley   /* Handle boundary values */
54963e9753d6SMatthew G. Knepley   /* Loop over domain faces */
54973e9753d6SMatthew G. Knepley   /*   Read out field, centroid, normal, volume for each side of face */
54983e9753d6SMatthew G. Knepley   /* Riemann solve over faces */
54993e9753d6SMatthew G. Knepley   /* Loop over domain faces */
55003e9753d6SMatthew G. Knepley   /*   Accumulate fluxes to cells */
55013e9753d6SMatthew G. Knepley   /* TODO Change printFEM to printDisc here */
55023e9753d6SMatthew G. Knepley   if (mesh->printFEM) {
55033e9753d6SMatthew G. Knepley     Vec          locFbc;
55043e9753d6SMatthew G. Knepley     PetscInt     pStart, pEnd, p, maxDof;
55053e9753d6SMatthew G. Knepley     PetscScalar *zeroes;
55063e9753d6SMatthew G. Knepley 
55079566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(locF, &locFbc));
55089566063dSJacob Faibussowitsch     PetscCall(VecCopy(locF, locFbc));
55099566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
55109566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetMaxDof(section, &maxDof));
55119566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(maxDof, &zeroes));
551248a46eb9SPierre Jolivet     for (p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, section, p, zeroes, INSERT_BC_VALUES));
55139566063dSJacob Faibussowitsch     PetscCall(PetscFree(zeroes));
55149566063dSJacob Faibussowitsch     PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc));
55159566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&locFbc));
55163e9753d6SMatthew G. Knepley   }
55173cc88e6aSStefano Zampini end:
55189566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0));
55193ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
55203e9753d6SMatthew G. Knepley }
55213e9753d6SMatthew G. Knepley 
5522754e4fbaSMatthew G. Knepley /*@
5523754e4fbaSMatthew G. Knepley   DMPlexComputeResidualHybridByKey - Compute the local residual over hybrid cells for terms matching the input key
55246528b96dSMatthew G. Knepley 
5525754e4fbaSMatthew G. Knepley   Collective
55266528b96dSMatthew G. Knepley 
5527754e4fbaSMatthew G. Knepley   Input Parameters:
5528754e4fbaSMatthew G. Knepley + dm     - The output `DM`
5529754e4fbaSMatthew G. Knepley . key    - The `PetscFormKey` array (left cell, right cell, cohesive cell) indicating what should be integrated
5530754e4fbaSMatthew G. Knepley . cellIS - The `IS` give a set of cells to integrate over
5531754e4fbaSMatthew G. Knepley . time   - The time, or `PETSC_MIN_REAL` to include implicit terms in a time-independent problems
5532754e4fbaSMatthew G. Knepley . locX   - The local solution
5533754e4fbaSMatthew G. Knepley . locX_t - The time derivative of the local solution, or `NULL` for time-independent problems
5534754e4fbaSMatthew G. Knepley . t      - The time
5535754e4fbaSMatthew G. Knepley - user   - An optional user context, passed to the pointwise functions
55366528b96dSMatthew G. Knepley 
5537754e4fbaSMatthew G. Knepley   Output Parameter:
5538754e4fbaSMatthew G. Knepley . locF - The local residual
5539754e4fbaSMatthew G. Knepley 
5540754e4fbaSMatthew G. Knepley   Level: developer
5541754e4fbaSMatthew G. Knepley 
5542754e4fbaSMatthew G. Knepley .seealso: `DMPlexComputeResidualByKey()`, `DMPlexComputeJacobianByKey()`, `DMPlexComputeJacobianHybridByKey()`, `PetscFormKey`
5543754e4fbaSMatthew G. Knepley @*/
5544754e4fbaSMatthew G. Knepley PetscErrorCode DMPlexComputeResidualHybridByKey(DM dm, PetscFormKey key[], IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
5545d71ae5a4SJacob Faibussowitsch {
55463e9753d6SMatthew G. Knepley   DM_Plex        *mesh       = (DM_Plex *)dm->data;
55473e9753d6SMatthew G. Knepley   const char     *name       = "Hybrid Residual";
554804c51a94SMatthew G. Knepley   DM              dmAux[3]   = {NULL, NULL, NULL};
55493e9753d6SMatthew G. Knepley   DMLabel         ghostLabel = NULL;
55506528b96dSMatthew G. Knepley   PetscDS         ds         = NULL;
555107218a29SMatthew G. Knepley   PetscDS         dsIn       = NULL;
55526528b96dSMatthew G. Knepley   PetscDS         dsAux[3]   = {NULL, NULL, NULL};
555304c51a94SMatthew G. Knepley   Vec             locA[3]    = {NULL, NULL, NULL};
555407218a29SMatthew G. Knepley   DM              dmScale[3] = {NULL, NULL, NULL};
555507218a29SMatthew G. Knepley   PetscDS         dsScale[3] = {NULL, NULL, NULL};
555607218a29SMatthew G. Knepley   Vec             locS[3]    = {NULL, NULL, NULL};
55573e9753d6SMatthew G. Knepley   PetscSection    section    = NULL;
55583e9753d6SMatthew G. Knepley   DMField         coordField = NULL;
555907218a29SMatthew G. Knepley   PetscScalar    *a[3]       = {NULL, NULL, NULL};
556007218a29SMatthew G. Knepley   PetscScalar    *s[3]       = {NULL, NULL, NULL};
5561b2ab40e6SMatthew G. Knepley   PetscScalar    *u          = NULL, *u_t;
556207218a29SMatthew G. Knepley   PetscScalar    *elemVecNeg, *elemVecPos, *elemVecCoh;
5563989fa639SBrad Aagaard   IS              chunkISF, chunkISN;
55643e9753d6SMatthew G. Knepley   const PetscInt *cells;
5565989fa639SBrad Aagaard   PetscInt       *faces, *neighbors;
55663e9753d6SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
55673e2b0218SMatthew G. Knepley   PetscInt        Nf, f, totDim, totDimIn, totDimAux[3], totDimScale[3], numChunks, cellChunkSize, chunk;
55681690c2aeSBarry Smith   PetscInt        maxDegree   = PETSC_INT_MAX;
5569989fa639SBrad Aagaard   PetscQuadrature affineQuadF = NULL, *quadsF = NULL;
5570989fa639SBrad Aagaard   PetscFEGeom    *affineGeomF = NULL, **geomsF = NULL;
5571989fa639SBrad Aagaard   PetscQuadrature affineQuadN = NULL, *quadsN = NULL;
5572989fa639SBrad Aagaard   PetscFEGeom    *affineGeomN = NULL, **geomsN = NULL;
55733e9753d6SMatthew G. Knepley 
55743e9753d6SMatthew G. Knepley   PetscFunctionBegin;
55753cc88e6aSStefano Zampini   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0));
55763cc88e6aSStefano Zampini   if (!cellIS) goto end;
5577437e83fbSMatthew G. Knepley   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
5578437e83fbSMatthew G. Knepley   PetscCall(ISGetLocalSize(cellIS, &numCells));
55793cc88e6aSStefano Zampini   if (cStart >= cEnd) goto end;
55805fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
55815fedec97SMatthew G. Knepley     const char *name;
55829566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name));
558363a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
55845fedec97SMatthew G. Knepley   }
55853e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
55863e9753d6SMatthew G. Knepley   /* FEM */
55873e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
5588b98a7184SJames Wright   PetscCall(DMGetLocalSection(dm, &section));
55899566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
559007218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn));
55919566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
55929566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
559307218a29SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(dsIn, &totDimIn));
55949566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
559504c51a94SMatthew G. Knepley   if (locA[2]) {
55961059d808SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
55971059d808SMatthew G. Knepley 
55989566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
559907218a29SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2], NULL));
56009566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
56016528b96dSMatthew G. Knepley     {
56026528b96dSMatthew G. Knepley       const PetscInt *cone;
56036528b96dSMatthew G. Knepley       PetscInt        c;
56046528b96dSMatthew G. Knepley 
56051059d808SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
56066528b96dSMatthew G. Knepley       for (c = 0; c < 2; ++c) {
56076528b96dSMatthew G. Knepley         const PetscInt *support;
56086528b96dSMatthew G. Knepley         PetscInt        ssize, s;
56096528b96dSMatthew G. Knepley 
56109566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
56119566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
56121059d808SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
56131059d808SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
56141059d808SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
56151059d808SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
56169566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
5617c75bfeddSPierre Jolivet         PetscCheck(locA[c], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Must have auxiliary vector for (%p, %" PetscInt_FMT ", %" PetscInt_FMT ")", (void *)key[c].label, key[c].value, key[c].part);
56189566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
5619ad540459SPierre Jolivet         else dmAux[c] = dmAux[2];
562007218a29SMatthew G. Knepley         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c], NULL));
56219566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
56226528b96dSMatthew G. Knepley       }
56236528b96dSMatthew G. Knepley     }
56243e9753d6SMatthew G. Knepley   }
562507218a29SMatthew G. Knepley   /* Handle mass matrix scaling
562607218a29SMatthew G. Knepley        The field in key[2] is the field to be scaled, and the scaling field is the first in the dsScale */
562707218a29SMatthew G. Knepley   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, -key[2].value, key[2].part, &locS[2]));
562807218a29SMatthew G. Knepley   if (locS[2]) {
56293e2b0218SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
563007218a29SMatthew G. Knepley     PetscInt       Nb, Nbs;
563107218a29SMatthew G. Knepley 
563207218a29SMatthew G. Knepley     PetscCall(VecGetDM(locS[2], &dmScale[2]));
56333e2b0218SMatthew G. Knepley     PetscCall(DMGetCellDS(dmScale[2], cellStart, &dsScale[2], NULL));
56343e2b0218SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(dsScale[2], &totDimScale[2]));
563507218a29SMatthew G. Knepley     // BRAD: This is not set correctly
563607218a29SMatthew G. Knepley     key[2].field = 2;
563707218a29SMatthew G. Knepley     PetscCall(PetscDSGetFieldSize(ds, key[2].field, &Nb));
563807218a29SMatthew G. Knepley     PetscCall(PetscDSGetFieldSize(dsScale[2], 0, &Nbs));
563907218a29SMatthew G. Knepley     PetscCheck(Nb == Nbs, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Field %" PetscInt_FMT " of size %" PetscInt_FMT " cannot be scaled by field of size %" PetscInt_FMT, key[2].field, Nb, Nbs);
56403e2b0218SMatthew G. Knepley     {
56413e2b0218SMatthew G. Knepley       const PetscInt *cone;
56423e2b0218SMatthew G. Knepley       PetscInt        c;
56433e2b0218SMatthew G. Knepley 
56443e2b0218SMatthew G. Knepley       locS[1] = locS[0] = locS[2];
56453e2b0218SMatthew G. Knepley       dmScale[1] = dmScale[0] = dmScale[2];
56463e2b0218SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
56473e2b0218SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
56483e2b0218SMatthew G. Knepley         const PetscInt *support;
56493e2b0218SMatthew G. Knepley         PetscInt        ssize, s;
56503e2b0218SMatthew G. Knepley 
56513e2b0218SMatthew G. Knepley         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
56523e2b0218SMatthew G. Knepley         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
56533e2b0218SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
56543e2b0218SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
56553e2b0218SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
56563e2b0218SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
56573e2b0218SMatthew G. Knepley         PetscCall(DMGetCellDS(dmScale[c], support[s], &dsScale[c], NULL));
56583e2b0218SMatthew G. Knepley         PetscCall(PetscDSGetTotalDimension(dsScale[c], &totDimScale[c]));
56593e2b0218SMatthew G. Knepley       }
56603e2b0218SMatthew G. Knepley     }
566107218a29SMatthew G. Knepley   }
56623e9753d6SMatthew G. Knepley   /* 2: Setup geometric data */
56639566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
56649566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
56653e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
5666989fa639SBrad Aagaard     PetscCall(PetscCalloc4(Nf, &quadsF, Nf, &geomsF, Nf, &quadsN, Nf, &geomsN));
56673e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
56683e9753d6SMatthew G. Knepley       PetscFE   fe;
5669989fa639SBrad Aagaard       PetscBool isCohesiveField;
56703e9753d6SMatthew G. Knepley 
56719566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
56723e9753d6SMatthew G. Knepley       if (fe) {
5673989fa639SBrad Aagaard         PetscCall(PetscFEGetQuadrature(fe, &quadsF[f]));
5674989fa639SBrad Aagaard         PetscCall(PetscObjectReference((PetscObject)quadsF[f]));
5675989fa639SBrad Aagaard       }
5676989fa639SBrad Aagaard       PetscCall(PetscDSGetDiscretization(dsIn, f, (PetscObject *)&fe));
5677989fa639SBrad Aagaard       PetscCall(PetscDSGetCohesive(dsIn, f, &isCohesiveField));
5678989fa639SBrad Aagaard       if (fe) {
5679989fa639SBrad Aagaard         if (isCohesiveField) {
5680989fa639SBrad Aagaard           for (PetscInt g = 0; g < Nf; ++g) {
5681989fa639SBrad Aagaard             PetscCall(PetscDSGetDiscretization(dsIn, g, (PetscObject *)&fe));
5682989fa639SBrad Aagaard             PetscCall(PetscDSGetCohesive(dsIn, g, &isCohesiveField));
5683989fa639SBrad Aagaard             if (!isCohesiveField) break;
5684989fa639SBrad Aagaard           }
5685989fa639SBrad Aagaard         }
5686989fa639SBrad Aagaard         PetscCall(PetscFEGetQuadrature(fe, &quadsN[f]));
5687989fa639SBrad Aagaard         PetscCall(PetscObjectReference((PetscObject)quadsN[f]));
56883e9753d6SMatthew G. Knepley       }
56893e9753d6SMatthew G. Knepley     }
56903e9753d6SMatthew G. Knepley   }
56913e9753d6SMatthew G. Knepley   /* Loop over chunks */
56923e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
56933e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize);
5694989fa639SBrad Aagaard   PetscCall(PetscCalloc2(2 * cellChunkSize, &faces, 2 * cellChunkSize, &neighbors));
5695989fa639SBrad Aagaard   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkISF));
5696989fa639SBrad Aagaard   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, neighbors, PETSC_USE_POINTER, &chunkISN));
56973e9753d6SMatthew G. Knepley   /* Extract field coefficients */
56983e9753d6SMatthew G. Knepley   /* NOTE This needs the end cap faces to have identical orientations */
569907218a29SMatthew G. Knepley   PetscCall(DMPlexGetHybridCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
570007218a29SMatthew G. Knepley   PetscCall(DMPlexGetHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a));
57013e2b0218SMatthew G. Knepley   PetscCall(DMPlexGetHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s));
570207218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecNeg));
570307218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecPos));
570407218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecCoh));
57053e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
57063e9753d6SMatthew G. Knepley     PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
57073e9753d6SMatthew G. Knepley 
570807218a29SMatthew G. Knepley     PetscCall(PetscArrayzero(elemVecNeg, cellChunkSize * totDim));
570907218a29SMatthew G. Knepley     PetscCall(PetscArrayzero(elemVecPos, cellChunkSize * totDim));
571007218a29SMatthew G. Knepley     PetscCall(PetscArrayzero(elemVecCoh, cellChunkSize * totDim));
5711989fa639SBrad Aagaard     /* Get faces and neighbors */
57123e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
57133e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
5714989fa639SBrad Aagaard       const PetscInt *cone, *support;
57159566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(dm, cell, &cone));
571607218a29SMatthew G. Knepley       faces[(c - cS) * 2 + 0] = cone[0];
571707218a29SMatthew G. Knepley       faces[(c - cS) * 2 + 1] = cone[1];
5718989fa639SBrad Aagaard       PetscCall(DMPlexGetSupport(dm, cone[0], &support));
5719989fa639SBrad Aagaard       neighbors[(c - cS) * 2 + 0] = support[0] == cell ? support[1] : support[0];
5720989fa639SBrad Aagaard       PetscCall(DMPlexGetSupport(dm, cone[1], &support));
5721989fa639SBrad Aagaard       neighbors[(c - cS) * 2 + 1] = support[0] == cell ? support[1] : support[0];
57223e9753d6SMatthew G. Knepley     }
5723989fa639SBrad Aagaard     PetscCall(ISGeneralSetIndices(chunkISF, 2 * cellChunkSize, faces, PETSC_USE_POINTER));
5724989fa639SBrad Aagaard     PetscCall(ISGeneralSetIndices(chunkISN, 2 * cellChunkSize, neighbors, PETSC_USE_POINTER));
57253e9753d6SMatthew G. Knepley     /* Get geometric data */
57263e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
5727989fa639SBrad Aagaard       if (!affineQuadF) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkISF, &affineQuadF));
5728989fa639SBrad Aagaard       if (affineQuadF) PetscCall(DMSNESGetFEGeom(coordField, chunkISF, affineQuadF, PETSC_FEGEOM_COHESIVE, &affineGeomF));
5729989fa639SBrad Aagaard       if (!affineQuadN) {
5730989fa639SBrad Aagaard         PetscInt dim;
5731989fa639SBrad Aagaard         PetscCall(PetscQuadratureGetData(affineQuadF, &dim, NULL, NULL, NULL, NULL));
5732989fa639SBrad Aagaard         PetscCall(DMFieldCreateDefaultFaceQuadrature(coordField, chunkISN, &affineQuadN));
5733989fa639SBrad Aagaard         PetscCall(PetscQuadratureSetData(affineQuadN, dim + 1, PETSC_DECIDE, PETSC_DECIDE, NULL, NULL));
5734989fa639SBrad Aagaard       }
5735989fa639SBrad Aagaard       if (affineQuadN) PetscCall(DMSNESGetFEGeom(coordField, chunkISN, affineQuadN, PETSC_FEGEOM_BASIC, &affineGeomN));
57363e9753d6SMatthew G. Knepley     } else {
57373e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
5738989fa639SBrad Aagaard         if (quadsF[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkISF, quadsF[f], PETSC_FEGEOM_COHESIVE, &geomsF[f]));
5739989fa639SBrad Aagaard         if (quadsN[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkISN, quadsN[f], PETSC_FEGEOM_BASIC, &geomsN[f]));
57403e9753d6SMatthew G. Knepley       }
57413e9753d6SMatthew G. Knepley     }
57423e9753d6SMatthew G. Knepley     /* Loop over fields */
57433e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
57443e9753d6SMatthew G. Knepley       PetscFE         fe;
5745989fa639SBrad Aagaard       PetscFEGeom    *geomF      = affineGeomF ? affineGeomF : geomsF[f];
5746989fa639SBrad Aagaard       PetscFEGeom    *chunkGeomF = NULL, *remGeomF = NULL;
5747989fa639SBrad Aagaard       PetscFEGeom    *geomN      = affineGeomN ? affineGeomN : geomsN[f];
5748989fa639SBrad Aagaard       PetscFEGeom    *chunkGeomN = NULL, *remGeomN = NULL;
5749989fa639SBrad Aagaard       PetscQuadrature quadF = affineQuadF ? affineQuadF : quadsF[f];
57503e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
57515fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
57523e9753d6SMatthew G. Knepley 
57539566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
57543e9753d6SMatthew G. Knepley       if (!fe) continue;
57559566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
5756989fa639SBrad Aagaard       PetscCall(PetscQuadratureGetData(quadF, NULL, NULL, &Nq, NULL, NULL));
57579566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
57583e9753d6SMatthew G. Knepley       blockSize = Nb;
57593e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
57609566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
57613e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches * batchSize);
57623e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
57633e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches * batchSize);
57643e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
5765989fa639SBrad Aagaard       PetscCall(PetscFEGeomGetChunk(geomF, 0, offset * 2, &chunkGeomF));
5766989fa639SBrad Aagaard       PetscCall(PetscFEGeomGetChunk(geomF, offset * 2, numCells * 2, &remGeomF));
5767989fa639SBrad Aagaard       PetscCall(PetscFEGeomGetChunk(geomN, 0, offset * 2, &chunkGeomN));
5768989fa639SBrad Aagaard       PetscCall(PetscFEGeomGetChunk(geomN, offset * 2, numCells * 2, &remGeomN));
57699566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, f, &isCohesiveField));
5770989fa639SBrad Aagaard       // TODO Do I need to set isCohesive on the chunks?
57716528b96dSMatthew G. Knepley       key[0].field = f;
57726528b96dSMatthew G. Knepley       key[1].field = f;
57735fedec97SMatthew G. Knepley       key[2].field = f;
5774989fa639SBrad Aagaard       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[0], 0, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[0], a[0], t, elemVecNeg));
5775989fa639SBrad Aagaard       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[0], 0, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[0], PetscSafePointerPlusOffset(a[0], offset * totDimAux[0]), t, &elemVecNeg[offset * totDim]));
5776989fa639SBrad Aagaard       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[1], 1, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[1], a[1], t, elemVecPos));
5777989fa639SBrad Aagaard       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[1], 1, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[1], PetscSafePointerPlusOffset(a[1], offset * totDimAux[1]), t, &elemVecPos[offset * totDim]));
5778989fa639SBrad Aagaard       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[2], 2, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[2], a[2], t, elemVecCoh));
5779989fa639SBrad Aagaard       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[2], 2, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[2], PetscSafePointerPlusOffset(a[2], offset * totDimAux[2]), t, &elemVecCoh[offset * totDim]));
5780989fa639SBrad Aagaard       PetscCall(PetscFEGeomRestoreChunk(geomF, offset, numCells, &remGeomF));
5781989fa639SBrad Aagaard       PetscCall(PetscFEGeomRestoreChunk(geomF, 0, offset, &chunkGeomF));
5782989fa639SBrad Aagaard       PetscCall(PetscFEGeomRestoreChunk(geomN, offset, numCells, &remGeomN));
5783989fa639SBrad Aagaard       PetscCall(PetscFEGeomRestoreChunk(geomN, 0, offset, &chunkGeomN));
57843e9753d6SMatthew G. Knepley     }
57853e9753d6SMatthew G. Knepley     /* Add elemVec to locX */
57863e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
57873e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
57883e9753d6SMatthew G. Knepley       const PetscInt cind = c - cStart;
578907218a29SMatthew G. Knepley       PetscInt       i;
57903e9753d6SMatthew G. Knepley 
579107218a29SMatthew G. Knepley       /* Scale element values */
579207218a29SMatthew G. Knepley       if (locS[0]) {
57933e2b0218SMatthew G. Knepley         PetscInt  Nb, off = cind * totDim, soff = cind * totDimScale[0];
579407218a29SMatthew G. Knepley         PetscBool cohesive;
579507218a29SMatthew G. Knepley 
579607218a29SMatthew G. Knepley         for (f = 0; f < Nf; ++f) {
579707218a29SMatthew G. Knepley           PetscCall(PetscDSGetFieldSize(ds, f, &Nb));
579807218a29SMatthew G. Knepley           PetscCall(PetscDSGetCohesive(ds, f, &cohesive));
579907218a29SMatthew G. Knepley           if (f == key[2].field) {
580007218a29SMatthew G. Knepley             PetscCheck(cohesive, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Scaling should not happen for face fields");
580107218a29SMatthew G. Knepley             // No cohesive scaling field is currently input
580207218a29SMatthew G. Knepley             for (i = 0; i < Nb; ++i) elemVecCoh[off + i] += s[0][soff + i] * elemVecNeg[off + i] + s[1][soff + i] * elemVecPos[off + i];
580307218a29SMatthew G. Knepley             off += Nb;
580407218a29SMatthew G. Knepley           } else {
580507218a29SMatthew G. Knepley             const PetscInt N = cohesive ? Nb : Nb * 2;
580607218a29SMatthew G. Knepley 
580707218a29SMatthew G. Knepley             for (i = 0; i < N; ++i) elemVecCoh[off + i] += elemVecNeg[off + i] + elemVecPos[off + i];
580807218a29SMatthew G. Knepley             off += N;
580907218a29SMatthew G. Knepley           }
581007218a29SMatthew G. Knepley         }
581107218a29SMatthew G. Knepley       } else {
581207218a29SMatthew G. Knepley         for (i = cind * totDim; i < (cind + 1) * totDim; ++i) elemVecCoh[i] += elemVecNeg[i] + elemVecPos[i];
581307218a29SMatthew G. Knepley       }
581407218a29SMatthew G. Knepley       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVecCoh[cind * totDim]));
58153e9753d6SMatthew G. Knepley       if (ghostLabel) {
58163e9753d6SMatthew G. Knepley         PetscInt ghostVal;
58173e9753d6SMatthew G. Knepley 
58189566063dSJacob Faibussowitsch         PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
58193e9753d6SMatthew G. Knepley         if (ghostVal > 0) continue;
58203e9753d6SMatthew G. Knepley       }
582107218a29SMatthew G. Knepley       PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVecCoh[cind * totDim], ADD_ALL_VALUES));
58223e9753d6SMatthew G. Knepley     }
58233e9753d6SMatthew G. Knepley   }
58249566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
582507218a29SMatthew G. Knepley   PetscCall(DMPlexRestoreHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a));
58263e2b0218SMatthew G. Knepley   PetscCall(DMPlexRestoreHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s));
582707218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecNeg));
582807218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecPos));
582907218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecCoh));
5830989fa639SBrad Aagaard   PetscCall(PetscFree2(faces, neighbors));
5831989fa639SBrad Aagaard   PetscCall(ISDestroy(&chunkISF));
5832989fa639SBrad Aagaard   PetscCall(ISDestroy(&chunkISN));
58339566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
58343e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
5835989fa639SBrad Aagaard     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuadF, PETSC_FALSE, &affineGeomF));
5836989fa639SBrad Aagaard     PetscCall(PetscQuadratureDestroy(&affineQuadF));
5837989fa639SBrad Aagaard     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuadN, PETSC_FALSE, &affineGeomN));
5838989fa639SBrad Aagaard     PetscCall(PetscQuadratureDestroy(&affineQuadN));
58393e9753d6SMatthew G. Knepley   } else {
58403e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
5841989fa639SBrad Aagaard       if (geomsF) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quadsF[f], PETSC_FALSE, &geomsF[f]));
5842989fa639SBrad Aagaard       if (quadsF) PetscCall(PetscQuadratureDestroy(&quadsF[f]));
5843989fa639SBrad Aagaard       if (geomsN) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quadsN[f], PETSC_FALSE, &geomsN[f]));
5844989fa639SBrad Aagaard       if (quadsN) PetscCall(PetscQuadratureDestroy(&quadsN[f]));
58453e9753d6SMatthew G. Knepley     }
5846989fa639SBrad Aagaard     PetscCall(PetscFree4(quadsF, geomsF, quadsN, geomsN));
58473e9753d6SMatthew G. Knepley   }
584885cc2951SMatthew G. Knepley   if (mesh->printFEM) {
584985cc2951SMatthew G. Knepley     Vec          locFbc;
585085cc2951SMatthew G. Knepley     PetscInt     pStart, pEnd, p, maxDof;
585185cc2951SMatthew G. Knepley     PetscScalar *zeroes;
585285cc2951SMatthew G. Knepley 
585385cc2951SMatthew G. Knepley     PetscCall(VecDuplicate(locF, &locFbc));
585485cc2951SMatthew G. Knepley     PetscCall(VecCopy(locF, locFbc));
585585cc2951SMatthew G. Knepley     PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
585685cc2951SMatthew G. Knepley     PetscCall(PetscSectionGetMaxDof(section, &maxDof));
585785cc2951SMatthew G. Knepley     PetscCall(PetscCalloc1(maxDof, &zeroes));
585885cc2951SMatthew G. Knepley     for (p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, section, p, zeroes, INSERT_BC_VALUES));
585985cc2951SMatthew G. Knepley     PetscCall(PetscFree(zeroes));
586085cc2951SMatthew G. Knepley     PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc));
586185cc2951SMatthew G. Knepley     PetscCall(VecDestroy(&locFbc));
586285cc2951SMatthew G. Knepley   }
58633cc88e6aSStefano Zampini end:
58649566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0));
58653ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
58663e9753d6SMatthew G. Knepley }
58673e9753d6SMatthew G. Knepley 
5868997bf629SMatthew G. Knepley /*@
5869997bf629SMatthew G. Knepley   DMPlexComputeBdJacobianSingleByLabel - Compute the local boundary Jacobian for terms matching the input label
5870997bf629SMatthew G. Knepley 
5871997bf629SMatthew G. Knepley   Not collective
5872997bf629SMatthew G. Knepley 
5873997bf629SMatthew G. Knepley   Input Parameters:
5874997bf629SMatthew G. Knepley + dm         - The output `DM`
5875997bf629SMatthew G. Knepley . wf         - The `PetscWeakForm` holding forms on this boundary
5876997bf629SMatthew G. Knepley . label      - The `DMLabel` indicating what faces should be integrated over
5877997bf629SMatthew G. Knepley . numValues  - The number of label values
5878997bf629SMatthew G. Knepley . values     - The array of label values
5879997bf629SMatthew G. Knepley . fieldI     - The test field for these integrals
5880997bf629SMatthew G. Knepley . facetIS    - The `IS` giving the set of possible faces to integrate over (intersected with the label)
5881997bf629SMatthew G. Knepley . locX       - The local solution
5882997bf629SMatthew G. Knepley . locX_t     - The time derivative of the local solution, or `NULL` for time-independent problems
5883997bf629SMatthew G. Knepley . t          - The time
5884997bf629SMatthew G. Knepley . coordField - The `DMField` object with coordinates for these faces
5885997bf629SMatthew G. Knepley - X_tShift   - The multiplier for dF/dxdot
5886997bf629SMatthew G. Knepley 
5887997bf629SMatthew G. Knepley   Output Parameters:
5888997bf629SMatthew G. Knepley + Jac  - The local Jacobian
5889997bf629SMatthew G. Knepley - JacP - The local Jacobian preconditioner
5890997bf629SMatthew G. Knepley 
5891997bf629SMatthew G. Knepley   Level: developer
5892997bf629SMatthew G. Knepley 
5893997bf629SMatthew G. Knepley .seealso: `DMPlexComputeBdJacobianSingle()`, `DMPlexComputeJacobianByKey()`, `DMPlexComputeResidualHybridByKey()`, `DMPlexComputeJacobianHybridByKey()`, `PetscFormKey`
5894997bf629SMatthew G. Knepley @*/
5895997bf629SMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobianSingleByLabel(DM dm, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, IS facetIS, Vec locX, Vec locX_t, PetscReal t, DMField coordField, PetscReal X_tShift, Mat Jac, Mat JacP)
5896d71ae5a4SJacob Faibussowitsch {
58973e9753d6SMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *)dm->data;
58983e9753d6SMatthew G. Knepley   DM              plex = NULL, plexA = NULL, tdm;
58993e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
5900be62b2b4SMatthew G. Knepley   PetscDS         ds, dsAux           = NULL;
59013e9753d6SMatthew G. Knepley   PetscSection    section, sectionAux = NULL;
5902e432b41dSStefano Zampini   PetscSection    globalSection;
59033e9753d6SMatthew G. Knepley   Vec             locA = NULL, tv;
5904be62b2b4SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL;
59053e9753d6SMatthew G. Knepley   PetscInt        v;
59063e9753d6SMatthew G. Knepley   PetscInt        Nf, totDim, totDimAux = 0;
5907be62b2b4SMatthew G. Knepley   PetscBool       hasJac = PETSC_FALSE, hasPrec = PETSC_FALSE, transform;
59083e9753d6SMatthew G. Knepley 
59093e9753d6SMatthew G. Knepley   PetscFunctionBegin;
59109566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
59119566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
59129566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
59139566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
5914be62b2b4SMatthew G. Knepley   PetscCall(DMGetDS(dm, &ds));
5915be62b2b4SMatthew G. Knepley   PetscCall(PetscDSGetNumFields(ds, &Nf));
5916be62b2b4SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
591721a4fad8SMatthew G. Knepley   PetscCall(PetscWeakFormHasBdJacobian(wf, &hasJac));
591821a4fad8SMatthew G. Knepley   PetscCall(PetscWeakFormHasBdJacobianPreconditioner(wf, &hasPrec));
591921a4fad8SMatthew G. Knepley   if (!hasJac && !hasPrec) PetscFunctionReturn(PETSC_SUCCESS);
592021a4fad8SMatthew G. Knepley   PetscCall(DMConvert(dm, DMPLEX, &plex));
59219566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, label, values[0], 0, &locA));
59223e9753d6SMatthew G. Knepley   if (locA) {
59233e9753d6SMatthew G. Knepley     DM dmAux;
59243e9753d6SMatthew G. Knepley 
59259566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
59269566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
59279566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
5928be62b2b4SMatthew G. Knepley     PetscCall(DMGetDS(plexA, &dsAux));
5929be62b2b4SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux));
59309566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
59313e9753d6SMatthew G. Knepley   }
59323e9753d6SMatthew G. Knepley 
59339566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
59343e9753d6SMatthew G. Knepley   for (v = 0; v < numValues; ++v) {
59353e9753d6SMatthew G. Knepley     PetscFEGeom    *fgeom;
59363e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
59373e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
59383e9753d6SMatthew G. Knepley     IS              pointIS;
59393e9753d6SMatthew G. Knepley     const PetscInt *points;
594006ad1575SMatthew G. Knepley     PetscFormKey    key;
59413e9753d6SMatthew G. Knepley     PetscInt        numFaces, face, Nq;
59423e9753d6SMatthew G. Knepley 
594345480ffeSMatthew G. Knepley     key.label = label;
594445480ffeSMatthew G. Knepley     key.value = values[v];
594506ad1575SMatthew G. Knepley     key.part  = 0;
59469566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
59473e9753d6SMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
59483e9753d6SMatthew G. Knepley     {
59493e9753d6SMatthew G. Knepley       IS isectIS;
59503e9753d6SMatthew G. Knepley 
59513e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a prior if one is a superset of the other */
59529566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
59539566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
59543e9753d6SMatthew G. Knepley       pointIS = isectIS;
59553e9753d6SMatthew G. Knepley     }
59569566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
59579566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
595832603206SJames Wright     PetscCall(PetscMalloc5(numFaces * totDim, &u, (locX_t ? (size_t)numFaces * totDim : 0), &u_t, (hasJac ? (size_t)numFaces * totDim * totDim : 0), &elemMat, (hasPrec ? (size_t)numFaces * totDim * totDim : 0), &elemMatP, (locA ? (size_t)numFaces * totDimAux : 0), &a));
59599566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
596048a46eb9SPierre Jolivet     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
59613e9753d6SMatthew G. Knepley     if (!qGeom) {
59623e9753d6SMatthew G. Knepley       PetscFE fe;
59633e9753d6SMatthew G. Knepley 
5964be62b2b4SMatthew G. Knepley       PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&fe));
59659566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
59669566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
59673e9753d6SMatthew G. Knepley     }
59689566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
5969ac9d17c7SMatthew G. Knepley     PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_FEGEOM_BOUNDARY, &fgeom));
59703e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
5971f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
59723e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
5973f15274beSMatthew Knepley       PetscInt       i;
59743e9753d6SMatthew G. Knepley 
59759566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
59769566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
59773e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i];
59789566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
59793e9753d6SMatthew G. Knepley       if (locX_t) {
59809566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
59813e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i];
59829566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
59833e9753d6SMatthew G. Knepley       }
59843e9753d6SMatthew G. Knepley       if (locA) {
59853e9753d6SMatthew G. Knepley         PetscInt subp;
59869566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
59879566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
59883e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i];
59899566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
59903e9753d6SMatthew G. Knepley       }
59913e9753d6SMatthew G. Knepley     }
599221a4fad8SMatthew G. Knepley     if (elemMat) PetscCall(PetscArrayzero(elemMat, numFaces * totDim * totDim));
599321a4fad8SMatthew G. Knepley     if (elemMatP) PetscCall(PetscArrayzero(elemMatP, numFaces * totDim * totDim));
59943e9753d6SMatthew G. Knepley     {
59953e9753d6SMatthew G. Knepley       PetscFE  fe;
59963e9753d6SMatthew G. Knepley       PetscInt Nb;
59973e9753d6SMatthew G. Knepley       /* Conforming batches */
59983e9753d6SMatthew G. Knepley       PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
59993e9753d6SMatthew G. Knepley       /* Remainder */
60003e9753d6SMatthew G. Knepley       PetscFEGeom *chunkGeom = NULL;
60013e9753d6SMatthew G. Knepley       PetscInt     fieldJ, Nr, offset;
60023e9753d6SMatthew G. Knepley 
6003be62b2b4SMatthew G. Knepley       PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&fe));
60049566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
60059566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
60063e9753d6SMatthew G. Knepley       blockSize = Nb;
60073e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
60089566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
60093e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches * batchSize);
60103e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
60113e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches * batchSize);
60123e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
60139566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom));
60143e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
601545480ffeSMatthew G. Knepley         key.field = fieldI * Nf + fieldJ;
6016be62b2b4SMatthew G. Knepley         if (hasJac) PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, dsAux, a, t, X_tShift, elemMat));
6017be62b2b4SMatthew G. Knepley         if (hasPrec) PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, dsAux, a, t, X_tShift, elemMatP));
60183e9753d6SMatthew G. Knepley       }
60199566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
60203e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
602145480ffeSMatthew G. Knepley         key.field = fieldI * Nf + fieldJ;
6022be62b2b4SMatthew G. Knepley         if (hasJac)
60238e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * totDim]));
6024be62b2b4SMatthew G. Knepley         if (hasPrec)
60258e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN_PRE, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatP[offset * totDim * totDim]));
60263e9753d6SMatthew G. Knepley       }
60279566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
60283e9753d6SMatthew G. Knepley     }
60293e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
60303e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
60313e9753d6SMatthew G. Knepley 
60323e9753d6SMatthew G. Knepley       /* Transform to global basis before insertion in Jacobian */
60339566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
603421a4fad8SMatthew G. Knepley       if (hasJac && transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMat[face * totDim * totDim]));
603521a4fad8SMatthew G. Knepley       if (hasPrec && transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMatP[face * totDim * totDim]));
6036be62b2b4SMatthew G. Knepley       if (hasPrec) {
6037be62b2b4SMatthew G. Knepley         if (hasJac) {
60389566063dSJacob Faibussowitsch           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face * totDim * totDim]));
603921a4fad8SMatthew G. Knepley           PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, Jac, support[0], &elemMat[face * totDim * totDim], ADD_VALUES));
60403e9753d6SMatthew G. Knepley         }
6041be62b2b4SMatthew G. Knepley         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMatP[face * totDim * totDim]));
6042be62b2b4SMatthew G. Knepley         PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, JacP, support[0], &elemMatP[face * totDim * totDim], ADD_VALUES));
6043be62b2b4SMatthew G. Knepley       } else {
6044be62b2b4SMatthew G. Knepley         if (hasJac) {
6045be62b2b4SMatthew G. Knepley           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face * totDim * totDim]));
604621a4fad8SMatthew G. Knepley           PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, Jac, support[0], &elemMat[face * totDim * totDim], ADD_VALUES));
6047be62b2b4SMatthew G. Knepley         }
6048be62b2b4SMatthew G. Knepley       }
6049be62b2b4SMatthew G. Knepley     }
60509566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
60519566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
60529566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
60539566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
6054be62b2b4SMatthew G. Knepley     PetscCall(PetscFree5(u, u_t, elemMat, elemMatP, a));
60553e9753d6SMatthew G. Knepley   }
60569566063dSJacob Faibussowitsch   if (plex) PetscCall(DMDestroy(&plex));
60579566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
60583ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
60593e9753d6SMatthew G. Knepley }
60603e9753d6SMatthew G. Knepley 
6061997bf629SMatthew G. Knepley /*@
6062997bf629SMatthew G. Knepley   DMPlexComputeBdJacobianSingle - Compute the local boundary Jacobian
6063997bf629SMatthew G. Knepley 
6064997bf629SMatthew G. Knepley   Not collective
6065997bf629SMatthew G. Knepley 
6066997bf629SMatthew G. Knepley   Input Parameters:
6067997bf629SMatthew G. Knepley + dm        - The output `DM`
6068997bf629SMatthew G. Knepley . wf        - The `PetscWeakForm` holding forms on this boundary
6069997bf629SMatthew G. Knepley . label     - The `DMLabel` indicating what faces should be integrated over
6070997bf629SMatthew G. Knepley . numValues - The number of label values
6071997bf629SMatthew G. Knepley . values    - The array of label values
6072997bf629SMatthew G. Knepley . fieldI    - The test field for these integrals
6073997bf629SMatthew G. Knepley . locX      - The local solution
6074997bf629SMatthew G. Knepley . locX_t    - The time derivative of the local solution, or `NULL` for time-independent problems
6075997bf629SMatthew G. Knepley . t         - The time
6076997bf629SMatthew G. Knepley - X_tShift  - The multiplier for dF/dxdot
6077997bf629SMatthew G. Knepley 
6078997bf629SMatthew G. Knepley   Output Parameters:
6079997bf629SMatthew G. Knepley + Jac  - The local Jacobian
6080997bf629SMatthew G. Knepley - JacP - The local Jacobian preconditioner
6081997bf629SMatthew G. Knepley 
6082997bf629SMatthew G. Knepley   Level: developer
6083997bf629SMatthew G. Knepley 
6084997bf629SMatthew G. Knepley .seealso: `DMPlexComputeBdJacobianSingleByLabel()`, `DMPlexComputeJacobianByKey()`, `DMPlexComputeResidualHybridByKey()`, `DMPlexComputeJacobianHybridByKey()`, `PetscFormKey`
6085997bf629SMatthew G. Knepley @*/
6086997bf629SMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobianSingle(DM dm, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP)
6087d71ae5a4SJacob Faibussowitsch {
60883e9753d6SMatthew G. Knepley   DMField  coordField;
60893e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
60903e9753d6SMatthew G. Knepley   IS       facetIS;
60913e9753d6SMatthew G. Knepley   PetscInt dim;
60923e9753d6SMatthew G. Knepley 
60933e9753d6SMatthew G. Knepley   PetscFunctionBegin;
60949566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
60959566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
60969566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
60979566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
6098997bf629SMatthew G. Knepley   PetscCall(DMPlexComputeBdJacobianSingleByLabel(dm, wf, label, numValues, values, fieldI, facetIS, locX, locX_t, t, coordField, X_tShift, Jac, JacP));
60999566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
61003ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
61013e9753d6SMatthew G. Knepley }
61023e9753d6SMatthew G. Knepley 
6103a4e35b19SJacob Faibussowitsch static PetscErrorCode DMPlexComputeBdJacobian_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP, void *user)
6104d71ae5a4SJacob Faibussowitsch {
61053e9753d6SMatthew G. Knepley   PetscDS  prob;
61063e9753d6SMatthew G. Knepley   PetscInt dim, numBd, bd;
61073e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
61083e9753d6SMatthew G. Knepley   DMField  coordField = NULL;
61093e9753d6SMatthew G. Knepley   IS       facetIS;
61103e9753d6SMatthew G. Knepley 
61113e9753d6SMatthew G. Knepley   PetscFunctionBegin;
61129566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
61139566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
61149566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
61159566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
61169566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
61179566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
61183e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
611945480ffeSMatthew G. Knepley     PetscWeakForm           wf;
61203e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
61213e9753d6SMatthew G. Knepley     DMLabel                 label;
61223e9753d6SMatthew G. Knepley     const PetscInt         *values;
61233e9753d6SMatthew G. Knepley     PetscInt                fieldI, numValues;
61243e9753d6SMatthew G. Knepley     PetscObject             obj;
61253e9753d6SMatthew G. Knepley     PetscClassId            id;
61263e9753d6SMatthew G. Knepley 
61279566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &fieldI, NULL, NULL, NULL, NULL, NULL));
61283d3e5d66SMatthew G. Knepley     if (type & DM_BC_ESSENTIAL) continue;
61299566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &obj));
61309566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
61313d3e5d66SMatthew G. Knepley     if (id != PETSCFE_CLASSID) continue;
6132997bf629SMatthew G. Knepley     PetscCall(DMPlexComputeBdJacobianSingleByLabel(dm, wf, label, numValues, values, fieldI, facetIS, locX, locX_t, t, coordField, X_tShift, Jac, JacP));
61333e9753d6SMatthew G. Knepley   }
61349566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
61353ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
61363e9753d6SMatthew G. Knepley }
61373e9753d6SMatthew G. Knepley 
6138754e4fbaSMatthew G. Knepley /*@
6139754e4fbaSMatthew G. Knepley   DMPlexComputeJacobianByKey - Compute the local Jacobian for terms matching the input key
6140754e4fbaSMatthew G. Knepley 
6141754e4fbaSMatthew G. Knepley   Collective
6142754e4fbaSMatthew G. Knepley 
6143754e4fbaSMatthew G. Knepley   Input Parameters:
6144754e4fbaSMatthew G. Knepley + dm       - The output `DM`
6145754e4fbaSMatthew G. Knepley . key      - The `PetscFormKey` indicating what should be integrated
6146754e4fbaSMatthew G. Knepley . cellIS   - The `IS` give a set of cells to integrate over
6147754e4fbaSMatthew G. Knepley . t        - The time
6148754e4fbaSMatthew G. Knepley . X_tShift - The multiplier for the Jacobian with respect to $X_t$
6149754e4fbaSMatthew G. Knepley . locX     - The local solution
6150754e4fbaSMatthew G. Knepley . locX_t   - The time derivative of the local solution, or `NULL` for time-independent problems
6151754e4fbaSMatthew G. Knepley - user     - An optional user context, passed to the pointwise functions
6152754e4fbaSMatthew G. Knepley 
6153754e4fbaSMatthew G. Knepley   Output Parameters:
6154754e4fbaSMatthew G. Knepley + Jac  - The local Jacobian
6155754e4fbaSMatthew G. Knepley - JacP - The local Jacobian preconditioner
6156754e4fbaSMatthew G. Knepley 
6157754e4fbaSMatthew G. Knepley   Level: developer
6158754e4fbaSMatthew G. Knepley 
6159754e4fbaSMatthew G. Knepley .seealso: `DMPlexComputeResidualByKey()`, `DMPlexComputeResidualHybridByKey()`, `DMPlexComputeJacobianHybridByKey()`, `PetscFormKey`
6160754e4fbaSMatthew G. Knepley @*/
6161754e4fbaSMatthew G. Knepley PetscErrorCode DMPlexComputeJacobianByKey(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user)
6162d71ae5a4SJacob Faibussowitsch {
61633e9753d6SMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *)dm->data;
61643e9753d6SMatthew G. Knepley   const char     *name  = "Jacobian";
61659a2a23afSMatthew G. Knepley   DM              dmAux = NULL, plex, tdm;
61663e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
61673e9753d6SMatthew G. Knepley   Vec             A, tv;
61683e9753d6SMatthew G. Knepley   DMField         coordField;
61693e9753d6SMatthew G. Knepley   PetscDS         prob, probAux = NULL;
6170e432b41dSStefano Zampini   PetscSection    section, globalSection, sectionAux;
61713e9753d6SMatthew G. Knepley   PetscScalar    *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL;
61723e9753d6SMatthew G. Knepley   const PetscInt *cells;
61733e9753d6SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
617428351e22SJed Brown   PetscInt        totDim, totDimAux = 0, cStart, cEnd, numCells, c;
6175e04ae0b4SMatthew G. Knepley   PetscBool       hasJac = PETSC_FALSE, hasPrec = PETSC_FALSE, hasDyn, hasFV = PETSC_FALSE, transform;
61763e9753d6SMatthew G. Knepley 
61773e9753d6SMatthew G. Knepley   PetscFunctionBegin;
6178af18b51aSmarkadams4   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
6179ae3cf2c1SStefano Zampini   PetscCall(DMGetLocalSection(dm, &section));
6180ae3cf2c1SStefano Zampini   PetscCall(DMGetGlobalSection(dm, &globalSection));
6181ae3cf2c1SStefano Zampini   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
6182ae3cf2c1SStefano Zampini   if (A) {
6183ae3cf2c1SStefano Zampini     PetscCall(VecGetDM(A, &dmAux));
6184ae3cf2c1SStefano Zampini     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
6185ae3cf2c1SStefano Zampini     PetscCall(DMConvert(dmAux, DMPLEX, &plex));
6186ae3cf2c1SStefano Zampini     PetscCall(DMGetLocalSection(plex, &sectionAux));
6187ae3cf2c1SStefano Zampini     PetscCall(DMGetDS(dmAux, &probAux));
6188ae3cf2c1SStefano Zampini     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
6189ae3cf2c1SStefano Zampini   }
6190ae3cf2c1SStefano Zampini   PetscCall(DMGetCoordinateField(dm, &coordField));
6191e04ae0b4SMatthew G. Knepley   if (!cellIS) goto end;
61929566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
6193437e83fbSMatthew G. Knepley   PetscCall(ISGetLocalSize(cellIS, &numCells));
6194437e83fbSMatthew G. Knepley   if (cStart >= cEnd) goto end;
61959566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
61969566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
61979566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
619807218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL));
61999566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
62009566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
62019566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
62029566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
62033e9753d6SMatthew G. Knepley   /* user passed in the same matrix, avoid double contributions and
62043e9753d6SMatthew G. Knepley      only assemble the Jacobian */
62053e9753d6SMatthew G. Knepley   if (hasJac && Jac == JacP) hasPrec = PETSC_FALSE;
62069566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
62073e9753d6SMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
6208754e4fbaSMatthew G. Knepley   PetscCall(PetscMalloc5(numCells * totDim, &u, (locX_t ? (size_t)numCells * totDim : 0), &u_t, (hasJac ? (size_t)numCells * totDim * totDim : 0), &elemMat, (hasPrec ? (size_t)numCells * totDim * totDim : 0), &elemMatP, (hasDyn ? (size_t)numCells * totDim * totDim : 0), &elemMatD));
62099566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
62103e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
62113e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
62123e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
62133e9753d6SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL;
62143e9753d6SMatthew G. Knepley     PetscInt       i;
62153e9753d6SMatthew G. Knepley 
6216754e4fbaSMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(dm, section, locX, cell, NULL, &x));
62173e9753d6SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i];
6218754e4fbaSMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cell, NULL, &x));
6219754e4fbaSMatthew G. Knepley     if (locX_t) {
6220754e4fbaSMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(dm, section, locX_t, cell, NULL, &x_t));
62213e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i];
6222754e4fbaSMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(dm, section, locX_t, cell, NULL, &x_t));
62233e9753d6SMatthew G. Knepley     }
62243e9753d6SMatthew G. Knepley     if (dmAux) {
62253e9753d6SMatthew G. Knepley       PetscInt subcell;
62269566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
62279566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x));
62283e9753d6SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i];
62299566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x));
62303e9753d6SMatthew G. Knepley     }
62313e9753d6SMatthew G. Knepley   }
62329566063dSJacob Faibussowitsch   if (hasJac) PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim));
62339566063dSJacob Faibussowitsch   if (hasPrec) PetscCall(PetscArrayzero(elemMatP, numCells * totDim * totDim));
62349566063dSJacob Faibussowitsch   if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim));
62353e9753d6SMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
62363e9753d6SMatthew G. Knepley     PetscClassId    id;
62373e9753d6SMatthew G. Knepley     PetscFE         fe;
62383e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
62393e9753d6SMatthew G. Knepley     PetscInt        Nb;
62403e9753d6SMatthew G. Knepley     /* Conforming batches */
62413e9753d6SMatthew G. Knepley     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
62423e9753d6SMatthew G. Knepley     /* Remainder */
62433e9753d6SMatthew G. Knepley     PetscInt     Nr, offset, Nq;
62443e9753d6SMatthew G. Knepley     PetscInt     maxDegree;
62453e9753d6SMatthew G. Knepley     PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
62463e9753d6SMatthew G. Knepley 
62479566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
62489566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
62499371c9d4SSatish Balay     if (id == PETSCFV_CLASSID) {
62509371c9d4SSatish Balay       hasFV = PETSC_TRUE;
62519371c9d4SSatish Balay       continue;
62529371c9d4SSatish Balay     }
62539566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
62549566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
62559566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
625648a46eb9SPierre Jolivet     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
62573e9753d6SMatthew G. Knepley     if (!qGeom) {
62589566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &qGeom));
62599566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
62603e9753d6SMatthew G. Knepley     }
62619566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
6262ac9d17c7SMatthew G. Knepley     PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FEGEOM_BASIC, &cgeomFEM));
62633e9753d6SMatthew G. Knepley     blockSize = Nb;
62643e9753d6SMatthew G. Knepley     batchSize = numBlocks * blockSize;
62659566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
62663e9753d6SMatthew G. Knepley     numChunks = numCells / (numBatches * batchSize);
62673e9753d6SMatthew G. Knepley     Ne        = numChunks * numBatches * batchSize;
62683e9753d6SMatthew G. Knepley     Nr        = numCells % (numBatches * batchSize);
62693e9753d6SMatthew G. Knepley     offset    = numCells - Nr;
62709566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
62719566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom));
62723e9753d6SMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
62736528b96dSMatthew G. Knepley       key.field = fieldI * Nf + fieldJ;
62743e9753d6SMatthew G. Knepley       if (hasJac) {
62754561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
62764561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * totDim]));
62773e9753d6SMatthew G. Knepley       }
62783e9753d6SMatthew G. Knepley       if (hasPrec) {
62794561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatP));
62804561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN_PRE, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatP[offset * totDim * totDim]));
62813e9753d6SMatthew G. Knepley       }
62823e9753d6SMatthew G. Knepley       if (hasDyn) {
62834561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
62844561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatD[offset * totDim * totDim]));
62853e9753d6SMatthew G. Knepley       }
62863e9753d6SMatthew G. Knepley     }
62879566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom));
62889566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom));
62899566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
62909566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
62913e9753d6SMatthew G. Knepley   }
62923e9753d6SMatthew G. Knepley   /*   Add contribution from X_t */
62939371c9d4SSatish Balay   if (hasDyn) {
62949371c9d4SSatish Balay     for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
62959371c9d4SSatish Balay   }
62963e9753d6SMatthew G. Knepley   if (hasFV) {
62973e9753d6SMatthew G. Knepley     PetscClassId id;
62983e9753d6SMatthew G. Knepley     PetscFV      fv;
62993e9753d6SMatthew G. Knepley     PetscInt     offsetI, NcI, NbI = 1, fc, f;
63003e9753d6SMatthew G. Knepley 
63013e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
63029566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv));
63039566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, fieldI, &offsetI));
63049566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId((PetscObject)fv, &id));
63053e9753d6SMatthew G. Knepley       if (id != PETSCFV_CLASSID) continue;
63062f86f8c5SMatthew G. Knepley       /* Put in the weighted identity */
63079566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &NcI));
63083e9753d6SMatthew G. Knepley       for (c = cStart; c < cEnd; ++c) {
63093e9753d6SMatthew G. Knepley         const PetscInt cind    = c - cStart;
63103e9753d6SMatthew G. Knepley         const PetscInt eOffset = cind * totDim * totDim;
63112f86f8c5SMatthew G. Knepley         PetscReal      vol;
63122f86f8c5SMatthew G. Knepley 
63132f86f8c5SMatthew G. Knepley         PetscCall(DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL));
63143e9753d6SMatthew G. Knepley         for (fc = 0; fc < NcI; ++fc) {
63153e9753d6SMatthew G. Knepley           for (f = 0; f < NbI; ++f) {
63163e9753d6SMatthew G. Knepley             const PetscInt i = offsetI + f * NcI + fc;
63173e9753d6SMatthew G. Knepley             if (hasPrec) {
63182f86f8c5SMatthew G. Knepley               if (hasJac) elemMat[eOffset + i * totDim + i] = vol;
63192f86f8c5SMatthew G. Knepley               elemMatP[eOffset + i * totDim + i] = vol;
63209371c9d4SSatish Balay             } else {
63212f86f8c5SMatthew G. Knepley               elemMat[eOffset + i * totDim + i] = vol;
63229371c9d4SSatish Balay             }
63233e9753d6SMatthew G. Knepley           }
63243e9753d6SMatthew G. Knepley         }
63253e9753d6SMatthew G. Knepley       }
63263e9753d6SMatthew G. Knepley     }
63273e9753d6SMatthew G. Knepley     /* No allocated space for FV stuff, so ignore the zero entries */
63289566063dSJacob Faibussowitsch     PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE));
63293e9753d6SMatthew G. Knepley   }
63303e9753d6SMatthew G. Knepley   /* Insert values into matrix */
63313e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
63323e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
63333e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
63343e9753d6SMatthew G. Knepley 
63353e9753d6SMatthew G. Knepley     /* Transform to global basis before insertion in Jacobian */
63369566063dSJacob Faibussowitsch     if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, cell, PETSC_TRUE, totDim, &elemMat[cind * totDim * totDim]));
63373e9753d6SMatthew G. Knepley     if (hasPrec) {
63383e9753d6SMatthew G. Knepley       if (hasJac) {
63399566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
6340e8e188d2SZach Atkins         PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, Jac, cell, &elemMat[cind * totDim * totDim], ADD_VALUES));
63413e9753d6SMatthew G. Knepley       }
63429566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind * totDim * totDim]));
6343be62b2b4SMatthew G. Knepley       PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, JacP, cell, &elemMatP[cind * totDim * totDim], ADD_VALUES));
63443e9753d6SMatthew G. Knepley     } else {
63453e9753d6SMatthew G. Knepley       if (hasJac) {
63469566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
6347e8e188d2SZach Atkins         PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, JacP, cell, &elemMat[cind * totDim * totDim], ADD_VALUES));
63483e9753d6SMatthew G. Knepley       }
63493e9753d6SMatthew G. Knepley     }
63503e9753d6SMatthew G. Knepley   }
63519566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
63529566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
63539566063dSJacob Faibussowitsch   PetscCall(PetscFree5(u, u_t, elemMat, elemMatP, elemMatD));
6354d9302694SMatthew G. Knepley   if (dmAux) PetscCall(PetscFree(a));
63553e9753d6SMatthew G. Knepley   /* Compute boundary integrals */
6356754e4fbaSMatthew G. Knepley   PetscCall(DMPlexComputeBdJacobian_Internal(dm, locX, locX_t, t, X_tShift, Jac, JacP, user));
63573e9753d6SMatthew G. Knepley   /* Assemble matrix */
63589371c9d4SSatish Balay end: {
6359e04ae0b4SMatthew G. Knepley   PetscBool assOp = hasJac && hasPrec ? PETSC_TRUE : PETSC_FALSE, gassOp;
6360e04ae0b4SMatthew G. Knepley 
6361d9302694SMatthew G. Knepley   if (dmAux) PetscCall(DMDestroy(&plex));
6362462c564dSBarry Smith   PetscCallMPI(MPIU_Allreduce(&assOp, &gassOp, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
63633e9753d6SMatthew G. Knepley   if (hasJac && hasPrec) {
63649566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
63659566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
63663e9753d6SMatthew G. Knepley   }
6367e04ae0b4SMatthew G. Knepley }
63689566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
63699566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
63709566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
63713ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
63723e9753d6SMatthew G. Knepley }
63733e9753d6SMatthew G. Knepley 
63744561e6c9SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobianByKeyGeneral(DM dmr, DM dmc, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user)
63754561e6c9SMatthew G. Knepley {
63764561e6c9SMatthew G. Knepley   DM_Plex        *mesh     = (DM_Plex *)dmr->data;
63774561e6c9SMatthew G. Knepley   const char     *name     = "Jacobian";
63784561e6c9SMatthew G. Knepley   DM              dmAux    = NULL, plex, tdm;
63794561e6c9SMatthew G. Knepley   PetscInt        printFEM = mesh->printFEM;
63804561e6c9SMatthew G. Knepley   PetscBool       clPerm   = mesh->useMatClPerm;
63814561e6c9SMatthew G. Knepley   DMEnclosureType encAux;
63824561e6c9SMatthew G. Knepley   Vec             A, tv;
63834561e6c9SMatthew G. Knepley   DMField         coordField;
63844561e6c9SMatthew G. Knepley   PetscDS         rds, cds, dsAux = NULL;
63854561e6c9SMatthew G. Knepley   PetscSection    rsection, rglobalSection, csection, cglobalSection, sectionAux;
63864561e6c9SMatthew G. Knepley   PetscScalar    *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL;
63874561e6c9SMatthew G. Knepley   const PetscInt *cells;
63884561e6c9SMatthew G. Knepley   PetscInt        Nf, cNf;
63894561e6c9SMatthew G. Knepley   PetscInt        totDim, ctotDim, totDimAux = 0, cStart, cEnd, numCells;
63904561e6c9SMatthew G. Knepley   PetscBool       hasJac = PETSC_FALSE, hasPrec = PETSC_FALSE, hasDyn, hasFV = PETSC_FALSE, transform;
63914561e6c9SMatthew G. Knepley   MPI_Comm        comm;
63924561e6c9SMatthew G. Knepley 
63934561e6c9SMatthew G. Knepley   PetscFunctionBegin;
63944561e6c9SMatthew G. Knepley   PetscCall(PetscObjectGetComm((PetscObject)dmr, &comm));
63954561e6c9SMatthew G. Knepley   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dmr, 0, 0, 0));
63964561e6c9SMatthew G. Knepley   PetscCall(DMGetLocalSection(dmr, &rsection));
63974561e6c9SMatthew G. Knepley   PetscCall(DMGetGlobalSection(dmr, &rglobalSection));
63984561e6c9SMatthew G. Knepley   PetscCall(DMGetLocalSection(dmc, &csection));
63994561e6c9SMatthew G. Knepley   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
64004561e6c9SMatthew G. Knepley   PetscCall(DMGetAuxiliaryVec(dmr, key.label, key.value, key.part, &A));
64014561e6c9SMatthew G. Knepley   if (A) {
64024561e6c9SMatthew G. Knepley     PetscCall(VecGetDM(A, &dmAux));
64034561e6c9SMatthew G. Knepley     PetscCall(DMGetEnclosureRelation(dmAux, dmr, &encAux));
64044561e6c9SMatthew G. Knepley     PetscCall(DMConvert(dmAux, DMPLEX, &plex));
64054561e6c9SMatthew G. Knepley     PetscCall(DMGetLocalSection(plex, &sectionAux));
64064561e6c9SMatthew G. Knepley     PetscCall(DMGetDS(dmAux, &dsAux));
64074561e6c9SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux));
64084561e6c9SMatthew G. Knepley   }
64094561e6c9SMatthew G. Knepley   PetscCall(DMGetCoordinateField(dmr, &coordField));
64104561e6c9SMatthew G. Knepley   if (!cellIS) goto end;
64114561e6c9SMatthew G. Knepley   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
64124561e6c9SMatthew G. Knepley   PetscCall(ISGetLocalSize(cellIS, &numCells));
64134561e6c9SMatthew G. Knepley   if (cStart >= cEnd) goto end;
64144561e6c9SMatthew G. Knepley   PetscCall(DMHasBasisTransform(dmr, &transform));
64154561e6c9SMatthew G. Knepley   PetscCall(DMGetBasisTransformDM_Internal(dmr, &tdm));
64164561e6c9SMatthew G. Knepley   PetscCall(DMGetBasisTransformVec_Internal(dmr, &tv));
64174561e6c9SMatthew G. Knepley   PetscCall(DMGetCellDS(dmr, cells ? cells[cStart] : cStart, &rds, NULL));
64184561e6c9SMatthew G. Knepley   PetscCall(DMGetCellDS(dmc, cells ? cells[cStart] : cStart, &cds, NULL));
64194561e6c9SMatthew G. Knepley   PetscCall(PetscDSGetNumFields(rds, &Nf));
64204561e6c9SMatthew G. Knepley   PetscCall(PetscDSGetNumFields(cds, &cNf));
64214561e6c9SMatthew G. Knepley   PetscCheck(Nf == cNf, comm, PETSC_ERR_ARG_WRONG, "Number of row fields %" PetscInt_FMT " != %" PetscInt_FMT " number of columns field", Nf, cNf);
64224561e6c9SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(rds, &totDim));
64234561e6c9SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(cds, &ctotDim));
64244561e6c9SMatthew G. Knepley   PetscCall(PetscDSHasJacobian(rds, &hasJac));
64254561e6c9SMatthew G. Knepley   PetscCall(PetscDSHasJacobianPreconditioner(rds, &hasPrec));
64264561e6c9SMatthew G. Knepley   /* user passed in the same matrix, avoid double contributions and
64274561e6c9SMatthew G. Knepley      only assemble the Jacobian */
64284561e6c9SMatthew G. Knepley   if (hasJac && Jac == JacP) hasPrec = PETSC_FALSE;
64294561e6c9SMatthew G. Knepley   PetscCall(PetscDSHasDynamicJacobian(rds, &hasDyn));
64304561e6c9SMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
64314561e6c9SMatthew G. Knepley   PetscCall(PetscMalloc5(numCells * totDim, &u, (locX_t ? (size_t)numCells * totDim : 0), &u_t, (hasJac ? (size_t)numCells * totDim * ctotDim : 0), &elemMat, (hasPrec ? (size_t)numCells * totDim * ctotDim : 0), &elemMatP, (hasDyn ? (size_t)numCells * totDim * ctotDim : 0), &elemMatD));
64324561e6c9SMatthew G. Knepley   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
64334561e6c9SMatthew G. Knepley   for (PetscInt c = cStart; c < cEnd; ++c) {
64344561e6c9SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
64354561e6c9SMatthew G. Knepley     const PetscInt cind = c - cStart;
64364561e6c9SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL;
64374561e6c9SMatthew G. Knepley     PetscInt       i;
64384561e6c9SMatthew G. Knepley 
64394561e6c9SMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(dmr, rsection, locX, cell, NULL, &x));
64404561e6c9SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i];
64414561e6c9SMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(dmr, rsection, locX, cell, NULL, &x));
64424561e6c9SMatthew G. Knepley     if (locX_t) {
64434561e6c9SMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(dmr, rsection, locX_t, cell, NULL, &x_t));
64444561e6c9SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i];
64454561e6c9SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(dmr, rsection, locX_t, cell, NULL, &x_t));
64464561e6c9SMatthew G. Knepley     }
64474561e6c9SMatthew G. Knepley     if (dmAux) {
64484561e6c9SMatthew G. Knepley       PetscInt subcell;
64494561e6c9SMatthew G. Knepley       PetscCall(DMGetEnclosurePoint(dmAux, dmr, encAux, cell, &subcell));
64504561e6c9SMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x));
64514561e6c9SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i];
64524561e6c9SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x));
64534561e6c9SMatthew G. Knepley     }
64544561e6c9SMatthew G. Knepley   }
64554561e6c9SMatthew G. Knepley   if (hasJac) PetscCall(PetscArrayzero(elemMat, numCells * totDim * ctotDim));
64564561e6c9SMatthew G. Knepley   if (hasPrec) PetscCall(PetscArrayzero(elemMatP, numCells * totDim * ctotDim));
64574561e6c9SMatthew G. Knepley   if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * ctotDim));
64584561e6c9SMatthew G. Knepley   for (PetscInt fieldI = 0; fieldI < Nf; ++fieldI) {
64594561e6c9SMatthew G. Knepley     PetscClassId    id;
64604561e6c9SMatthew G. Knepley     PetscFE         fe;
64614561e6c9SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
64624561e6c9SMatthew G. Knepley     PetscInt        Nb;
64634561e6c9SMatthew G. Knepley     /* Conforming batches */
64644561e6c9SMatthew G. Knepley     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
64654561e6c9SMatthew G. Knepley     /* Remainder */
64664561e6c9SMatthew G. Knepley     PetscInt     Nr, offset, Nq;
64674561e6c9SMatthew G. Knepley     PetscInt     maxDegree;
64684561e6c9SMatthew G. Knepley     PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
64694561e6c9SMatthew G. Knepley 
64704561e6c9SMatthew G. Knepley     PetscCall(PetscDSGetDiscretization(rds, fieldI, (PetscObject *)&fe));
64714561e6c9SMatthew G. Knepley     PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
64724561e6c9SMatthew G. Knepley     if (id == PETSCFV_CLASSID) {
64734561e6c9SMatthew G. Knepley       hasFV = PETSC_TRUE;
64744561e6c9SMatthew G. Knepley       continue;
64754561e6c9SMatthew G. Knepley     }
64764561e6c9SMatthew G. Knepley     PetscCall(PetscFEGetDimension(fe, &Nb));
64774561e6c9SMatthew G. Knepley     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
64784561e6c9SMatthew G. Knepley     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
64794561e6c9SMatthew G. Knepley     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
64804561e6c9SMatthew G. Knepley     if (!qGeom) {
64814561e6c9SMatthew G. Knepley       PetscCall(PetscFEGetQuadrature(fe, &qGeom));
64824561e6c9SMatthew G. Knepley       PetscCall(PetscObjectReference((PetscObject)qGeom));
64834561e6c9SMatthew G. Knepley     }
64844561e6c9SMatthew G. Knepley     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
64854561e6c9SMatthew G. Knepley     PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FEGEOM_BASIC, &cgeomFEM));
64864561e6c9SMatthew G. Knepley     blockSize = Nb;
64874561e6c9SMatthew G. Knepley     batchSize = numBlocks * blockSize;
64884561e6c9SMatthew G. Knepley     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
64894561e6c9SMatthew G. Knepley     numChunks = numCells / (numBatches * batchSize);
64904561e6c9SMatthew G. Knepley     Ne        = numChunks * numBatches * batchSize;
64914561e6c9SMatthew G. Knepley     Nr        = numCells % (numBatches * batchSize);
64924561e6c9SMatthew G. Knepley     offset    = numCells - Nr;
64934561e6c9SMatthew G. Knepley     PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
64944561e6c9SMatthew G. Knepley     PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom));
64954561e6c9SMatthew G. Knepley     for (PetscInt fieldJ = 0; fieldJ < Nf; ++fieldJ) {
64964561e6c9SMatthew G. Knepley       key.field = fieldI * Nf + fieldJ;
64974561e6c9SMatthew G. Knepley       if (hasJac) {
64984561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(rds, cds, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, dsAux, a, t, X_tShift, elemMat));
64994561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(rds, cds, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * ctotDim]));
65004561e6c9SMatthew G. Knepley       }
65014561e6c9SMatthew G. Knepley       if (hasPrec) {
65024561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(rds, cds, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, dsAux, a, t, X_tShift, elemMatP));
65034561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(rds, cds, PETSCFE_JACOBIAN_PRE, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatP[offset * totDim * ctotDim]));
65044561e6c9SMatthew G. Knepley       }
65054561e6c9SMatthew G. Knepley       if (hasDyn) {
65064561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(rds, cds, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, dsAux, a, t, X_tShift, elemMatD));
65074561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(rds, cds, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatD[offset * totDim * ctotDim]));
65084561e6c9SMatthew G. Knepley       }
65094561e6c9SMatthew G. Knepley     }
65104561e6c9SMatthew G. Knepley     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom));
65114561e6c9SMatthew G. Knepley     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom));
65124561e6c9SMatthew G. Knepley     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
65134561e6c9SMatthew G. Knepley     PetscCall(PetscQuadratureDestroy(&qGeom));
65144561e6c9SMatthew G. Knepley   }
65154561e6c9SMatthew G. Knepley   /*   Add contribution from X_t */
65164561e6c9SMatthew G. Knepley   if (hasDyn) {
65174561e6c9SMatthew G. Knepley     for (PetscInt c = 0; c < numCells * totDim * ctotDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
65184561e6c9SMatthew G. Knepley   }
65194561e6c9SMatthew G. Knepley   if (hasFV) {
65204561e6c9SMatthew G. Knepley     PetscClassId id;
65214561e6c9SMatthew G. Knepley     PetscFV      fv;
65224561e6c9SMatthew G. Knepley     PetscInt     offsetI, NcI, NbI = 1;
65234561e6c9SMatthew G. Knepley 
65244561e6c9SMatthew G. Knepley     for (PetscInt fieldI = 0; fieldI < Nf; ++fieldI) {
65254561e6c9SMatthew G. Knepley       PetscCall(PetscDSGetDiscretization(rds, fieldI, (PetscObject *)&fv));
65264561e6c9SMatthew G. Knepley       PetscCall(PetscDSGetFieldOffset(rds, fieldI, &offsetI));
65274561e6c9SMatthew G. Knepley       PetscCall(PetscObjectGetClassId((PetscObject)fv, &id));
65284561e6c9SMatthew G. Knepley       if (id != PETSCFV_CLASSID) continue;
65294561e6c9SMatthew G. Knepley       /* Put in the weighted identity */
65304561e6c9SMatthew G. Knepley       PetscCall(PetscFVGetNumComponents(fv, &NcI));
65314561e6c9SMatthew G. Knepley       for (PetscInt c = cStart; c < cEnd; ++c) {
65324561e6c9SMatthew G. Knepley         const PetscInt cind    = c - cStart;
65334561e6c9SMatthew G. Knepley         const PetscInt eOffset = cind * totDim * ctotDim;
65344561e6c9SMatthew G. Knepley         PetscReal      vol;
65354561e6c9SMatthew G. Knepley 
65364561e6c9SMatthew G. Knepley         PetscCall(DMPlexComputeCellGeometryFVM(dmr, c, &vol, NULL, NULL));
65374561e6c9SMatthew G. Knepley         for (PetscInt fc = 0; fc < NcI; ++fc) {
65384561e6c9SMatthew G. Knepley           for (PetscInt f = 0; f < NbI; ++f) {
65394561e6c9SMatthew G. Knepley             const PetscInt i = offsetI + f * NcI + fc;
65404561e6c9SMatthew G. Knepley             if (hasPrec) {
65414561e6c9SMatthew G. Knepley               if (hasJac) elemMat[eOffset + i * ctotDim + i] = vol;
65424561e6c9SMatthew G. Knepley               elemMatP[eOffset + i * ctotDim + i] = vol;
65434561e6c9SMatthew G. Knepley             } else {
65444561e6c9SMatthew G. Knepley               elemMat[eOffset + i * ctotDim + i] = vol;
65454561e6c9SMatthew G. Knepley             }
65464561e6c9SMatthew G. Knepley           }
65474561e6c9SMatthew G. Knepley         }
65484561e6c9SMatthew G. Knepley       }
65494561e6c9SMatthew G. Knepley     }
65504561e6c9SMatthew G. Knepley     /* No allocated space for FV stuff, so ignore the zero entries */
65514561e6c9SMatthew G. Knepley     PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE));
65524561e6c9SMatthew G. Knepley   }
65534561e6c9SMatthew G. Knepley   /* Insert values into matrix */
65544561e6c9SMatthew G. Knepley   for (PetscInt c = cStart; c < cEnd; ++c) {
65554561e6c9SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
65564561e6c9SMatthew G. Knepley     const PetscInt cind = c - cStart;
65574561e6c9SMatthew G. Knepley 
65584561e6c9SMatthew G. Knepley     /* Transform to global basis before insertion in Jacobian */
65594561e6c9SMatthew G. Knepley     if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dmr, tdm, tv, cell, PETSC_TRUE, totDim, &elemMat[cind * totDim * ctotDim]));
65604561e6c9SMatthew G. Knepley     if (hasPrec) {
65614561e6c9SMatthew G. Knepley       if (hasJac) {
65624561e6c9SMatthew G. Knepley         if (printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, ctotDim, &elemMat[cind * totDim * ctotDim]));
65634561e6c9SMatthew G. Knepley         PetscCall(DMPlexMatSetClosureGeneral(dmr, rsection, rglobalSection, clPerm, dmc, csection, cglobalSection, clPerm, Jac, cell, &elemMat[cind * totDim * ctotDim], ADD_VALUES));
65644561e6c9SMatthew G. Knepley       }
65654561e6c9SMatthew G. Knepley       if (printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, ctotDim, &elemMatP[cind * totDim * ctotDim]));
65664561e6c9SMatthew G. Knepley       PetscCall(DMPlexMatSetClosureGeneral(dmr, rsection, rglobalSection, clPerm, dmc, csection, cglobalSection, clPerm, JacP, cell, &elemMatP[cind * totDim * ctotDim], ADD_VALUES));
65674561e6c9SMatthew G. Knepley     } else {
65684561e6c9SMatthew G. Knepley       if (hasJac) {
65694561e6c9SMatthew G. Knepley         if (printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, ctotDim, &elemMat[cind * totDim * ctotDim]));
65704561e6c9SMatthew G. Knepley         PetscCall(DMPlexMatSetClosureGeneral(dmr, rsection, rglobalSection, clPerm, dmc, csection, cglobalSection, clPerm, JacP, cell, &elemMat[cind * totDim * ctotDim], ADD_VALUES));
65714561e6c9SMatthew G. Knepley       }
65724561e6c9SMatthew G. Knepley     }
65734561e6c9SMatthew G. Knepley   }
65744561e6c9SMatthew G. Knepley   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
65754561e6c9SMatthew G. Knepley   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
65764561e6c9SMatthew G. Knepley   PetscCall(PetscFree5(u, u_t, elemMat, elemMatP, elemMatD));
65774561e6c9SMatthew G. Knepley   if (dmAux) PetscCall(PetscFree(a));
65784561e6c9SMatthew G. Knepley   /* Compute boundary integrals */
65794561e6c9SMatthew G. Knepley   PetscCall(DMPlexComputeBdJacobian_Internal(dmr, locX, locX_t, t, X_tShift, Jac, JacP, user));
65804561e6c9SMatthew G. Knepley   /* Assemble matrix */
65814561e6c9SMatthew G. Knepley end: {
65824561e6c9SMatthew G. Knepley   PetscBool assOp = hasJac && hasPrec ? PETSC_TRUE : PETSC_FALSE, gassOp;
65834561e6c9SMatthew G. Knepley 
65844561e6c9SMatthew G. Knepley   if (dmAux) PetscCall(DMDestroy(&plex));
65854561e6c9SMatthew G. Knepley   PetscCallMPI(MPIU_Allreduce(&assOp, &gassOp, 1, MPIU_BOOL, MPI_LOR, comm));
65864561e6c9SMatthew G. Knepley   if (hasJac && hasPrec) {
65874561e6c9SMatthew G. Knepley     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
65884561e6c9SMatthew G. Knepley     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
65894561e6c9SMatthew G. Knepley   }
65904561e6c9SMatthew G. Knepley }
65914561e6c9SMatthew G. Knepley   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
65924561e6c9SMatthew G. Knepley   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
65934561e6c9SMatthew G. Knepley   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dmr, 0, 0, 0));
65944561e6c9SMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
65954561e6c9SMatthew G. Knepley }
65964561e6c9SMatthew G. Knepley 
6597754e4fbaSMatthew G. Knepley /*@
6598754e4fbaSMatthew G. Knepley   DMPlexComputeJacobianHybridByKey - Compute the local Jacobian over hybrid cells for terms matching the input key
6599754e4fbaSMatthew G. Knepley 
6600754e4fbaSMatthew G. Knepley   Collective
6601754e4fbaSMatthew G. Knepley 
6602754e4fbaSMatthew G. Knepley   Input Parameters:
6603754e4fbaSMatthew G. Knepley + dm       - The output `DM`
6604754e4fbaSMatthew G. Knepley . key      - The `PetscFormKey` array (left cell, right cell, cohesive cell) indicating what should be integrated
6605754e4fbaSMatthew G. Knepley . cellIS   - The `IS` give a set of cells to integrate over
6606754e4fbaSMatthew G. Knepley . t        - The time
6607754e4fbaSMatthew G. Knepley . X_tShift - The multiplier for the Jacobian with respect to $X_t$
6608754e4fbaSMatthew G. Knepley . locX     - The local solution
6609754e4fbaSMatthew G. Knepley . locX_t   - The time derivative of the local solution, or `NULL` for time-independent problems
6610754e4fbaSMatthew G. Knepley - user     - An optional user context, passed to the pointwise functions
6611754e4fbaSMatthew G. Knepley 
6612754e4fbaSMatthew G. Knepley   Output Parameters:
6613754e4fbaSMatthew G. Knepley + Jac  - The local Jacobian
6614754e4fbaSMatthew G. Knepley - JacP - The local Jacobian preconditioner
6615754e4fbaSMatthew G. Knepley 
6616754e4fbaSMatthew G. Knepley   Level: developer
6617754e4fbaSMatthew G. Knepley 
6618754e4fbaSMatthew G. Knepley .seealso: `DMPlexComputeResidualByKey()`, `DMPlexComputeJacobianByKey()`, `DMPlexComputeResidualHybridByKey()`, `PetscFormKey`
6619754e4fbaSMatthew G. Knepley @*/
6620754e4fbaSMatthew G. Knepley PetscErrorCode DMPlexComputeJacobianHybridByKey(DM dm, PetscFormKey key[], IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user)
6621d71ae5a4SJacob Faibussowitsch {
66223e9753d6SMatthew G. Knepley   DM_Plex        *mesh          = (DM_Plex *)dm->data;
66233e9753d6SMatthew G. Knepley   const char     *name          = "Hybrid Jacobian";
6624148442b3SMatthew G. Knepley   DM              dmAux[3]      = {NULL, NULL, NULL};
6625148442b3SMatthew G. Knepley   DMLabel         ghostLabel    = NULL;
66263e9753d6SMatthew G. Knepley   DM              plex          = NULL;
66273e9753d6SMatthew G. Knepley   DM              plexA         = NULL;
6628148442b3SMatthew G. Knepley   PetscDS         ds            = NULL;
662907218a29SMatthew G. Knepley   PetscDS         dsIn          = NULL;
6630148442b3SMatthew G. Knepley   PetscDS         dsAux[3]      = {NULL, NULL, NULL};
6631148442b3SMatthew G. Knepley   Vec             locA[3]       = {NULL, NULL, NULL};
663207218a29SMatthew G. Knepley   DM              dmScale[3]    = {NULL, NULL, NULL};
663307218a29SMatthew G. Knepley   PetscDS         dsScale[3]    = {NULL, NULL, NULL};
663407218a29SMatthew G. Knepley   Vec             locS[3]       = {NULL, NULL, NULL};
66353e9753d6SMatthew G. Knepley   PetscSection    section       = NULL;
6636148442b3SMatthew G. Knepley   PetscSection    sectionAux[3] = {NULL, NULL, NULL};
66373e9753d6SMatthew G. Knepley   DMField         coordField    = NULL;
663807218a29SMatthew G. Knepley   PetscScalar    *a[3]          = {NULL, NULL, NULL};
663907218a29SMatthew G. Knepley   PetscScalar    *s[3]          = {NULL, NULL, NULL};
664007218a29SMatthew G. Knepley   PetscScalar    *u             = NULL, *u_t;
664107218a29SMatthew G. Knepley   PetscScalar    *elemMatNeg, *elemMatPos, *elemMatCoh;
664207218a29SMatthew G. Knepley   PetscScalar    *elemMatNegP, *elemMatPosP, *elemMatCohP;
6643e432b41dSStefano Zampini   PetscSection    globalSection;
6644989fa639SBrad Aagaard   IS              chunkISF, chunkISN;
66453e9753d6SMatthew G. Knepley   const PetscInt *cells;
6646989fa639SBrad Aagaard   PetscInt       *faces, *neighbors;
66473e9753d6SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
66483e2b0218SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ, totDim, totDimIn, totDimAux[3], totDimScale[3], numChunks, cellChunkSize, chunk;
66491690c2aeSBarry Smith   PetscInt        maxDegree   = PETSC_INT_MAX;
6650989fa639SBrad Aagaard   PetscQuadrature affineQuadF = NULL, *quadsF = NULL;
6651989fa639SBrad Aagaard   PetscFEGeom    *affineGeomF = NULL, **geomsF = NULL;
6652989fa639SBrad Aagaard   PetscQuadrature affineQuadN = NULL;
6653989fa639SBrad Aagaard   PetscFEGeom    *affineGeomN = NULL;
6654e432b41dSStefano Zampini   PetscBool       hasBdJac, hasBdPrec;
66553e9753d6SMatthew G. Knepley 
66563e9753d6SMatthew G. Knepley   PetscFunctionBegin;
66573cc88e6aSStefano Zampini   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
66583cc88e6aSStefano Zampini   if (!cellIS) goto end;
6659437e83fbSMatthew G. Knepley   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
6660437e83fbSMatthew G. Knepley   PetscCall(ISGetLocalSize(cellIS, &numCells));
66613cc88e6aSStefano Zampini   if (cStart >= cEnd) goto end;
66625fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
66635fedec97SMatthew G. Knepley     const char *name;
66649566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name));
666563a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
66665fedec97SMatthew G. Knepley   }
66679566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
6668b98a7184SJames Wright   PetscCall(DMGetLocalSection(dm, &section));
66699566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
66709566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
667107218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn));
66729566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
66739566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
667407218a29SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(dsIn, &totDimIn));
66759566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobian(ds, &hasBdJac));
66769566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobianPreconditioner(ds, &hasBdPrec));
66779566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
6678148442b3SMatthew G. Knepley   if (locA[2]) {
66791059d808SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
66801059d808SMatthew G. Knepley 
66819566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
66829566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux[2], DMPLEX, &plexA));
6683b98a7184SJames Wright     PetscCall(DMGetLocalSection(dmAux[2], &sectionAux[2]));
668407218a29SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2], NULL));
66859566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
6686148442b3SMatthew G. Knepley     {
6687148442b3SMatthew G. Knepley       const PetscInt *cone;
6688148442b3SMatthew G. Knepley       PetscInt        c;
6689148442b3SMatthew G. Knepley 
66901059d808SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
6691148442b3SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
6692148442b3SMatthew G. Knepley         const PetscInt *support;
6693148442b3SMatthew G. Knepley         PetscInt        ssize, s;
6694148442b3SMatthew G. Knepley 
66959566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
66969566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
66971059d808SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
66981059d808SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
66991059d808SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
67001059d808SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
67019566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
67029566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
6703ad540459SPierre Jolivet         else dmAux[c] = dmAux[2];
670407218a29SMatthew G. Knepley         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c], NULL));
67059566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
6706148442b3SMatthew G. Knepley       }
6707148442b3SMatthew G. Knepley     }
67083e9753d6SMatthew G. Knepley   }
670907218a29SMatthew G. Knepley   /* Handle mass matrix scaling
671007218a29SMatthew G. Knepley        The field in key[2] is the field to be scaled, and the scaling field is the first in the dsScale */
671107218a29SMatthew G. Knepley   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, -key[2].value, key[2].part, &locS[2]));
671207218a29SMatthew G. Knepley   if (locS[2]) {
67133e2b0218SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
671407218a29SMatthew G. Knepley     PetscInt       Nb, Nbs;
671507218a29SMatthew G. Knepley 
671607218a29SMatthew G. Knepley     PetscCall(VecGetDM(locS[2], &dmScale[2]));
671707218a29SMatthew G. Knepley     PetscCall(DMGetCellDS(dmScale[2], cells ? cells[cStart] : cStart, &dsScale[2], NULL));
67183e2b0218SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(dsScale[2], &totDimScale[2]));
671907218a29SMatthew G. Knepley     // BRAD: This is not set correctly
672007218a29SMatthew G. Knepley     key[2].field = 2;
672107218a29SMatthew G. Knepley     PetscCall(PetscDSGetFieldSize(ds, key[2].field, &Nb));
672207218a29SMatthew G. Knepley     PetscCall(PetscDSGetFieldSize(dsScale[2], 0, &Nbs));
672307218a29SMatthew G. Knepley     PetscCheck(Nb == Nbs, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Field %" PetscInt_FMT " of size %" PetscInt_FMT " cannot be scaled by field of size %" PetscInt_FMT, key[2].field, Nb, Nbs);
67243e2b0218SMatthew G. Knepley     {
67253e2b0218SMatthew G. Knepley       const PetscInt *cone;
67263e2b0218SMatthew G. Knepley       PetscInt        c;
67273e2b0218SMatthew G. Knepley 
67283e2b0218SMatthew G. Knepley       locS[1] = locS[0] = locS[2];
67293e2b0218SMatthew G. Knepley       dmScale[1] = dmScale[0] = dmScale[2];
67303e2b0218SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
67313e2b0218SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
67323e2b0218SMatthew G. Knepley         const PetscInt *support;
67333e2b0218SMatthew G. Knepley         PetscInt        ssize, s;
67343e2b0218SMatthew G. Knepley 
67353e2b0218SMatthew G. Knepley         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
67363e2b0218SMatthew G. Knepley         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
67373e2b0218SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
67383e2b0218SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
67393e2b0218SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
67403e2b0218SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
67413e2b0218SMatthew G. Knepley         PetscCall(DMGetCellDS(dmScale[c], support[s], &dsScale[c], NULL));
67423e2b0218SMatthew G. Knepley         PetscCall(PetscDSGetTotalDimension(dsScale[c], &totDimScale[c]));
67433e2b0218SMatthew G. Knepley       }
67443e2b0218SMatthew G. Knepley     }
674507218a29SMatthew G. Knepley   }
674607218a29SMatthew G. Knepley   /* 2: Setup geometric data */
67479566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
67489566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
67493e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
67503e9753d6SMatthew G. Knepley     PetscInt f;
6751989fa639SBrad Aagaard     PetscCall(PetscCalloc2(Nf, &quadsF, Nf, &geomsF));
67523e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
67533e9753d6SMatthew G. Knepley       PetscFE fe;
67543e9753d6SMatthew G. Knepley 
67559566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
67563e9753d6SMatthew G. Knepley       if (fe) {
6757989fa639SBrad Aagaard         PetscCall(PetscFEGetQuadrature(fe, &quadsF[f]));
6758989fa639SBrad Aagaard         PetscCall(PetscObjectReference((PetscObject)quadsF[f]));
67593e9753d6SMatthew G. Knepley       }
67603e9753d6SMatthew G. Knepley     }
67613e9753d6SMatthew G. Knepley   }
676207218a29SMatthew G. Knepley   /* Loop over chunks */
67633e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
67643e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize);
6765989fa639SBrad Aagaard   PetscCall(PetscCalloc2(2 * cellChunkSize, &faces, 2 * cellChunkSize, &neighbors));
6766989fa639SBrad Aagaard   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkISF));
6767989fa639SBrad Aagaard   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, neighbors, PETSC_USE_POINTER, &chunkISN));
676807218a29SMatthew G. Knepley   /* Extract field coefficients */
676907218a29SMatthew G. Knepley   /* NOTE This needs the end cap faces to have identical orientations */
677007218a29SMatthew G. Knepley   PetscCall(DMPlexGetHybridCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
677107218a29SMatthew G. Knepley   PetscCall(DMPlexGetHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a));
67728bf0a22dSMatthew G. Knepley   PetscCall(DMPlexGetHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s));
677307218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNeg));
677407218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPos));
677507218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCoh));
677607218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNegP));
677707218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPosP));
677807218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCohP));
67793e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
67803e9753d6SMatthew G. Knepley     PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
67813e9753d6SMatthew G. Knepley 
678207218a29SMatthew G. Knepley     if (hasBdJac) {
678307218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatNeg, cellChunkSize * totDim * totDim));
678407218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatPos, cellChunkSize * totDim * totDim));
678507218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatCoh, cellChunkSize * totDim * totDim));
678607218a29SMatthew G. Knepley     }
678707218a29SMatthew G. Knepley     if (hasBdPrec) {
678807218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatNegP, cellChunkSize * totDim * totDim));
678907218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatPosP, cellChunkSize * totDim * totDim));
679007218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatCohP, cellChunkSize * totDim * totDim));
679107218a29SMatthew G. Knepley     }
67923e9753d6SMatthew G. Knepley     /* Get faces */
67933e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
67943e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
6795989fa639SBrad Aagaard       const PetscInt *cone, *support;
67969566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(plex, cell, &cone));
679707218a29SMatthew G. Knepley       faces[(c - cS) * 2 + 0] = cone[0];
679807218a29SMatthew G. Knepley       faces[(c - cS) * 2 + 1] = cone[1];
6799989fa639SBrad Aagaard       PetscCall(DMPlexGetSupport(dm, cone[0], &support));
6800989fa639SBrad Aagaard       neighbors[(c - cS) * 2 + 0] = support[0] == cell ? support[1] : support[0];
6801989fa639SBrad Aagaard       PetscCall(DMPlexGetSupport(dm, cone[1], &support));
6802989fa639SBrad Aagaard       neighbors[(c - cS) * 2 + 1] = support[0] == cell ? support[1] : support[0];
68033e9753d6SMatthew G. Knepley     }
6804989fa639SBrad Aagaard     PetscCall(ISGeneralSetIndices(chunkISF, 2 * cellChunkSize, faces, PETSC_USE_POINTER));
6805989fa639SBrad Aagaard     PetscCall(ISGeneralSetIndices(chunkISN, 2 * cellChunkSize, neighbors, PETSC_USE_POINTER));
68063e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
6807989fa639SBrad Aagaard       if (!affineQuadF) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkISF, &affineQuadF));
6808989fa639SBrad Aagaard       if (affineQuadF) PetscCall(DMSNESGetFEGeom(coordField, chunkISF, affineQuadF, PETSC_FEGEOM_COHESIVE, &affineGeomF));
6809989fa639SBrad Aagaard       if (!affineQuadN) {
6810989fa639SBrad Aagaard         PetscInt dim;
6811989fa639SBrad Aagaard         PetscCall(PetscQuadratureGetData(affineQuadF, &dim, NULL, NULL, NULL, NULL));
6812989fa639SBrad Aagaard         PetscCall(DMFieldCreateDefaultFaceQuadrature(coordField, chunkISN, &affineQuadN));
6813989fa639SBrad Aagaard         PetscCall(PetscQuadratureSetData(affineQuadN, dim + 1, PETSC_DECIDE, PETSC_DECIDE, NULL, NULL));
6814989fa639SBrad Aagaard       }
6815989fa639SBrad Aagaard       if (affineQuadN) PetscCall(DMSNESGetFEGeom(coordField, chunkISN, affineQuadN, PETSC_FEGEOM_BASIC, &affineGeomN));
68163e9753d6SMatthew G. Knepley     } else {
68173e9753d6SMatthew G. Knepley       PetscInt f;
68183e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
6819989fa639SBrad Aagaard         if (quadsF[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkISF, quadsF[f], PETSC_FEGEOM_COHESIVE, &geomsF[f]));
68203e9753d6SMatthew G. Knepley       }
68213e9753d6SMatthew G. Knepley     }
68223e9753d6SMatthew G. Knepley 
68233e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
68243e9753d6SMatthew G. Knepley       PetscFE         feI;
6825989fa639SBrad Aagaard       PetscFEGeom    *geomF      = affineGeomF ? affineGeomF : geomsF[fieldI];
6826989fa639SBrad Aagaard       PetscFEGeom    *chunkGeomF = NULL, *remGeomF = NULL;
6827989fa639SBrad Aagaard       PetscFEGeom    *geomN      = affineGeomN ? affineGeomN : geomsF[fieldI];
6828989fa639SBrad Aagaard       PetscFEGeom    *chunkGeomN = NULL, *remGeomN = NULL;
6829989fa639SBrad Aagaard       PetscQuadrature quadF = affineQuadF ? affineQuadF : quadsF[fieldI];
68303e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
68315fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
68323e9753d6SMatthew G. Knepley 
68339566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&feI));
68343e9753d6SMatthew G. Knepley       if (!feI) continue;
68359566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(feI, NULL, &numBlocks, NULL, &numBatches));
6836989fa639SBrad Aagaard       PetscCall(PetscQuadratureGetData(quadF, NULL, NULL, &Nq, NULL, NULL));
68379566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feI, &Nb));
68383e9753d6SMatthew G. Knepley       blockSize = Nb;
68393e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
68409566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(feI, blockSize, numBlocks, batchSize, numBatches));
68413e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches * batchSize);
68423e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
68433e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches * batchSize);
68443e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
6845989fa639SBrad Aagaard       PetscCall(PetscFEGeomGetChunk(geomF, 0, offset * 2, &chunkGeomF));
6846989fa639SBrad Aagaard       PetscCall(PetscFEGeomGetChunk(geomF, offset * 2, numCells * 2, &remGeomF));
6847989fa639SBrad Aagaard       PetscCall(PetscFEGeomGetChunk(geomN, 0, offset * 2, &chunkGeomN));
6848989fa639SBrad Aagaard       PetscCall(PetscFEGeomGetChunk(geomN, offset * 2, numCells * 2, &remGeomN));
68499566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, fieldI, &isCohesiveField));
68503e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
68513e9753d6SMatthew G. Knepley         PetscFE feJ;
68523e9753d6SMatthew G. Knepley 
68539566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, fieldJ, (PetscObject *)&feJ));
68543e9753d6SMatthew G. Knepley         if (!feJ) continue;
6855148442b3SMatthew G. Knepley         key[0].field = fieldI * Nf + fieldJ;
6856148442b3SMatthew G. Knepley         key[1].field = fieldI * Nf + fieldJ;
68575fedec97SMatthew G. Knepley         key[2].field = fieldI * Nf + fieldJ;
6858148442b3SMatthew G. Knepley         if (hasBdJac) {
6859989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[0], 0, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatNeg));
6860989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[0], 0, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[0], PetscSafePointerPlusOffset(a[0], offset * totDimAux[0]), t, X_tShift, &elemMatNeg[offset * totDim * totDim]));
6861989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[1], 1, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatPos));
6862989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[1], 1, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[1], PetscSafePointerPlusOffset(a[1], offset * totDimAux[1]), t, X_tShift, &elemMatPos[offset * totDim * totDim]));
6863148442b3SMatthew G. Knepley         }
6864148442b3SMatthew G. Knepley         if (hasBdPrec) {
6865989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[0], 0, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatNegP));
6866989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[0], 0, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[0], &a[0][offset * totDimAux[0]], t, X_tShift, &elemMatNegP[offset * totDim * totDim]));
6867989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[1], 1, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatPosP));
6868989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[1], 1, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[1], &a[1][offset * totDimAux[1]], t, X_tShift, &elemMatPosP[offset * totDim * totDim]));
6869148442b3SMatthew G. Knepley         }
68705fedec97SMatthew G. Knepley         if (hasBdJac) {
6871989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[2], 2, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatCoh));
6872989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[2], 2, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[2], PetscSafePointerPlusOffset(a[2], offset * totDimAux[2]), t, X_tShift, &elemMatCoh[offset * totDim * totDim]));
6873148442b3SMatthew G. Knepley         }
68745fedec97SMatthew G. Knepley         if (hasBdPrec) {
6875989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[2], 2, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatCohP));
6876989fa639SBrad Aagaard           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[2], 2, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[2], &a[2][offset * totDimAux[2]], t, X_tShift, &elemMatCohP[offset * totDim * totDim]));
68773e9753d6SMatthew G. Knepley         }
68783e9753d6SMatthew G. Knepley       }
6879989fa639SBrad Aagaard       PetscCall(PetscFEGeomRestoreChunk(geomF, offset, numCells, &remGeomF));
6880989fa639SBrad Aagaard       PetscCall(PetscFEGeomRestoreChunk(geomF, 0, offset, &chunkGeomF));
6881989fa639SBrad Aagaard       PetscCall(PetscFEGeomRestoreChunk(geomN, offset, numCells, &remGeomN));
6882989fa639SBrad Aagaard       PetscCall(PetscFEGeomRestoreChunk(geomN, 0, offset, &chunkGeomN));
68833e9753d6SMatthew G. Knepley     }
68843e9753d6SMatthew G. Knepley     /* Insert values into matrix */
68853e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
68863e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
688707218a29SMatthew G. Knepley       const PetscInt cind = c - cS, coff = cind * totDim * totDim;
688807218a29SMatthew G. Knepley       PetscInt       i, j;
68893e9753d6SMatthew G. Knepley 
689007218a29SMatthew G. Knepley       /* Scale element values */
689107218a29SMatthew G. Knepley       if (locS[0]) {
68923e2b0218SMatthew G. Knepley         PetscInt  Nb, soff = cind * totDimScale[0], off = 0;
689307218a29SMatthew G. Knepley         PetscBool cohesive;
689407218a29SMatthew G. Knepley 
689507218a29SMatthew G. Knepley         for (fieldI = 0; fieldI < Nf; ++fieldI) {
689607218a29SMatthew G. Knepley           PetscCall(PetscDSGetFieldSize(ds, fieldI, &Nb));
689707218a29SMatthew G. Knepley           PetscCall(PetscDSGetCohesive(ds, fieldI, &cohesive));
689807218a29SMatthew G. Knepley 
689907218a29SMatthew G. Knepley           if (fieldI == key[2].field) {
690007218a29SMatthew G. Knepley             PetscCheck(cohesive, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Scaling should not happen for face fields");
69018bf0a22dSMatthew G. Knepley             for (i = 0; i < Nb; ++i) {
690207218a29SMatthew G. Knepley               for (j = 0; j < totDim; ++j) elemMatCoh[coff + (off + i) * totDim + j] += s[0][soff + i] * elemMatNeg[coff + (off + i) * totDim + j] + s[1][soff + i] * elemMatPos[coff + (off + i) * totDim + j];
69038bf0a22dSMatthew G. Knepley               if (hasBdPrec)
69048bf0a22dSMatthew G. Knepley                 for (j = 0; j < totDim; ++j) elemMatCohP[coff + (off + i) * totDim + j] += s[0][soff + i] * elemMatNegP[coff + (off + i) * totDim + j] + s[1][soff + i] * elemMatPosP[coff + (off + i) * totDim + j];
69058bf0a22dSMatthew G. Knepley             }
690607218a29SMatthew G. Knepley             off += Nb;
690707218a29SMatthew G. Knepley           } else {
690807218a29SMatthew G. Knepley             const PetscInt N = cohesive ? Nb : Nb * 2;
690907218a29SMatthew G. Knepley 
69108bf0a22dSMatthew G. Knepley             for (i = 0; i < N; ++i) {
691107218a29SMatthew G. Knepley               for (j = 0; j < totDim; ++j) elemMatCoh[coff + (off + i) * totDim + j] += elemMatNeg[coff + (off + i) * totDim + j] + elemMatPos[coff + (off + i) * totDim + j];
69128bf0a22dSMatthew G. Knepley               if (hasBdPrec)
69138bf0a22dSMatthew G. Knepley                 for (j = 0; j < totDim; ++j) elemMatCohP[coff + (off + i) * totDim + j] += elemMatNegP[coff + (off + i) * totDim + j] + elemMatPosP[coff + (off + i) * totDim + j];
69148bf0a22dSMatthew G. Knepley             }
691507218a29SMatthew G. Knepley             off += N;
691607218a29SMatthew G. Knepley           }
691707218a29SMatthew G. Knepley         }
691807218a29SMatthew G. Knepley       } else {
691907218a29SMatthew G. Knepley         for (i = 0; i < totDim * totDim; ++i) elemMatCoh[coff + i] += elemMatNeg[coff + i] + elemMatPos[coff + i];
69208bf0a22dSMatthew G. Knepley         if (hasBdPrec)
69218bf0a22dSMatthew G. Knepley           for (i = 0; i < totDim * totDim; ++i) elemMatCohP[coff + i] += elemMatNegP[coff + i] + elemMatPosP[coff + i];
692207218a29SMatthew G. Knepley       }
69233e9753d6SMatthew G. Knepley       if (hasBdPrec) {
69243e9753d6SMatthew G. Knepley         if (hasBdJac) {
692507218a29SMatthew G. Knepley           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCoh[cind * totDim * totDim]));
6926e8e188d2SZach Atkins           PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, Jac, cell, &elemMatCoh[cind * totDim * totDim], ADD_VALUES));
69273e9753d6SMatthew G. Knepley         }
692807218a29SMatthew G. Knepley         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCohP[cind * totDim * totDim]));
692907218a29SMatthew G. Knepley         PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMatCohP[cind * totDim * totDim], ADD_VALUES));
69303e9753d6SMatthew G. Knepley       } else if (hasBdJac) {
693107218a29SMatthew G. Knepley         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCoh[cind * totDim * totDim]));
6932e8e188d2SZach Atkins         PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, JacP, cell, &elemMatCoh[cind * totDim * totDim], ADD_VALUES));
69333e9753d6SMatthew G. Knepley       }
69343e9753d6SMatthew G. Knepley     }
69353e9753d6SMatthew G. Knepley   }
69369566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
693707218a29SMatthew G. Knepley   PetscCall(DMPlexRestoreHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a));
693807218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNeg));
693907218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPos));
694007218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCoh));
694107218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNegP));
694207218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPosP));
694307218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCohP));
6944989fa639SBrad Aagaard   PetscCall(PetscFree2(faces, neighbors));
6945989fa639SBrad Aagaard   PetscCall(ISDestroy(&chunkISF));
6946989fa639SBrad Aagaard   PetscCall(ISDestroy(&chunkISN));
69479566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
69483e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
6949989fa639SBrad Aagaard     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuadF, PETSC_FALSE, &affineGeomF));
6950989fa639SBrad Aagaard     PetscCall(PetscQuadratureDestroy(&affineQuadF));
6951989fa639SBrad Aagaard     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuadN, PETSC_FALSE, &affineGeomN));
6952989fa639SBrad Aagaard     PetscCall(PetscQuadratureDestroy(&affineQuadN));
69533e9753d6SMatthew G. Knepley   } else {
69543e9753d6SMatthew G. Knepley     PetscInt f;
69553e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
6956989fa639SBrad Aagaard       if (geomsF) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quadsF[f], PETSC_FALSE, &geomsF[f]));
6957989fa639SBrad Aagaard       if (quadsF) PetscCall(PetscQuadratureDestroy(&quadsF[f]));
69583e9753d6SMatthew G. Knepley     }
6959989fa639SBrad Aagaard     PetscCall(PetscFree2(quadsF, geomsF));
69603e9753d6SMatthew G. Knepley   }
69619566063dSJacob Faibussowitsch   if (dmAux[2]) PetscCall(DMDestroy(&plexA));
69629566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
69633cc88e6aSStefano Zampini end:
69649566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
69653ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
69663e9753d6SMatthew G. Knepley }
69678e3a2eefSMatthew G. Knepley 
6968754e4fbaSMatthew G. Knepley /*@
6969754e4fbaSMatthew G. Knepley   DMPlexComputeJacobianActionByKey - Compute the local Jacobian for terms matching the input key
6970754e4fbaSMatthew G. Knepley 
6971754e4fbaSMatthew G. Knepley   Collective
69728e3a2eefSMatthew G. Knepley 
69738e3a2eefSMatthew G. Knepley   Input Parameters:
6974754e4fbaSMatthew G. Knepley + dm       - The output `DM`
6975754e4fbaSMatthew G. Knepley . key      - The `PetscFormKey` indicating what should be integrated
6976754e4fbaSMatthew G. Knepley . cellIS   - The `IS` give a set of cells to integrate over
69778e3a2eefSMatthew G. Knepley . t        - The time
6978754e4fbaSMatthew G. Knepley . X_tShift - The multiplier for the Jacobian with respect to $X_t$
6979754e4fbaSMatthew G. Knepley . locX     - The local solution
6980754e4fbaSMatthew G. Knepley . locX_t   - The time derivative of the local solution, or `NULL` for time-independent problems
6981754e4fbaSMatthew G. Knepley . locY     - The local vector acted on by J
6982754e4fbaSMatthew G. Knepley - user     - An optional user context, passed to the pointwise functions
69838e3a2eefSMatthew G. Knepley 
69848e3a2eefSMatthew G. Knepley   Output Parameter:
6985754e4fbaSMatthew G. Knepley . locF - The local residual F = J(X) Y
69868e3a2eefSMatthew G. Knepley 
6987754e4fbaSMatthew G. Knepley   Level: developer
6988754e4fbaSMatthew G. Knepley 
6989754e4fbaSMatthew G. Knepley .seealso: `DMPlexComputeResidualByKey()`, `DMPlexComputeJacobianByKey()`, `DMPlexComputeResidualHybridByKey()`, `DMPlexComputeJacobianHybridByKey()`, `PetscFormKey`
6990754e4fbaSMatthew G. Knepley @*/
6991754e4fbaSMatthew G. Knepley PetscErrorCode DMPlexComputeJacobianActionByKey(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Vec locY, Vec locF, void *user)
6992d71ae5a4SJacob Faibussowitsch {
69938e3a2eefSMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *)dm->data;
69948e3a2eefSMatthew G. Knepley   const char     *name  = "Jacobian";
69958e3a2eefSMatthew G. Knepley   DM              dmAux = NULL, plex, plexAux = NULL;
69968e3a2eefSMatthew G. Knepley   DMEnclosureType encAux;
69978e3a2eefSMatthew G. Knepley   Vec             A;
69988e3a2eefSMatthew G. Knepley   DMField         coordField;
69998e3a2eefSMatthew G. Knepley   PetscDS         prob, probAux = NULL;
70008e3a2eefSMatthew G. Knepley   PetscQuadrature quad;
70018e3a2eefSMatthew G. Knepley   PetscSection    section, globalSection, sectionAux;
70028e3a2eefSMatthew G. Knepley   PetscScalar    *elemMat, *elemMatD, *u, *u_t, *a = NULL, *y, *z;
70038e3a2eefSMatthew G. Knepley   const PetscInt *cells;
70048e3a2eefSMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
70058e3a2eefSMatthew G. Knepley   PetscInt        totDim, totDimAux = 0, cStart, cEnd, numCells, c;
70068e3a2eefSMatthew G. Knepley   PetscBool       hasDyn;
70078e3a2eefSMatthew G. Knepley 
70088e3a2eefSMatthew G. Knepley   PetscFunctionBegin;
70099566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
70109566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
70119566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
70129566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
70139566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
70149566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
701507218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL));
70169566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
70179566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
70189566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
70198e3a2eefSMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
70209566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
70218e3a2eefSMatthew G. Knepley   if (A) {
70229566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
70239566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
70249566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexAux));
70259566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexAux, &sectionAux));
70269566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
70279566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
70288e3a2eefSMatthew G. Knepley   }
7029754e4fbaSMatthew G. Knepley   PetscCall(VecSet(locF, 0.0));
7030754e4fbaSMatthew G. Knepley   PetscCall(PetscMalloc6(numCells * totDim, &u, (locX_t ? (size_t)numCells * totDim : 0), &u_t, numCells * totDim * totDim, &elemMat, (hasDyn ? (size_t)numCells * totDim * totDim : 0), &elemMatD, numCells * totDim, &y, totDim, &z));
70319566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
70329566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
70338e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
70348e3a2eefSMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
70358e3a2eefSMatthew G. Knepley     const PetscInt cind = c - cStart;
70368e3a2eefSMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL;
70378e3a2eefSMatthew G. Knepley     PetscInt       i;
70388e3a2eefSMatthew G. Knepley 
7039754e4fbaSMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x));
70408e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i];
7041754e4fbaSMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x));
7042754e4fbaSMatthew G. Knepley     if (locX_t) {
7043754e4fbaSMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t));
70448e3a2eefSMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i];
7045754e4fbaSMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t));
70468e3a2eefSMatthew G. Knepley     }
70478e3a2eefSMatthew G. Knepley     if (dmAux) {
70488e3a2eefSMatthew G. Knepley       PetscInt subcell;
70499566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
70509566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexAux, sectionAux, A, subcell, NULL, &x));
70518e3a2eefSMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i];
70529566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexAux, sectionAux, A, subcell, NULL, &x));
70538e3a2eefSMatthew G. Knepley     }
7054754e4fbaSMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(plex, section, locY, cell, NULL, &x));
70558e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) y[cind * totDim + i] = x[i];
7056754e4fbaSMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(plex, section, locY, cell, NULL, &x));
70578e3a2eefSMatthew G. Knepley   }
70589566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim));
70599566063dSJacob Faibussowitsch   if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim));
70608e3a2eefSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
70618e3a2eefSMatthew G. Knepley     PetscFE  fe;
70628e3a2eefSMatthew G. Knepley     PetscInt Nb;
70638e3a2eefSMatthew G. Knepley     /* Conforming batches */
70648e3a2eefSMatthew G. Knepley     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
70658e3a2eefSMatthew G. Knepley     /* Remainder */
70668e3a2eefSMatthew G. Knepley     PetscInt        Nr, offset, Nq;
70678e3a2eefSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
70688e3a2eefSMatthew G. Knepley     PetscInt        maxDegree;
70698e3a2eefSMatthew G. Knepley     PetscFEGeom    *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
70708e3a2eefSMatthew G. Knepley 
70719566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
70729566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
70739566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
70749566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
70759566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
70769566063dSJacob Faibussowitsch     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
70778e3a2eefSMatthew G. Knepley     if (!qGeom) {
70789566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &qGeom));
70799566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
70808e3a2eefSMatthew G. Knepley     }
70819566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
7082ac9d17c7SMatthew G. Knepley     PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FEGEOM_BASIC, &cgeomFEM));
70838e3a2eefSMatthew G. Knepley     blockSize = Nb;
70848e3a2eefSMatthew G. Knepley     batchSize = numBlocks * blockSize;
70859566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
70868e3a2eefSMatthew G. Knepley     numChunks = numCells / (numBatches * batchSize);
70878e3a2eefSMatthew G. Knepley     Ne        = numChunks * numBatches * batchSize;
70888e3a2eefSMatthew G. Knepley     Nr        = numCells % (numBatches * batchSize);
70898e3a2eefSMatthew G. Knepley     offset    = numCells - Nr;
70909566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
70919566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom));
70928e3a2eefSMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
70938e3a2eefSMatthew G. Knepley       key.field = fieldI * Nf + fieldJ;
70944561e6c9SMatthew G. Knepley       PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
70954561e6c9SMatthew G. Knepley       PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * totDim]));
70968e3a2eefSMatthew G. Knepley       if (hasDyn) {
70974561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
70984561e6c9SMatthew G. Knepley         PetscCall(PetscFEIntegrateJacobian(prob, prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, &a[offset * totDimAux], t, X_tShift, &elemMatD[offset * totDim * totDim]));
70998e3a2eefSMatthew G. Knepley       }
71008e3a2eefSMatthew G. Knepley     }
71019566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom));
71029566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom));
71039566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
71049566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
71058e3a2eefSMatthew G. Knepley   }
71068e3a2eefSMatthew G. Knepley   if (hasDyn) {
71078e3a2eefSMatthew G. Knepley     for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
71088e3a2eefSMatthew G. Knepley   }
71098e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
71108e3a2eefSMatthew G. Knepley     const PetscInt     cell = cells ? cells[c] : c;
71118e3a2eefSMatthew G. Knepley     const PetscInt     cind = c - cStart;
71126497c311SBarry Smith     const PetscBLASInt one  = 1;
71136497c311SBarry Smith     PetscBLASInt       M;
71148e3a2eefSMatthew G. Knepley     const PetscScalar  a = 1.0, b = 0.0;
71158e3a2eefSMatthew G. Knepley 
71166497c311SBarry Smith     PetscCall(PetscBLASIntCast(totDim, &M));
7117792fecdfSBarry Smith     PetscCallBLAS("BLASgemv", BLASgemv_("N", &M, &M, &a, &elemMat[cind * totDim * totDim], &M, &y[cind * totDim], &one, &b, z, &one));
71188e3a2eefSMatthew G. Knepley     if (mesh->printFEM > 1) {
71199566063dSJacob Faibussowitsch       PetscCall(DMPrintCellMatrix(c, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
71209566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Y", totDim, &y[cind * totDim]));
71219566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Z", totDim, z));
71228e3a2eefSMatthew G. Knepley     }
7123754e4fbaSMatthew G. Knepley     PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, z, ADD_VALUES));
71248e3a2eefSMatthew G. Knepley   }
71259566063dSJacob Faibussowitsch   PetscCall(PetscFree6(u, u_t, elemMat, elemMatD, y, z));
71268e3a2eefSMatthew G. Knepley   if (mesh->printFEM) {
7127754e4fbaSMatthew G. Knepley     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)locF), "Z:\n"));
7128754e4fbaSMatthew G. Knepley     PetscCall(VecView(locF, NULL));
71298e3a2eefSMatthew G. Knepley   }
71301059d808SMatthew G. Knepley   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
71319566063dSJacob Faibussowitsch   PetscCall(PetscFree(a));
71329566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexAux));
71339566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
71349566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
71353ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
71368e3a2eefSMatthew G. Knepley }
7137d52c2f21SMatthew G. Knepley 
7138d52c2f21SMatthew G. Knepley static void f0_1(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
7139d52c2f21SMatthew G. Knepley {
7140d52c2f21SMatthew G. Knepley   f0[0] = u[0];
7141d52c2f21SMatthew G. Knepley }
7142d52c2f21SMatthew G. Knepley 
7143d52c2f21SMatthew G. Knepley static void f0_x(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
7144d52c2f21SMatthew G. Knepley {
7145d52c2f21SMatthew G. Knepley   f0[0] = x[(int)PetscRealPart(constants[0])] * u[0];
7146d52c2f21SMatthew G. Knepley }
7147d52c2f21SMatthew G. Knepley 
7148d52c2f21SMatthew G. Knepley static void f0_x2(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
7149d52c2f21SMatthew G. Knepley {
7150d52c2f21SMatthew G. Knepley   PetscInt d;
7151d52c2f21SMatthew G. Knepley 
7152d52c2f21SMatthew G. Knepley   f0[0] = 0.0;
7153d52c2f21SMatthew G. Knepley   for (d = 0; d < dim; ++d) f0[0] += PetscSqr(x[d]) * u[0];
7154d52c2f21SMatthew G. Knepley }
7155d52c2f21SMatthew G. Knepley 
7156d52c2f21SMatthew G. Knepley /*@
7157d52c2f21SMatthew G. Knepley   DMPlexComputeMoments - Compute the first three moments for a field
7158d52c2f21SMatthew G. Knepley 
7159d52c2f21SMatthew G. Knepley   Noncollective
7160d52c2f21SMatthew G. Knepley 
7161d52c2f21SMatthew G. Knepley   Input Parameters:
7162d52c2f21SMatthew G. Knepley + dm - the `DMPLEX`
7163d52c2f21SMatthew G. Knepley - u  - the field
7164d52c2f21SMatthew G. Knepley 
7165d52c2f21SMatthew G. Knepley   Output Parameter:
7166d52c2f21SMatthew G. Knepley . moments - the field moments
7167d52c2f21SMatthew G. Knepley 
7168d52c2f21SMatthew G. Knepley   Level: intermediate
7169d52c2f21SMatthew G. Knepley 
7170d52c2f21SMatthew G. Knepley   Note:
717117314648SMatthew G. Knepley   The `moments` array should be of length cdim + 2, where cdim is the number of components for the coordinate field.
7172d52c2f21SMatthew G. Knepley 
7173d52c2f21SMatthew G. Knepley .seealso: `DM`, `DMPLEX`, `DMSwarmComputeMoments()`
7174d52c2f21SMatthew G. Knepley @*/
7175d52c2f21SMatthew G. Knepley PetscErrorCode DMPlexComputeMoments(DM dm, Vec u, PetscReal moments[])
7176d52c2f21SMatthew G. Knepley {
7177d52c2f21SMatthew G. Knepley   PetscDS            ds;
7178d52c2f21SMatthew G. Knepley   PetscScalar        mom, constants[1];
7179d52c2f21SMatthew G. Knepley   const PetscScalar *oldConstants;
718017314648SMatthew G. Knepley   PetscInt           cdim, Nf, field = 0, Ncon;
7181d52c2f21SMatthew G. Knepley   MPI_Comm           comm;
7182d52c2f21SMatthew G. Knepley   void              *user;
7183d52c2f21SMatthew G. Knepley 
7184d52c2f21SMatthew G. Knepley   PetscFunctionBeginUser;
7185d52c2f21SMatthew G. Knepley   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
718617314648SMatthew G. Knepley   PetscCall(DMGetCoordinateDim(dm, &cdim));
7187d52c2f21SMatthew G. Knepley   PetscCall(DMGetApplicationContext(dm, &user));
7188d52c2f21SMatthew G. Knepley   PetscCall(DMGetDS(dm, &ds));
7189d52c2f21SMatthew G. Knepley   PetscCall(PetscDSGetNumFields(ds, &Nf));
7190d52c2f21SMatthew G. Knepley   PetscCall(PetscDSGetConstants(ds, &Ncon, &oldConstants));
7191d52c2f21SMatthew G. Knepley   PetscCheck(Nf == 1, comm, PETSC_ERR_ARG_WRONG, "We currently only support 1 field, not %" PetscInt_FMT, Nf);
7192d52c2f21SMatthew G. Knepley   PetscCall(PetscDSSetObjective(ds, field, &f0_1));
7193d52c2f21SMatthew G. Knepley   PetscCall(DMPlexComputeIntegralFEM(dm, u, &mom, user));
7194d52c2f21SMatthew G. Knepley   moments[0] = PetscRealPart(mom);
719517314648SMatthew G. Knepley   for (PetscInt c = 0; c < cdim; ++c) {
7196d52c2f21SMatthew G. Knepley     constants[0] = c;
719717314648SMatthew G. Knepley     PetscCall(PetscDSSetConstants(ds, 1, constants));
7198d52c2f21SMatthew G. Knepley     PetscCall(PetscDSSetObjective(ds, field, &f0_x));
7199d52c2f21SMatthew G. Knepley     PetscCall(DMPlexComputeIntegralFEM(dm, u, &mom, user));
7200d52c2f21SMatthew G. Knepley     moments[c + 1] = PetscRealPart(mom);
7201d52c2f21SMatthew G. Knepley   }
7202d52c2f21SMatthew G. Knepley   PetscCall(PetscDSSetObjective(ds, field, &f0_x2));
7203d52c2f21SMatthew G. Knepley   PetscCall(DMPlexComputeIntegralFEM(dm, u, &mom, user));
720417314648SMatthew G. Knepley   moments[cdim + 1] = PetscRealPart(mom);
7205d52c2f21SMatthew G. Knepley   PetscCall(PetscDSSetConstants(ds, Ncon, (PetscScalar *)oldConstants));
7206d52c2f21SMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
7207d52c2f21SMatthew G. Knepley }
7208