xref: /petsc/src/dm/impls/plex/plexfem.c (revision b98a7184b7290cc8e940d31f2351436254fc494c)
1af0996ceSBarry Smith #include <petsc/private/dmpleximpl.h> /*I      "petscdmplex.h"   I*/
2da97024aSMatthew G. Knepley #include <petscsf.h>
3cb1e1211SMatthew G Knepley 
48e3a2eefSMatthew G. Knepley #include <petscblaslapack.h>
5e8f14785SLisandro Dalcin #include <petsc/private/hashsetij.h>
6af0996ceSBarry Smith #include <petsc/private/petscfeimpl.h>
7af0996ceSBarry Smith #include <petsc/private/petscfvimpl.h>
8a0845e3aSMatthew G. Knepley 
95f0b18bfSMatthew G. Knepley PetscBool  Clementcite       = PETSC_FALSE;
105f0b18bfSMatthew G. Knepley const char ClementCitation[] = "@article{clement1975approximation,\n"
115f0b18bfSMatthew G. Knepley                                "  title   = {Approximation by finite element functions using local regularization},\n"
125f0b18bfSMatthew G. Knepley                                "  author  = {Philippe Cl{\\'e}ment},\n"
135f0b18bfSMatthew G. Knepley                                "  journal = {Revue fran{\\c{c}}aise d'automatique, informatique, recherche op{\\'e}rationnelle. Analyse num{\\'e}rique},\n"
145f0b18bfSMatthew G. Knepley                                "  volume  = {9},\n"
155f0b18bfSMatthew G. Knepley                                "  number  = {R2},\n"
165f0b18bfSMatthew G. Knepley                                "  pages   = {77--84},\n"
175f0b18bfSMatthew G. Knepley                                "  year    = {1975}\n}\n";
185f0b18bfSMatthew G. Knepley 
19d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexConvertPlex(DM dm, DM *plex, PetscBool copy)
20d71ae5a4SJacob Faibussowitsch {
212f856554SMatthew G. Knepley   PetscBool isPlex;
222f856554SMatthew G. Knepley 
232f856554SMatthew G. Knepley   PetscFunctionBegin;
249566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex));
252f856554SMatthew G. Knepley   if (isPlex) {
262f856554SMatthew G. Knepley     *plex = dm;
279566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)dm));
282f856554SMatthew G. Knepley   } else {
299566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex));
302f856554SMatthew G. Knepley     if (!*plex) {
319566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm, DMPLEX, plex));
329566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex));
33cbf8eb3cSStefano Zampini     } else {
34cbf8eb3cSStefano Zampini       PetscCall(PetscObjectReference((PetscObject)*plex));
35cbf8eb3cSStefano Zampini     }
362f856554SMatthew G. Knepley     if (copy) {
372f856554SMatthew G. Knepley       DMSubDomainHookLink link;
389a2a23afSMatthew G. Knepley 
39bb4b53efSMatthew G. Knepley       PetscCall(DMCopyDS(dm, PETSC_DETERMINE, PETSC_DETERMINE, *plex));
409566063dSJacob Faibussowitsch       PetscCall(DMCopyAuxiliaryVec(dm, *plex));
419a2a23afSMatthew G. Knepley       /* Run the subdomain hook (this will copy the DMSNES/DMTS) */
422f856554SMatthew G. Knepley       for (link = dm->subdomainhook; link; link = link->next) {
439566063dSJacob Faibussowitsch         if (link->ddhook) PetscCall((*link->ddhook)(dm, *plex, link->ctx));
442f856554SMatthew G. Knepley       }
452f856554SMatthew G. Knepley     }
462f856554SMatthew G. Knepley   }
473ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
482f856554SMatthew G. Knepley }
492f856554SMatthew G. Knepley 
5049abdd8aSBarry Smith static PetscErrorCode PetscContainerCtxDestroy_PetscFEGeom(void **ctx)
51d71ae5a4SJacob Faibussowitsch {
5249abdd8aSBarry Smith   PetscFEGeom *geom = (PetscFEGeom *)*ctx;
539b6f715bSMatthew G. Knepley 
549b6f715bSMatthew G. Knepley   PetscFunctionBegin;
559566063dSJacob Faibussowitsch   PetscCall(PetscFEGeomDestroy(&geom));
563ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
579b6f715bSMatthew G. Knepley }
589b6f715bSMatthew G. Knepley 
59d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
60d71ae5a4SJacob Faibussowitsch {
619b6f715bSMatthew G. Knepley   char           composeStr[33] = {0};
629b6f715bSMatthew G. Knepley   PetscObjectId  id;
639b6f715bSMatthew G. Knepley   PetscContainer container;
649b6f715bSMatthew G. Knepley 
659b6f715bSMatthew G. Knepley   PetscFunctionBegin;
669566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad, &id));
6763a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMPlexGetFEGeom_%" PetscInt64_FMT "\n", id));
689566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container));
699b6f715bSMatthew G. Knepley   if (container) {
709566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **)geom));
719b6f715bSMatthew G. Knepley   } else {
729566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom));
739566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container));
749566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *)*geom));
7549abdd8aSBarry Smith     PetscCall(PetscContainerSetCtxDestroy(container, PetscContainerCtxDestroy_PetscFEGeom));
769566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container));
779566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
789b6f715bSMatthew G. Knepley   }
793ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
809b6f715bSMatthew G. Knepley }
819b6f715bSMatthew G. Knepley 
82d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
83d71ae5a4SJacob Faibussowitsch {
849b6f715bSMatthew G. Knepley   PetscFunctionBegin;
859b6f715bSMatthew G. Knepley   *geom = NULL;
863ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
879b6f715bSMatthew G. Knepley }
889b6f715bSMatthew G. Knepley 
8946fa42a0SMatthew G. Knepley /*@
9046fa42a0SMatthew G. Knepley   DMPlexGetScale - Get the scale for the specified fundamental unit
9146fa42a0SMatthew G. Knepley 
9220f4b53cSBarry Smith   Not Collective
9346fa42a0SMatthew G. Knepley 
944165533cSJose E. Roman   Input Parameters:
95a1cb98faSBarry Smith + dm   - the `DM`
9646fa42a0SMatthew G. Knepley - unit - The SI unit
9746fa42a0SMatthew G. Knepley 
984165533cSJose E. Roman   Output Parameter:
9946fa42a0SMatthew G. Knepley . scale - The value used to scale all quantities with this unit
10046fa42a0SMatthew G. Knepley 
10146fa42a0SMatthew G. Knepley   Level: advanced
10246fa42a0SMatthew G. Knepley 
1031cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetScale()`, `PetscUnit`
10446fa42a0SMatthew G. Knepley @*/
105d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetScale(DM dm, PetscUnit unit, PetscReal *scale)
106d71ae5a4SJacob Faibussowitsch {
107cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
108cb1e1211SMatthew G Knepley 
109cb1e1211SMatthew G Knepley   PetscFunctionBegin;
110cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1114f572ea9SToby Isaac   PetscAssertPointer(scale, 3);
112cb1e1211SMatthew G Knepley   *scale = mesh->scale[unit];
1133ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
114cb1e1211SMatthew G Knepley }
115cb1e1211SMatthew G Knepley 
11646fa42a0SMatthew G. Knepley /*@
11746fa42a0SMatthew G. Knepley   DMPlexSetScale - Set the scale for the specified fundamental unit
11846fa42a0SMatthew G. Knepley 
11920f4b53cSBarry Smith   Not Collective
12046fa42a0SMatthew G. Knepley 
1214165533cSJose E. Roman   Input Parameters:
122a1cb98faSBarry Smith + dm    - the `DM`
12346fa42a0SMatthew G. Knepley . unit  - The SI unit
12446fa42a0SMatthew G. Knepley - scale - The value used to scale all quantities with this unit
12546fa42a0SMatthew G. Knepley 
12646fa42a0SMatthew G. Knepley   Level: advanced
12746fa42a0SMatthew G. Knepley 
1281cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetScale()`, `PetscUnit`
12946fa42a0SMatthew G. Knepley @*/
130d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexSetScale(DM dm, PetscUnit unit, PetscReal scale)
131d71ae5a4SJacob Faibussowitsch {
132cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
133cb1e1211SMatthew G Knepley 
134cb1e1211SMatthew G Knepley   PetscFunctionBegin;
135cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
136cb1e1211SMatthew G Knepley   mesh->scale[unit] = scale;
1373ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
138cb1e1211SMatthew G Knepley }
139cb1e1211SMatthew G Knepley 
140d2b2dc1eSMatthew G. Knepley PetscErrorCode DMPlexGetUseCeed_Plex(DM dm, PetscBool *useCeed)
141d2b2dc1eSMatthew G. Knepley {
142d2b2dc1eSMatthew G. Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
143d2b2dc1eSMatthew G. Knepley 
144d2b2dc1eSMatthew G. Knepley   PetscFunctionBegin;
145d2b2dc1eSMatthew G. Knepley   *useCeed = mesh->useCeed;
146d2b2dc1eSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
147d2b2dc1eSMatthew G. Knepley }
148d2b2dc1eSMatthew G. Knepley PetscErrorCode DMPlexSetUseCeed_Plex(DM dm, PetscBool useCeed)
149d2b2dc1eSMatthew G. Knepley {
150d2b2dc1eSMatthew G. Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
151d2b2dc1eSMatthew G. Knepley 
152d2b2dc1eSMatthew G. Knepley   PetscFunctionBegin;
153d2b2dc1eSMatthew G. Knepley   mesh->useCeed = useCeed;
154d2b2dc1eSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
155d2b2dc1eSMatthew G. Knepley }
156d2b2dc1eSMatthew G. Knepley 
157d2b2dc1eSMatthew G. Knepley /*@
158d2b2dc1eSMatthew G. Knepley   DMPlexGetUseCeed - Get flag for using the LibCEED backend
159d2b2dc1eSMatthew G. Knepley 
160d2b2dc1eSMatthew G. Knepley   Not collective
161d2b2dc1eSMatthew G. Knepley 
162d2b2dc1eSMatthew G. Knepley   Input Parameter:
163d2b2dc1eSMatthew G. Knepley . dm - The `DM`
164d2b2dc1eSMatthew G. Knepley 
165d2b2dc1eSMatthew G. Knepley   Output Parameter:
166d2b2dc1eSMatthew G. Knepley . useCeed - The flag
167d2b2dc1eSMatthew G. Knepley 
168d2b2dc1eSMatthew G. Knepley   Level: intermediate
169d2b2dc1eSMatthew G. Knepley 
170d2b2dc1eSMatthew G. Knepley .seealso: `DMPlexSetUseCeed()`
171d2b2dc1eSMatthew G. Knepley @*/
172d2b2dc1eSMatthew G. Knepley PetscErrorCode DMPlexGetUseCeed(DM dm, PetscBool *useCeed)
173d2b2dc1eSMatthew G. Knepley {
174d2b2dc1eSMatthew G. Knepley   PetscFunctionBegin;
175d2b2dc1eSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
176d2b2dc1eSMatthew G. Knepley   PetscAssertPointer(useCeed, 2);
177d2b2dc1eSMatthew G. Knepley   *useCeed = PETSC_FALSE;
178d2b2dc1eSMatthew G. Knepley   PetscTryMethod(dm, "DMPlexGetUseCeed_C", (DM, PetscBool *), (dm, useCeed));
179d2b2dc1eSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
180d2b2dc1eSMatthew G. Knepley }
181d2b2dc1eSMatthew G. Knepley 
182d2b2dc1eSMatthew G. Knepley /*@
183d2b2dc1eSMatthew G. Knepley   DMPlexSetUseCeed - Set flag for using the LibCEED backend
184d2b2dc1eSMatthew G. Knepley 
185d2b2dc1eSMatthew G. Knepley   Not collective
186d2b2dc1eSMatthew G. Knepley 
187d2b2dc1eSMatthew G. Knepley   Input Parameters:
188d2b2dc1eSMatthew G. Knepley + dm      - The `DM`
189d2b2dc1eSMatthew G. Knepley - useCeed - The flag
190d2b2dc1eSMatthew G. Knepley 
191d2b2dc1eSMatthew G. Knepley   Level: intermediate
192d2b2dc1eSMatthew G. Knepley 
193fe8e7dddSPierre Jolivet .seealso: `DMPlexGetUseCeed()`
194d2b2dc1eSMatthew G. Knepley @*/
195d2b2dc1eSMatthew G. Knepley PetscErrorCode DMPlexSetUseCeed(DM dm, PetscBool useCeed)
196d2b2dc1eSMatthew G. Knepley {
197d2b2dc1eSMatthew G. Knepley   PetscFunctionBegin;
198d2b2dc1eSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
199d2b2dc1eSMatthew G. Knepley   PetscValidLogicalCollectiveBool(dm, useCeed, 2);
200d2b2dc1eSMatthew G. Knepley   PetscUseMethod(dm, "DMPlexSetUseCeed_C", (DM, PetscBool), (dm, useCeed));
201d2b2dc1eSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
202d2b2dc1eSMatthew G. Knepley }
203d2b2dc1eSMatthew G. Knepley 
204e8e188d2SZach Atkins /*@
205e8e188d2SZach Atkins   DMPlexGetUseMatClosurePermutation - Get flag for using a closure permutation for matrix insertion
206e8e188d2SZach Atkins 
207e8e188d2SZach Atkins   Not collective
208e8e188d2SZach Atkins 
209e8e188d2SZach Atkins   Input Parameter:
210e8e188d2SZach Atkins . dm - The `DM`
211e8e188d2SZach Atkins 
212e8e188d2SZach Atkins   Output Parameter:
213e8e188d2SZach Atkins . useClPerm - The flag
214e8e188d2SZach Atkins 
215e8e188d2SZach Atkins   Level: intermediate
216e8e188d2SZach Atkins 
217e8e188d2SZach Atkins .seealso: `DMPlexSetUseMatClosurePermutation()`
218e8e188d2SZach Atkins @*/
219e8e188d2SZach Atkins PetscErrorCode DMPlexGetUseMatClosurePermutation(DM dm, PetscBool *useClPerm)
220e8e188d2SZach Atkins {
221e8e188d2SZach Atkins   DM_Plex *mesh = (DM_Plex *)dm->data;
222e8e188d2SZach Atkins 
223e8e188d2SZach Atkins   PetscFunctionBegin;
224e8e188d2SZach Atkins   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
225e8e188d2SZach Atkins   PetscAssertPointer(useClPerm, 2);
226e8e188d2SZach Atkins   *useClPerm = mesh->useMatClPerm;
227e8e188d2SZach Atkins   PetscFunctionReturn(PETSC_SUCCESS);
228e8e188d2SZach Atkins }
229e8e188d2SZach Atkins 
230e8e188d2SZach Atkins /*@
231e8e188d2SZach Atkins   DMPlexSetUseMatClosurePermutation - Set flag for using a closure permutation for matrix insertion
232e8e188d2SZach Atkins 
233e8e188d2SZach Atkins   Not collective
234e8e188d2SZach Atkins 
235e8e188d2SZach Atkins   Input Parameters:
236e8e188d2SZach Atkins + dm        - The `DM`
237e8e188d2SZach Atkins - useClPerm - The flag
238e8e188d2SZach Atkins 
239e8e188d2SZach Atkins   Level: intermediate
240e8e188d2SZach Atkins 
241e8e188d2SZach Atkins .seealso: `DMPlexGetUseMatClosurePermutation()`
242e8e188d2SZach Atkins @*/
243e8e188d2SZach Atkins PetscErrorCode DMPlexSetUseMatClosurePermutation(DM dm, PetscBool useClPerm)
244e8e188d2SZach Atkins {
245e8e188d2SZach Atkins   DM_Plex *mesh = (DM_Plex *)dm->data;
246e8e188d2SZach Atkins 
247e8e188d2SZach Atkins   PetscFunctionBegin;
248e8e188d2SZach Atkins   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
249e8e188d2SZach Atkins   PetscValidLogicalCollectiveBool(dm, useClPerm, 2);
250e8e188d2SZach Atkins   mesh->useMatClPerm = useClPerm;
251e8e188d2SZach Atkins   PetscFunctionReturn(PETSC_SUCCESS);
252e8e188d2SZach Atkins }
253e8e188d2SZach Atkins 
254d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexProjectRigidBody_Private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nc, PetscScalar *mode, void *ctx)
255d71ae5a4SJacob Faibussowitsch {
2569371c9d4SSatish Balay   const PetscInt eps[3][3][3] = {
2579371c9d4SSatish Balay     {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2589371c9d4SSatish Balay     {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2599371c9d4SSatish Balay     {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2609371c9d4SSatish Balay   };
261026175e5SToby Isaac   PetscInt *ctxInt = (PetscInt *)ctx;
262ad917190SMatthew G. Knepley   PetscInt  dim2   = ctxInt[0];
263026175e5SToby Isaac   PetscInt  d      = ctxInt[1];
264026175e5SToby Isaac   PetscInt  i, j, k = dim > 2 ? d - dim : d;
265026175e5SToby Isaac 
266ad917190SMatthew G. Knepley   PetscFunctionBegin;
26763a3b9bcSJacob Faibussowitsch   PetscCheck(dim == dim2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Input dimension %" PetscInt_FMT " does not match context dimension %" PetscInt_FMT, dim, dim2);
268026175e5SToby Isaac   for (i = 0; i < dim; i++) mode[i] = 0.;
269026175e5SToby Isaac   if (d < dim) {
27012adca46SMatthew G. Knepley     mode[d] = 1.; /* Translation along axis d */
271ad917190SMatthew G. Knepley   } else {
272026175e5SToby Isaac     for (i = 0; i < dim; i++) {
2739371c9d4SSatish Balay       for (j = 0; j < dim; j++) { mode[j] += eps[i][j][k] * X[i]; /* Rotation about axis d */ }
274026175e5SToby Isaac     }
275026175e5SToby Isaac   }
2763ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
277026175e5SToby Isaac }
278026175e5SToby Isaac 
279cc4e42d9SMartin Diehl /*@
28012adca46SMatthew G. Knepley   DMPlexCreateRigidBody - For the default global section, create rigid body modes by function space interpolation
281cb1e1211SMatthew G Knepley 
28220f4b53cSBarry Smith   Collective
283cb1e1211SMatthew G Knepley 
2844165533cSJose E. Roman   Input Parameters:
285a1cb98faSBarry Smith + dm    - the `DM`
28656cf3b9cSMatthew G. Knepley - field - The field number for the rigid body space, or 0 for the default
287cb1e1211SMatthew G Knepley 
2884165533cSJose E. Roman   Output Parameter:
289cb1e1211SMatthew G Knepley . sp - the null space
290cb1e1211SMatthew G Knepley 
291cb1e1211SMatthew G Knepley   Level: advanced
292cb1e1211SMatthew G Knepley 
293a1cb98faSBarry Smith   Note:
294a1cb98faSBarry Smith   This is necessary to provide a suitable coarse space for algebraic multigrid
295a1cb98faSBarry Smith 
2961cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `MatNullSpaceCreate()`, `PCGAMG`
297cb1e1211SMatthew G Knepley @*/
298d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexCreateRigidBody(DM dm, PetscInt field, MatNullSpace *sp)
299d71ae5a4SJacob Faibussowitsch {
30056cf3b9cSMatthew G. Knepley   PetscErrorCode (**func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *);
301cb1e1211SMatthew G Knepley   MPI_Comm     comm;
302026175e5SToby Isaac   Vec          mode[6];
303026175e5SToby Isaac   PetscSection section, globalSection;
30456cf3b9cSMatthew G. Knepley   PetscInt     dim, dimEmbed, Nf, n, m, mmin, d, i, j;
305db14aad5SMatthew G. Knepley   void       **ctxs;
306cb1e1211SMatthew G Knepley 
307cb1e1211SMatthew G Knepley   PetscFunctionBegin;
3089566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3099566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
3109566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
3119566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
31263a3b9bcSJacob Faibussowitsch   PetscCheck(!Nf || !(field < 0 || field >= Nf), comm, PETSC_ERR_ARG_OUTOFRANGE, "Field %" PetscInt_FMT " is not in [0, %" PetscInt_FMT ")", field, Nf);
31356cf3b9cSMatthew G. Knepley   if (dim == 1 && Nf < 2) {
3149566063dSJacob Faibussowitsch     PetscCall(MatNullSpaceCreate(comm, PETSC_TRUE, 0, NULL, sp));
3153ba16761SJacob Faibussowitsch     PetscFunctionReturn(PETSC_SUCCESS);
316cb1e1211SMatthew G Knepley   }
3179566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
3189566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
3199566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
320db14aad5SMatthew G. Knepley   PetscCall(PetscCalloc2(Nf, &func, Nf, &ctxs));
321b247467aSMatthew G. Knepley   m = (dim * (dim + 1)) / 2;
3229566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
3239566063dSJacob Faibussowitsch   PetscCall(VecSetType(mode[0], dm->vectype));
3249566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
3259566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
3269566063dSJacob Faibussowitsch   PetscCall(VecGetSize(mode[0], &n));
327b247467aSMatthew G. Knepley   mmin        = PetscMin(m, n);
32856cf3b9cSMatthew G. Knepley   func[field] = DMPlexProjectRigidBody_Private;
3299566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
330026175e5SToby Isaac   for (d = 0; d < m; d++) {
331330485fdSToby Isaac     PetscInt ctx[2];
332cb1e1211SMatthew G Knepley 
333db14aad5SMatthew G. Knepley     ctxs[field] = (void *)(&ctx[0]);
3349d8fbdeaSMatthew G. Knepley     ctx[0]      = dimEmbed;
335330485fdSToby Isaac     ctx[1]      = d;
336db14aad5SMatthew G. Knepley     PetscCall(DMProjectFunction(dm, 0.0, func, ctxs, INSERT_VALUES, mode[d]));
337cb1e1211SMatthew G Knepley   }
3383b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
339b50a2c0aSJacob Faibussowitsch   for (i = 0; i < mmin; ++i) {
340b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
341b50a2c0aSJacob Faibussowitsch 
3429566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
3439566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], mmin - i - 1, mode + i + 1, dots + i + 1));
344b50a2c0aSJacob Faibussowitsch     for (j = i + 1; j < mmin; ++j) {
345b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
3469566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
347b50a2c0aSJacob Faibussowitsch     }
348cb1e1211SMatthew G Knepley   }
3499566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, mmin, mode, sp));
3509566063dSJacob Faibussowitsch   for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i]));
351db14aad5SMatthew G. Knepley   PetscCall(PetscFree2(func, ctxs));
3523ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
353cb1e1211SMatthew G Knepley }
354cb1e1211SMatthew G Knepley 
355cc4e42d9SMartin Diehl /*@
35612adca46SMatthew G. Knepley   DMPlexCreateRigidBodies - For the default global section, create rigid body modes by function space interpolation
35712adca46SMatthew G. Knepley 
35820f4b53cSBarry Smith   Collective
35912adca46SMatthew G. Knepley 
3604165533cSJose E. Roman   Input Parameters:
361a1cb98faSBarry Smith + dm    - the `DM`
36212adca46SMatthew G. Knepley . nb    - The number of bodies
363a1cb98faSBarry Smith . label - The `DMLabel` marking each domain
36412adca46SMatthew G. Knepley . nids  - The number of ids per body
36512adca46SMatthew G. Knepley - ids   - An array of the label ids in sequence for each domain
36612adca46SMatthew G. Knepley 
3674165533cSJose E. Roman   Output Parameter:
36812adca46SMatthew G. Knepley . sp - the null space
36912adca46SMatthew G. Knepley 
37012adca46SMatthew G. Knepley   Level: advanced
37112adca46SMatthew G. Knepley 
372a1cb98faSBarry Smith   Note:
373a1cb98faSBarry Smith   This is necessary to provide a suitable coarse space for algebraic multigrid
374a1cb98faSBarry Smith 
3751cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `MatNullSpaceCreate()`
37612adca46SMatthew G. Knepley @*/
377d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexCreateRigidBodies(DM dm, PetscInt nb, DMLabel label, const PetscInt nids[], const PetscInt ids[], MatNullSpace *sp)
378d71ae5a4SJacob Faibussowitsch {
37912adca46SMatthew G. Knepley   MPI_Comm     comm;
38012adca46SMatthew G. Knepley   PetscSection section, globalSection;
38112adca46SMatthew G. Knepley   Vec         *mode;
38212adca46SMatthew G. Knepley   PetscScalar *dots;
38312adca46SMatthew G. Knepley   PetscInt     dim, dimEmbed, n, m, b, d, i, j, off;
38412adca46SMatthew G. Knepley 
38512adca46SMatthew G. Knepley   PetscFunctionBegin;
3869566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3879566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
3889566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
3899566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
3909566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
3919566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
39212adca46SMatthew G. Knepley   m = nb * (dim * (dim + 1)) / 2;
3939566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(m, &mode, m, &dots));
3949566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
3959566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
3969566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
3979566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
39812adca46SMatthew G. Knepley   for (b = 0, off = 0; b < nb; ++b) {
39912adca46SMatthew G. Knepley     for (d = 0; d < m / nb; ++d) {
40012adca46SMatthew G. Knepley       PetscInt ctx[2];
40112adca46SMatthew G. Knepley       PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private;
40212adca46SMatthew G. Knepley       void *voidctx                                                                                   = (void *)(&ctx[0]);
40312adca46SMatthew G. Knepley 
40412adca46SMatthew G. Knepley       ctx[0] = dimEmbed;
40512adca46SMatthew G. Knepley       ctx[1] = d;
4069566063dSJacob Faibussowitsch       PetscCall(DMProjectFunctionLabel(dm, 0.0, label, nids[b], &ids[off], 0, NULL, &func, &voidctx, INSERT_VALUES, mode[d]));
40712adca46SMatthew G. Knepley       off += nids[b];
40812adca46SMatthew G. Knepley     }
40912adca46SMatthew G. Knepley   }
4103b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
411606c1a1cSJacob Faibussowitsch   for (i = 0; i < m; ++i) {
412b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
4135a0e29b9SJacob Faibussowitsch 
4149566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
4159566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], m - i - 1, mode + i + 1, dots + i + 1));
4165a0e29b9SJacob Faibussowitsch     for (j = i + 1; j < m; ++j) {
417b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
4189566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
4195a0e29b9SJacob Faibussowitsch     }
42012adca46SMatthew G. Knepley   }
4219566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, m, mode, sp));
4229566063dSJacob Faibussowitsch   for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i]));
4239566063dSJacob Faibussowitsch   PetscCall(PetscFree2(mode, dots));
4243ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
42512adca46SMatthew G. Knepley }
42612adca46SMatthew G. Knepley 
427b29cfa1cSToby Isaac /*@
428b29cfa1cSToby Isaac   DMPlexSetMaxProjectionHeight - In DMPlexProjectXXXLocal() functions, the projected values of a basis function's dofs
429b29cfa1cSToby Isaac   are computed by associating the basis function with one of the mesh points in its transitively-closed support, and
430a4e35b19SJacob Faibussowitsch   evaluating the dual space basis of that point.
431b29cfa1cSToby Isaac 
432b29cfa1cSToby Isaac   Input Parameters:
433a1cb98faSBarry Smith + dm     - the `DMPLEX` object
434b29cfa1cSToby Isaac - height - the maximum projection height >= 0
435b29cfa1cSToby Isaac 
436b29cfa1cSToby Isaac   Level: advanced
437b29cfa1cSToby Isaac 
438a4e35b19SJacob Faibussowitsch   Notes:
439a4e35b19SJacob Faibussowitsch   A basis function is associated with the point in its transitively-closed support whose mesh
440a4e35b19SJacob Faibussowitsch   height is highest (w.r.t. DAG height), but not greater than the maximum projection height,
441a4e35b19SJacob Faibussowitsch   which is set with this function.  By default, the maximum projection height is zero, which
442a4e35b19SJacob Faibussowitsch   means that only mesh cells are used to project basis functions.  A height of one, for
443a4e35b19SJacob Faibussowitsch   example, evaluates a cell-interior basis functions using its cells dual space basis, but all
444a4e35b19SJacob Faibussowitsch   other basis functions with the dual space basis of a face.
445a4e35b19SJacob Faibussowitsch 
4461cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()`
447b29cfa1cSToby Isaac @*/
448d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexSetMaxProjectionHeight(DM dm, PetscInt height)
449d71ae5a4SJacob Faibussowitsch {
450b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *)dm->data;
451b29cfa1cSToby Isaac 
452b29cfa1cSToby Isaac   PetscFunctionBegin;
453b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
454b29cfa1cSToby Isaac   plex->maxProjectionHeight = height;
4553ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
456b29cfa1cSToby Isaac }
457b29cfa1cSToby Isaac 
458b29cfa1cSToby Isaac /*@
459b29cfa1cSToby Isaac   DMPlexGetMaxProjectionHeight - Get the maximum height (w.r.t. DAG) of mesh points used to evaluate dual bases in
460b29cfa1cSToby Isaac   DMPlexProjectXXXLocal() functions.
461b29cfa1cSToby Isaac 
4622fe279fdSBarry Smith   Input Parameter:
463a1cb98faSBarry Smith . dm - the `DMPLEX` object
464b29cfa1cSToby Isaac 
4652fe279fdSBarry Smith   Output Parameter:
466b29cfa1cSToby Isaac . height - the maximum projection height
467b29cfa1cSToby Isaac 
468b29cfa1cSToby Isaac   Level: intermediate
469b29cfa1cSToby Isaac 
4701cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()`
471b29cfa1cSToby Isaac @*/
472d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetMaxProjectionHeight(DM dm, PetscInt *height)
473d71ae5a4SJacob Faibussowitsch {
474b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *)dm->data;
475b29cfa1cSToby Isaac 
476b29cfa1cSToby Isaac   PetscFunctionBegin;
477b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
478b29cfa1cSToby Isaac   *height = plex->maxProjectionHeight;
4793ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
480b29cfa1cSToby Isaac }
481b29cfa1cSToby Isaac 
482ca3d3a14SMatthew G. Knepley typedef struct {
483ca3d3a14SMatthew G. Knepley   PetscReal    alpha; /* The first Euler angle, and in 2D the only one */
484ca3d3a14SMatthew G. Knepley   PetscReal    beta;  /* The second Euler angle */
485ca3d3a14SMatthew G. Knepley   PetscReal    gamma; /* The third Euler angle */
486ca3d3a14SMatthew G. Knepley   PetscInt     dim;   /* The dimension of R */
487ca3d3a14SMatthew G. Knepley   PetscScalar *R;     /* The rotation matrix, transforming a vector in the local basis to the global basis */
488ca3d3a14SMatthew G. Knepley   PetscScalar *RT;    /* The transposed rotation matrix, transforming a vector in the global basis to the local basis */
489ca3d3a14SMatthew G. Knepley } RotCtx;
490ca3d3a14SMatthew G. Knepley 
491ca3d3a14SMatthew G. Knepley /*
492ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
493ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
494ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
495ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
496ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
497ca3d3a14SMatthew G. Knepley */
498d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformSetUp_Rotation_Internal(DM dm, void *ctx)
499d71ae5a4SJacob Faibussowitsch {
500ca3d3a14SMatthew G. Knepley   RotCtx   *rc  = (RotCtx *)ctx;
501ca3d3a14SMatthew G. Knepley   PetscInt  dim = rc->dim;
502ca3d3a14SMatthew G. Knepley   PetscReal c1, s1, c2, s2, c3, s3;
503ca3d3a14SMatthew G. Knepley 
504ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5059566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(PetscSqr(dim), &rc->R, PetscSqr(dim), &rc->RT));
506ca3d3a14SMatthew G. Knepley   switch (dim) {
507ca3d3a14SMatthew G. Knepley   case 2:
5089371c9d4SSatish Balay     c1       = PetscCosReal(rc->alpha);
5099371c9d4SSatish Balay     s1       = PetscSinReal(rc->alpha);
5109371c9d4SSatish Balay     rc->R[0] = c1;
5119371c9d4SSatish Balay     rc->R[1] = s1;
5129371c9d4SSatish Balay     rc->R[2] = -s1;
5139371c9d4SSatish Balay     rc->R[3] = c1;
5149566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
515b458e8f1SJose E. Roman     DMPlex_Transpose2D_Internal(rc->RT);
516ca3d3a14SMatthew G. Knepley     break;
517ca3d3a14SMatthew G. Knepley   case 3:
5189371c9d4SSatish Balay     c1       = PetscCosReal(rc->alpha);
5199371c9d4SSatish Balay     s1       = PetscSinReal(rc->alpha);
5209371c9d4SSatish Balay     c2       = PetscCosReal(rc->beta);
5219371c9d4SSatish Balay     s2       = PetscSinReal(rc->beta);
5229371c9d4SSatish Balay     c3       = PetscCosReal(rc->gamma);
5239371c9d4SSatish Balay     s3       = PetscSinReal(rc->gamma);
5249371c9d4SSatish Balay     rc->R[0] = c1 * c3 - c2 * s1 * s3;
5259371c9d4SSatish Balay     rc->R[1] = c3 * s1 + c1 * c2 * s3;
5269371c9d4SSatish Balay     rc->R[2] = s2 * s3;
5279371c9d4SSatish Balay     rc->R[3] = -c1 * s3 - c2 * c3 * s1;
5289371c9d4SSatish Balay     rc->R[4] = c1 * c2 * c3 - s1 * s3;
5299371c9d4SSatish Balay     rc->R[5] = c3 * s2;
5309371c9d4SSatish Balay     rc->R[6] = s1 * s2;
5319371c9d4SSatish Balay     rc->R[7] = -c1 * s2;
5329371c9d4SSatish Balay     rc->R[8] = c2;
5339566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
534b458e8f1SJose E. Roman     DMPlex_Transpose3D_Internal(rc->RT);
535ca3d3a14SMatthew G. Knepley     break;
536d71ae5a4SJacob Faibussowitsch   default:
537d71ae5a4SJacob Faibussowitsch     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " not supported", dim);
538ca3d3a14SMatthew G. Knepley   }
5393ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
540ca3d3a14SMatthew G. Knepley }
541ca3d3a14SMatthew G. Knepley 
542d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformDestroy_Rotation_Internal(DM dm, void *ctx)
543d71ae5a4SJacob Faibussowitsch {
544ca3d3a14SMatthew G. Knepley   RotCtx *rc = (RotCtx *)ctx;
545ca3d3a14SMatthew G. Knepley 
546ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5479566063dSJacob Faibussowitsch   PetscCall(PetscFree2(rc->R, rc->RT));
5489566063dSJacob Faibussowitsch   PetscCall(PetscFree(rc));
5493ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
550ca3d3a14SMatthew G. Knepley }
551ca3d3a14SMatthew G. Knepley 
552d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformGetMatrix_Rotation_Internal(DM dm, const PetscReal x[], PetscBool l2g, const PetscScalar **A, void *ctx)
553d71ae5a4SJacob Faibussowitsch {
554ca3d3a14SMatthew G. Knepley   RotCtx *rc = (RotCtx *)ctx;
555ca3d3a14SMatthew G. Knepley 
556ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
5574f572ea9SToby Isaac   PetscAssertPointer(ctx, 5);
5589371c9d4SSatish Balay   if (l2g) {
5599371c9d4SSatish Balay     *A = rc->R;
5609371c9d4SSatish Balay   } else {
5619371c9d4SSatish Balay     *A = rc->RT;
5629371c9d4SSatish Balay   }
5633ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
564ca3d3a14SMatthew G. Knepley }
565ca3d3a14SMatthew G. Knepley 
566d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexBasisTransformApplyReal_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscReal *y, PetscReal *z, void *ctx)
567d71ae5a4SJacob Faibussowitsch {
568ec277c0fSMatthew G. Knepley   PetscFunctionBegin;
569ab6a9622SMatthew G. Knepley #if defined(PETSC_USE_COMPLEX)
570ab6a9622SMatthew G. Knepley   switch (dim) {
5719371c9d4SSatish Balay   case 2: {
57227104ee2SJacob Faibussowitsch     PetscScalar yt[2] = {y[0], y[1]}, zt[2] = {0.0, 0.0};
573ab6a9622SMatthew G. Knepley 
5749566063dSJacob Faibussowitsch     PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
5759371c9d4SSatish Balay     z[0] = PetscRealPart(zt[0]);
5769371c9d4SSatish Balay     z[1] = PetscRealPart(zt[1]);
5779371c9d4SSatish Balay   } break;
5789371c9d4SSatish Balay   case 3: {
57927104ee2SJacob Faibussowitsch     PetscScalar yt[3] = {y[0], y[1], y[2]}, zt[3] = {0.0, 0.0, 0.0};
580ab6a9622SMatthew G. Knepley 
5819566063dSJacob Faibussowitsch     PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
5829371c9d4SSatish Balay     z[0] = PetscRealPart(zt[0]);
5839371c9d4SSatish Balay     z[1] = PetscRealPart(zt[1]);
5849371c9d4SSatish Balay     z[2] = PetscRealPart(zt[2]);
5859371c9d4SSatish Balay   } break;
586ab6a9622SMatthew G. Knepley   }
587ab6a9622SMatthew G. Knepley #else
5889566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, y, z, ctx));
589ab6a9622SMatthew G. Knepley #endif
5903ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
591ab6a9622SMatthew G. Knepley }
592ab6a9622SMatthew G. Knepley 
593d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexBasisTransformApply_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscScalar *y, PetscScalar *z, void *ctx)
594d71ae5a4SJacob Faibussowitsch {
595ca3d3a14SMatthew G. Knepley   const PetscScalar *A;
596ca3d3a14SMatthew G. Knepley 
597ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
5989566063dSJacob Faibussowitsch   PetscCall((*dm->transformGetMatrix)(dm, x, l2g, &A, ctx));
599ca3d3a14SMatthew G. Knepley   switch (dim) {
600d71ae5a4SJacob Faibussowitsch   case 2:
601d71ae5a4SJacob Faibussowitsch     DMPlex_Mult2D_Internal(A, 1, y, z);
602d71ae5a4SJacob Faibussowitsch     break;
603d71ae5a4SJacob Faibussowitsch   case 3:
604d71ae5a4SJacob Faibussowitsch     DMPlex_Mult3D_Internal(A, 1, y, z);
605d71ae5a4SJacob Faibussowitsch     break;
606ca3d3a14SMatthew G. Knepley   }
6073ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
608ca3d3a14SMatthew G. Knepley }
609ca3d3a14SMatthew G. Knepley 
610d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformField_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscInt f, PetscBool l2g, PetscScalar *a)
611d71ae5a4SJacob Faibussowitsch {
612ca3d3a14SMatthew G. Knepley   PetscSection       ts;
613ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tva;
614ca3d3a14SMatthew G. Knepley   PetscInt           dof;
615ca3d3a14SMatthew G. Knepley 
616ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
6179566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
6189566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, p, f, &dof));
6199566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
6209566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, p, f, ta, &tva));
621ca3d3a14SMatthew G. Knepley   if (l2g) {
622ca3d3a14SMatthew G. Knepley     switch (dof) {
623d71ae5a4SJacob Faibussowitsch     case 4:
624d71ae5a4SJacob Faibussowitsch       DMPlex_Mult2D_Internal(tva, 1, a, a);
625d71ae5a4SJacob Faibussowitsch       break;
626d71ae5a4SJacob Faibussowitsch     case 9:
627d71ae5a4SJacob Faibussowitsch       DMPlex_Mult3D_Internal(tva, 1, a, a);
628d71ae5a4SJacob Faibussowitsch       break;
629ca3d3a14SMatthew G. Knepley     }
630ca3d3a14SMatthew G. Knepley   } else {
631ca3d3a14SMatthew G. Knepley     switch (dof) {
632d71ae5a4SJacob Faibussowitsch     case 4:
633d71ae5a4SJacob Faibussowitsch       DMPlex_MultTranspose2D_Internal(tva, 1, a, a);
634d71ae5a4SJacob Faibussowitsch       break;
635d71ae5a4SJacob Faibussowitsch     case 9:
636d71ae5a4SJacob Faibussowitsch       DMPlex_MultTranspose3D_Internal(tva, 1, a, a);
637d71ae5a4SJacob Faibussowitsch       break;
638ca3d3a14SMatthew G. Knepley     }
639ca3d3a14SMatthew G. Knepley   }
6409566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
6413ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
642ca3d3a14SMatthew G. Knepley }
643ca3d3a14SMatthew G. Knepley 
644d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransformFieldTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt pf, PetscInt f, PetscInt pg, PetscInt g, PetscBool l2g, PetscInt lda, PetscScalar *a)
645d71ae5a4SJacob Faibussowitsch {
646ca3d3a14SMatthew G. Knepley   PetscSection       s, ts;
647ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tvaf, *tvag;
648ca3d3a14SMatthew G. Knepley   PetscInt           fdof, gdof, fpdof, gpdof;
649ca3d3a14SMatthew G. Knepley 
650ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
6519566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
6529566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
6539566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pf, f, &fpdof));
6549566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pg, g, &gpdof));
6559566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pf, f, &fdof));
6569566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pg, g, &gdof));
6579566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
6589566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pf, f, ta, &tvaf));
6599566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pg, g, ta, &tvag));
660ca3d3a14SMatthew G. Knepley   if (l2g) {
661ca3d3a14SMatthew G. Knepley     switch (fdof) {
662d71ae5a4SJacob Faibussowitsch     case 4:
663d71ae5a4SJacob Faibussowitsch       DMPlex_MatMult2D_Internal(tvaf, gpdof, lda, a, a);
664d71ae5a4SJacob Faibussowitsch       break;
665d71ae5a4SJacob Faibussowitsch     case 9:
666d71ae5a4SJacob Faibussowitsch       DMPlex_MatMult3D_Internal(tvaf, gpdof, lda, a, a);
667d71ae5a4SJacob Faibussowitsch       break;
668ca3d3a14SMatthew G. Knepley     }
669ca3d3a14SMatthew G. Knepley     switch (gdof) {
670d71ae5a4SJacob Faibussowitsch     case 4:
671d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultTransposeLeft2D_Internal(tvag, fpdof, lda, a, a);
672d71ae5a4SJacob Faibussowitsch       break;
673d71ae5a4SJacob Faibussowitsch     case 9:
674d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultTransposeLeft3D_Internal(tvag, fpdof, lda, a, a);
675d71ae5a4SJacob Faibussowitsch       break;
676ca3d3a14SMatthew G. Knepley     }
677ca3d3a14SMatthew G. Knepley   } else {
678ca3d3a14SMatthew G. Knepley     switch (fdof) {
679d71ae5a4SJacob Faibussowitsch     case 4:
680d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultTranspose2D_Internal(tvaf, gpdof, lda, a, a);
681d71ae5a4SJacob Faibussowitsch       break;
682d71ae5a4SJacob Faibussowitsch     case 9:
683d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultTranspose3D_Internal(tvaf, gpdof, lda, a, a);
684d71ae5a4SJacob Faibussowitsch       break;
685ca3d3a14SMatthew G. Knepley     }
686ca3d3a14SMatthew G. Knepley     switch (gdof) {
687d71ae5a4SJacob Faibussowitsch     case 4:
688d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultLeft2D_Internal(tvag, fpdof, lda, a, a);
689d71ae5a4SJacob Faibussowitsch       break;
690d71ae5a4SJacob Faibussowitsch     case 9:
691d71ae5a4SJacob Faibussowitsch       DMPlex_MatMultLeft3D_Internal(tvag, fpdof, lda, a, a);
692d71ae5a4SJacob Faibussowitsch       break;
693ca3d3a14SMatthew G. Knepley     }
694ca3d3a14SMatthew G. Knepley   }
6959566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
6963ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
697ca3d3a14SMatthew G. Knepley }
698ca3d3a14SMatthew G. Knepley 
699d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexBasisTransformPoint_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool fieldActive[], PetscBool l2g, PetscScalar *a)
700d71ae5a4SJacob Faibussowitsch {
701ca3d3a14SMatthew G. Knepley   PetscSection    s;
702ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
703ca3d3a14SMatthew G. Knepley   IS              clPoints;
704ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
705ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
706ca3d3a14SMatthew G. Knepley   PetscInt        Nf, f, Np, cp, dof, d = 0;
707ca3d3a14SMatthew G. Knepley 
708ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
7099566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
7109566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
71107218a29SMatthew G. Knepley   PetscCall(DMPlexGetCompressedClosure(dm, s, p, 0, &Np, &points, &clSection, &clPoints, &clp));
712ca3d3a14SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
713ca3d3a14SMatthew G. Knepley     for (cp = 0; cp < Np * 2; cp += 2) {
7149566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cp], f, &dof));
715ca3d3a14SMatthew G. Knepley       if (!dof) continue;
7169566063dSJacob Faibussowitsch       if (fieldActive[f]) PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, points[cp], f, l2g, &a[d]));
717ca3d3a14SMatthew G. Knepley       d += dof;
718ca3d3a14SMatthew G. Knepley     }
719ca3d3a14SMatthew G. Knepley   }
7209566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
7213ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
722ca3d3a14SMatthew G. Knepley }
723ca3d3a14SMatthew G. Knepley 
724d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexBasisTransformPointTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool l2g, PetscInt lda, PetscScalar *a)
725d71ae5a4SJacob Faibussowitsch {
726ca3d3a14SMatthew G. Knepley   PetscSection    s;
727ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
728ca3d3a14SMatthew G. Knepley   IS              clPoints;
729ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
730ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
7318bdb3c71SMatthew G. Knepley   PetscInt        Nf, f, g, Np, cpf, cpg, fdof, gdof, r, c = 0;
732ca3d3a14SMatthew G. Knepley 
733ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
7349566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
7359566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
73607218a29SMatthew G. Knepley   PetscCall(DMPlexGetCompressedClosure(dm, s, p, 0, &Np, &points, &clSection, &clPoints, &clp));
737ca3d3a14SMatthew G. Knepley   for (f = 0, r = 0; f < Nf; ++f) {
738ca3d3a14SMatthew G. Knepley     for (cpf = 0; cpf < Np * 2; cpf += 2) {
7399566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cpf], f, &fdof));
740ca3d3a14SMatthew G. Knepley       for (g = 0, c = 0; g < Nf; ++g) {
741ca3d3a14SMatthew G. Knepley         for (cpg = 0; cpg < Np * 2; cpg += 2) {
7429566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldDof(s, points[cpg], g, &gdof));
7439566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformFieldTensor_Internal(dm, tdm, tv, points[cpf], f, points[cpg], g, l2g, lda, &a[r * lda + c]));
744ca3d3a14SMatthew G. Knepley           c += gdof;
745ca3d3a14SMatthew G. Knepley         }
746ca3d3a14SMatthew G. Knepley       }
74763a3b9bcSJacob Faibussowitsch       PetscCheck(c == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of columns %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
748ca3d3a14SMatthew G. Knepley       r += fdof;
749ca3d3a14SMatthew G. Knepley     }
750ca3d3a14SMatthew G. Knepley   }
75163a3b9bcSJacob Faibussowitsch   PetscCheck(r == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of rows %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
7529566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
7533ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
754ca3d3a14SMatthew G. Knepley }
755ca3d3a14SMatthew G. Knepley 
756d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexBasisTransform_Internal(DM dm, Vec lv, PetscBool l2g)
757d71ae5a4SJacob Faibussowitsch {
758ca3d3a14SMatthew G. Knepley   DM                 tdm;
759ca3d3a14SMatthew G. Knepley   Vec                tv;
760ca3d3a14SMatthew G. Knepley   PetscSection       ts, s;
761ca3d3a14SMatthew G. Knepley   const PetscScalar *ta;
762ca3d3a14SMatthew G. Knepley   PetscScalar       *a, *va;
763ca3d3a14SMatthew G. Knepley   PetscInt           pStart, pEnd, p, Nf, f;
764ca3d3a14SMatthew G. Knepley 
765ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
7669566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
7679566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
7689566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
7699566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
7709566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetChart(s, &pStart, &pEnd));
7719566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
7729566063dSJacob Faibussowitsch   PetscCall(VecGetArray(lv, &a));
7739566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
774ca3d3a14SMatthew G. Knepley   for (p = pStart; p < pEnd; ++p) {
775ca3d3a14SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
7769566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalFieldRef(dm, p, f, a, &va));
7779566063dSJacob Faibussowitsch       PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, p, f, l2g, va));
778ca3d3a14SMatthew G. Knepley     }
779ca3d3a14SMatthew G. Knepley   }
7809566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(lv, &a));
7819566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
7823ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
783ca3d3a14SMatthew G. Knepley }
784ca3d3a14SMatthew G. Knepley 
785ca3d3a14SMatthew G. Knepley /*@
786ca3d3a14SMatthew G. Knepley   DMPlexGlobalToLocalBasis - Transform the values in the given local vector from the global basis to the local basis
787ca3d3a14SMatthew G. Knepley 
788ca3d3a14SMatthew G. Knepley   Input Parameters:
789a1cb98faSBarry Smith + dm - The `DM`
790ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the global basis
791ca3d3a14SMatthew G. Knepley 
7922fe279fdSBarry Smith   Output Parameter:
793ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the local basis
794ca3d3a14SMatthew G. Knepley 
795ca3d3a14SMatthew G. Knepley   Level: developer
796ca3d3a14SMatthew G. Knepley 
797a1cb98faSBarry Smith   Note:
798a1cb98faSBarry Smith   This method is only intended to be called inside `DMGlobalToLocal()`. It is unlikely that a user will have a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
799a1cb98faSBarry Smith 
8001cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLocalToGlobalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()`
801ca3d3a14SMatthew G. Knepley @*/
802d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGlobalToLocalBasis(DM dm, Vec lv)
803d71ae5a4SJacob Faibussowitsch {
804ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
805ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
806ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
8079566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_FALSE));
8083ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
809ca3d3a14SMatthew G. Knepley }
810ca3d3a14SMatthew G. Knepley 
811ca3d3a14SMatthew G. Knepley /*@
812ca3d3a14SMatthew G. Knepley   DMPlexLocalToGlobalBasis - Transform the values in the given local vector from the local basis to the global basis
813ca3d3a14SMatthew G. Knepley 
814ca3d3a14SMatthew G. Knepley   Input Parameters:
815a1cb98faSBarry Smith + dm - The `DM`
816ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the local basis
817ca3d3a14SMatthew G. Knepley 
8182fe279fdSBarry Smith   Output Parameter:
819ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the global basis
820ca3d3a14SMatthew G. Knepley 
821ca3d3a14SMatthew G. Knepley   Level: developer
822ca3d3a14SMatthew G. Knepley 
823a1cb98faSBarry Smith   Note:
824a1cb98faSBarry Smith   This method is only intended to be called inside `DMGlobalToLocal()`. It is unlikely that a user would want a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
825a1cb98faSBarry Smith 
8261cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGlobalToLocalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()`
827ca3d3a14SMatthew G. Knepley @*/
828d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexLocalToGlobalBasis(DM dm, Vec lv)
829d71ae5a4SJacob Faibussowitsch {
830ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
831ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
832ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
8339566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_TRUE));
8343ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
835ca3d3a14SMatthew G. Knepley }
836ca3d3a14SMatthew G. Knepley 
837ca3d3a14SMatthew G. Knepley /*@
838ca3d3a14SMatthew G. Knepley   DMPlexCreateBasisRotation - Create an internal transformation from the global basis, used to specify boundary conditions
839ca3d3a14SMatthew G. Knepley   and global solutions, to a local basis, appropriate for discretization integrals and assembly.
840ca3d3a14SMatthew G. Knepley 
841ca3d3a14SMatthew G. Knepley   Input Parameters:
842a1cb98faSBarry Smith + dm    - The `DM`
843ca3d3a14SMatthew G. Knepley . alpha - The first Euler angle, and in 2D the only one
844ca3d3a14SMatthew G. Knepley . beta  - The second Euler angle
845f0fc11ceSJed Brown - gamma - The third Euler angle
846ca3d3a14SMatthew G. Knepley 
847ca3d3a14SMatthew G. Knepley   Level: developer
848ca3d3a14SMatthew G. Knepley 
849a1cb98faSBarry Smith   Note:
850a1cb98faSBarry Smith   Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
851a1cb98faSBarry Smith   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows
852a1cb98faSBarry Smith .vb
853a1cb98faSBarry Smith    The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
854a1cb98faSBarry Smith    The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
855a1cb98faSBarry Smith    The XYZ system rotates a third time about the z axis by gamma.
856a1cb98faSBarry Smith .ve
857a1cb98faSBarry Smith 
8581cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGlobalToLocalBasis()`, `DMPlexLocalToGlobalBasis()`
859ca3d3a14SMatthew G. Knepley @*/
860d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexCreateBasisRotation(DM dm, PetscReal alpha, PetscReal beta, PetscReal gamma)
861d71ae5a4SJacob Faibussowitsch {
862ca3d3a14SMatthew G. Knepley   RotCtx  *rc;
863ca3d3a14SMatthew G. Knepley   PetscInt cdim;
864ca3d3a14SMatthew G. Knepley 
865362febeeSStefano Zampini   PetscFunctionBegin;
8669566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
8679566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(1, &rc));
868ca3d3a14SMatthew G. Knepley   dm->transformCtx       = rc;
869ca3d3a14SMatthew G. Knepley   dm->transformSetUp     = DMPlexBasisTransformSetUp_Rotation_Internal;
870ca3d3a14SMatthew G. Knepley   dm->transformDestroy   = DMPlexBasisTransformDestroy_Rotation_Internal;
871ca3d3a14SMatthew G. Knepley   dm->transformGetMatrix = DMPlexBasisTransformGetMatrix_Rotation_Internal;
872ca3d3a14SMatthew G. Knepley   rc->dim                = cdim;
873ca3d3a14SMatthew G. Knepley   rc->alpha              = alpha;
874ca3d3a14SMatthew G. Knepley   rc->beta               = beta;
875ca3d3a14SMatthew G. Knepley   rc->gamma              = gamma;
8769566063dSJacob Faibussowitsch   PetscCall((*dm->transformSetUp)(dm, dm->transformCtx));
8779566063dSJacob Faibussowitsch   PetscCall(DMConstructBasisTransform_Internal(dm));
8783ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
879ca3d3a14SMatthew G. Knepley }
880ca3d3a14SMatthew G. Knepley 
881b278463cSMatthew G. Knepley /*@C
882ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssential - Insert boundary values into a local vector using a function of the coordinates
883b278463cSMatthew G. Knepley 
884b278463cSMatthew G. Knepley   Input Parameters:
885a1cb98faSBarry Smith + dm     - The `DM`, with a `PetscDS` that matches the problem being constrained
886b278463cSMatthew G. Knepley . time   - The time
887b278463cSMatthew G. Knepley . field  - The field to constrain
8881c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
88920f4b53cSBarry Smith . comps  - An array of constrained component numbers, or `NULL` for all components
890a1cb98faSBarry Smith . label  - The `DMLabel` defining constrained points
891a1cb98faSBarry Smith . numids - The number of `DMLabel` ids for constrained points
892b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
893b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
894b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
895b278463cSMatthew G. Knepley 
896b278463cSMatthew G. Knepley   Output Parameter:
897b278463cSMatthew G. Knepley . locX - A local vector to receives the boundary values
898b278463cSMatthew G. Knepley 
899b278463cSMatthew G. Knepley   Level: developer
900b278463cSMatthew G. Knepley 
9011cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLabel`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()`
902b278463cSMatthew G. Knepley @*/
903d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValuesEssential(DM dm, PetscReal time, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void *ctx, Vec locX)
904d71ae5a4SJacob Faibussowitsch {
9050163fd50SMatthew G. Knepley   PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx);
90655f2e967SMatthew G. Knepley   void   **ctxs;
907d7ddef95SMatthew G. Knepley   PetscInt numFields;
908d7ddef95SMatthew G. Knepley 
909d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
9109566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
9119566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs));
912d7ddef95SMatthew G. Knepley   funcs[field] = func;
913d7ddef95SMatthew G. Knepley   ctxs[field]  = ctx;
9149566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_BC_VALUES, locX));
9159566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs, ctxs));
9163ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
917d7ddef95SMatthew G. Knepley }
918d7ddef95SMatthew G. Knepley 
919b278463cSMatthew G. Knepley /*@C
920ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialField - Insert boundary values into a local vector using a function of the coordinates and field data
921b278463cSMatthew G. Knepley 
922b278463cSMatthew G. Knepley   Input Parameters:
923a1cb98faSBarry Smith + dm     - The `DM`, with a `PetscDS` that matches the problem being constrained
924b278463cSMatthew G. Knepley . time   - The time
925b278463cSMatthew G. Knepley . locU   - A local vector with the input solution values
926b278463cSMatthew G. Knepley . field  - The field to constrain
9271c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
92820f4b53cSBarry Smith . comps  - An array of constrained component numbers, or `NULL` for all components
929a1cb98faSBarry Smith . label  - The `DMLabel` defining constrained points
930a1cb98faSBarry Smith . numids - The number of `DMLabel` ids for constrained points
931b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
932b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
933b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
934b278463cSMatthew G. Knepley 
935b278463cSMatthew G. Knepley   Output Parameter:
936b278463cSMatthew G. Knepley . locX - A local vector to receives the boundary values
937b278463cSMatthew G. Knepley 
938b278463cSMatthew G. Knepley   Level: developer
939b278463cSMatthew G. Knepley 
9401cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()`
941b278463cSMatthew G. Knepley @*/
942d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValuesEssentialField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX)
943d71ae5a4SJacob Faibussowitsch {
9449371c9d4SSatish Balay   void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
945c60e475cSMatthew G. Knepley   void   **ctxs;
946c60e475cSMatthew G. Knepley   PetscInt numFields;
947c60e475cSMatthew G. Knepley 
948c60e475cSMatthew G. Knepley   PetscFunctionBegin;
9499566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
9509566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs));
951c60e475cSMatthew G. Knepley   funcs[field] = func;
952c60e475cSMatthew G. Knepley   ctxs[field]  = ctx;
9539566063dSJacob Faibussowitsch   PetscCall(DMProjectFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
9549566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs, ctxs));
9553ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
956c60e475cSMatthew G. Knepley }
957c60e475cSMatthew G. Knepley 
958b278463cSMatthew G. Knepley /*@C
959d5b43468SJose E. Roman   DMPlexInsertBoundaryValuesEssentialBdField - Insert boundary values into a local vector using a function of the coordinates and boundary field data
960ece3a9fcSMatthew G. Knepley 
96120f4b53cSBarry Smith   Collective
962ece3a9fcSMatthew G. Knepley 
963ece3a9fcSMatthew G. Knepley   Input Parameters:
964a1cb98faSBarry Smith + dm     - The `DM`, with a `PetscDS` that matches the problem being constrained
965ece3a9fcSMatthew G. Knepley . time   - The time
966ece3a9fcSMatthew G. Knepley . locU   - A local vector with the input solution values
967ece3a9fcSMatthew G. Knepley . field  - The field to constrain
968ece3a9fcSMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
96920f4b53cSBarry Smith . comps  - An array of constrained component numbers, or `NULL` for all components
970a1cb98faSBarry Smith . label  - The `DMLabel` defining constrained points
971a1cb98faSBarry Smith . numids - The number of `DMLabel` ids for constrained points
972ece3a9fcSMatthew G. Knepley . ids    - An array of ids for constrained points
97320f4b53cSBarry Smith . func   - A pointwise function giving boundary values, the calling sequence is given in `DMProjectBdFieldLabelLocal()`
97420f4b53cSBarry Smith - ctx    - An optional user context for `func`
975ece3a9fcSMatthew G. Knepley 
976ece3a9fcSMatthew G. Knepley   Output Parameter:
977ece3a9fcSMatthew G. Knepley . locX - A local vector to receive the boundary values
978ece3a9fcSMatthew G. Knepley 
979ece3a9fcSMatthew G. Knepley   Level: developer
980ece3a9fcSMatthew G. Knepley 
9811cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectBdFieldLabelLocal()`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()`
982ece3a9fcSMatthew G. Knepley @*/
983d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValuesEssentialBdField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX)
984d71ae5a4SJacob Faibussowitsch {
9859371c9d4SSatish Balay   void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
986ece3a9fcSMatthew G. Knepley   void   **ctxs;
987ece3a9fcSMatthew G. Knepley   PetscInt numFields;
988ece3a9fcSMatthew G. Knepley 
989ece3a9fcSMatthew G. Knepley   PetscFunctionBegin;
9909566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
9919566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs));
992ece3a9fcSMatthew G. Knepley   funcs[field] = func;
993ece3a9fcSMatthew G. Knepley   ctxs[field]  = ctx;
9949566063dSJacob Faibussowitsch   PetscCall(DMProjectBdFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
9959566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs, ctxs));
9963ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
997ece3a9fcSMatthew G. Knepley }
998ece3a9fcSMatthew G. Knepley 
999ece3a9fcSMatthew G. Knepley /*@C
1000b278463cSMatthew G. Knepley   DMPlexInsertBoundaryValuesRiemann - Insert boundary values into a local vector
1001b278463cSMatthew G. Knepley 
1002b278463cSMatthew G. Knepley   Input Parameters:
1003a1cb98faSBarry Smith + dm           - The `DM`, with a `PetscDS` that matches the problem being constrained
1004b278463cSMatthew G. Knepley . time         - The time
1005b278463cSMatthew G. Knepley . faceGeometry - A vector with the FVM face geometry information
1006b278463cSMatthew G. Knepley . cellGeometry - A vector with the FVM cell geometry information
1007b278463cSMatthew G. Knepley . Grad         - A vector with the FVM cell gradient information
1008b278463cSMatthew G. Knepley . field        - The field to constrain
10091c531cf8SMatthew G. Knepley . Nc           - The number of constrained field components, or 0 for all components
101020f4b53cSBarry Smith . comps        - An array of constrained component numbers, or `NULL` for all components
1011a1cb98faSBarry Smith . label        - The `DMLabel` defining constrained points
1012a1cb98faSBarry Smith . numids       - The number of `DMLabel` ids for constrained points
1013b278463cSMatthew G. Knepley . ids          - An array of ids for constrained points
1014b278463cSMatthew G. Knepley . func         - A pointwise function giving boundary values
1015b278463cSMatthew G. Knepley - ctx          - An optional user context for bcFunc
1016b278463cSMatthew G. Knepley 
1017b278463cSMatthew G. Knepley   Output Parameter:
1018b278463cSMatthew G. Knepley . locX - A local vector to receives the boundary values
1019b278463cSMatthew G. Knepley 
1020b278463cSMatthew G. Knepley   Level: developer
1021b278463cSMatthew G. Knepley 
1022a1cb98faSBarry Smith   Note:
1023a1cb98faSBarry Smith   This implementation currently ignores the numcomps/comps argument from `DMAddBoundary()`
1024a1cb98faSBarry Smith 
10251cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()`
1026b278463cSMatthew G. Knepley @*/
1027d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValuesRiemann(DM dm, PetscReal time, Vec faceGeometry, Vec cellGeometry, Vec Grad, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *), void *ctx, Vec locX)
1028d71ae5a4SJacob Faibussowitsch {
102961f58d28SMatthew G. Knepley   PetscDS            prob;
1030da97024aSMatthew G. Knepley   PetscSF            sf;
1031d7ddef95SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad;
103220369375SToby Isaac   const PetscScalar *facegeom, *cellgeom = NULL, *grad;
1033da97024aSMatthew G. Knepley   const PetscInt    *leaves;
1034d7ddef95SMatthew G. Knepley   PetscScalar       *x, *fx;
1035520b3818SMatthew G. Knepley   PetscInt           dim, nleaves, loc, fStart, fEnd, pdim, i;
10363ba16761SJacob Faibussowitsch   PetscErrorCode     ierru = PETSC_SUCCESS;
1037d7ddef95SMatthew G. Knepley 
1038d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
10399566063dSJacob Faibussowitsch   PetscCall(DMGetPointSF(dm, &sf));
10409566063dSJacob Faibussowitsch   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
1041da97024aSMatthew G. Knepley   nleaves = PetscMax(0, nleaves);
10429566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
10439566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
10449566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
10459566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
10469566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
104720369375SToby Isaac   if (cellGeometry) {
10489566063dSJacob Faibussowitsch     PetscCall(VecGetDM(cellGeometry, &dmCell));
10499566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
105020369375SToby Isaac   }
1051d7ddef95SMatthew G. Knepley   if (Grad) {
1052c0a6632aSMatthew G. Knepley     PetscFV fv;
1053c0a6632aSMatthew G. Knepley 
10549566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fv));
10559566063dSJacob Faibussowitsch     PetscCall(VecGetDM(Grad, &dmGrad));
10569566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(Grad, &grad));
10579566063dSJacob Faibussowitsch     PetscCall(PetscFVGetNumComponents(fv, &pdim));
10589566063dSJacob Faibussowitsch     PetscCall(DMGetWorkArray(dm, pdim, MPIU_SCALAR, &fx));
1059d7ddef95SMatthew G. Knepley   }
10609566063dSJacob Faibussowitsch   PetscCall(VecGetArray(locX, &x));
1061d7ddef95SMatthew G. Knepley   for (i = 0; i < numids; ++i) {
1062d7ddef95SMatthew G. Knepley     IS              faceIS;
1063d7ddef95SMatthew G. Knepley     const PetscInt *faces;
1064d7ddef95SMatthew G. Knepley     PetscInt        numFaces, f;
1065d7ddef95SMatthew G. Knepley 
10669566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, ids[i], &faceIS));
1067d7ddef95SMatthew G. Knepley     if (!faceIS) continue; /* No points with that id on this process */
10689566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(faceIS, &numFaces));
10699566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(faceIS, &faces));
1070d7ddef95SMatthew G. Knepley     for (f = 0; f < numFaces; ++f) {
1071d7ddef95SMatthew G. Knepley       const PetscInt   face = faces[f], *cells;
1072640bce14SSatish Balay       PetscFVFaceGeom *fg;
1073d7ddef95SMatthew G. Knepley 
1074d7ddef95SMatthew G. Knepley       if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */
10759566063dSJacob Faibussowitsch       PetscCall(PetscFindInt(face, nleaves, (PetscInt *)leaves, &loc));
1076da97024aSMatthew G. Knepley       if (loc >= 0) continue;
10779566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
10789566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, face, &cells));
1079d7ddef95SMatthew G. Knepley       if (Grad) {
1080640bce14SSatish Balay         PetscFVCellGeom *cg;
1081640bce14SSatish Balay         PetscScalar     *cx, *cgrad;
1082d7ddef95SMatthew G. Knepley         PetscScalar     *xG;
1083d7ddef95SMatthew G. Knepley         PetscReal        dx[3];
1084d7ddef95SMatthew G. Knepley         PetscInt         d;
1085d7ddef95SMatthew G. Knepley 
10869566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg));
10879566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &cx));
10889566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad));
10899566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
1090d7ddef95SMatthew G. Knepley         DMPlex_WaxpyD_Internal(dim, -1, cg->centroid, fg->centroid, dx);
1091d7ddef95SMatthew G. Knepley         for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DMPlex_DotD_Internal(dim, &cgrad[d * dim], dx);
10929566063dSJacob Faibussowitsch         PetscCall((*func)(time, fg->centroid, fg->normal, fx, xG, ctx));
1093d7ddef95SMatthew G. Knepley       } else {
1094640bce14SSatish Balay         PetscScalar *xI;
1095d7ddef95SMatthew G. Knepley         PetscScalar *xG;
1096d7ddef95SMatthew G. Knepley 
10979566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &xI));
10989566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
1099e735a8a9SMatthew G. Knepley         ierru = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx);
1100e735a8a9SMatthew G. Knepley         if (ierru) {
11019566063dSJacob Faibussowitsch           PetscCall(ISRestoreIndices(faceIS, &faces));
11029566063dSJacob Faibussowitsch           PetscCall(ISDestroy(&faceIS));
1103e735a8a9SMatthew G. Knepley           goto cleanup;
1104e735a8a9SMatthew G. Knepley         }
1105d7ddef95SMatthew G. Knepley       }
1106d7ddef95SMatthew G. Knepley     }
11079566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(faceIS, &faces));
11089566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&faceIS));
1109d7ddef95SMatthew G. Knepley   }
1110e735a8a9SMatthew G. Knepley cleanup:
11119566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(locX, &x));
1112d7ddef95SMatthew G. Knepley   if (Grad) {
11139566063dSJacob Faibussowitsch     PetscCall(DMRestoreWorkArray(dm, pdim, MPIU_SCALAR, &fx));
11149566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(Grad, &grad));
1115d7ddef95SMatthew G. Knepley   }
11169566063dSJacob Faibussowitsch   if (cellGeometry) PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
11179566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
11189566063dSJacob Faibussowitsch   PetscCall(ierru);
11193ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1120d7ddef95SMatthew G. Knepley }
1121d7ddef95SMatthew G. Knepley 
1122d71ae5a4SJacob Faibussowitsch static PetscErrorCode zero(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nc, PetscScalar *u, void *ctx)
1123d71ae5a4SJacob Faibussowitsch {
11240c364540SMatthew G. Knepley   PetscInt c;
11250c364540SMatthew G. Knepley   for (c = 0; c < Nc; ++c) u[c] = 0.0;
11263ba16761SJacob Faibussowitsch   return PETSC_SUCCESS;
11270c364540SMatthew G. Knepley }
11280c364540SMatthew G. Knepley 
1129d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1130d71ae5a4SJacob Faibussowitsch {
11310c364540SMatthew G. Knepley   PetscObject isZero;
1132e5e52638SMatthew G. Knepley   PetscDS     prob;
1133d7ddef95SMatthew G. Knepley   PetscInt    numBd, b;
113455f2e967SMatthew G. Knepley 
113555f2e967SMatthew G. Knepley   PetscFunctionBegin;
11369566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
11379566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
11389566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero));
1139450c7a9fSMatthew G. Knepley   PetscCall(PetscDSUpdateBoundaryLabels(prob, dm));
114055f2e967SMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
114145480ffeSMatthew G. Knepley     PetscWeakForm           wf;
1142f971fd6bSMatthew G. Knepley     DMBoundaryConditionType type;
114345480ffeSMatthew G. Knepley     const char             *name;
1144d7ddef95SMatthew G. Knepley     DMLabel                 label;
11451c531cf8SMatthew G. Knepley     PetscInt                field, Nc;
11461c531cf8SMatthew G. Knepley     const PetscInt         *comps;
1147d7ddef95SMatthew G. Knepley     PetscObject             obj;
1148d7ddef95SMatthew G. Knepley     PetscClassId            id;
114945480ffeSMatthew G. Knepley     void (*bvfunc)(void);
1150d7ddef95SMatthew G. Knepley     PetscInt        numids;
1151d7ddef95SMatthew G. Knepley     const PetscInt *ids;
115255f2e967SMatthew G. Knepley     void           *ctx;
115355f2e967SMatthew G. Knepley 
11549566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, &bvfunc, NULL, &ctx));
1155f971fd6bSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
11569566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
11579566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1158d7ddef95SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1159c60e475cSMatthew G. Knepley       switch (type) {
1160c60e475cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
11619371c9d4SSatish Balay       case DM_BC_ESSENTIAL: {
11628434afd1SBarry Smith         PetscSimplePointFn *func = (PetscSimplePointFn *)bvfunc;
116345480ffeSMatthew G. Knepley 
116445480ffeSMatthew G. Knepley         if (isZero) func = zero;
11659566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
11669566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func, ctx, locX));
11679566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
11689371c9d4SSatish Balay       } break;
11699371c9d4SSatish Balay       case DM_BC_ESSENTIAL_FIELD: {
117045480ffeSMatthew G. Knepley         PetscPointFunc func = (PetscPointFunc)bvfunc;
117145480ffeSMatthew G. Knepley 
11729566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
11739566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func, ctx, locX));
11749566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
11759371c9d4SSatish Balay       } break;
1176d71ae5a4SJacob Faibussowitsch       default:
1177d71ae5a4SJacob Faibussowitsch         break;
1178c60e475cSMatthew G. Knepley       }
1179d7ddef95SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
118045480ffeSMatthew G. Knepley       {
118145480ffeSMatthew G. Knepley         PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *) = (PetscErrorCode (*)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *))bvfunc;
118245480ffeSMatthew G. Knepley 
118343ea7facSMatthew G. Knepley         if (!faceGeomFVM) continue;
11849566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesRiemann(dm, time, faceGeomFVM, cellGeomFVM, gradFVM, field, Nc, comps, label, numids, ids, func, ctx, locX));
118545480ffeSMatthew G. Knepley       }
118663a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
118755f2e967SMatthew G. Knepley   }
11883ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
118955f2e967SMatthew G. Knepley }
119055f2e967SMatthew G. Knepley 
1191d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1192d71ae5a4SJacob Faibussowitsch {
119356cf3b9cSMatthew G. Knepley   PetscObject isZero;
119456cf3b9cSMatthew G. Knepley   PetscDS     prob;
119556cf3b9cSMatthew G. Knepley   PetscInt    numBd, b;
119656cf3b9cSMatthew G. Knepley 
119756cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
11983ba16761SJacob Faibussowitsch   if (!locX) PetscFunctionReturn(PETSC_SUCCESS);
11999566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
12009566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
12019566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero));
120256cf3b9cSMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
120345480ffeSMatthew G. Knepley     PetscWeakForm           wf;
120456cf3b9cSMatthew G. Knepley     DMBoundaryConditionType type;
120545480ffeSMatthew G. Knepley     const char             *name;
120656cf3b9cSMatthew G. Knepley     DMLabel                 label;
120756cf3b9cSMatthew G. Knepley     PetscInt                field, Nc;
120856cf3b9cSMatthew G. Knepley     const PetscInt         *comps;
120956cf3b9cSMatthew G. Knepley     PetscObject             obj;
121056cf3b9cSMatthew G. Knepley     PetscClassId            id;
121156cf3b9cSMatthew G. Knepley     PetscInt                numids;
121256cf3b9cSMatthew G. Knepley     const PetscInt         *ids;
121345480ffeSMatthew G. Knepley     void (*bvfunc)(void);
121456cf3b9cSMatthew G. Knepley     void *ctx;
121556cf3b9cSMatthew G. Knepley 
12169566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, NULL, &bvfunc, &ctx));
121756cf3b9cSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
12189566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
12199566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
122056cf3b9cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
122156cf3b9cSMatthew G. Knepley       switch (type) {
122256cf3b9cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
12239371c9d4SSatish Balay       case DM_BC_ESSENTIAL: {
12248434afd1SBarry Smith         PetscSimplePointFn *func_t = (PetscSimplePointFn *)bvfunc;
122545480ffeSMatthew G. Knepley 
122645480ffeSMatthew G. Knepley         if (isZero) func_t = zero;
12279566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
12289566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
12299566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
12309371c9d4SSatish Balay       } break;
12319371c9d4SSatish Balay       case DM_BC_ESSENTIAL_FIELD: {
123245480ffeSMatthew G. Knepley         PetscPointFunc func_t = (PetscPointFunc)bvfunc;
123345480ffeSMatthew G. Knepley 
12349566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
12359566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
12369566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
12379371c9d4SSatish Balay       } break;
1238d71ae5a4SJacob Faibussowitsch       default:
1239d71ae5a4SJacob Faibussowitsch         break;
124056cf3b9cSMatthew G. Knepley       }
124163a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
124256cf3b9cSMatthew G. Knepley   }
12433ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
124456cf3b9cSMatthew G. Knepley }
124556cf3b9cSMatthew G. Knepley 
1246f1d73a7aSMatthew G. Knepley /*@
1247f1d73a7aSMatthew G. Knepley   DMPlexInsertBoundaryValues - Puts coefficients which represent boundary values into the local solution vector
1248f1d73a7aSMatthew G. Knepley 
1249ed808b8fSJed Brown   Not Collective
1250ed808b8fSJed Brown 
1251f1d73a7aSMatthew G. Knepley   Input Parameters:
1252a1cb98faSBarry Smith + dm              - The `DM`
1253f1d73a7aSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
1254f1d73a7aSMatthew G. Knepley . time            - The time
1255f1d73a7aSMatthew G. Knepley . faceGeomFVM     - Face geometry data for FV discretizations
1256f1d73a7aSMatthew G. Knepley . cellGeomFVM     - Cell geometry data for FV discretizations
1257f1d73a7aSMatthew G. Knepley - gradFVM         - Gradient reconstruction data for FV discretizations
1258f1d73a7aSMatthew G. Knepley 
12592fe279fdSBarry Smith   Output Parameter:
1260f1d73a7aSMatthew G. Knepley . locX - Solution updated with boundary values
1261f1d73a7aSMatthew G. Knepley 
1262ed808b8fSJed Brown   Level: intermediate
1263f1d73a7aSMatthew G. Knepley 
12641cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunctionLabelLocal()`, `DMAddBoundary()`
1265f1d73a7aSMatthew G. Knepley @*/
1266d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertBoundaryValues(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1267d71ae5a4SJacob Faibussowitsch {
1268f1d73a7aSMatthew G. Knepley   PetscFunctionBegin;
1269f1d73a7aSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1270064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
1271ad540459SPierre Jolivet   if (faceGeomFVM) PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);
1272ad540459SPierre Jolivet   if (cellGeomFVM) PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);
1273ad540459SPierre Jolivet   if (gradFVM) PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);
1274cac4c232SBarry Smith   PetscTryMethod(dm, "DMPlexInsertBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX, time, faceGeomFVM, cellGeomFVM, gradFVM));
12753ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1276f1d73a7aSMatthew G. Knepley }
1277f1d73a7aSMatthew G. Knepley 
127856cf3b9cSMatthew G. Knepley /*@
1279a5b23f4aSJose E. Roman   DMPlexInsertTimeDerivativeBoundaryValues - Puts coefficients which represent boundary values of the time derivative into the local solution vector
128056cf3b9cSMatthew G. Knepley 
128156cf3b9cSMatthew G. Knepley   Input Parameters:
1282a1cb98faSBarry Smith + dm              - The `DM`
128356cf3b9cSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
128456cf3b9cSMatthew G. Knepley . time            - The time
128556cf3b9cSMatthew G. Knepley . faceGeomFVM     - Face geometry data for FV discretizations
128656cf3b9cSMatthew G. Knepley . cellGeomFVM     - Cell geometry data for FV discretizations
128756cf3b9cSMatthew G. Knepley - gradFVM         - Gradient reconstruction data for FV discretizations
128856cf3b9cSMatthew G. Knepley 
12892fe279fdSBarry Smith   Output Parameter:
129056cf3b9cSMatthew G. Knepley . locX_t - Solution updated with boundary values
129156cf3b9cSMatthew G. Knepley 
129256cf3b9cSMatthew G. Knepley   Level: developer
129356cf3b9cSMatthew G. Knepley 
12941cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunctionLabelLocal()`
129556cf3b9cSMatthew G. Knepley @*/
1296d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues(DM dm, PetscBool insertEssential, Vec locX_t, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1297d71ae5a4SJacob Faibussowitsch {
129856cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
129956cf3b9cSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1300ad540459SPierre Jolivet   if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 3);
1301ad540459SPierre Jolivet   if (faceGeomFVM) PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);
1302ad540459SPierre Jolivet   if (cellGeomFVM) PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);
1303ad540459SPierre Jolivet   if (gradFVM) PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);
13046c51210dSStefano Zampini   PetscTryMethod(dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX_t, time, faceGeomFVM, cellGeomFVM, gradFVM));
13053ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
130656cf3b9cSMatthew G. Knepley }
130756cf3b9cSMatthew G. Knepley 
13085962854dSMatthew G. Knepley // Handle non-essential (e.g. outflow) boundary values
13095962854dSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesFVM(DM dm, PetscFV fv, Vec locX, PetscReal time, Vec *locGradient)
13105962854dSMatthew G. Knepley {
13115962854dSMatthew G. Knepley   DM  dmGrad;
13125962854dSMatthew G. Knepley   Vec cellGeometryFVM, faceGeometryFVM, locGrad = NULL;
13135962854dSMatthew G. Knepley 
13145962854dSMatthew G. Knepley   PetscFunctionBegin;
13155962854dSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
13165962854dSMatthew G. Knepley   PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);
13175962854dSMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
13185962854dSMatthew G. Knepley   if (locGradient) {
13195962854dSMatthew G. Knepley     PetscAssertPointer(locGradient, 5);
13205962854dSMatthew G. Knepley     *locGradient = NULL;
13215962854dSMatthew G. Knepley   }
13225962854dSMatthew G. Knepley   PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL));
13235962854dSMatthew G. Knepley   /* Reconstruct and limit cell gradients */
13245962854dSMatthew G. Knepley   PetscCall(DMPlexGetGradientDM(dm, fv, &dmGrad));
13255962854dSMatthew G. Knepley   if (dmGrad) {
13265962854dSMatthew G. Knepley     Vec      grad;
13275962854dSMatthew G. Knepley     PetscInt fStart, fEnd;
13285962854dSMatthew G. Knepley 
13295962854dSMatthew G. Knepley     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
13305962854dSMatthew G. Knepley     PetscCall(DMGetGlobalVector(dmGrad, &grad));
13315962854dSMatthew G. Knepley     PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
13325962854dSMatthew G. Knepley     /* Communicate gradient values */
13335962854dSMatthew G. Knepley     PetscCall(DMGetLocalVector(dmGrad, &locGrad));
13345962854dSMatthew G. Knepley     PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
13355962854dSMatthew G. Knepley     PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
13365962854dSMatthew G. Knepley     PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
13375962854dSMatthew G. Knepley   }
13385962854dSMatthew G. Knepley   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, time, faceGeometryFVM, cellGeometryFVM, locGrad));
13395962854dSMatthew G. Knepley   if (locGradient) *locGradient = locGrad;
13405962854dSMatthew G. Knepley   else if (locGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
13415962854dSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
13425962854dSMatthew G. Knepley }
13435962854dSMatthew G. Knepley 
1344d71ae5a4SJacob Faibussowitsch PetscErrorCode DMComputeL2Diff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
1345d71ae5a4SJacob Faibussowitsch {
1346574a98acSMatthew G. Knepley   Vec localX;
1347574a98acSMatthew G. Knepley 
1348574a98acSMatthew G. Knepley   PetscFunctionBegin;
13499566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
13509566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, localX, time, NULL, NULL, NULL));
13519566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
13529566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
13539566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeL2DiffLocal(dm, time, funcs, ctxs, localX, diff));
13549566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
13553ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1356574a98acSMatthew G. Knepley }
1357574a98acSMatthew G. Knepley 
1358574a98acSMatthew G. Knepley /*@C
135960225df5SJacob Faibussowitsch   DMPlexComputeL2DiffLocal - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h.
1360574a98acSMatthew G. Knepley 
136120f4b53cSBarry Smith   Collective
1362c0f8e1fdSMatthew G. Knepley 
1363574a98acSMatthew G. Knepley   Input Parameters:
1364a1cb98faSBarry Smith + dm     - The `DM`
1365574a98acSMatthew G. Knepley . time   - The time
1366574a98acSMatthew G. Knepley . funcs  - The functions to evaluate for each field component
136720f4b53cSBarry Smith . ctxs   - Optional array of contexts to pass to each function, or `NULL`.
1368574a98acSMatthew G. Knepley - localX - The coefficient vector u_h, a local vector
1369574a98acSMatthew G. Knepley 
1370574a98acSMatthew G. Knepley   Output Parameter:
1371574a98acSMatthew G. Knepley . diff - The diff ||u - u_h||_2
1372574a98acSMatthew G. Knepley 
1373574a98acSMatthew G. Knepley   Level: developer
1374574a98acSMatthew G. Knepley 
13751cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
1376574a98acSMatthew G. Knepley @*/
1377d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeL2DiffLocal(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec localX, PetscReal *diff)
1378d71ae5a4SJacob Faibussowitsch {
13790f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex *)dm->data)->printL2;
1380ca3d3a14SMatthew G. Knepley   DM               tdm;
1381ca3d3a14SMatthew G. Knepley   Vec              tv;
1382cb1e1211SMatthew G Knepley   PetscSection     section;
1383c5bbbd5bSMatthew G. Knepley   PetscQuadrature  quad;
13844bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
138515496722SMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
13864bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
1387cb1e1211SMatthew G Knepley   PetscReal        localDiff = 0.0;
13887318780aSToby Isaac   const PetscReal *quadWeights;
1389412e9a14SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cellHeight, cStart, cEnd, c, field, fieldOffset;
1390ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1391cb1e1211SMatthew G Knepley 
1392cb1e1211SMatthew G Knepley   PetscFunctionBegin;
13939566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
13949566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
13952a4e142eSMatthew G. Knepley   fegeom.dimEmbed = coordDim;
13969566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
13979566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
13989566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
13999566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
14009566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1401eae3dc7dSJacob Faibussowitsch   PetscCheck(numFields, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
1402cb1e1211SMatthew G Knepley   for (field = 0; field < numFields; ++field) {
140315496722SMatthew G. Knepley     PetscObject  obj;
140415496722SMatthew G. Knepley     PetscClassId id;
1405c5bbbd5bSMatthew G. Knepley     PetscInt     Nc;
1406c5bbbd5bSMatthew G. Knepley 
14079566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
14089566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
140915496722SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
141015496722SMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
141115496722SMatthew G. Knepley 
14129566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
14139566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
141415496722SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
141515496722SMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
141615496722SMatthew G. Knepley 
14179566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
14189566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
141963a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1420c5bbbd5bSMatthew G. Knepley     numComponents += Nc;
1421cb1e1211SMatthew G Knepley   }
14229566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
142363a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
1424c9cb6370SYANG Zongze   PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * (Nq + 1), &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ));
14259566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
14269566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
1427cb1e1211SMatthew G Knepley   for (c = cStart; c < cEnd; ++c) {
1428a1e44745SMatthew G. Knepley     PetscScalar *x        = NULL;
1429cb1e1211SMatthew G Knepley     PetscReal    elemDiff = 0.0;
14309c3cf19fSMatthew G. Knepley     PetscInt     qc       = 0;
1431cb1e1211SMatthew G Knepley 
14329566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
1433e8e188d2SZach Atkins     PetscCall(DMPlexVecGetOrientedClosure_Internal(dm, NULL, PETSC_FALSE, localX, c, 0, NULL, &x));
1434cb1e1211SMatthew G Knepley 
143515496722SMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
143615496722SMatthew G. Knepley       PetscObject  obj;
143715496722SMatthew G. Knepley       PetscClassId id;
1438c110b1eeSGeoffrey Irving       void *const  ctx = ctxs ? ctxs[field] : NULL;
143915496722SMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1440cb1e1211SMatthew G Knepley 
14419566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
14429566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
14439371c9d4SSatish Balay       if (id == PETSCFE_CLASSID) {
14449371c9d4SSatish Balay         PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
14459371c9d4SSatish Balay         PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
14469371c9d4SSatish Balay       } else if (id == PETSCFV_CLASSID) {
14479371c9d4SSatish Balay         PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
14489371c9d4SSatish Balay         Nb = 1;
14499371c9d4SSatish Balay       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1450cb1e1211SMatthew G Knepley       if (debug) {
1451cb1e1211SMatthew G Knepley         char title[1024];
145263a3b9bcSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
14539566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1454cb1e1211SMatthew G Knepley       }
14557318780aSToby Isaac       for (q = 0; q < Nq; ++q) {
14562a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1457d0609cedSBarry Smith         PetscErrorCode ierr;
14582a4e142eSMatthew G. Knepley 
14592a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
14602a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q * coordDim * coordDim];
14612a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
14622a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
146363a3b9bcSJacob Faibussowitsch         PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", point %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1464d3a7d86cSMatthew G. Knepley         if (transform) {
1465d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * Nq];
14669566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx));
1467d3a7d86cSMatthew G. Knepley         } else {
1468d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * q];
1469d3a7d86cSMatthew G. Knepley         }
14709566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal, Nc));
1471ca3d3a14SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, Nc, funcVal, ctx);
1472e735a8a9SMatthew G. Knepley         if (ierr) {
14739566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
14749566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
14759566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
1476e735a8a9SMatthew G. Knepley         }
14779566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
14789566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant));
14799566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant));
14802df84da0SMatthew G. Knepley         else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
148115496722SMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1482beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
14839371c9d4SSatish Balay           if (debug)
1484835f2295SStefano Zampini             PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g (%g, %g)\n", c, field, fc, (double)(coordDim > 0 ? coords[coordDim * q] : 0), (double)(coordDim > 1 ? coords[coordDim * q + 1] : 0), (double)(coordDim > 2 ? coords[coordDim * q + 2] : 0),
14859371c9d4SSatish Balay                                   (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]), (double)PetscRealPart(interpolant[fc]), (double)PetscRealPart(funcVal[fc])));
14864bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
1487cb1e1211SMatthew G Knepley         }
1488cb1e1211SMatthew G Knepley       }
14899c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
1490beaa55a6SMatthew G. Knepley       qc += Nc;
1491cb1e1211SMatthew G Knepley     }
14929566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
14939566063dSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
1494cb1e1211SMatthew G Knepley     localDiff += elemDiff;
1495cb1e1211SMatthew G Knepley   }
14969566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
1497462c564dSBarry Smith   PetscCallMPI(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
1498cb1e1211SMatthew G Knepley   *diff = PetscSqrtReal(*diff);
14993ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1500cb1e1211SMatthew G Knepley }
1501cb1e1211SMatthew G Knepley 
1502d71ae5a4SJacob Faibussowitsch PetscErrorCode DMComputeL2GradientDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, const PetscReal n[], PetscReal *diff)
1503d71ae5a4SJacob Faibussowitsch {
15040f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex *)dm->data)->printL2;
1505ca3d3a14SMatthew G. Knepley   DM               tdm;
1506cb1e1211SMatthew G Knepley   PetscSection     section;
150740e14135SMatthew G. Knepley   PetscQuadrature  quad;
1508ca3d3a14SMatthew G. Knepley   Vec              localX, tv;
15099c3cf19fSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
15102a4e142eSMatthew G. Knepley   const PetscReal *quadWeights;
15114bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
15124bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
151340e14135SMatthew G. Knepley   PetscReal        localDiff = 0.0;
1514485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, qNc = 0, Nq = 0, numFields, numComponents = 0, cStart, cEnd, c, field, fieldOffset;
1515ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1516cb1e1211SMatthew G Knepley 
1517cb1e1211SMatthew G Knepley   PetscFunctionBegin;
15189566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
15199566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
15204bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
15219566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
15229566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
15239566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
15249566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
15259566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
15269566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
15279566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
15289566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1529652b88e8SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
15300f2d7e86SMatthew G. Knepley     PetscFE  fe;
153140e14135SMatthew G. Knepley     PetscInt Nc;
1532652b88e8SMatthew G. Knepley 
15339566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe));
15349566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
15359566063dSJacob Faibussowitsch     PetscCall(PetscFEGetNumComponents(fe, &Nc));
153640e14135SMatthew G. Knepley     numComponents += Nc;
1537652b88e8SMatthew G. Knepley   }
15389566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
153963a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
15409566063dSJacob Faibussowitsch   /* PetscCall(DMProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX)); */
1541c9cb6370SYANG Zongze   PetscCall(PetscMalloc6(numComponents, &funcVal, coordDim * (Nq + 1), &coords, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ, numComponents * coordDim, &interpolant, Nq, &fegeom.detJ));
15429566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
154340e14135SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
154440e14135SMatthew G. Knepley     PetscScalar *x        = NULL;
154540e14135SMatthew G. Knepley     PetscReal    elemDiff = 0.0;
15469c3cf19fSMatthew G. Knepley     PetscInt     qc       = 0;
1547652b88e8SMatthew G. Knepley 
15489566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
1549e8e188d2SZach Atkins     PetscCall(DMPlexVecGetOrientedClosure_Internal(dm, NULL, PETSC_FALSE, localX, c, 0, NULL, &x));
155040e14135SMatthew G. Knepley 
15519c3cf19fSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
15520f2d7e86SMatthew G. Knepley       PetscFE     fe;
155351259fa3SMatthew G. Knepley       void *const ctx = ctxs ? ctxs[field] : NULL;
15549c3cf19fSMatthew G. Knepley       PetscInt    Nb, Nc, q, fc;
155540e14135SMatthew G. Knepley 
15569566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe));
15579566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
15589566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
155940e14135SMatthew G. Knepley       if (debug) {
156040e14135SMatthew G. Knepley         char title[1024];
15619566063dSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
15629566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1563652b88e8SMatthew G. Knepley       }
15649c3cf19fSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
15652a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1566d0609cedSBarry Smith         PetscErrorCode ierr;
15672a4e142eSMatthew G. Knepley 
15682a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
15692a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q * coordDim * coordDim];
15702a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
15712a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
15722df84da0SMatthew G. Knepley         PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1573d3a7d86cSMatthew G. Knepley         if (transform) {
1574d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * Nq];
15759566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx));
1576d3a7d86cSMatthew G. Knepley         } else {
1577d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * q];
1578d3a7d86cSMatthew G. Knepley         }
15799566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal, Nc));
15804bee2e38SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, n, Nc, funcVal, ctx);
1581e735a8a9SMatthew G. Knepley         if (ierr) {
15829566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
15839566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
15849566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ));
1585e735a8a9SMatthew G. Knepley         }
15869566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
15879566063dSJacob Faibussowitsch         PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[fieldOffset], &qgeom, q, interpolant));
15884bee2e38SMatthew G. Knepley         /* Overwrite with the dot product if the normal is given */
15894bee2e38SMatthew G. Knepley         if (n) {
15904bee2e38SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
15914bee2e38SMatthew G. Knepley             PetscScalar sum = 0.0;
15924bee2e38SMatthew G. Knepley             PetscInt    d;
15934bee2e38SMatthew G. Knepley             for (d = 0; d < dim; ++d) sum += interpolant[fc * dim + d] * n[d];
15944bee2e38SMatthew G. Knepley             interpolant[fc] = sum;
15954bee2e38SMatthew G. Knepley           }
15964bee2e38SMatthew G. Knepley         }
15979c3cf19fSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1598beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
159963a3b9bcSJacob Faibussowitsch           if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " fieldDer %" PetscInt_FMT ",%" PetscInt_FMT " diff %g\n", c, field, fc, (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q])));
16004bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
160140e14135SMatthew G. Knepley         }
160240e14135SMatthew G. Knepley       }
16039c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
16049c3cf19fSMatthew G. Knepley       qc += Nc;
160540e14135SMatthew G. Knepley     }
16069566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
160763a3b9bcSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
160840e14135SMatthew G. Knepley     localDiff += elemDiff;
160940e14135SMatthew G. Knepley   }
16109566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ));
16119566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
1612462c564dSBarry Smith   PetscCallMPI(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
161340e14135SMatthew G. Knepley   *diff = PetscSqrtReal(*diff);
16143ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1615cb1e1211SMatthew G Knepley }
1616cb1e1211SMatthew G Knepley 
1617d71ae5a4SJacob Faibussowitsch PetscErrorCode DMComputeL2FieldDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
1618d71ae5a4SJacob Faibussowitsch {
16190f09c10fSMatthew G. Knepley   const PetscInt debug = ((DM_Plex *)dm->data)->printL2;
1620ca3d3a14SMatthew G. Knepley   DM             tdm;
1621083401c6SMatthew G. Knepley   DMLabel        depthLabel;
162273d901b8SMatthew G. Knepley   PetscSection   section;
1623ca3d3a14SMatthew G. Knepley   Vec            localX, tv;
162473d901b8SMatthew G. Knepley   PetscReal     *localDiff;
1625083401c6SMatthew G. Knepley   PetscInt       dim, depth, dE, Nf, f, Nds, s;
1626ca3d3a14SMatthew G. Knepley   PetscBool      transform;
1627835f2295SStefano Zampini   PetscMPIInt    Nfi;
162873d901b8SMatthew G. Knepley 
162973d901b8SMatthew G. Knepley   PetscFunctionBegin;
16309566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
16319566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dE));
16329566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
16339566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
16349566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
16359566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
16369566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
16379566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
16389566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
16399566063dSJacob Faibussowitsch   PetscCall(DMLabelGetNumValues(depthLabel, &depth));
1640083401c6SMatthew G. Knepley 
16419566063dSJacob Faibussowitsch   PetscCall(VecSet(localX, 0.0));
16429566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
16439566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
16449566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
16459566063dSJacob Faibussowitsch   PetscCall(DMGetNumDS(dm, &Nds));
16469566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(Nf, &localDiff));
1647083401c6SMatthew G. Knepley   for (s = 0; s < Nds; ++s) {
1648083401c6SMatthew G. Knepley     PetscDS          ds;
1649083401c6SMatthew G. Knepley     DMLabel          label;
1650083401c6SMatthew G. Knepley     IS               fieldIS, pointIS;
1651083401c6SMatthew G. Knepley     const PetscInt  *fields, *points = NULL;
1652083401c6SMatthew G. Knepley     PetscQuadrature  quad;
1653083401c6SMatthew G. Knepley     const PetscReal *quadPoints, *quadWeights;
1654083401c6SMatthew G. Knepley     PetscFEGeom      fegeom;
1655083401c6SMatthew G. Knepley     PetscReal       *coords, *gcoords;
1656083401c6SMatthew G. Knepley     PetscScalar     *funcVal, *interpolant;
16575fedec97SMatthew G. Knepley     PetscBool        isCohesive;
1658083401c6SMatthew G. Knepley     PetscInt         qNc, Nq, totNc, cStart = 0, cEnd, c, dsNf;
165973d901b8SMatthew G. Knepley 
166007218a29SMatthew G. Knepley     PetscCall(DMGetRegionNumDS(dm, s, &label, &fieldIS, &ds, NULL));
16619566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(fieldIS, &fields));
16629566063dSJacob Faibussowitsch     PetscCall(PetscDSIsCohesive(ds, &isCohesive));
16639566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(ds, &dsNf));
16649566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalComponents(ds, &totNc));
16659566063dSJacob Faibussowitsch     PetscCall(PetscDSGetQuadrature(ds, &quad));
16669566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
166763a3b9bcSJacob Faibussowitsch     PetscCheck(!(qNc != 1) || !(qNc != totNc), PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, totNc);
16689566063dSJacob Faibussowitsch     PetscCall(PetscCalloc6(totNc, &funcVal, totNc, &interpolant, dE * (Nq + 1), &coords, Nq, &fegeom.detJ, dE * dE * Nq, &fegeom.J, dE * dE * Nq, &fegeom.invJ));
1669083401c6SMatthew G. Knepley     if (!label) {
16709566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1671083401c6SMatthew G. Knepley     } else {
16729566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(label, 1, &pointIS));
16739566063dSJacob Faibussowitsch       PetscCall(ISGetLocalSize(pointIS, &cEnd));
16749566063dSJacob Faibussowitsch       PetscCall(ISGetIndices(pointIS, &points));
1675083401c6SMatthew G. Knepley     }
167673d901b8SMatthew G. Knepley     for (c = cStart; c < cEnd; ++c) {
1677083401c6SMatthew G. Knepley       const PetscInt  cell = points ? points[c] : c;
167873d901b8SMatthew G. Knepley       PetscScalar    *x    = NULL;
16795fedec97SMatthew G. Knepley       const PetscInt *cone;
16805fedec97SMatthew G. Knepley       PetscInt        qc = 0, fOff = 0, dep;
168173d901b8SMatthew G. Knepley 
16829566063dSJacob Faibussowitsch       PetscCall(DMLabelGetValue(depthLabel, cell, &dep));
1683083401c6SMatthew G. Knepley       if (dep != depth - 1) continue;
16845fedec97SMatthew G. Knepley       if (isCohesive) {
16859566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cell, &cone));
16869566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cone[0], quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
168796959cd1SMatthew G. Knepley       } else {
16889566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
168996959cd1SMatthew G. Knepley       }
1690e8e188d2SZach Atkins       PetscCall(DMPlexVecGetOrientedClosure_Internal(dm, NULL, PETSC_FALSE, localX, cell, 0, NULL, &x));
16915fedec97SMatthew G. Knepley       for (f = 0; f < dsNf; ++f) {
169215496722SMatthew G. Knepley         PetscObject  obj;
169315496722SMatthew G. Knepley         PetscClassId id;
1694083401c6SMatthew G. Knepley         void *const  ctx = ctxs ? ctxs[fields[f]] : NULL;
169515496722SMatthew G. Knepley         PetscInt     Nb, Nc, q, fc;
169615496722SMatthew G. Knepley         PetscReal    elemDiff = 0.0;
16975fedec97SMatthew G. Knepley         PetscBool    cohesive;
169815496722SMatthew G. Knepley 
16999566063dSJacob Faibussowitsch         PetscCall(PetscDSGetCohesive(ds, f, &cohesive));
17009566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
17019566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
17029371c9d4SSatish Balay         if (id == PETSCFE_CLASSID) {
17039371c9d4SSatish Balay           PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
17049371c9d4SSatish Balay           PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
17059371c9d4SSatish Balay         } else if (id == PETSCFV_CLASSID) {
17069371c9d4SSatish Balay           PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
17079371c9d4SSatish Balay           Nb = 1;
17089371c9d4SSatish Balay         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
1709c82db6dbSMatthew G. Knepley         if (isCohesive && !cohesive) {
1710c82db6dbSMatthew G. Knepley           fOff += Nb * 2;
1711c82db6dbSMatthew G. Knepley           qc += Nc;
1712c82db6dbSMatthew G. Knepley           continue;
1713c82db6dbSMatthew G. Knepley         }
171473d901b8SMatthew G. Knepley         if (debug) {
171573d901b8SMatthew G. Knepley           char title[1024];
171663a3b9bcSJacob Faibussowitsch           PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, fields[f]));
17179566063dSJacob Faibussowitsch           PetscCall(DMPrintCellVector(cell, title, Nb, &x[fOff]));
171873d901b8SMatthew G. Knepley         }
17197318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
17202a4e142eSMatthew G. Knepley           PetscFEGeom    qgeom;
1721d0609cedSBarry Smith           PetscErrorCode ierr;
17222a4e142eSMatthew G. Knepley 
17232a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
1724083401c6SMatthew G. Knepley           qgeom.J        = &fegeom.J[q * dE * dE];
1725083401c6SMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * dE * dE];
17262a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
172763a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for cell %" PetscInt_FMT ", quadrature point %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
1728d3a7d86cSMatthew G. Knepley           if (transform) {
1729083401c6SMatthew G. Knepley             gcoords = &coords[dE * Nq];
17309566063dSJacob Faibussowitsch             PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[dE * q], PETSC_TRUE, dE, &coords[dE * q], gcoords, dm->transformCtx));
1731d3a7d86cSMatthew G. Knepley           } else {
1732083401c6SMatthew G. Knepley             gcoords = &coords[dE * q];
1733d3a7d86cSMatthew G. Knepley           }
17342df84da0SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) funcVal[fc] = 0.;
1735083401c6SMatthew G. Knepley           ierr = (*funcs[fields[f]])(dE, time, gcoords, Nc, funcVal, ctx);
1736e735a8a9SMatthew G. Knepley           if (ierr) {
17379566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
17389566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
17399566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
1740e735a8a9SMatthew G. Knepley           }
17419566063dSJacob Faibussowitsch           if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[dE * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
174296959cd1SMatthew G. Knepley           /* Call once for each face, except for lagrange field */
17439566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fOff], &qgeom, q, interpolant));
17449566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fOff], q, interpolant));
174563a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
174615496722SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1747beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
17489371c9d4SSatish Balay             if (debug)
1749835f2295SStefano Zampini               PetscCall(PetscPrintf(PETSC_COMM_SELF, "    cell %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g\n", cell, fields[f], fc, (double)(dE > 0 ? coords[dE * q] : 0), (double)(dE > 1 ? coords[dE * q + 1] : 0), (double)(dE > 2 ? coords[dE * q + 2] : 0),
17509371c9d4SSatish Balay                                     (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q])));
17514bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
175273d901b8SMatthew G. Knepley           }
175373d901b8SMatthew G. Knepley         }
1754083401c6SMatthew G. Knepley         fOff += Nb;
17559c3cf19fSMatthew G. Knepley         qc += Nc;
1756083401c6SMatthew G. Knepley         localDiff[fields[f]] += elemDiff;
175763a3b9bcSJacob Faibussowitsch         if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  cell %" PetscInt_FMT " field %" PetscInt_FMT " cum diff %g\n", cell, fields[f], (double)localDiff[fields[f]]));
175873d901b8SMatthew G. Knepley       }
17599566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
1760083401c6SMatthew G. Knepley     }
1761083401c6SMatthew G. Knepley     if (label) {
17629566063dSJacob Faibussowitsch       PetscCall(ISRestoreIndices(pointIS, &points));
17639566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
1764083401c6SMatthew G. Knepley     }
17659566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(fieldIS, &fields));
17669566063dSJacob Faibussowitsch     PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
176773d901b8SMatthew G. Knepley   }
17689566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
1769835f2295SStefano Zampini   PetscCall(PetscMPIIntCast(Nf, &Nfi));
1770835f2295SStefano Zampini   PetscCallMPI(MPIU_Allreduce(localDiff, diff, Nfi, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
17719566063dSJacob Faibussowitsch   PetscCall(PetscFree(localDiff));
1772083401c6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) diff[f] = PetscSqrtReal(diff[f]);
17733ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
177473d901b8SMatthew G. Knepley }
177573d901b8SMatthew G. Knepley 
1776e729f68cSMatthew G. Knepley /*@C
1777e729f68cSMatthew G. Knepley   DMPlexComputeL2DiffVec - This function computes the cellwise L_2 difference between a function u and an FEM interpolant solution u_h, and stores it in a Vec.
1778e729f68cSMatthew G. Knepley 
177920f4b53cSBarry Smith   Collective
1780c0f8e1fdSMatthew G. Knepley 
1781e729f68cSMatthew G. Knepley   Input Parameters:
1782a1cb98faSBarry Smith + dm    - The `DM`
17830163fd50SMatthew G. Knepley . time  - The time
178420f4b53cSBarry Smith . funcs - The functions to evaluate for each field component: `NULL` means that component does not contribute to error calculation
178520f4b53cSBarry Smith . ctxs  - Optional array of contexts to pass to each function, or `NULL`.
1786e729f68cSMatthew G. Knepley - X     - The coefficient vector u_h
1787e729f68cSMatthew G. Knepley 
1788e729f68cSMatthew G. Knepley   Output Parameter:
178920f4b53cSBarry Smith . D - A `Vec` which holds the difference ||u - u_h||_2 for each cell
1790e729f68cSMatthew G. Knepley 
1791e729f68cSMatthew G. Knepley   Level: developer
1792e729f68cSMatthew G. Knepley 
17931cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
1794e729f68cSMatthew G. Knepley @*/
1795d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeL2DiffVec(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, Vec D)
1796d71ae5a4SJacob Faibussowitsch {
1797e729f68cSMatthew G. Knepley   PetscSection     section;
1798e729f68cSMatthew G. Knepley   PetscQuadrature  quad;
1799e729f68cSMatthew G. Knepley   Vec              localX;
18004bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
1801e729f68cSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
18024bee2e38SMatthew G. Knepley   PetscReal       *coords;
1803e729f68cSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1804485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, c, field, fieldOffset;
1805e729f68cSMatthew G. Knepley 
1806e729f68cSMatthew G. Knepley   PetscFunctionBegin;
18079566063dSJacob Faibussowitsch   PetscCall(VecSet(D, 0.0));
18089566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
18099566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
18109566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
18119566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
18129566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
18139566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
18149566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
18159566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
1816e729f68cSMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
1817e729f68cSMatthew G. Knepley     PetscObject  obj;
1818e729f68cSMatthew G. Knepley     PetscClassId id;
1819e729f68cSMatthew G. Knepley     PetscInt     Nc;
1820e729f68cSMatthew G. Knepley 
18219566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
18229566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1823e729f68cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1824e729f68cSMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
1825e729f68cSMatthew G. Knepley 
18269566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
18279566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
1828e729f68cSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
1829e729f68cSMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
1830e729f68cSMatthew G. Knepley 
18319566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
18329566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
183363a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1834e729f68cSMatthew G. Knepley     numComponents += Nc;
1835e729f68cSMatthew G. Knepley   }
18369566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
183763a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
18389566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ));
18399566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1840e729f68cSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
1841e729f68cSMatthew G. Knepley     PetscScalar *x        = NULL;
18426f288a59SMatthew G. Knepley     PetscScalar  elemDiff = 0.0;
18439c3cf19fSMatthew G. Knepley     PetscInt     qc       = 0;
1844e729f68cSMatthew G. Knepley 
18459566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
1846e8e188d2SZach Atkins     PetscCall(DMPlexVecGetOrientedClosure_Internal(dm, NULL, PETSC_FALSE, localX, c, 0, NULL, &x));
1847e729f68cSMatthew G. Knepley 
1848e729f68cSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
1849e729f68cSMatthew G. Knepley       PetscObject  obj;
1850e729f68cSMatthew G. Knepley       PetscClassId id;
1851e729f68cSMatthew G. Knepley       void *const  ctx = ctxs ? ctxs[field] : NULL;
1852e729f68cSMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1853e729f68cSMatthew G. Knepley 
18549566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
18559566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
18569371c9d4SSatish Balay       if (id == PETSCFE_CLASSID) {
18579371c9d4SSatish Balay         PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
18589371c9d4SSatish Balay         PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
18599371c9d4SSatish Balay       } else if (id == PETSCFV_CLASSID) {
18609371c9d4SSatish Balay         PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
18619371c9d4SSatish Balay         Nb = 1;
18629371c9d4SSatish Balay       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
186323f34ed2SToby Isaac       if (funcs[field]) {
18647318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
18652a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
18662a4e142eSMatthew G. Knepley 
18672a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
18682a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q * coordDim * coordDim];
18692a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
18702a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
187163a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
18729566063dSJacob Faibussowitsch           PetscCall((*funcs[field])(coordDim, time, &coords[q * coordDim], Nc, funcVal, ctx));
1873c3e24edfSBarry Smith #if defined(needs_fix_with_return_code_argument)
1874e735a8a9SMatthew G. Knepley           if (ierr) {
18759566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
18769566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
18779566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
1878e735a8a9SMatthew G. Knepley           }
1879c3e24edfSBarry Smith #endif
18809566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant));
18819566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant));
188263a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1883e729f68cSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1884beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
18854bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
1886e729f68cSMatthew G. Knepley           }
1887e729f68cSMatthew G. Knepley         }
188823f34ed2SToby Isaac       }
1889beaa55a6SMatthew G. Knepley       fieldOffset += Nb;
18909c3cf19fSMatthew G. Knepley       qc += Nc;
1891e729f68cSMatthew G. Knepley     }
18929566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
18939566063dSJacob Faibussowitsch     PetscCall(VecSetValue(D, c - cStart, elemDiff, INSERT_VALUES));
1894e729f68cSMatthew G. Knepley   }
18959566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
18969566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
18979566063dSJacob Faibussowitsch   PetscCall(VecSqrtAbs(D));
18983ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
1899e729f68cSMatthew G. Knepley }
1900e729f68cSMatthew G. Knepley 
19015f0b18bfSMatthew G. Knepley /*@
19022353bfffSMatthew G. Knepley   DMPlexComputeL2FluxDiffVecLocal - This function computes the integral of the difference between the gradient of field `f`in `u` and field `mf` in `mu`
19032353bfffSMatthew G. Knepley 
19042353bfffSMatthew G. Knepley   Collective
19052353bfffSMatthew G. Knepley 
19062353bfffSMatthew G. Knepley   Input Parameters:
19072353bfffSMatthew G. Knepley + lu  - The local `Vec` containing the primal solution
19082353bfffSMatthew G. Knepley . f   - The field number for the potential
19092353bfffSMatthew G. Knepley . lmu - The local `Vec` containing the mixed solution
19102353bfffSMatthew G. Knepley - mf  - The field number for the flux
19112353bfffSMatthew G. Knepley 
19122353bfffSMatthew G. Knepley   Output Parameter:
19132353bfffSMatthew G. Knepley . eFlux - A global `Vec` which holds $||\nabla u_f - \mu_{mf}||$
19142353bfffSMatthew G. Knepley 
19152353bfffSMatthew G. Knepley   Level: advanced
19162353bfffSMatthew G. Knepley 
19172353bfffSMatthew G. Knepley   Notes:
19182353bfffSMatthew G. Knepley   We assume that the `DM` for each solution has the same topology, geometry, and quadrature.
19192353bfffSMatthew G. Knepley 
19202353bfffSMatthew G. Knepley   This is usually used to get an error estimate for the primal solution, using the flux from a mixed solution.
19212353bfffSMatthew G. Knepley 
19222353bfffSMatthew G. Knepley .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeL2FluxDiffVec()`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
19232353bfffSMatthew G. Knepley @*/
19242353bfffSMatthew G. Knepley PetscErrorCode DMPlexComputeL2FluxDiffVecLocal(Vec lu, PetscInt f, Vec lmu, PetscInt mf, Vec eFlux)
19252353bfffSMatthew G. Knepley {
19262353bfffSMatthew G. Knepley   DM               dm, mdm, edm;
19272353bfffSMatthew G. Knepley   PetscFE          fe, mfe;
19282353bfffSMatthew G. Knepley   PetscFEGeom      fegeom;
19292353bfffSMatthew G. Knepley   PetscQuadrature  quad;
19302353bfffSMatthew G. Knepley   const PetscReal *quadWeights;
19312353bfffSMatthew G. Knepley   PetscReal       *coords;
19322353bfffSMatthew G. Knepley   PetscScalar     *interpolant, *minterpolant, *earray;
19332353bfffSMatthew G. Knepley   PetscInt         cdim, mcdim, cStart, cEnd, Nc, mNc, qNc, Nq;
19342353bfffSMatthew G. Knepley   MPI_Comm         comm;
19352353bfffSMatthew G. Knepley 
19362353bfffSMatthew G. Knepley   PetscFunctionBegin;
19372353bfffSMatthew G. Knepley   PetscCall(VecGetDM(lu, &dm));
19382353bfffSMatthew G. Knepley   PetscCall(VecGetDM(lmu, &mdm));
19392353bfffSMatthew G. Knepley   PetscCall(VecGetDM(eFlux, &edm));
19402353bfffSMatthew G. Knepley   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
19412353bfffSMatthew G. Knepley   PetscCall(VecSet(eFlux, 0.0));
19422353bfffSMatthew G. Knepley 
19432353bfffSMatthew G. Knepley   // Check if the both problems are on the same mesh
19442353bfffSMatthew G. Knepley   PetscCall(DMGetCoordinateDim(dm, &cdim));
19452353bfffSMatthew G. Knepley   PetscCall(DMGetCoordinateDim(mdm, &mcdim));
19462353bfffSMatthew G. Knepley   PetscCheck(cdim == mcdim, comm, PETSC_ERR_ARG_SIZ, "primal coordinate Dim %" PetscInt_FMT " != %" PetscInt_FMT " mixed coordinate Dim", cdim, mcdim);
19472353bfffSMatthew G. Knepley   fegeom.dimEmbed = cdim;
19482353bfffSMatthew G. Knepley 
19492353bfffSMatthew G. Knepley   PetscCall(DMGetField(dm, f, NULL, (PetscObject *)&fe));
19502353bfffSMatthew G. Knepley   PetscCall(DMGetField(mdm, mf, NULL, (PetscObject *)&mfe));
19512353bfffSMatthew G. Knepley   PetscCall(PetscFEGetNumComponents(fe, &Nc));
19522353bfffSMatthew G. Knepley   PetscCall(PetscFEGetNumComponents(mfe, &mNc));
19532353bfffSMatthew G. Knepley   PetscCall(PetscFEGetQuadrature(fe, &quad));
19542353bfffSMatthew G. Knepley   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
19552353bfffSMatthew G. Knepley   PetscCheck(qNc == 1 || qNc == mNc, comm, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, mNc);
19562353bfffSMatthew G. Knepley 
19572353bfffSMatthew G. Knepley   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
19582353bfffSMatthew G. Knepley   PetscCall(VecGetArrayWrite(eFlux, &earray));
19592353bfffSMatthew G. Knepley   PetscCall(PetscMalloc6(Nc * cdim, &interpolant, mNc * cdim, &minterpolant, cdim * (Nq + 1), &coords, cdim * cdim * Nq, &fegeom.J, cdim * cdim * Nq, &fegeom.invJ, Nq, &fegeom.detJ));
19602353bfffSMatthew G. Knepley   for (PetscInt c = cStart; c < cEnd; ++c) {
19612353bfffSMatthew G. Knepley     PetscScalar *x            = NULL;
19622353bfffSMatthew G. Knepley     PetscScalar *mx           = NULL;
19632353bfffSMatthew G. Knepley     PetscScalar *eval         = NULL;
19642353bfffSMatthew G. Knepley     PetscReal    fluxElemDiff = 0.0;
19652353bfffSMatthew G. Knepley 
19662353bfffSMatthew G. Knepley     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
19672353bfffSMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(dm, NULL, lu, c, NULL, &x));
19682353bfffSMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(mdm, NULL, lmu, c, NULL, &mx));
19692353bfffSMatthew G. Knepley 
19702353bfffSMatthew G. Knepley     for (PetscInt q = 0; q < Nq; ++q) {
19712353bfffSMatthew G. Knepley       PetscFEGeom qgeom;
19722353bfffSMatthew G. Knepley 
19732353bfffSMatthew G. Knepley       qgeom.dimEmbed = fegeom.dimEmbed;
19742353bfffSMatthew G. Knepley       qgeom.J        = &fegeom.J[q * cdim * cdim];
19752353bfffSMatthew G. Knepley       qgeom.invJ     = &fegeom.invJ[q * cdim * cdim];
19762353bfffSMatthew G. Knepley       qgeom.detJ     = &fegeom.detJ[q];
19772353bfffSMatthew G. Knepley 
19782353bfffSMatthew G. Knepley       PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
19792353bfffSMatthew G. Knepley 
19802353bfffSMatthew G. Knepley       PetscCall(PetscFEInterpolate_Static(mfe, &mx[0], &qgeom, q, minterpolant));
19812353bfffSMatthew G. Knepley       PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[0], &qgeom, q, interpolant));
19822353bfffSMatthew G. Knepley 
19832353bfffSMatthew G. Knepley       /* Now take the elementwise difference and store that in a vector. */
19842353bfffSMatthew G. Knepley       for (PetscInt fc = 0; fc < mNc; ++fc) {
19852353bfffSMatthew G. Knepley         const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : fc)];
19862353bfffSMatthew G. Knepley         fluxElemDiff += PetscSqr(PetscRealPart(interpolant[fc] - minterpolant[fc])) * wt * fegeom.detJ[q];
19872353bfffSMatthew G. Knepley       }
19882353bfffSMatthew G. Knepley     }
19892353bfffSMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(dm, NULL, lu, c, NULL, &x));
19902353bfffSMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(mdm, NULL, lmu, c, NULL, &mx));
19912353bfffSMatthew G. Knepley     PetscCall(DMPlexPointGlobalRef(edm, c, earray, (void *)&eval));
19922353bfffSMatthew G. Knepley     if (eval) eval[0] = fluxElemDiff;
19932353bfffSMatthew G. Knepley   }
19942353bfffSMatthew G. Knepley   PetscCall(PetscFree6(interpolant, minterpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
19952353bfffSMatthew G. Knepley   PetscCall(VecRestoreArrayWrite(eFlux, &earray));
19962353bfffSMatthew G. Knepley 
19972353bfffSMatthew G. Knepley   PetscCall(VecAssemblyBegin(eFlux));
19982353bfffSMatthew G. Knepley   PetscCall(VecAssemblyEnd(eFlux));
19992353bfffSMatthew G. Knepley   PetscCall(VecSqrtAbs(eFlux));
20002353bfffSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
20012353bfffSMatthew G. Knepley }
20022353bfffSMatthew G. Knepley 
20032353bfffSMatthew G. Knepley /*@
20042353bfffSMatthew G. Knepley   DMPlexComputeL2FluxDiffVec - This function computes the integral of the difference between the gradient of field `f`in `u` and field `mf` in `mu`
20052353bfffSMatthew G. Knepley 
20062353bfffSMatthew G. Knepley   Collective
20072353bfffSMatthew G. Knepley 
20082353bfffSMatthew G. Knepley   Input Parameters:
20092353bfffSMatthew G. Knepley + u  - The global `Vec` containing the primal solution
20102353bfffSMatthew G. Knepley . f  - The field number for the potential
20112353bfffSMatthew G. Knepley . mu - The global `Vec` containing the mixed solution
20122353bfffSMatthew G. Knepley - mf - The field number for the flux
20132353bfffSMatthew G. Knepley 
20142353bfffSMatthew G. Knepley   Output Parameter:
20152353bfffSMatthew G. Knepley . eFlux - A global `Vec` which holds $||\nabla u_f - \mu_{mf}||$
20162353bfffSMatthew G. Knepley 
20172353bfffSMatthew G. Knepley   Level: advanced
20182353bfffSMatthew G. Knepley 
20192353bfffSMatthew G. Knepley   Notes:
20202353bfffSMatthew G. Knepley   We assume that the `DM` for each solution has the same topology, geometry, and quadrature.
20212353bfffSMatthew G. Knepley 
20222353bfffSMatthew G. Knepley   This is usually used to get an error estimate for the primal solution, using the flux from a mixed solution.
20232353bfffSMatthew G. Knepley 
20242353bfffSMatthew G. Knepley .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeL2FluxDiffVecLocal()`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
20252353bfffSMatthew G. Knepley @*/
20262353bfffSMatthew G. Knepley PetscErrorCode DMPlexComputeL2FluxDiffVec(Vec u, PetscInt f, Vec mu, PetscInt mf, Vec eFlux)
20272353bfffSMatthew G. Knepley {
20282353bfffSMatthew G. Knepley   DM  dm, mdm;
20292353bfffSMatthew G. Knepley   Vec lu, lmu;
20302353bfffSMatthew G. Knepley 
20312353bfffSMatthew G. Knepley   PetscFunctionBegin;
20322353bfffSMatthew G. Knepley   PetscCall(VecGetDM(u, &dm));
20332353bfffSMatthew G. Knepley   PetscCall(DMGetLocalVector(dm, &lu));
20342353bfffSMatthew G. Knepley   PetscCall(DMGlobalToLocal(dm, u, INSERT_VALUES, lu));
20352353bfffSMatthew G. Knepley   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, lu, 0.0, NULL, NULL, NULL));
20362353bfffSMatthew G. Knepley 
20372353bfffSMatthew G. Knepley   PetscCall(VecGetDM(mu, &mdm));
20382353bfffSMatthew G. Knepley   PetscCall(DMGetLocalVector(mdm, &lmu));
20392353bfffSMatthew G. Knepley   PetscCall(DMGlobalToLocal(mdm, mu, INSERT_VALUES, lmu));
20402353bfffSMatthew G. Knepley   PetscCall(DMPlexInsertBoundaryValues(mdm, PETSC_TRUE, lmu, 0.0, NULL, NULL, NULL));
20412353bfffSMatthew G. Knepley 
20422353bfffSMatthew G. Knepley   PetscCall(DMPlexComputeL2FluxDiffVecLocal(lu, f, lmu, mf, eFlux));
20432353bfffSMatthew G. Knepley 
20442353bfffSMatthew G. Knepley   PetscCall(DMRestoreLocalVector(dm, &lu));
20452353bfffSMatthew G. Knepley   PetscCall(DMRestoreLocalVector(mdm, &lmu));
20462353bfffSMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
20472353bfffSMatthew G. Knepley }
20482353bfffSMatthew G. Knepley 
20492353bfffSMatthew G. Knepley /*@
205020f4b53cSBarry Smith   DMPlexComputeClementInterpolant - This function computes the L2 projection of the cellwise values of a function u onto P1
20515f0b18bfSMatthew G. Knepley 
205220f4b53cSBarry Smith   Collective
20535f0b18bfSMatthew G. Knepley 
20545f0b18bfSMatthew G. Knepley   Input Parameters:
2055a1cb98faSBarry Smith + dm   - The `DM`
20565f0b18bfSMatthew G. Knepley - locX - The coefficient vector u_h
20575f0b18bfSMatthew G. Knepley 
20585f0b18bfSMatthew G. Knepley   Output Parameter:
2059a1cb98faSBarry Smith . locC - A `Vec` which holds the Clement interpolant of the function
20605f0b18bfSMatthew G. Knepley 
20615f0b18bfSMatthew G. Knepley   Level: developer
20625f0b18bfSMatthew G. Knepley 
2063a1cb98faSBarry Smith   Note:
2064a1cb98faSBarry Smith   $ u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| $ where $ |T_i| $ is the cell volume
2065a1cb98faSBarry Smith 
20661cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
20675f0b18bfSMatthew G. Knepley @*/
2068d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeClementInterpolant(DM dm, Vec locX, Vec locC)
2069d71ae5a4SJacob Faibussowitsch {
20705f0b18bfSMatthew G. Knepley   PetscInt         debug = ((DM_Plex *)dm->data)->printFEM;
20715f0b18bfSMatthew G. Knepley   DM               dmc;
20725f0b18bfSMatthew G. Knepley   PetscQuadrature  quad;
20735f0b18bfSMatthew G. Knepley   PetscScalar     *interpolant, *valsum;
20745f0b18bfSMatthew G. Knepley   PetscFEGeom      fegeom;
20755f0b18bfSMatthew G. Knepley   PetscReal       *coords;
20765f0b18bfSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
20775f0b18bfSMatthew G. Knepley   PetscInt         dim, cdim, Nf, f, Nc = 0, Nq, qNc, cStart, cEnd, vStart, vEnd, v;
20785f0b18bfSMatthew G. Knepley 
20795f0b18bfSMatthew G. Knepley   PetscFunctionBegin;
20809566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
20819566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmc));
20829566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
20839566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
20849566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
20855f0b18bfSMatthew G. Knepley   fegeom.dimEmbed = cdim;
20869566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
20875f0b18bfSMatthew G. Knepley   PetscCheck(Nf > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
20885f0b18bfSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
20895f0b18bfSMatthew G. Knepley     PetscObject  obj;
20905f0b18bfSMatthew G. Knepley     PetscClassId id;
20915f0b18bfSMatthew G. Knepley     PetscInt     fNc;
20925f0b18bfSMatthew G. Knepley 
20939566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, f, NULL, &obj));
20949566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
20955f0b18bfSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
20965f0b18bfSMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
20975f0b18bfSMatthew G. Knepley 
20989566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
20999566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &fNc));
21005f0b18bfSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
21015f0b18bfSMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
21025f0b18bfSMatthew G. Knepley 
21039566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
21049566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &fNc));
210563a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
21065f0b18bfSMatthew G. Knepley     Nc += fNc;
21075f0b18bfSMatthew G. Knepley   }
21089566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
210963a3b9bcSJacob Faibussowitsch   PetscCheck(qNc == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " > 1", qNc);
21109566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(Nc * 2, &valsum, Nc, &interpolant, cdim * Nq, &coords, Nq, &fegeom.detJ, cdim * cdim * Nq, &fegeom.J, cdim * cdim * Nq, &fegeom.invJ));
21119566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
21129566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
21135f0b18bfSMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
21145f0b18bfSMatthew G. Knepley     PetscScalar volsum = 0.0;
21155f0b18bfSMatthew G. Knepley     PetscInt   *star   = NULL;
21165f0b18bfSMatthew G. Knepley     PetscInt    starSize, st, fc;
21175f0b18bfSMatthew G. Knepley 
21189566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(valsum, Nc));
21199566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
21205f0b18bfSMatthew G. Knepley     for (st = 0; st < starSize * 2; st += 2) {
21215f0b18bfSMatthew G. Knepley       const PetscInt cell = star[st];
21225f0b18bfSMatthew G. Knepley       PetscScalar   *val  = &valsum[Nc];
21235f0b18bfSMatthew G. Knepley       PetscScalar   *x    = NULL;
21245f0b18bfSMatthew G. Knepley       PetscReal      vol  = 0.0;
21255f0b18bfSMatthew G. Knepley       PetscInt       foff = 0;
21265f0b18bfSMatthew G. Knepley 
21275f0b18bfSMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
21289566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
21299566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
21305f0b18bfSMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
21315f0b18bfSMatthew G. Knepley         PetscObject  obj;
21325f0b18bfSMatthew G. Knepley         PetscClassId id;
21335f0b18bfSMatthew G. Knepley         PetscInt     Nb, fNc, q;
21345f0b18bfSMatthew G. Knepley 
21359566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(val, Nc));
21369566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, f, NULL, &obj));
21379566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
21389371c9d4SSatish Balay         if (id == PETSCFE_CLASSID) {
21399371c9d4SSatish Balay           PetscCall(PetscFEGetNumComponents((PetscFE)obj, &fNc));
21409371c9d4SSatish Balay           PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
21419371c9d4SSatish Balay         } else if (id == PETSCFV_CLASSID) {
21429371c9d4SSatish Balay           PetscCall(PetscFVGetNumComponents((PetscFV)obj, &fNc));
21439371c9d4SSatish Balay           Nb = 1;
21449371c9d4SSatish Balay         } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
21455f0b18bfSMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
21465f0b18bfSMatthew G. Knepley           const PetscReal wt = quadWeights[q] * fegeom.detJ[q];
21475f0b18bfSMatthew G. Knepley           PetscFEGeom     qgeom;
21485f0b18bfSMatthew G. Knepley 
21495f0b18bfSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
21505f0b18bfSMatthew G. Knepley           qgeom.J        = &fegeom.J[q * cdim * cdim];
21515f0b18bfSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * cdim * cdim];
21525f0b18bfSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
215363a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
21549566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[foff], &qgeom, q, interpolant));
215563a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
21565f0b18bfSMatthew G. Knepley           for (fc = 0; fc < fNc; ++fc) val[foff + fc] += interpolant[fc] * wt;
21575f0b18bfSMatthew G. Knepley           vol += wt;
21585f0b18bfSMatthew G. Knepley         }
21595f0b18bfSMatthew G. Knepley         foff += Nb;
21605f0b18bfSMatthew G. Knepley       }
21619566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
21625f0b18bfSMatthew G. Knepley       for (fc = 0; fc < Nc; ++fc) valsum[fc] += val[fc];
21635f0b18bfSMatthew G. Knepley       volsum += vol;
21645f0b18bfSMatthew G. Knepley       if (debug) {
21659566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " value: [", v, cell));
21665f0b18bfSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
21679566063dSJacob Faibussowitsch           if (fc) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
21689566063dSJacob Faibussowitsch           PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(val[fc])));
21695f0b18bfSMatthew G. Knepley         }
21709566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
21715f0b18bfSMatthew G. Knepley       }
21725f0b18bfSMatthew G. Knepley     }
21735f0b18bfSMatthew G. Knepley     for (fc = 0; fc < Nc; ++fc) valsum[fc] /= volsum;
21749566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
21759566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmc, NULL, locC, v, valsum, INSERT_VALUES));
21765f0b18bfSMatthew G. Knepley   }
21779566063dSJacob Faibussowitsch   PetscCall(PetscFree6(valsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
21783ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
21795f0b18bfSMatthew G. Knepley }
21805f0b18bfSMatthew G. Knepley 
21815f0b18bfSMatthew G. Knepley /*@
218220f4b53cSBarry Smith   DMPlexComputeGradientClementInterpolant - This function computes the L2 projection of the cellwise gradient of a function u onto P1
21831555c271SMatthew G. Knepley 
218420f4b53cSBarry Smith   Collective
2185c0f8e1fdSMatthew G. Knepley 
21861555c271SMatthew G. Knepley   Input Parameters:
2187a1cb98faSBarry Smith + dm   - The `DM`
21885f0b18bfSMatthew G. Knepley - locX - The coefficient vector u_h
21891555c271SMatthew G. Knepley 
21901555c271SMatthew G. Knepley   Output Parameter:
2191a1cb98faSBarry Smith . locC - A `Vec` which holds the Clement interpolant of the gradient
21921555c271SMatthew G. Knepley 
21931555c271SMatthew G. Knepley   Level: developer
21941555c271SMatthew G. Knepley 
2195a1cb98faSBarry Smith   Note:
2196a1cb98faSBarry Smith   $\nabla u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| \nabla u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| $ where $ |T_i| $ is the cell volume
2197a1cb98faSBarry Smith 
21981cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
21991555c271SMatthew G. Knepley @*/
2200d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeGradientClementInterpolant(DM dm, Vec locX, Vec locC)
2201d71ae5a4SJacob Faibussowitsch {
2202db1066baSMatthew G. Knepley   DM_Plex         *mesh  = (DM_Plex *)dm->data;
2203db1066baSMatthew G. Knepley   PetscInt         debug = mesh->printFEM;
22041555c271SMatthew G. Knepley   DM               dmC;
22051555c271SMatthew G. Knepley   PetscQuadrature  quad;
22061555c271SMatthew G. Knepley   PetscScalar     *interpolant, *gradsum;
22074bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
22084bee2e38SMatthew G. Knepley   PetscReal       *coords;
22091555c271SMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
2210485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, vStart, vEnd, v, field, fieldOffset;
22111555c271SMatthew G. Knepley 
22121555c271SMatthew G. Knepley   PetscFunctionBegin;
22139566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
22149566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmC));
22159566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
22169566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
22179566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
22184bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
22199566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
22205f80ce2aSJacob Faibussowitsch   PetscCheck(numFields, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
22211555c271SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
22221555c271SMatthew G. Knepley     PetscObject  obj;
22231555c271SMatthew G. Knepley     PetscClassId id;
22241555c271SMatthew G. Knepley     PetscInt     Nc;
22251555c271SMatthew G. Knepley 
22269566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
22279566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
22281555c271SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
22291555c271SMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
22301555c271SMatthew G. Knepley 
22319566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
22329566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
22331555c271SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
22341555c271SMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
22351555c271SMatthew G. Knepley 
22369566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
22379566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
223863a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
22391555c271SMatthew G. Knepley     numComponents += Nc;
22401555c271SMatthew G. Knepley   }
22419566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
224263a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
22439566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(coordDim * numComponents * 2, &gradsum, coordDim * numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ));
22449566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
22459566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
22461555c271SMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
22471555c271SMatthew G. Knepley     PetscScalar volsum = 0.0;
22481555c271SMatthew G. Knepley     PetscInt   *star   = NULL;
22491555c271SMatthew G. Knepley     PetscInt    starSize, st, d, fc;
22501555c271SMatthew G. Knepley 
22519566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(gradsum, coordDim * numComponents));
22529566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
22531555c271SMatthew G. Knepley     for (st = 0; st < starSize * 2; st += 2) {
22541555c271SMatthew G. Knepley       const PetscInt cell = star[st];
22551555c271SMatthew G. Knepley       PetscScalar   *grad = &gradsum[coordDim * numComponents];
22561555c271SMatthew G. Knepley       PetscScalar   *x    = NULL;
22571555c271SMatthew G. Knepley       PetscReal      vol  = 0.0;
22581555c271SMatthew G. Knepley 
22591555c271SMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
22609566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
22619566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
22621555c271SMatthew G. Knepley       for (field = 0, fieldOffset = 0; field < numFields; ++field) {
22631555c271SMatthew G. Knepley         PetscObject  obj;
22641555c271SMatthew G. Knepley         PetscClassId id;
22651555c271SMatthew G. Knepley         PetscInt     Nb, Nc, q, qc = 0;
22661555c271SMatthew G. Knepley 
22679566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(grad, coordDim * numComponents));
22689566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, field, NULL, &obj));
22699566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
22709371c9d4SSatish Balay         if (id == PETSCFE_CLASSID) {
22719371c9d4SSatish Balay           PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
22729371c9d4SSatish Balay           PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
22739371c9d4SSatish Balay         } else if (id == PETSCFV_CLASSID) {
22749371c9d4SSatish Balay           PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
22759371c9d4SSatish Balay           Nb = 1;
22769371c9d4SSatish Balay         } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
22771555c271SMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
22782a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
22792a4e142eSMatthew G. Knepley 
22802a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
22812a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q * coordDim * coordDim];
22822a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
22832a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
228463a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
22859566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolateGradient_Static((PetscFE)obj, 1, &x[fieldOffset], &qgeom, q, interpolant));
228663a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
22871555c271SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
228853d2db2dSJoe Wallwork             const PetscReal wt = quadWeights[q * qNc + qc];
22891555c271SMatthew G. Knepley 
22904bee2e38SMatthew G. Knepley             for (d = 0; d < coordDim; ++d) grad[fc * coordDim + d] += interpolant[fc * dim + d] * wt * fegeom.detJ[q];
22911555c271SMatthew G. Knepley           }
22924bee2e38SMatthew G. Knepley           vol += quadWeights[q * qNc] * fegeom.detJ[q];
22931555c271SMatthew G. Knepley         }
22941555c271SMatthew G. Knepley         fieldOffset += Nb;
22951555c271SMatthew G. Knepley         qc += Nc;
22961555c271SMatthew G. Knepley       }
22979566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
2298f8527842SMatthew G. Knepley       for (fc = 0; fc < numComponents; ++fc) {
2299ad540459SPierre Jolivet         for (d = 0; d < coordDim; ++d) gradsum[fc * coordDim + d] += grad[fc * coordDim + d];
2300f8527842SMatthew G. Knepley       }
2301f8527842SMatthew G. Knepley       volsum += vol;
2302db1066baSMatthew G. Knepley       if (debug) {
23039566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " gradient: [", v, cell));
23041555c271SMatthew G. Knepley         for (fc = 0; fc < numComponents; ++fc) {
23051555c271SMatthew G. Knepley           for (d = 0; d < coordDim; ++d) {
23069566063dSJacob Faibussowitsch             if (fc || d > 0) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
23079566063dSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(grad[fc * coordDim + d])));
23081555c271SMatthew G. Knepley           }
23091555c271SMatthew G. Knepley         }
23109566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
2311db1066baSMatthew G. Knepley       }
23121555c271SMatthew G. Knepley     }
23131555c271SMatthew G. Knepley     for (fc = 0; fc < numComponents; ++fc) {
23141555c271SMatthew G. Knepley       for (d = 0; d < coordDim; ++d) gradsum[fc * coordDim + d] /= volsum;
23151555c271SMatthew G. Knepley     }
23169566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
23179566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmC, NULL, locC, v, gradsum, INSERT_VALUES));
23181555c271SMatthew G. Knepley   }
23199566063dSJacob Faibussowitsch   PetscCall(PetscFree6(gradsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
23203ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
23211555c271SMatthew G. Knepley }
23221555c271SMatthew G. Knepley 
23236493148fSStefano Zampini PetscErrorCode DMPlexComputeIntegral_Internal(DM dm, Vec locX, PetscInt cStart, PetscInt cEnd, PetscScalar *cintegral, void *user)
2324d71ae5a4SJacob Faibussowitsch {
2325cbf8eb3cSStefano Zampini   DM           dmAux = NULL, plexA = NULL;
232661aaff12SToby Isaac   PetscDS      prob, probAux       = NULL;
232773d901b8SMatthew G. Knepley   PetscSection section, sectionAux;
23286493148fSStefano Zampini   Vec          locA;
2329c330f8ffSToby Isaac   PetscInt     dim, numCells = cEnd - cStart, c, f;
2330c330f8ffSToby Isaac   PetscBool    useFVM = PETSC_FALSE;
2331338f77d5SMatthew G. Knepley   /* DS */
2332338f77d5SMatthew G. Knepley   PetscInt           Nf, totDim, *uOff, *uOff_x, numConstants;
2333338f77d5SMatthew G. Knepley   PetscInt           NfAux, totDimAux, *aOff;
23348e3a54c0SPierre Jolivet   PetscScalar       *u, *a = NULL;
2335338f77d5SMatthew G. Knepley   const PetscScalar *constants;
2336338f77d5SMatthew G. Knepley   /* Geometry */
2337c330f8ffSToby Isaac   PetscFEGeom       *cgeomFEM;
2338338f77d5SMatthew G. Knepley   DM                 dmGrad;
2339c330f8ffSToby Isaac   PetscQuadrature    affineQuad      = NULL;
2340338f77d5SMatthew G. Knepley   Vec                cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL;
2341b5a3613cSMatthew G. Knepley   PetscFVCellGeom   *cgeomFVM;
2342338f77d5SMatthew G. Knepley   const PetscScalar *lgrad;
2343b7260050SToby Isaac   PetscInt           maxDegree;
2344c330f8ffSToby Isaac   DMField            coordField;
2345c330f8ffSToby Isaac   IS                 cellIS;
234673d901b8SMatthew G. Knepley 
234773d901b8SMatthew G. Knepley   PetscFunctionBegin;
23489566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
23499566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
23509566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
23519566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
2352338f77d5SMatthew G. Knepley   /* Determine which discretizations we have */
2353b5a3613cSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
2354b5a3613cSMatthew G. Knepley     PetscObject  obj;
2355b5a3613cSMatthew G. Knepley     PetscClassId id;
2356b5a3613cSMatthew G. Knepley 
23579566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
23589566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
2359338f77d5SMatthew G. Knepley     if (id == PETSCFV_CLASSID) useFVM = PETSC_TRUE;
2360338f77d5SMatthew G. Knepley   }
2361338f77d5SMatthew G. Knepley   /* Read DS information */
23629566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
23639566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
23649566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
23659566063dSJacob Faibussowitsch   PetscCall(ISCreateStride(PETSC_COMM_SELF, numCells, cStart, 1, &cellIS));
23669566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
2367338f77d5SMatthew G. Knepley   /* Read Auxiliary DS information */
23689566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
23699a2a23afSMatthew G. Knepley   if (locA) {
23709566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
2371cbf8eb3cSStefano Zampini     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
23729566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
23739566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
23749566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
23759566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
23769566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
2377338f77d5SMatthew G. Knepley   }
2378338f77d5SMatthew G. Knepley   /* Allocate data  arrays */
23799566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(numCells * totDim, &u));
23809566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
2381338f77d5SMatthew G. Knepley   /* Read out geometry */
23829566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
23839566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
2384b7260050SToby Isaac   if (maxDegree <= 1) {
23859566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad));
238648a46eb9SPierre Jolivet     if (affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &cgeomFEM));
2387b5a3613cSMatthew G. Knepley   }
2388b5a3613cSMatthew G. Knepley   if (useFVM) {
2389338f77d5SMatthew G. Knepley     PetscFV   fv = NULL;
2390b5a3613cSMatthew G. Knepley     Vec       grad;
2391b5a3613cSMatthew G. Knepley     PetscInt  fStart, fEnd;
2392b5a3613cSMatthew G. Knepley     PetscBool compGrad;
2393b5a3613cSMatthew G. Knepley 
2394338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
2395338f77d5SMatthew G. Knepley       PetscObject  obj;
2396338f77d5SMatthew G. Knepley       PetscClassId id;
2397338f77d5SMatthew G. Knepley 
23989566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
23999566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
24009371c9d4SSatish Balay       if (id == PETSCFV_CLASSID) {
24019371c9d4SSatish Balay         fv = (PetscFV)obj;
24029371c9d4SSatish Balay         break;
24039371c9d4SSatish Balay       }
2404338f77d5SMatthew G. Knepley     }
24059566063dSJacob Faibussowitsch     PetscCall(PetscFVGetComputeGradients(fv, &compGrad));
24069566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, PETSC_TRUE));
24079566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGeometryFVM(dm, &cellGeometryFVM, &faceGeometryFVM));
24089566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGradientFVM(dm, fv, faceGeometryFVM, cellGeometryFVM, &dmGrad));
24099566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, compGrad));
24109566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM));
2411b5a3613cSMatthew G. Knepley     /* Reconstruct and limit cell gradients */
24129566063dSJacob Faibussowitsch     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
24139566063dSJacob Faibussowitsch     PetscCall(DMGetGlobalVector(dmGrad, &grad));
24149566063dSJacob Faibussowitsch     PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
2415b5a3613cSMatthew G. Knepley     /* Communicate gradient values */
24169566063dSJacob Faibussowitsch     PetscCall(DMGetLocalVector(dmGrad, &locGrad));
24179566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
24189566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
24199566063dSJacob Faibussowitsch     PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
2420b5a3613cSMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
24219566063dSJacob Faibussowitsch     PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, 0.0, faceGeometryFVM, cellGeometryFVM, locGrad));
24229566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
2423b5a3613cSMatthew G. Knepley   }
2424338f77d5SMatthew G. Knepley   /* Read out data from inputs */
242573d901b8SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
242673d901b8SMatthew G. Knepley     PetscScalar *x = NULL;
242773d901b8SMatthew G. Knepley     PetscInt     i;
242873d901b8SMatthew G. Knepley 
24299566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, section, locX, c, NULL, &x));
24300f2d7e86SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i];
24319566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, section, locX, c, NULL, &x));
243273d901b8SMatthew G. Knepley     if (dmAux) {
2433cbf8eb3cSStefano Zampini       PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, c, NULL, &x));
24340f2d7e86SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i];
2435cbf8eb3cSStefano Zampini       PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, c, NULL, &x));
243673d901b8SMatthew G. Knepley     }
243773d901b8SMatthew G. Knepley   }
2438338f77d5SMatthew G. Knepley   /* Do integration for each field */
243973d901b8SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
2440c1f031eeSMatthew G. Knepley     PetscObject  obj;
2441c1f031eeSMatthew G. Knepley     PetscClassId id;
2442c1f031eeSMatthew G. Knepley     PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
244373d901b8SMatthew G. Knepley 
24449566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
24459566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
2446c1f031eeSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
2447c1f031eeSMatthew G. Knepley       PetscFE         fe = (PetscFE)obj;
2448c1f031eeSMatthew G. Knepley       PetscQuadrature q;
2449c330f8ffSToby Isaac       PetscFEGeom    *chunkGeom = NULL;
2450c1f031eeSMatthew G. Knepley       PetscInt        Nq, Nb;
2451c1f031eeSMatthew G. Knepley 
24529566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
24539566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &q));
24549566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
24559566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
2456c1f031eeSMatthew G. Knepley       blockSize = Nb * Nq;
245773d901b8SMatthew G. Knepley       batchSize = numBlocks * blockSize;
24589566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
245973d901b8SMatthew G. Knepley       numChunks = numCells / (numBatches * batchSize);
246073d901b8SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
246173d901b8SMatthew G. Knepley       Nr        = numCells % (numBatches * batchSize);
246273d901b8SMatthew G. Knepley       offset    = numCells - Nr;
246348a46eb9SPierre Jolivet       if (!affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, q, PETSC_FALSE, &cgeomFEM));
24649566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
24659566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrate(prob, f, Ne, chunkGeom, u, probAux, a, cintegral));
24669566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &chunkGeom));
24678e3a54c0SPierre Jolivet       PetscCall(PetscFEIntegrate(prob, f, Nr, chunkGeom, &u[offset * totDim], probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), &cintegral[offset * Nf]));
24689566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &chunkGeom));
246948a46eb9SPierre Jolivet       if (!affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM));
2470c1f031eeSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
2471c1f031eeSMatthew G. Knepley       PetscInt       foff;
2472420e96edSMatthew G. Knepley       PetscPointFunc obj_func;
2473c1f031eeSMatthew G. Knepley 
24749566063dSJacob Faibussowitsch       PetscCall(PetscDSGetObjective(prob, f, &obj_func));
24759566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, f, &foff));
2476c1f031eeSMatthew G. Knepley       if (obj_func) {
2477c1f031eeSMatthew G. Knepley         for (c = 0; c < numCells; ++c) {
2478b5a3613cSMatthew G. Knepley           PetscScalar *u_x;
2479d627b919SMatthew G. Knepley           PetscScalar  lint = 0.;
2480b5a3613cSMatthew G. Knepley 
24819566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, c, lgrad, &u_x));
248255bffe1bSPierre Jolivet           obj_func(dim, Nf, NfAux, uOff, uOff_x, &u[totDim * c + foff], NULL, u_x, aOff, NULL, PetscSafePointerPlusOffset(a, totDimAux * c), NULL, NULL, 0.0, cgeomFVM[c].centroid, numConstants, constants, &lint);
2483338f77d5SMatthew G. Knepley           cintegral[c * Nf + f] += PetscRealPart(lint) * cgeomFVM[c].volume;
248473d901b8SMatthew G. Knepley         }
2485c1f031eeSMatthew G. Knepley       }
248663a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
2487c1f031eeSMatthew G. Knepley   }
2488338f77d5SMatthew G. Knepley   /* Cleanup data arrays */
2489b5a3613cSMatthew G. Knepley   if (useFVM) {
24909566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
24919566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM));
24929566063dSJacob Faibussowitsch     PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
24939566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&faceGeometryFVM));
24949566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&cellGeometryFVM));
24959566063dSJacob Faibussowitsch     PetscCall(DMDestroy(&dmGrad));
2496b5a3613cSMatthew G. Knepley   }
24979566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscFree(a));
2498cbf8eb3cSStefano Zampini   PetscCall(DMDestroy(&plexA));
24999566063dSJacob Faibussowitsch   PetscCall(PetscFree(u));
2500338f77d5SMatthew G. Knepley   /* Cleanup */
250148a46eb9SPierre Jolivet   if (affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM));
25029566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&affineQuad));
25039566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cellIS));
25043ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
2505338f77d5SMatthew G. Knepley }
2506338f77d5SMatthew G. Knepley 
2507338f77d5SMatthew G. Knepley /*@
2508338f77d5SMatthew G. Knepley   DMPlexComputeIntegralFEM - Form the integral over the domain from the global input X using pointwise functions specified by the user
2509338f77d5SMatthew G. Knepley 
2510338f77d5SMatthew G. Knepley   Input Parameters:
2511338f77d5SMatthew G. Knepley + dm   - The mesh
2512338f77d5SMatthew G. Knepley . X    - Global input vector
2513338f77d5SMatthew G. Knepley - user - The user context
2514338f77d5SMatthew G. Knepley 
2515338f77d5SMatthew G. Knepley   Output Parameter:
2516338f77d5SMatthew G. Knepley . integral - Integral for each field
2517338f77d5SMatthew G. Knepley 
2518338f77d5SMatthew G. Knepley   Level: developer
2519338f77d5SMatthew G. Knepley 
25201cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSNESComputeResidualFEM()`
2521338f77d5SMatthew G. Knepley @*/
2522d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeIntegralFEM(DM dm, Vec X, PetscScalar *integral, void *user)
2523d71ae5a4SJacob Faibussowitsch {
2524cbf8eb3cSStefano Zampini   PetscInt     printFEM;
2525b8feb594SMatthew G. Knepley   PetscScalar *cintegral, *lintegral;
2526412e9a14SMatthew G. Knepley   PetscInt     Nf, f, cellHeight, cStart, cEnd, cell;
25276493148fSStefano Zampini   Vec          locX;
2528835f2295SStefano Zampini   PetscMPIInt  Nfi;
2529338f77d5SMatthew G. Knepley 
2530338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2531338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2532338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
25334f572ea9SToby Isaac   PetscAssertPointer(integral, 3);
25349566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0));
2535cbf8eb3cSStefano Zampini   PetscCall(DMPlexConvertPlex(dm, &dm, PETSC_TRUE));
25369566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
25379566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
25389566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2539338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
25409566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &lintegral, (cEnd - cStart) * Nf, &cintegral));
25416493148fSStefano Zampini   /* Get local solution with boundary values */
25426493148fSStefano Zampini   PetscCall(DMGetLocalVector(dm, &locX));
25436493148fSStefano Zampini   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
25446493148fSStefano Zampini   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
25456493148fSStefano Zampini   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
25466493148fSStefano Zampini   PetscCall(DMPlexComputeIntegral_Internal(dm, locX, cStart, cEnd, cintegral, user));
25476493148fSStefano Zampini   PetscCall(DMRestoreLocalVector(dm, &locX));
2548cbf8eb3cSStefano Zampini   printFEM = ((DM_Plex *)dm->data)->printFEM;
2549338f77d5SMatthew G. Knepley   /* Sum up values */
2550338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2551338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2552338f77d5SMatthew G. Knepley 
2553cbf8eb3cSStefano Zampini     if (printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf]));
2554b8feb594SMatthew G. Knepley     for (f = 0; f < Nf; ++f) lintegral[f] += cintegral[c * Nf + f];
2555338f77d5SMatthew G. Knepley   }
2556835f2295SStefano Zampini   PetscCall(PetscMPIIntCast(Nf, &Nfi));
2557835f2295SStefano Zampini   PetscCallMPI(MPIU_Allreduce(lintegral, integral, Nfi, MPIU_SCALAR, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
2558cbf8eb3cSStefano Zampini   if (printFEM) {
25599566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "Integral:"));
25609566063dSJacob Faibussowitsch     for (f = 0; f < Nf; ++f) PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), " %g", (double)PetscRealPart(integral[f])));
25619566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "\n"));
256273d901b8SMatthew G. Knepley   }
25639566063dSJacob Faibussowitsch   PetscCall(PetscFree2(lintegral, cintegral));
25649566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0));
2565cbf8eb3cSStefano Zampini   PetscCall(DMDestroy(&dm));
25663ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
2567338f77d5SMatthew G. Knepley }
2568338f77d5SMatthew G. Knepley 
2569338f77d5SMatthew G. Knepley /*@
2570338f77d5SMatthew G. Knepley   DMPlexComputeCellwiseIntegralFEM - Form the vector of cellwise integrals F from the global input X using pointwise functions specified by the user
2571338f77d5SMatthew G. Knepley 
2572338f77d5SMatthew G. Knepley   Input Parameters:
2573338f77d5SMatthew G. Knepley + dm   - The mesh
2574338f77d5SMatthew G. Knepley . X    - Global input vector
2575338f77d5SMatthew G. Knepley - user - The user context
2576338f77d5SMatthew G. Knepley 
2577338f77d5SMatthew G. Knepley   Output Parameter:
257860225df5SJacob Faibussowitsch . F - Cellwise integrals for each field
2579338f77d5SMatthew G. Knepley 
2580338f77d5SMatthew G. Knepley   Level: developer
2581338f77d5SMatthew G. Knepley 
25821cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSNESComputeResidualFEM()`
2583338f77d5SMatthew G. Knepley @*/
2584d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeCellwiseIntegralFEM(DM dm, Vec X, Vec F, void *user)
2585d71ae5a4SJacob Faibussowitsch {
2586cbf8eb3cSStefano Zampini   PetscInt     printFEM;
2587338f77d5SMatthew G. Knepley   DM           dmF;
25889c8ab049SStefano Zampini   PetscSection sectionF = NULL;
2589338f77d5SMatthew G. Knepley   PetscScalar *cintegral, *af;
25909c8ab049SStefano Zampini   PetscInt     Nf, f, cellHeight, cStart, cEnd, cell, n;
25916493148fSStefano Zampini   Vec          locX;
2592338f77d5SMatthew G. Knepley 
2593338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2594338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2595338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2596338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(F, VEC_CLASSID, 3);
25979566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0));
2598cbf8eb3cSStefano Zampini   PetscCall(DMPlexConvertPlex(dm, &dm, PETSC_TRUE));
25999566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
26009566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
26019566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2602338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
26039566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1((cEnd - cStart) * Nf, &cintegral));
26046493148fSStefano Zampini   /* Get local solution with boundary values */
26056493148fSStefano Zampini   PetscCall(DMGetLocalVector(dm, &locX));
26066493148fSStefano Zampini   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
26076493148fSStefano Zampini   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
26086493148fSStefano Zampini   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
26096493148fSStefano Zampini   PetscCall(DMPlexComputeIntegral_Internal(dm, locX, cStart, cEnd, cintegral, user));
26106493148fSStefano Zampini   PetscCall(DMRestoreLocalVector(dm, &locX));
2611338f77d5SMatthew G. Knepley   /* Put values in F */
26129566063dSJacob Faibussowitsch   PetscCall(VecGetArray(F, &af));
26139c8ab049SStefano Zampini   PetscCall(VecGetDM(F, &dmF));
26149c8ab049SStefano Zampini   if (dmF) PetscCall(DMGetLocalSection(dmF, &sectionF));
26159c8ab049SStefano Zampini   PetscCall(VecGetLocalSize(F, &n));
26169c8ab049SStefano Zampini   PetscCheck(n >= (cEnd - cStart) * Nf, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Vector size %" PetscInt_FMT " < %" PetscInt_FMT, n, (cEnd - cStart) * Nf);
2617cbf8eb3cSStefano Zampini   printFEM = ((DM_Plex *)dm->data)->printFEM;
2618338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2619338f77d5SMatthew G. Knepley     const PetscInt c   = cell - cStart;
26209c8ab049SStefano Zampini     PetscInt       dof = Nf, off = c * Nf;
2621338f77d5SMatthew G. Knepley 
2622cbf8eb3cSStefano Zampini     if (printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf]));
26239c8ab049SStefano Zampini     if (sectionF) {
26249566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetDof(sectionF, cell, &dof));
26259566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetOffset(sectionF, cell, &off));
26269c8ab049SStefano Zampini     }
262763a3b9bcSJacob Faibussowitsch     PetscCheck(dof == Nf, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of cell dofs %" PetscInt_FMT " != %" PetscInt_FMT, dof, Nf);
2628338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) af[off + f] = cintegral[c * Nf + f];
2629338f77d5SMatthew G. Knepley   }
26309566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(F, &af));
26319566063dSJacob Faibussowitsch   PetscCall(PetscFree(cintegral));
26329566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0));
2633cbf8eb3cSStefano Zampini   PetscCall(DMDestroy(&dm));
26343ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
263573d901b8SMatthew G. Knepley }
263673d901b8SMatthew G. Knepley 
263779ab67a3SMatthew G. Knepley static PetscErrorCode DMPlexComputeBdIntegral_Internal(DM dm, Vec locX, IS pointIS, void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *fintegral, void *user)
2638d71ae5a4SJacob Faibussowitsch {
26399b6f715bSMatthew G. Knepley   DM                 plex = NULL, plexA = NULL;
2640a6e0b375SMatthew G. Knepley   DMEnclosureType    encAux;
26419b6f715bSMatthew G. Knepley   PetscDS            prob, probAux       = NULL;
26429b6f715bSMatthew G. Knepley   PetscSection       section, sectionAux = NULL;
26439b6f715bSMatthew G. Knepley   Vec                locA = NULL;
26449b6f715bSMatthew G. Knepley   DMField            coordField;
26459b6f715bSMatthew G. Knepley   PetscInt           Nf, totDim, *uOff, *uOff_x;
26469b6f715bSMatthew G. Knepley   PetscInt           NfAux = 0, totDimAux = 0, *aOff = NULL;
26479b6f715bSMatthew G. Knepley   PetscScalar       *u, *a = NULL;
264864c72086SMatthew G. Knepley   const PetscScalar *constants;
26499b6f715bSMatthew G. Knepley   PetscInt           numConstants, f;
265064c72086SMatthew G. Knepley 
265164c72086SMatthew G. Knepley   PetscFunctionBegin;
26529566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
26539566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
26549566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
26559566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
26569566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
265764c72086SMatthew G. Knepley   /* Determine which discretizations we have */
26589b6f715bSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
265964c72086SMatthew G. Knepley     PetscObject  obj;
266064c72086SMatthew G. Knepley     PetscClassId id;
266164c72086SMatthew G. Knepley 
26629566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
26639566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
266463a3b9bcSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not supported for FVM (field %" PetscInt_FMT ")", f);
266564c72086SMatthew G. Knepley   }
266664c72086SMatthew G. Knepley   /* Read DS information */
26679566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
26689566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
26699566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
26709566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
267164c72086SMatthew G. Knepley   /* Read Auxiliary DS information */
26729566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
26739b6f715bSMatthew G. Knepley   if (locA) {
26749b6f715bSMatthew G. Knepley     DM dmAux;
26759b6f715bSMatthew G. Knepley 
26769566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
26779566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
26789566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
26799566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
26809566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
26819566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
26829566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
26839566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
268464c72086SMatthew G. Knepley   }
26859b6f715bSMatthew G. Knepley   /* Integrate over points */
26869b6f715bSMatthew G. Knepley   {
26879b6f715bSMatthew G. Knepley     PetscFEGeom    *fgeom, *chunkGeom = NULL;
2688b7260050SToby Isaac     PetscInt        maxDegree;
26899b6f715bSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
26909b6f715bSMatthew G. Knepley     const PetscInt *points;
26919b6f715bSMatthew G. Knepley     PetscInt        numFaces, face, Nq, field;
26929b6f715bSMatthew G. Knepley     PetscInt        numChunks, chunkSize, chunk, Nr, offset;
269364c72086SMatthew G. Knepley 
26949566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
26959566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
269632603206SJames Wright     PetscCall(PetscCalloc2(numFaces * totDim, &u, (locA ? (size_t)numFaces * totDimAux : 0), &a));
26979566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
269879ab67a3SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
269979ab67a3SMatthew G. Knepley       const PetscInt point = points[face], *support;
270079ab67a3SMatthew G. Knepley       PetscScalar   *x     = NULL;
270179ab67a3SMatthew G. Knepley 
270279ab67a3SMatthew G. Knepley       PetscCall(DMPlexGetSupport(dm, point, &support));
270379ab67a3SMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
270479ab67a3SMatthew G. Knepley       for (PetscInt i = 0; i < totDim; ++i) u[face * totDim + i] = x[i];
270579ab67a3SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
270679ab67a3SMatthew G. Knepley       if (locA) {
270779ab67a3SMatthew G. Knepley         PetscInt subp;
270879ab67a3SMatthew G. Knepley         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
270979ab67a3SMatthew G. Knepley         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
271079ab67a3SMatthew G. Knepley         for (PetscInt i = 0; i < totDimAux; ++i) a[f * totDimAux + i] = x[i];
271179ab67a3SMatthew G. Knepley         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
271279ab67a3SMatthew G. Knepley       }
271379ab67a3SMatthew G. Knepley     }
271464c72086SMatthew G. Knepley     for (field = 0; field < Nf; ++field) {
271564c72086SMatthew G. Knepley       PetscFE fe;
271664c72086SMatthew G. Knepley 
27179566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fe));
27189566063dSJacob Faibussowitsch       if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
27199b6f715bSMatthew G. Knepley       if (!qGeom) {
27209566063dSJacob Faibussowitsch         PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
27219566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)qGeom));
27229b6f715bSMatthew G. Knepley       }
27239566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
27249566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
27259b6f715bSMatthew G. Knepley       /* Get blocking */
27269b6f715bSMatthew G. Knepley       {
27279b6f715bSMatthew G. Knepley         PetscQuadrature q;
27289b6f715bSMatthew G. Knepley         PetscInt        numBatches, batchSize, numBlocks, blockSize;
27299b6f715bSMatthew G. Knepley         PetscInt        Nq, Nb;
27309b6f715bSMatthew G. Knepley 
27319566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
27329566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &q));
27339566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
27349566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
273564c72086SMatthew G. Knepley         blockSize = Nb * Nq;
273664c72086SMatthew G. Knepley         batchSize = numBlocks * blockSize;
27379b6f715bSMatthew G. Knepley         chunkSize = numBatches * batchSize;
27389566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
27399b6f715bSMatthew G. Knepley         numChunks = numFaces / chunkSize;
27409b6f715bSMatthew G. Knepley         Nr        = numFaces % chunkSize;
274164c72086SMatthew G. Knepley         offset    = numFaces - Nr;
274264c72086SMatthew G. Knepley       }
27439b6f715bSMatthew G. Knepley       /* Do integration for each field */
27449b6f715bSMatthew G. Knepley       for (chunk = 0; chunk < numChunks; ++chunk) {
27459566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(fgeom, chunk * chunkSize, (chunk + 1) * chunkSize, &chunkGeom));
274679ab67a3SMatthew G. Knepley         PetscCall(PetscFEIntegrateBd(prob, field, funcs[field], chunkSize, chunkGeom, &u[chunk * chunkSize * totDim], probAux, PetscSafePointerPlusOffset(a, chunk * chunkSize * totDimAux), &fintegral[chunk * chunkSize * Nf]));
27479566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
274864c72086SMatthew G. Knepley       }
27499566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
275079ab67a3SMatthew G. Knepley       PetscCall(PetscFEIntegrateBd(prob, field, funcs[field], Nr, chunkGeom, &u[offset * totDim], probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), &fintegral[offset * Nf]));
27519566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
275264c72086SMatthew G. Knepley       /* Cleanup data arrays */
27539566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
27549566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&qGeom));
275579ab67a3SMatthew G. Knepley     }
27569566063dSJacob Faibussowitsch     PetscCall(PetscFree2(u, a));
27579566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
275864c72086SMatthew G. Knepley   }
27599566063dSJacob Faibussowitsch   if (plex) PetscCall(DMDestroy(&plex));
27609566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
27613ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
27629b6f715bSMatthew G. Knepley }
27639b6f715bSMatthew G. Knepley 
276460225df5SJacob Faibussowitsch /*@C
27659b6f715bSMatthew G. Knepley   DMPlexComputeBdIntegral - Form the integral over the specified boundary from the global input X using pointwise functions specified by the user
27669b6f715bSMatthew G. Knepley 
27679b6f715bSMatthew G. Knepley   Input Parameters:
27689b6f715bSMatthew G. Knepley + dm      - The mesh
27699b6f715bSMatthew G. Knepley . X       - Global input vector
2770a1cb98faSBarry Smith . label   - The boundary `DMLabel`
2771a1cb98faSBarry Smith . numVals - The number of label values to use, or `PETSC_DETERMINE` for all values
2772a1cb98faSBarry Smith . vals    - The label values to use, or NULL for all values
277379ab67a3SMatthew G. Knepley . funcs   - The functions to integrate along the boundary for each field
27749b6f715bSMatthew G. Knepley - user    - The user context
27759b6f715bSMatthew G. Knepley 
27769b6f715bSMatthew G. Knepley   Output Parameter:
27779b6f715bSMatthew G. Knepley . integral - Integral for each field
27789b6f715bSMatthew G. Knepley 
27799b6f715bSMatthew G. Knepley   Level: developer
27809b6f715bSMatthew G. Knepley 
27811cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeIntegralFEM()`, `DMPlexComputeBdResidualFEM()`
27829b6f715bSMatthew G. Knepley @*/
278379ab67a3SMatthew G. Knepley PetscErrorCode DMPlexComputeBdIntegral(DM dm, Vec X, DMLabel label, PetscInt numVals, const PetscInt vals[], void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *integral, void *user)
2784d71ae5a4SJacob Faibussowitsch {
27859b6f715bSMatthew G. Knepley   Vec          locX;
27869b6f715bSMatthew G. Knepley   PetscSection section;
27879b6f715bSMatthew G. Knepley   DMLabel      depthLabel;
27889b6f715bSMatthew G. Knepley   IS           facetIS;
27899b6f715bSMatthew G. Knepley   PetscInt     dim, Nf, f, v;
27909b6f715bSMatthew G. Knepley 
27919b6f715bSMatthew G. Knepley   PetscFunctionBegin;
27929b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
27939b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2794292bffcbSToby Isaac   PetscValidHeaderSpecific(label, DMLABEL_CLASSID, 3);
27954f572ea9SToby Isaac   if (vals) PetscAssertPointer(vals, 5);
27964f572ea9SToby Isaac   PetscAssertPointer(integral, 7);
27979566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0));
27989566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
27999566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
28009566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
28019566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
28029566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
28039b6f715bSMatthew G. Knepley   /* Get local solution with boundary values */
28049566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &locX));
28059566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
28069566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
28079566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
28089b6f715bSMatthew G. Knepley   /* Loop over label values */
28099566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(integral, Nf));
28109b6f715bSMatthew G. Knepley   for (v = 0; v < numVals; ++v) {
28119b6f715bSMatthew G. Knepley     IS           pointIS;
28129b6f715bSMatthew G. Knepley     PetscInt     numFaces, face;
28139b6f715bSMatthew G. Knepley     PetscScalar *fintegral;
28149b6f715bSMatthew G. Knepley 
28159566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, vals[v], &pointIS));
28169b6f715bSMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
28179b6f715bSMatthew G. Knepley     {
28189b6f715bSMatthew G. Knepley       IS isectIS;
28199b6f715bSMatthew G. Knepley 
28209b6f715bSMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
28219566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
28229566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
28239b6f715bSMatthew G. Knepley       pointIS = isectIS;
28249b6f715bSMatthew G. Knepley     }
28259566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
28269566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(numFaces * Nf, &fintegral));
282779ab67a3SMatthew G. Knepley     PetscCall(DMPlexComputeBdIntegral_Internal(dm, locX, pointIS, funcs, fintegral, user));
28289b6f715bSMatthew G. Knepley     /* Sum point contributions into integral */
28299371c9d4SSatish Balay     for (f = 0; f < Nf; ++f)
28309371c9d4SSatish Balay       for (face = 0; face < numFaces; ++face) integral[f] += fintegral[face * Nf + f];
28319566063dSJacob Faibussowitsch     PetscCall(PetscFree(fintegral));
28329566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
28339b6f715bSMatthew G. Knepley   }
28349566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &locX));
28359566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
28369566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0));
28373ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
283864c72086SMatthew G. Knepley }
283964c72086SMatthew G. Knepley 
2840d69c5d34SMatthew G. Knepley /*@
28410318f8a0SStefano Zampini   DMPlexComputeInterpolatorNested - Form the local portion of the interpolation matrix from the coarse `DM` to a uniformly refined `DM`.
2842d69c5d34SMatthew G. Knepley 
2843d69c5d34SMatthew G. Knepley   Input Parameters:
2844cf51de39SMatthew G. Knepley + dmc       - The coarse mesh
2845cf51de39SMatthew G. Knepley . dmf       - The fine mesh
2846cf51de39SMatthew G. Knepley . isRefined - Flag indicating regular refinement, rather than the same topology
2847d69c5d34SMatthew G. Knepley - user      - The user context
2848d69c5d34SMatthew G. Knepley 
2849d69c5d34SMatthew G. Knepley   Output Parameter:
2850934789fcSMatthew G. Knepley . In - The interpolation matrix
2851d69c5d34SMatthew G. Knepley 
2852d69c5d34SMatthew G. Knepley   Level: developer
2853d69c5d34SMatthew G. Knepley 
2854ae3cf2c1SStefano Zampini .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorGeneral()`
2855d69c5d34SMatthew G. Knepley @*/
2856d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeInterpolatorNested(DM dmc, DM dmf, PetscBool isRefined, Mat In, void *user)
2857d71ae5a4SJacob Faibussowitsch {
2858d69c5d34SMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dmc->data;
2859d69c5d34SMatthew G. Knepley   const char  *name = "Interpolator";
2860d69c5d34SMatthew G. Knepley   PetscFE     *feRef;
286197c42addSMatthew G. Knepley   PetscFV     *fvRef;
2862d69c5d34SMatthew G. Knepley   PetscSection fsection, fglobalSection;
2863d69c5d34SMatthew G. Knepley   PetscSection csection, cglobalSection;
2864d69c5d34SMatthew G. Knepley   PetscScalar *elemMat;
2865485ad865SMatthew G. Knepley   PetscInt     dim, Nf, f, fieldI, fieldJ, offsetI, offsetJ, cStart, cEnd, c;
28662ea9c922SToby Isaac   PetscInt     cTotDim = 0, rTotDim = 0;
2867d69c5d34SMatthew G. Knepley 
2868d69c5d34SMatthew G. Knepley   PetscFunctionBegin;
28699566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
28709566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
28719566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
28729566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
28739566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
28749566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
28759566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
28769566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
28779566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &feRef, Nf, &fvRef));
2878d69c5d34SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
28792ea9c922SToby Isaac     PetscObject  obj, objc;
28802ea9c922SToby Isaac     PetscClassId id, idc;
28812ea9c922SToby Isaac     PetscInt     rNb = 0, Nc = 0, cNb = 0;
2882d69c5d34SMatthew G. Knepley 
28839566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmf, f, NULL, &obj));
28849566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
288597c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
288697c42addSMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
288797c42addSMatthew G. Knepley 
2888cf51de39SMatthew G. Knepley       if (isRefined) {
28899566063dSJacob Faibussowitsch         PetscCall(PetscFERefine(fe, &feRef[f]));
2890cf51de39SMatthew G. Knepley       } else {
28919566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)fe));
2892cf51de39SMatthew G. Knepley         feRef[f] = fe;
2893cf51de39SMatthew G. Knepley       }
28949566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &rNb));
28959566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
289697c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
289797c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV)obj;
289897c42addSMatthew G. Knepley       PetscDualSpace Q;
289997c42addSMatthew G. Knepley 
2900cf51de39SMatthew G. Knepley       if (isRefined) {
29019566063dSJacob Faibussowitsch         PetscCall(PetscFVRefine(fv, &fvRef[f]));
2902cf51de39SMatthew G. Knepley       } else {
29039566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)fv));
2904cf51de39SMatthew G. Knepley         fvRef[f] = fv;
2905cf51de39SMatthew G. Knepley       }
29069566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
29079566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &rNb));
29089566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
29099566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
291097c42addSMatthew G. Knepley     }
29119566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmc, f, NULL, &objc));
29129566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(objc, &idc));
29132ea9c922SToby Isaac     if (idc == PETSCFE_CLASSID) {
29142ea9c922SToby Isaac       PetscFE fe = (PetscFE)objc;
29152ea9c922SToby Isaac 
29169566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &cNb));
29172ea9c922SToby Isaac     } else if (id == PETSCFV_CLASSID) {
29182ea9c922SToby Isaac       PetscFV        fv = (PetscFV)obj;
29192ea9c922SToby Isaac       PetscDualSpace Q;
29202ea9c922SToby Isaac 
29219566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
29229566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &cNb));
2923d69c5d34SMatthew G. Knepley     }
29242ea9c922SToby Isaac     rTotDim += rNb;
29252ea9c922SToby Isaac     cTotDim += cNb;
29262ea9c922SToby Isaac   }
29279566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(rTotDim * cTotDim, &elemMat));
29289566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, rTotDim * cTotDim));
2929d69c5d34SMatthew G. Knepley   for (fieldI = 0, offsetI = 0; fieldI < Nf; ++fieldI) {
2930d69c5d34SMatthew G. Knepley     PetscDualSpace   Qref;
2931d69c5d34SMatthew G. Knepley     PetscQuadrature  f;
2932d69c5d34SMatthew G. Knepley     const PetscReal *qpoints, *qweights;
2933d69c5d34SMatthew G. Knepley     PetscReal       *points;
2934d69c5d34SMatthew G. Knepley     PetscInt         npoints = 0, Nc, Np, fpdim, i, k, p, d;
2935d69c5d34SMatthew G. Knepley 
2936d69c5d34SMatthew G. Knepley     /* Compose points from all dual basis functionals */
293797c42addSMatthew G. Knepley     if (feRef[fieldI]) {
29389566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[fieldI], &Qref));
29399566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[fieldI], &Nc));
294097c42addSMatthew G. Knepley     } else {
29419566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[fieldI], &Qref));
29429566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[fieldI], &Nc));
294397c42addSMatthew G. Knepley     }
29449566063dSJacob Faibussowitsch     PetscCall(PetscDualSpaceGetDimension(Qref, &fpdim));
2945d69c5d34SMatthew G. Knepley     for (i = 0; i < fpdim; ++i) {
29469566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
29479566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, NULL, NULL));
2948d69c5d34SMatthew G. Knepley       npoints += Np;
2949d69c5d34SMatthew G. Knepley     }
29509566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(npoints * dim, &points));
2951d69c5d34SMatthew G. Knepley     for (i = 0, k = 0; i < fpdim; ++i) {
29529566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
29539566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL));
29549371c9d4SSatish Balay       for (p = 0; p < Np; ++p, ++k)
29559371c9d4SSatish Balay         for (d = 0; d < dim; ++d) points[k * dim + d] = qpoints[p * dim + d];
2956d69c5d34SMatthew G. Knepley     }
2957d69c5d34SMatthew G. Knepley 
2958d69c5d34SMatthew G. Knepley     for (fieldJ = 0, offsetJ = 0; fieldJ < Nf; ++fieldJ) {
295997c42addSMatthew G. Knepley       PetscObject  obj;
296097c42addSMatthew G. Knepley       PetscClassId id;
29619c3cf19fSMatthew G. Knepley       PetscInt     NcJ = 0, cpdim = 0, j, qNc;
2962d69c5d34SMatthew G. Knepley 
29639566063dSJacob Faibussowitsch       PetscCall(DMGetField(dmc, fieldJ, NULL, &obj));
29649566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
296597c42addSMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
296697c42addSMatthew G. Knepley         PetscFE         fe = (PetscFE)obj;
2967ef0bb6c7SMatthew G. Knepley         PetscTabulation T  = NULL;
2968d69c5d34SMatthew G. Knepley 
2969d69c5d34SMatthew G. Knepley         /* Evaluate basis at points */
29709566063dSJacob Faibussowitsch         PetscCall(PetscFEGetNumComponents(fe, &NcJ));
29719566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &cpdim));
2972ffe73a53SMatthew G. Knepley         /* For now, fields only interpolate themselves */
2973ffe73a53SMatthew G. Knepley         if (fieldI == fieldJ) {
297463a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
29759566063dSJacob Faibussowitsch           PetscCall(PetscFECreateTabulation(fe, 1, npoints, points, 0, &T));
2976d69c5d34SMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
29779566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
29789566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
297963a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
2980d69c5d34SMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
298136a6d9c0SMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
2982d172c84bSMatthew G. Knepley                 /*
2983d172c84bSMatthew G. Knepley                    cTotDim:            Total columns in element interpolation matrix, sum of number of dual basis functionals in each field
2984d172c84bSMatthew G. Knepley                    offsetI, offsetJ:   Offsets into the larger element interpolation matrix for different fields
2985d172c84bSMatthew G. Knepley                    fpdim, i, cpdim, j: Dofs for fine and coarse grids, correspond to dual space basis functionals
2986d172c84bSMatthew G. Knepley                    qNC, Nc, Ncj, c:    Number of components in this field
2987d172c84bSMatthew G. Knepley                    Np, p:              Number of quad points in the fine grid functional i
2988d172c84bSMatthew G. Knepley                    k:                  i*Np + p, overall point number for the interpolation
2989d172c84bSMatthew G. Knepley                 */
2990ef0bb6c7SMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += T->T[0][k * cpdim * NcJ + j * Nc + c] * qweights[p * qNc + c];
299136a6d9c0SMatthew G. Knepley               }
2992d69c5d34SMatthew G. Knepley             }
2993d69c5d34SMatthew G. Knepley           }
29949566063dSJacob Faibussowitsch           PetscCall(PetscTabulationDestroy(&T));
2995ffe73a53SMatthew G. Knepley         }
299697c42addSMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
299797c42addSMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
299897c42addSMatthew G. Knepley 
299997c42addSMatthew G. Knepley         /* Evaluate constant function at points */
30009566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &NcJ));
300197c42addSMatthew G. Knepley         cpdim = 1;
300297c42addSMatthew G. Knepley         /* For now, fields only interpolate themselves */
300397c42addSMatthew G. Knepley         if (fieldI == fieldJ) {
300463a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
300597c42addSMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
30069566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
30079566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
300863a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
300997c42addSMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
301097c42addSMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
3011458eb97cSMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += 1.0 * qweights[p * qNc + c];
301297c42addSMatthew G. Knepley               }
301397c42addSMatthew G. Knepley             }
301497c42addSMatthew G. Knepley           }
301597c42addSMatthew G. Knepley         }
301697c42addSMatthew G. Knepley       }
3017d172c84bSMatthew G. Knepley       offsetJ += cpdim;
3018d69c5d34SMatthew G. Knepley     }
3019d172c84bSMatthew G. Knepley     offsetI += fpdim;
30209566063dSJacob Faibussowitsch     PetscCall(PetscFree(points));
3021d69c5d34SMatthew G. Knepley   }
30229566063dSJacob Faibussowitsch   if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(0, name, rTotDim, cTotDim, elemMat));
30237f5b169aSMatthew G. Knepley   /* Preallocate matrix */
30247f5b169aSMatthew G. Knepley   {
3025c094ef40SMatthew G. Knepley     Mat          preallocator;
3026c094ef40SMatthew G. Knepley     PetscScalar *vals;
3027c094ef40SMatthew G. Knepley     PetscInt    *cellCIndices, *cellFIndices;
3028c094ef40SMatthew G. Knepley     PetscInt     locRows, locCols, cell;
30297f5b169aSMatthew G. Knepley 
30309566063dSJacob Faibussowitsch     PetscCall(MatGetLocalSize(In, &locRows, &locCols));
30319566063dSJacob Faibussowitsch     PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &preallocator));
30329566063dSJacob Faibussowitsch     PetscCall(MatSetType(preallocator, MATPREALLOCATOR));
30339566063dSJacob Faibussowitsch     PetscCall(MatSetSizes(preallocator, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE));
30349566063dSJacob Faibussowitsch     PetscCall(MatSetUp(preallocator));
30359566063dSJacob Faibussowitsch     PetscCall(PetscCalloc3(rTotDim * cTotDim, &vals, cTotDim, &cellCIndices, rTotDim, &cellFIndices));
30367f5b169aSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3037cf51de39SMatthew G. Knepley       if (isRefined) {
30389566063dSJacob Faibussowitsch         PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, cell, cellCIndices, cellFIndices));
30399566063dSJacob Faibussowitsch         PetscCall(MatSetValues(preallocator, rTotDim, cellFIndices, cTotDim, cellCIndices, vals, INSERT_VALUES));
3040cf51de39SMatthew G. Knepley       } else {
3041e8e188d2SZach Atkins         PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, PETSC_FALSE, dmc, csection, cglobalSection, PETSC_FALSE, preallocator, cell, vals, INSERT_VALUES));
3042cf51de39SMatthew G. Knepley       }
30437f5b169aSMatthew G. Knepley     }
30449566063dSJacob Faibussowitsch     PetscCall(PetscFree3(vals, cellCIndices, cellFIndices));
30459566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(preallocator, MAT_FINAL_ASSEMBLY));
30469566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(preallocator, MAT_FINAL_ASSEMBLY));
30479566063dSJacob Faibussowitsch     PetscCall(MatPreallocatorPreallocate(preallocator, PETSC_TRUE, In));
30489566063dSJacob Faibussowitsch     PetscCall(MatDestroy(&preallocator));
30497f5b169aSMatthew G. Knepley   }
30507f5b169aSMatthew G. Knepley   /* Fill matrix */
30519566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(In));
3052d69c5d34SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
3053cf51de39SMatthew G. Knepley     if (isRefined) {
30549566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosureRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES));
3055cf51de39SMatthew G. Knepley     } else {
3056e8e188d2SZach Atkins       PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, PETSC_FALSE, dmc, csection, cglobalSection, PETSC_FALSE, In, c, elemMat, INSERT_VALUES));
3057cf51de39SMatthew G. Knepley     }
3058d69c5d34SMatthew G. Knepley   }
30599566063dSJacob Faibussowitsch   for (f = 0; f < Nf; ++f) PetscCall(PetscFEDestroy(&feRef[f]));
30609566063dSJacob Faibussowitsch   PetscCall(PetscFree2(feRef, fvRef));
30619566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
30629566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
30639566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
30649318fe57SMatthew G. Knepley   if (mesh->printFEM > 1) {
30659566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)In), "%s:\n", name));
30662ce66baaSPierre Jolivet     PetscCall(MatFilter(In, 1.0e-10, PETSC_FALSE, PETSC_FALSE));
30679566063dSJacob Faibussowitsch     PetscCall(MatView(In, NULL));
3068d69c5d34SMatthew G. Knepley   }
30699566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
30703ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
3071d69c5d34SMatthew G. Knepley }
30726c73c22cSMatthew G. Knepley 
3073d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeMassMatrixNested(DM dmc, DM dmf, Mat mass, void *user)
3074d71ae5a4SJacob Faibussowitsch {
3075bd041c0cSMatthew G. Knepley   SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_SUP, "Laziness");
3076bd041c0cSMatthew G. Knepley }
3077bd041c0cSMatthew G. Knepley 
307868132eb9SMatthew G. Knepley /*@
30790318f8a0SStefano Zampini   DMPlexComputeInterpolatorGeneral - Form the local portion of the interpolation matrix from the coarse `DM` to a non-nested fine `DM`.
308068132eb9SMatthew G. Knepley 
308168132eb9SMatthew G. Knepley   Input Parameters:
308268132eb9SMatthew G. Knepley + dmf  - The fine mesh
308368132eb9SMatthew G. Knepley . dmc  - The coarse mesh
308468132eb9SMatthew G. Knepley - user - The user context
308568132eb9SMatthew G. Knepley 
308668132eb9SMatthew G. Knepley   Output Parameter:
308768132eb9SMatthew G. Knepley . In - The interpolation matrix
308868132eb9SMatthew G. Knepley 
308968132eb9SMatthew G. Knepley   Level: developer
309068132eb9SMatthew G. Knepley 
3091ae3cf2c1SStefano Zampini .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorNested()`
309268132eb9SMatthew G. Knepley @*/
3093d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeInterpolatorGeneral(DM dmc, DM dmf, Mat In, void *user)
3094d71ae5a4SJacob Faibussowitsch {
309564e98e1dSMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dmf->data;
309664e98e1dSMatthew G. Knepley   const char  *name = "Interpolator";
30974ef9d792SMatthew G. Knepley   PetscDS      prob;
3098a0806964SMatthew G. Knepley   Mat          interp;
3099a0806964SMatthew G. Knepley   PetscSection fsection, globalFSection;
3100a0806964SMatthew G. Knepley   PetscSection csection, globalCSection;
3101a0806964SMatthew G. Knepley   PetscInt     locRows, locCols;
31024ef9d792SMatthew G. Knepley   PetscReal   *x, *v0, *J, *invJ, detJ;
31034ef9d792SMatthew G. Knepley   PetscReal   *v0c, *Jc, *invJc, detJc;
31044ef9d792SMatthew G. Knepley   PetscScalar *elemMat;
3105a0806964SMatthew G. Knepley   PetscInt     dim, Nf, field, totDim, cStart, cEnd, cell, ccell, s;
31064ef9d792SMatthew G. Knepley 
31074ef9d792SMatthew G. Knepley   PetscFunctionBegin;
31089566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
31099566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
31109566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
31119566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
31129566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
31139566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ));
31149566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc));
31159566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
31169566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
31179566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
31189566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
3119a0806964SMatthew G. Knepley   PetscCall(DMPlexGetSimplexOrBoxCells(dmf, 0, &cStart, &cEnd));
31209566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
31219566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
31224ef9d792SMatthew G. Knepley 
3123a0806964SMatthew G. Knepley   PetscCall(MatGetLocalSize(In, &locRows, &locCols));
3124a0806964SMatthew G. Knepley   PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &interp));
3125a0806964SMatthew G. Knepley   PetscCall(MatSetType(interp, MATPREALLOCATOR));
3126a0806964SMatthew G. Knepley   PetscCall(MatSetSizes(interp, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE));
3127a0806964SMatthew G. Knepley   PetscCall(MatSetUp(interp));
3128a0806964SMatthew G. Knepley   for (s = 0; s < 2; ++s) {
31294ef9d792SMatthew G. Knepley     for (field = 0; field < Nf; ++field) {
31304ef9d792SMatthew G. Knepley       PetscObject      obj;
31314ef9d792SMatthew G. Knepley       PetscClassId     id;
3132c0d7054bSMatthew G. Knepley       PetscDualSpace   Q = NULL;
3133ef0bb6c7SMatthew G. Knepley       PetscTabulation  T = NULL;
31344ef9d792SMatthew G. Knepley       PetscQuadrature  f;
31354ef9d792SMatthew G. Knepley       const PetscReal *qpoints, *qweights;
3136d0f6233fSMatthew G. Knepley       PetscInt         Nc, qNc, Np, fpdim, off, i, d;
31374ef9d792SMatthew G. Knepley 
3138d0f6233fSMatthew G. Knepley       PetscCall(PetscDSGetFieldOffset(prob, field, &off));
31399566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, &obj));
31409566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
31414ef9d792SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
31424ef9d792SMatthew G. Knepley         PetscFE fe = (PetscFE)obj;
31434ef9d792SMatthew G. Knepley 
31449566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDualSpace(fe, &Q));
31459566063dSJacob Faibussowitsch         PetscCall(PetscFEGetNumComponents(fe, &Nc));
3146a0806964SMatthew G. Knepley         if (s) PetscCall(PetscFECreateTabulation(fe, 1, 1, x, 0, &T));
31474ef9d792SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
31484ef9d792SMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
31494ef9d792SMatthew G. Knepley 
31509566063dSJacob Faibussowitsch         PetscCall(PetscFVGetDualSpace(fv, &Q));
31514ef9d792SMatthew G. Knepley         Nc = 1;
315263a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
31539566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &fpdim));
31544ef9d792SMatthew G. Knepley       /* For each fine grid cell */
31554ef9d792SMatthew G. Knepley       for (cell = cStart; cell < cEnd; ++cell) {
31564ef9d792SMatthew G. Knepley         PetscInt *findices, *cindices;
31574ef9d792SMatthew G. Knepley         PetscInt  numFIndices, numCIndices;
31584ef9d792SMatthew G. Knepley 
31599566063dSJacob Faibussowitsch         PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
31609566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3161d0f6233fSMatthew G. Knepley         PetscCheck(numFIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %" PetscInt_FMT " != %" PetscInt_FMT " dual basis vecs", numFIndices, totDim);
31624ef9d792SMatthew G. Knepley         for (i = 0; i < fpdim; ++i) {
31634ef9d792SMatthew G. Knepley           Vec                pointVec;
31644ef9d792SMatthew G. Knepley           PetscScalar       *pV;
316512111d7cSToby Isaac           PetscSF            coarseCellSF = NULL;
31663a93e3b7SToby Isaac           const PetscSFNode *coarseCells;
3167d0f6233fSMatthew G. Knepley           PetscInt           numCoarseCells, cpdim, row = findices[i + off], q, c, j;
31684ef9d792SMatthew G. Knepley 
31694ef9d792SMatthew G. Knepley           /* Get points from the dual basis functional quadrature */
31709566063dSJacob Faibussowitsch           PetscCall(PetscDualSpaceGetFunctional(Q, i, &f));
31719566063dSJacob Faibussowitsch           PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, &qpoints, &qweights));
317263a3b9bcSJacob Faibussowitsch           PetscCheck(qNc == Nc, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, Nc);
31739566063dSJacob Faibussowitsch           PetscCall(VecCreateSeq(PETSC_COMM_SELF, Np * dim, &pointVec));
31749566063dSJacob Faibussowitsch           PetscCall(VecSetBlockSize(pointVec, dim));
31759566063dSJacob Faibussowitsch           PetscCall(VecGetArray(pointVec, &pV));
31764ef9d792SMatthew G. Knepley           for (q = 0; q < Np; ++q) {
3177c330f8ffSToby Isaac             const PetscReal xi0[3] = {-1., -1., -1.};
3178c330f8ffSToby Isaac 
31794ef9d792SMatthew G. Knepley             /* Transform point to real space */
3180c330f8ffSToby Isaac             CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x);
31814ef9d792SMatthew G. Knepley             for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d];
31824ef9d792SMatthew G. Knepley           }
31839566063dSJacob Faibussowitsch           PetscCall(VecRestoreArray(pointVec, &pV));
31844ef9d792SMatthew G. Knepley           /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
31851555c271SMatthew G. Knepley           /* OPT: Read this out from preallocation information */
31869566063dSJacob Faibussowitsch           PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
31874ef9d792SMatthew G. Knepley           /* Update preallocation info */
31889566063dSJacob Faibussowitsch           PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
31895f80ce2aSJacob Faibussowitsch           PetscCheck(numCoarseCells == Np, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located");
31909566063dSJacob Faibussowitsch           PetscCall(VecGetArray(pointVec, &pV));
31914ef9d792SMatthew G. Knepley           for (ccell = 0; ccell < numCoarseCells; ++ccell) {
3192826eb36dSMatthew G. Knepley             PetscReal       pVReal[3];
3193c330f8ffSToby Isaac             const PetscReal xi0[3] = {-1., -1., -1.};
3194826eb36dSMatthew G. Knepley 
31959566063dSJacob Faibussowitsch             PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3196a0806964SMatthew G. Knepley             if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetDimension((PetscFE)obj, &cpdim));
3197a0806964SMatthew G. Knepley             else cpdim = 1;
3198a0806964SMatthew G. Knepley 
3199a0806964SMatthew G. Knepley             if (s) {
32004ef9d792SMatthew G. Knepley               /* Transform points from real space to coarse reference space */
32019566063dSJacob Faibussowitsch               PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
3202e2d86523SMatthew G. Knepley               for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]);
3203c330f8ffSToby Isaac               CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
32044ef9d792SMatthew G. Knepley 
32054ef9d792SMatthew G. Knepley               if (id == PETSCFE_CLASSID) {
32064ef9d792SMatthew G. Knepley                 /* Evaluate coarse basis on contained point */
3207a0806964SMatthew G. Knepley                 PetscCall(PetscFEComputeTabulation((PetscFE)obj, 1, x, 0, T));
32089566063dSJacob Faibussowitsch                 PetscCall(PetscArrayzero(elemMat, cpdim));
32094ef9d792SMatthew G. Knepley                 /* Get elemMat entries by multiplying by weight */
32104ef9d792SMatthew G. Knepley                 for (j = 0; j < cpdim; ++j) {
3211ef0bb6c7SMatthew G. Knepley                   for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * qweights[ccell * qNc + c];
32124ef9d792SMatthew G. Knepley                 }
32134ef9d792SMatthew G. Knepley               } else {
32144ef9d792SMatthew G. Knepley                 for (j = 0; j < cpdim; ++j) {
32159c3cf19fSMatthew G. Knepley                   for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * qweights[ccell * qNc + c];
32164ef9d792SMatthew G. Knepley                 }
32174ef9d792SMatthew G. Knepley               }
32189566063dSJacob Faibussowitsch               if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
3219a0806964SMatthew G. Knepley             }
3220a0806964SMatthew G. Knepley             /* Update interpolator */
3221d0f6233fSMatthew G. Knepley             PetscCheck(numCIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, totDim);
3222a0806964SMatthew G. Knepley             PetscCall(MatSetValues(interp, 1, &row, cpdim, &cindices[off], elemMat, INSERT_VALUES));
32239566063dSJacob Faibussowitsch             PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
32244ef9d792SMatthew G. Knepley           }
32259566063dSJacob Faibussowitsch           PetscCall(VecRestoreArray(pointVec, &pV));
32269566063dSJacob Faibussowitsch           PetscCall(PetscSFDestroy(&coarseCellSF));
32279566063dSJacob Faibussowitsch           PetscCall(VecDestroy(&pointVec));
32284ef9d792SMatthew G. Knepley         }
32299566063dSJacob Faibussowitsch         PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
32304ef9d792SMatthew G. Knepley       }
3231a0806964SMatthew G. Knepley       if (s && id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
3232a0806964SMatthew G. Knepley     }
3233a0806964SMatthew G. Knepley     if (!s) {
3234a0806964SMatthew G. Knepley       PetscCall(MatAssemblyBegin(interp, MAT_FINAL_ASSEMBLY));
3235a0806964SMatthew G. Knepley       PetscCall(MatAssemblyEnd(interp, MAT_FINAL_ASSEMBLY));
3236a0806964SMatthew G. Knepley       PetscCall(MatPreallocatorPreallocate(interp, PETSC_TRUE, In));
3237a0806964SMatthew G. Knepley       PetscCall(MatDestroy(&interp));
3238a0806964SMatthew G. Knepley       interp = In;
3239a0806964SMatthew G. Knepley     }
32404ef9d792SMatthew G. Knepley   }
32419566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0, J, invJ));
32429566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c, Jc, invJc));
32439566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
32449566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
32459566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
32469566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
32473ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
32484ef9d792SMatthew G. Knepley }
32494ef9d792SMatthew G. Knepley 
325046fa42a0SMatthew G. Knepley /*@
32510318f8a0SStefano Zampini   DMPlexComputeMassMatrixGeneral - Form the local portion of the mass matrix from the coarse `DM` to a non-nested fine `DM`.
3252bd041c0cSMatthew G. Knepley 
3253bd041c0cSMatthew G. Knepley   Input Parameters:
3254bd041c0cSMatthew G. Knepley + dmf  - The fine mesh
3255bd041c0cSMatthew G. Knepley . dmc  - The coarse mesh
3256bd041c0cSMatthew G. Knepley - user - The user context
3257bd041c0cSMatthew G. Knepley 
3258bd041c0cSMatthew G. Knepley   Output Parameter:
3259bd041c0cSMatthew G. Knepley . mass - The mass matrix
3260bd041c0cSMatthew G. Knepley 
3261bd041c0cSMatthew G. Knepley   Level: developer
3262bd041c0cSMatthew G. Knepley 
3263ae3cf2c1SStefano Zampini .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeMassMatrixNested()`, `DMPlexComputeInterpolatorNested()`, `DMPlexComputeInterpolatorGeneral()`
3264bd041c0cSMatthew G. Knepley @*/
3265d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeMassMatrixGeneral(DM dmc, DM dmf, Mat mass, void *user)
3266d71ae5a4SJacob Faibussowitsch {
3267bd041c0cSMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dmf->data;
3268bd041c0cSMatthew G. Knepley   const char  *name = "Mass Matrix";
3269bd041c0cSMatthew G. Knepley   PetscDS      prob;
3270bd041c0cSMatthew G. Knepley   PetscSection fsection, csection, globalFSection, globalCSection;
3271e8f14785SLisandro Dalcin   PetscHSetIJ  ht;
3272bd041c0cSMatthew G. Knepley   PetscLayout  rLayout;
3273bd041c0cSMatthew G. Knepley   PetscInt    *dnz, *onz;
3274bd041c0cSMatthew G. Knepley   PetscInt     locRows, rStart, rEnd;
3275bd041c0cSMatthew G. Knepley   PetscReal   *x, *v0, *J, *invJ, detJ;
3276bd041c0cSMatthew G. Knepley   PetscReal   *v0c, *Jc, *invJc, detJc;
3277bd041c0cSMatthew G. Knepley   PetscScalar *elemMat;
3278bd041c0cSMatthew G. Knepley   PetscInt     dim, Nf, field, totDim, cStart, cEnd, cell, ccell;
3279bd041c0cSMatthew G. Knepley 
3280bd041c0cSMatthew G. Knepley   PetscFunctionBegin;
32819566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
32829566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
32839566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
32849566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
32859566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ));
32869566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc));
32879566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
32889566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
32899566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
32909566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
32919566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd));
32929566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
32939566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
3294bd041c0cSMatthew G. Knepley 
32959566063dSJacob Faibussowitsch   PetscCall(MatGetLocalSize(mass, &locRows, NULL));
32969566063dSJacob Faibussowitsch   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)mass), &rLayout));
32979566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetLocalSize(rLayout, locRows));
32989566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetBlockSize(rLayout, 1));
32999566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetUp(rLayout));
33009566063dSJacob Faibussowitsch   PetscCall(PetscLayoutGetRange(rLayout, &rStart, &rEnd));
33019566063dSJacob Faibussowitsch   PetscCall(PetscLayoutDestroy(&rLayout));
33029566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(locRows, &dnz, locRows, &onz));
33039566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJCreate(&ht));
3304bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
3305bd041c0cSMatthew G. Knepley     PetscObject      obj;
3306bd041c0cSMatthew G. Knepley     PetscClassId     id;
3307bd041c0cSMatthew G. Knepley     PetscQuadrature  quad;
3308bd041c0cSMatthew G. Knepley     const PetscReal *qpoints;
3309bd041c0cSMatthew G. Knepley     PetscInt         Nq, Nc, i, d;
3310bd041c0cSMatthew G. Knepley 
33119566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
33129566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
33139566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad));
33149566063dSJacob Faibussowitsch     else PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad));
33159566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, NULL));
3316bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
3317bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3318bd041c0cSMatthew G. Knepley       Vec                pointVec;
3319bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
3320bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
3321bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
3322bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, q, c;
3323bd041c0cSMatthew G. Knepley       PetscInt          *findices, *cindices;
3324bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
3325bd041c0cSMatthew G. Knepley 
33269566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
33279566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3328bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
33299566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec));
33309566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
33319566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3332bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
3333c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3334c330f8ffSToby Isaac 
3335bd041c0cSMatthew G. Knepley         /* Transform point to real space */
3336c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x);
3337bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d];
3338bd041c0cSMatthew G. Knepley       }
33399566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
3340bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
33419566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
33429566063dSJacob Faibussowitsch       PetscCall(PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view"));
3343bd041c0cSMatthew G. Knepley       /* Update preallocation info */
33449566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
33455f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located");
3346bd041c0cSMatthew G. Knepley       {
3347e8f14785SLisandro Dalcin         PetscHashIJKey key;
3348e8f14785SLisandro Dalcin         PetscBool      missing;
3349bd041c0cSMatthew G. Knepley 
3350bd041c0cSMatthew G. Knepley         for (i = 0; i < numFIndices; ++i) {
3351e8f14785SLisandro Dalcin           key.i = findices[i];
3352e8f14785SLisandro Dalcin           if (key.i >= 0) {
3353bd041c0cSMatthew G. Knepley             /* Get indices for coarse elements */
3354bd041c0cSMatthew G. Knepley             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
33559566063dSJacob Faibussowitsch               PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3356bd041c0cSMatthew G. Knepley               for (c = 0; c < numCIndices; ++c) {
3357e8f14785SLisandro Dalcin                 key.j = cindices[c];
3358e8f14785SLisandro Dalcin                 if (key.j < 0) continue;
33599566063dSJacob Faibussowitsch                 PetscCall(PetscHSetIJQueryAdd(ht, key, &missing));
3360bd041c0cSMatthew G. Knepley                 if (missing) {
3361e8f14785SLisandro Dalcin                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i - rStart];
3362e8f14785SLisandro Dalcin                   else ++onz[key.i - rStart];
3363bd041c0cSMatthew G. Knepley                 }
3364bd041c0cSMatthew G. Knepley               }
33659566063dSJacob Faibussowitsch               PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3366bd041c0cSMatthew G. Knepley             }
3367bd041c0cSMatthew G. Knepley           }
3368bd041c0cSMatthew G. Knepley         }
3369bd041c0cSMatthew G. Knepley       }
33709566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
33719566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
33729566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
3373bd041c0cSMatthew G. Knepley     }
3374bd041c0cSMatthew G. Knepley   }
33759566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJDestroy(&ht));
33769566063dSJacob Faibussowitsch   PetscCall(MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL));
33779566063dSJacob Faibussowitsch   PetscCall(MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));
33789566063dSJacob Faibussowitsch   PetscCall(PetscFree2(dnz, onz));
3379bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
3380bd041c0cSMatthew G. Knepley     PetscObject      obj;
3381bd041c0cSMatthew G. Knepley     PetscClassId     id;
3382ef0bb6c7SMatthew G. Knepley     PetscTabulation  T, Tfine;
3383bd041c0cSMatthew G. Knepley     PetscQuadrature  quad;
3384bd041c0cSMatthew G. Knepley     const PetscReal *qpoints, *qweights;
3385bd041c0cSMatthew G. Knepley     PetscInt         Nq, Nc, i, d;
3386bd041c0cSMatthew G. Knepley 
33879566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
33889566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
3389ef0bb6c7SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
33909566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad));
33919566063dSJacob Faibussowitsch       PetscCall(PetscFEGetCellTabulation((PetscFE)obj, 1, &Tfine));
33929566063dSJacob Faibussowitsch       PetscCall(PetscFECreateTabulation((PetscFE)obj, 1, 1, x, 0, &T));
3393ef0bb6c7SMatthew G. Knepley     } else {
33949566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad));
3395ef0bb6c7SMatthew G. Knepley     }
33969566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, &qweights));
3397bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
3398bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3399bd041c0cSMatthew G. Knepley       Vec                pointVec;
3400bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
3401bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
3402bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
3403bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, cpdim, q, c, j;
3404bd041c0cSMatthew G. Knepley       PetscInt          *findices, *cindices;
3405bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
3406bd041c0cSMatthew G. Knepley 
34079566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
34089566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3409bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
34109566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec));
34119566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
34129566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3413bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
3414c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3415c330f8ffSToby Isaac 
3416bd041c0cSMatthew G. Knepley         /* Transform point to real space */
3417c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x);
3418bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d];
3419bd041c0cSMatthew G. Knepley       }
34209566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
3421bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
34229566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
3423bd041c0cSMatthew G. Knepley       /* Update matrix */
34249566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
34255f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located");
34269566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3427bd041c0cSMatthew G. Knepley       for (ccell = 0; ccell < numCoarseCells; ++ccell) {
3428bd041c0cSMatthew G. Knepley         PetscReal       pVReal[3];
3429c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3430c330f8ffSToby Isaac 
34319566063dSJacob Faibussowitsch         PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3432bd041c0cSMatthew G. Knepley         /* Transform points from real space to coarse reference space */
34339566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
3434bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]);
3435c330f8ffSToby Isaac         CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
3436bd041c0cSMatthew G. Knepley 
3437bd041c0cSMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
3438bd041c0cSMatthew G. Knepley           PetscFE fe = (PetscFE)obj;
3439bd041c0cSMatthew G. Knepley 
3440bd041c0cSMatthew G. Knepley           /* Evaluate coarse basis on contained point */
34419566063dSJacob Faibussowitsch           PetscCall(PetscFEGetDimension(fe, &cpdim));
34429566063dSJacob Faibussowitsch           PetscCall(PetscFEComputeTabulation(fe, 1, x, 0, T));
3443bd041c0cSMatthew G. Knepley           /* Get elemMat entries by multiplying by weight */
3444bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
34459566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3446bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3447ef0bb6c7SMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * Tfine->T[0][(ccell * numFIndices + i) * Nc + c] * qweights[ccell * Nc + c] * detJ;
3448bd041c0cSMatthew G. Knepley             }
3449bd041c0cSMatthew G. Knepley             /* Update interpolator */
34509566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
345163a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
34529566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3453bd041c0cSMatthew G. Knepley           }
3454bd041c0cSMatthew G. Knepley         } else {
3455bd041c0cSMatthew G. Knepley           cpdim = 1;
3456bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
34579566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3458bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3459bd041c0cSMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * 1.0 * qweights[ccell * Nc + c] * detJ;
3460bd041c0cSMatthew G. Knepley             }
3461bd041c0cSMatthew G. Knepley             /* Update interpolator */
34629566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
346363a3b9bcSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "Nq: %" PetscInt_FMT " %" PetscInt_FMT " Nf: %" PetscInt_FMT " %" PetscInt_FMT " Nc: %" PetscInt_FMT " %" PetscInt_FMT "\n", ccell, Nq, i, numFIndices, j, numCIndices));
346463a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
34659566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3466bd041c0cSMatthew G. Knepley           }
3467bd041c0cSMatthew G. Knepley         }
34689566063dSJacob Faibussowitsch         PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3469bd041c0cSMatthew G. Knepley       }
34709566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
34719566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
34729566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
34739566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
3474bd041c0cSMatthew G. Knepley     }
34759566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
3476bd041c0cSMatthew G. Knepley   }
34779566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0, J, invJ));
34789566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c, Jc, invJc));
34799566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
34809566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY));
34819566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY));
34823ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
3483bd041c0cSMatthew G. Knepley }
3484bd041c0cSMatthew G. Knepley 
3485bd041c0cSMatthew G. Knepley /*@
348646fa42a0SMatthew G. Knepley   DMPlexComputeInjectorFEM - Compute a mapping from coarse unknowns to fine unknowns
348746fa42a0SMatthew G. Knepley 
348846fa42a0SMatthew G. Knepley   Input Parameters:
348946fa42a0SMatthew G. Knepley + dmc  - The coarse mesh
349060225df5SJacob Faibussowitsch . dmf  - The fine mesh
349146fa42a0SMatthew G. Knepley - user - The user context
349246fa42a0SMatthew G. Knepley 
349346fa42a0SMatthew G. Knepley   Output Parameter:
349446fa42a0SMatthew G. Knepley . sc - The mapping
349546fa42a0SMatthew G. Knepley 
349646fa42a0SMatthew G. Knepley   Level: developer
349746fa42a0SMatthew G. Knepley 
3498ae3cf2c1SStefano Zampini .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorNested()`
349946fa42a0SMatthew G. Knepley @*/
3500d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeInjectorFEM(DM dmc, DM dmf, VecScatter *sc, void *user)
3501d71ae5a4SJacob Faibussowitsch {
3502e9d4ef1bSMatthew G. Knepley   PetscDS      prob;
35037c927364SMatthew G. Knepley   PetscFE     *feRef;
350497c42addSMatthew G. Knepley   PetscFV     *fvRef;
35057c927364SMatthew G. Knepley   Vec          fv, cv;
35067c927364SMatthew G. Knepley   IS           fis, cis;
35077c927364SMatthew G. Knepley   PetscSection fsection, fglobalSection, csection, cglobalSection;
35087c927364SMatthew G. Knepley   PetscInt    *cmap, *cellCIndices, *cellFIndices, *cindices, *findices;
3509485ad865SMatthew G. Knepley   PetscInt     cTotDim, fTotDim = 0, Nf, f, field, cStart, cEnd, c, dim, d, startC, endC, offsetC, offsetF, m;
35106f3d3cbcSMatthew G. Knepley   PetscBool   *needAvg;
35117c927364SMatthew G. Knepley 
35127c927364SMatthew G. Knepley   PetscFunctionBegin;
35139566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InjectorFEM, dmc, dmf, 0, 0));
35149566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
35159566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
35169566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
35179566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
35189566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
35199566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
35209566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
35219566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
35229566063dSJacob Faibussowitsch   PetscCall(PetscCalloc3(Nf, &feRef, Nf, &fvRef, Nf, &needAvg));
35237c927364SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
352497c42addSMatthew G. Knepley     PetscObject  obj;
352597c42addSMatthew G. Knepley     PetscClassId id;
3526aa7890ccSMatthew G. Knepley     PetscInt     fNb = 0, Nc = 0;
35277c927364SMatthew G. Knepley 
35289566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
35299566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
353097c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
353197c42addSMatthew G. Knepley       PetscFE    fe = (PetscFE)obj;
35326f3d3cbcSMatthew G. Knepley       PetscSpace sp;
35339b2fc754SMatthew G. Knepley       PetscInt   maxDegree;
353497c42addSMatthew G. Knepley 
35359566063dSJacob Faibussowitsch       PetscCall(PetscFERefine(fe, &feRef[f]));
35369566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &fNb));
35379566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
35389566063dSJacob Faibussowitsch       PetscCall(PetscFEGetBasisSpace(fe, &sp));
35399566063dSJacob Faibussowitsch       PetscCall(PetscSpaceGetDegree(sp, NULL, &maxDegree));
35409b2fc754SMatthew G. Knepley       if (!maxDegree) needAvg[f] = PETSC_TRUE;
354197c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
354297c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV)obj;
354397c42addSMatthew G. Knepley       PetscDualSpace Q;
354497c42addSMatthew G. Knepley 
35459566063dSJacob Faibussowitsch       PetscCall(PetscFVRefine(fv, &fvRef[f]));
35469566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
35479566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &fNb));
35489566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
35496f3d3cbcSMatthew G. Knepley       needAvg[f] = PETSC_TRUE;
355097c42addSMatthew G. Knepley     }
3551d172c84bSMatthew G. Knepley     fTotDim += fNb;
35527c927364SMatthew G. Knepley   }
35539566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &cTotDim));
35549566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(cTotDim, &cmap));
35557c927364SMatthew G. Knepley   for (field = 0, offsetC = 0, offsetF = 0; field < Nf; ++field) {
35567c927364SMatthew G. Knepley     PetscFE        feC;
355797c42addSMatthew G. Knepley     PetscFV        fvC;
35587c927364SMatthew G. Knepley     PetscDualSpace QF, QC;
3559d172c84bSMatthew G. Knepley     PetscInt       order = -1, NcF, NcC, fpdim, cpdim;
35607c927364SMatthew G. Knepley 
356197c42addSMatthew G. Knepley     if (feRef[field]) {
35629566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&feC));
35639566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feC, &NcC));
35649566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[field], &NcF));
35659566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[field], &QF));
35669566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetOrder(QF, &order));
35679566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
35689566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feC, &QC));
35699566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
357097c42addSMatthew G. Knepley     } else {
35719566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fvC));
35729566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvC, &NcC));
35739566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[field], &NcF));
35749566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[field], &QF));
35759566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
35769566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvC, &QC));
35779566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
357897c42addSMatthew G. Knepley     }
357963a3b9bcSJacob Faibussowitsch     PetscCheck(NcF == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, NcF, NcC);
35807c927364SMatthew G. Knepley     for (c = 0; c < cpdim; ++c) {
35817c927364SMatthew G. Knepley       PetscQuadrature  cfunc;
3582d172c84bSMatthew G. Knepley       const PetscReal *cqpoints, *cqweights;
3583d172c84bSMatthew G. Knepley       PetscInt         NqcC, NpC;
358497c42addSMatthew G. Knepley       PetscBool        found = PETSC_FALSE;
35857c927364SMatthew G. Knepley 
35869566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(QC, c, &cfunc));
35879566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(cfunc, NULL, &NqcC, &NpC, &cqpoints, &cqweights));
358863a3b9bcSJacob Faibussowitsch       PetscCheck(NqcC == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcC, NcC);
35891dca8a05SBarry Smith       PetscCheck(NpC == 1 || !feRef[field], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Do not know how to do injection for moments");
35907c927364SMatthew G. Knepley       for (f = 0; f < fpdim; ++f) {
35917c927364SMatthew G. Knepley         PetscQuadrature  ffunc;
3592d172c84bSMatthew G. Knepley         const PetscReal *fqpoints, *fqweights;
35937c927364SMatthew G. Knepley         PetscReal        sum = 0.0;
3594d172c84bSMatthew G. Knepley         PetscInt         NqcF, NpF;
35957c927364SMatthew G. Knepley 
35969566063dSJacob Faibussowitsch         PetscCall(PetscDualSpaceGetFunctional(QF, f, &ffunc));
35979566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(ffunc, NULL, &NqcF, &NpF, &fqpoints, &fqweights));
359863a3b9bcSJacob Faibussowitsch         PetscCheck(NqcF == NcF, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcF, NcF);
35997c927364SMatthew G. Knepley         if (NpC != NpF) continue;
36007c927364SMatthew G. Knepley         for (d = 0; d < dim; ++d) sum += PetscAbsReal(cqpoints[d] - fqpoints[d]);
36017c927364SMatthew G. Knepley         if (sum > 1.0e-9) continue;
3602d172c84bSMatthew G. Knepley         for (d = 0; d < NcC; ++d) sum += PetscAbsReal(cqweights[d] * fqweights[d]);
3603d172c84bSMatthew G. Knepley         if (sum < 1.0e-9) continue;
3604d172c84bSMatthew G. Knepley         cmap[offsetC + c] = offsetF + f;
360597c42addSMatthew G. Knepley         found             = PETSC_TRUE;
36067c927364SMatthew G. Knepley         break;
36077c927364SMatthew G. Knepley       }
360897c42addSMatthew G. Knepley       if (!found) {
360997c42addSMatthew G. Knepley         /* TODO We really want the average here, but some asshole put VecScatter in the interface */
3610d172c84bSMatthew G. Knepley         if (fvRef[field] || (feRef[field] && order == 0)) {
3611d172c84bSMatthew G. Knepley           cmap[offsetC + c] = offsetF + 0;
361297c42addSMatthew G. Knepley         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate matching functional for injection");
361397c42addSMatthew G. Knepley       }
36147c927364SMatthew G. Knepley     }
3615d172c84bSMatthew G. Knepley     offsetC += cpdim;
3616d172c84bSMatthew G. Knepley     offsetF += fpdim;
36177c927364SMatthew G. Knepley   }
36189371c9d4SSatish Balay   for (f = 0; f < Nf; ++f) {
36199371c9d4SSatish Balay     PetscCall(PetscFEDestroy(&feRef[f]));
36209371c9d4SSatish Balay     PetscCall(PetscFVDestroy(&fvRef[f]));
36219371c9d4SSatish Balay   }
36229566063dSJacob Faibussowitsch   PetscCall(PetscFree3(feRef, fvRef, needAvg));
36237c927364SMatthew G. Knepley 
36249566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmf, &fv));
36259566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmc, &cv));
36269566063dSJacob Faibussowitsch   PetscCall(VecGetOwnershipRange(cv, &startC, &endC));
36279566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(cglobalSection, &m));
36289566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(cTotDim, &cellCIndices, fTotDim, &cellFIndices));
36299566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m, &cindices));
36309566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m, &findices));
36317c927364SMatthew G. Knepley   for (d = 0; d < m; ++d) cindices[d] = findices[d] = -1;
36327c927364SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
36339566063dSJacob Faibussowitsch     PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, c, cellCIndices, cellFIndices));
36347c927364SMatthew G. Knepley     for (d = 0; d < cTotDim; ++d) {
36350bd915a7SMatthew G. Knepley       if ((cellCIndices[d] < startC) || (cellCIndices[d] >= endC)) continue;
3636d2457c26SMatthew G. Knepley       PetscCheck(!(findices[cellCIndices[d] - startC] >= 0) || !(findices[cellCIndices[d] - startC] != cellFIndices[cmap[d]]), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " Coarse dof %" PetscInt_FMT " maps to both %" PetscInt_FMT " and %" PetscInt_FMT, c, cindices[cellCIndices[d] - startC], findices[cellCIndices[d] - startC], cellFIndices[cmap[d]]);
36377c927364SMatthew G. Knepley       cindices[cellCIndices[d] - startC] = cellCIndices[d];
36387c927364SMatthew G. Knepley       findices[cellCIndices[d] - startC] = cellFIndices[cmap[d]];
36397c927364SMatthew G. Knepley     }
36407c927364SMatthew G. Knepley   }
36419566063dSJacob Faibussowitsch   PetscCall(PetscFree(cmap));
36429566063dSJacob Faibussowitsch   PetscCall(PetscFree2(cellCIndices, cellFIndices));
36437c927364SMatthew G. Knepley 
36449566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis));
36459566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis));
36469566063dSJacob Faibussowitsch   PetscCall(VecScatterCreate(cv, cis, fv, fis, sc));
36479566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cis));
36489566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&fis));
36499566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmf, &fv));
36509566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmc, &cv));
36519566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InjectorFEM, dmc, dmf, 0, 0));
36523ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
3653cb1e1211SMatthew G Knepley }
3654a1cf66bbSMatthew G. Knepley 
36552f856554SMatthew G. Knepley /*@C
36562f856554SMatthew G. Knepley   DMPlexGetCellFields - Retrieve the field values values for a chunk of cells
36572f856554SMatthew G. Knepley 
36582f856554SMatthew G. Knepley   Input Parameters:
3659a1cb98faSBarry Smith + dm     - The `DM`
36602f856554SMatthew G. Knepley . cellIS - The cells to include
36612f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
36622f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
36632f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
36642f856554SMatthew G. Knepley 
36652f856554SMatthew G. Knepley   Output Parameters:
36662f856554SMatthew G. Knepley + u   - The field coefficients
36672f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
36682f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
36692f856554SMatthew G. Knepley 
36702f856554SMatthew G. Knepley   Level: developer
36712f856554SMatthew G. Knepley 
36721cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()`
36732f856554SMatthew G. Knepley @*/
3674d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
3675d71ae5a4SJacob Faibussowitsch {
36762f856554SMatthew G. Knepley   DM              plex, plexA = NULL;
3677a6e0b375SMatthew G. Knepley   DMEnclosureType encAux;
36782f856554SMatthew G. Knepley   PetscSection    section, sectionAux;
36792f856554SMatthew G. Knepley   PetscDS         prob;
36802f856554SMatthew G. Knepley   const PetscInt *cells;
36812f856554SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, totDim, totDimAux, c;
36822f856554SMatthew G. Knepley 
36832f856554SMatthew G. Knepley   PetscFunctionBegin;
36842f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3685064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
3686ad540459SPierre Jolivet   if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4);
3687ad540459SPierre Jolivet   if (locA) PetscValidHeaderSpecific(locA, VEC_CLASSID, 5);
36884f572ea9SToby Isaac   PetscAssertPointer(u, 6);
36894f572ea9SToby Isaac   PetscAssertPointer(u_t, 7);
36904f572ea9SToby Isaac   PetscAssertPointer(a, 8);
36919566063dSJacob Faibussowitsch   PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE));
36929566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
36939566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
369407218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL));
36959566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
36962f856554SMatthew G. Knepley   if (locA) {
36972f856554SMatthew G. Knepley     DM      dmAux;
36982f856554SMatthew G. Knepley     PetscDS probAux;
36992f856554SMatthew G. Knepley 
37009566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
37019566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
37029566063dSJacob Faibussowitsch     PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE));
37039566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
37049566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
37059566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
37062f856554SMatthew G. Knepley   }
37072f856554SMatthew G. Knepley   numCells = cEnd - cStart;
37089566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u));
37099371c9d4SSatish Balay   if (locX_t) PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u_t));
3710ad540459SPierre Jolivet   else *u_t = NULL;
37119371c9d4SSatish Balay   if (locA) PetscCall(DMGetWorkArray(dm, numCells * totDimAux, MPIU_SCALAR, a));
3712ad540459SPierre Jolivet   else *a = NULL;
37132f856554SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
37142f856554SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
37152f856554SMatthew G. Knepley     const PetscInt cind = c - cStart;
37162f856554SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL, *ul = *u, *ul_t = *u_t, *al = *a;
37172f856554SMatthew G. Knepley     PetscInt       i;
37182f856554SMatthew G. Knepley 
37199566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x));
37202f856554SMatthew G. Knepley     for (i = 0; i < totDim; ++i) ul[cind * totDim + i] = x[i];
37219566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x));
37222f856554SMatthew G. Knepley     if (locX_t) {
37239566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t));
37242f856554SMatthew G. Knepley       for (i = 0; i < totDim; ++i) ul_t[cind * totDim + i] = x_t[i];
37259566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t));
37262f856554SMatthew G. Knepley     }
37272f856554SMatthew G. Knepley     if (locA) {
37282f856554SMatthew G. Knepley       PetscInt subcell;
37299566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell));
37309566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, NULL, &x));
37312f856554SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) al[cind * totDimAux + i] = x[i];
37329566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, NULL, &x));
37332f856554SMatthew G. Knepley     }
37342f856554SMatthew G. Knepley   }
37359566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
37369566063dSJacob Faibussowitsch   if (locA) PetscCall(DMDestroy(&plexA));
37379566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
37383ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
37392f856554SMatthew G. Knepley }
37402f856554SMatthew G. Knepley 
37412f856554SMatthew G. Knepley /*@C
37422f856554SMatthew G. Knepley   DMPlexRestoreCellFields - Restore the field values values for a chunk of cells
37432f856554SMatthew G. Knepley 
37442f856554SMatthew G. Knepley   Input Parameters:
3745a1cb98faSBarry Smith + dm     - The `DM`
37462f856554SMatthew G. Knepley . cellIS - The cells to include
37472f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
37482f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
37492f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
37502f856554SMatthew G. Knepley 
37512f856554SMatthew G. Knepley   Output Parameters:
37522f856554SMatthew G. Knepley + u   - The field coefficients
37532f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
37542f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
37552f856554SMatthew G. Knepley 
37562f856554SMatthew G. Knepley   Level: developer
37572f856554SMatthew G. Knepley 
37581cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()`
37592f856554SMatthew G. Knepley @*/
3760d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexRestoreCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
3761d71ae5a4SJacob Faibussowitsch {
37622f856554SMatthew G. Knepley   PetscFunctionBegin;
37639566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u));
37649566063dSJacob Faibussowitsch   if (locX_t) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u_t));
37659566063dSJacob Faibussowitsch   if (locA) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, a));
37663ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
37672f856554SMatthew G. Knepley }
37682f856554SMatthew G. Knepley 
3769a4e35b19SJacob Faibussowitsch static PetscErrorCode DMPlexGetHybridCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
3770d71ae5a4SJacob Faibussowitsch {
377107218a29SMatthew G. Knepley   DM              plex, plexA = NULL;
377207218a29SMatthew G. Knepley   DMEnclosureType encAux;
377307218a29SMatthew G. Knepley   PetscSection    section, sectionAux;
377407218a29SMatthew G. Knepley   PetscDS         ds, dsIn;
37756528b96dSMatthew G. Knepley   const PetscInt *cells;
377607218a29SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, c, totDim, totDimAux, Nf, f;
37776528b96dSMatthew G. Knepley 
37786528b96dSMatthew G. Knepley   PetscFunctionBegin;
377907218a29SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
378007218a29SMatthew G. Knepley   PetscValidHeaderSpecific(cellIS, IS_CLASSID, 2);
378107218a29SMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
378207218a29SMatthew G. Knepley   if (locX_t) { PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4); }
378307218a29SMatthew G. Knepley   if (locA) { PetscValidHeaderSpecific(locA, VEC_CLASSID, 5); }
37844f572ea9SToby Isaac   PetscAssertPointer(u, 6);
37854f572ea9SToby Isaac   PetscAssertPointer(u_t, 7);
37864f572ea9SToby Isaac   PetscAssertPointer(a, 8);
378707218a29SMatthew G. Knepley   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
378807218a29SMatthew G. Knepley   numCells = cEnd - cStart;
378907218a29SMatthew G. Knepley   PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE));
379007218a29SMatthew G. Knepley   PetscCall(DMGetLocalSection(dm, &section));
379107218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn));
379207218a29SMatthew G. Knepley   PetscCall(PetscDSGetNumFields(dsIn, &Nf));
379307218a29SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(dsIn, &totDim));
379407218a29SMatthew G. Knepley   if (locA) {
379507218a29SMatthew G. Knepley     DM      dmAux;
379607218a29SMatthew G. Knepley     PetscDS probAux;
379707218a29SMatthew G. Knepley 
379807218a29SMatthew G. Knepley     PetscCall(VecGetDM(locA, &dmAux));
379907218a29SMatthew G. Knepley     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
380007218a29SMatthew G. Knepley     PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE));
380107218a29SMatthew G. Knepley     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
380207218a29SMatthew G. Knepley     PetscCall(DMGetDS(dmAux, &probAux));
380307218a29SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
380407218a29SMatthew G. Knepley   }
380507218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u));
380607218a29SMatthew G. Knepley   if (locX_t) PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u_t));
380707218a29SMatthew G. Knepley   else {
380807218a29SMatthew G. Knepley     *u_t = NULL;
380907218a29SMatthew G. Knepley   }
381007218a29SMatthew G. Knepley   if (locA) PetscCall(DMGetWorkArray(dm, numCells * totDimAux, MPIU_SCALAR, a));
381107218a29SMatthew G. Knepley   else {
381207218a29SMatthew G. Knepley     *a = NULL;
381307218a29SMatthew G. Knepley   }
381407218a29SMatthew G. Knepley   // Loop over cohesive cells
381507218a29SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
381607218a29SMatthew G. Knepley     const PetscInt  cell = cells ? cells[c] : c;
381707218a29SMatthew G. Knepley     const PetscInt  cind = c - cStart;
381874a0c561SMatthew G. Knepley     PetscScalar    *xf = NULL, *xc = NULL, *x = NULL, *xf_t = NULL, *xc_t = NULL;
38198e3a54c0SPierre Jolivet     PetscScalar    *ul = &(*u)[cind * totDim], *ul_t = PetscSafePointerPlusOffset(*u_t, cind * totDim);
382007218a29SMatthew G. Knepley     const PetscInt *cone, *ornt;
382107218a29SMatthew G. Knepley     PetscInt        Nx = 0, Nxf, s;
382207218a29SMatthew G. Knepley 
382307218a29SMatthew G. Knepley     PetscCall(DMPlexGetCone(dm, cell, &cone));
382407218a29SMatthew G. Knepley     PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
382507218a29SMatthew G. Knepley     // Put in cohesive unknowns
382607218a29SMatthew G. Knepley     PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, &Nxf, &xf));
382774a0c561SMatthew G. Knepley     if (locX_t) PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &xf_t));
382807218a29SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
382907218a29SMatthew G. Knepley       PetscInt  fdofIn, foff, foffIn;
383007218a29SMatthew G. Knepley       PetscBool cohesive;
383107218a29SMatthew G. Knepley 
383207218a29SMatthew G. Knepley       PetscCall(PetscDSGetCohesive(dsIn, f, &cohesive));
383307218a29SMatthew G. Knepley       if (!cohesive) continue;
383407218a29SMatthew G. Knepley       PetscCall(PetscDSGetFieldSize(dsIn, f, &fdofIn));
383507218a29SMatthew G. Knepley       PetscCall(PetscDSGetFieldOffsetCohesive(ds, f, &foff));
383607218a29SMatthew G. Knepley       PetscCall(PetscDSGetFieldOffsetCohesive(dsIn, f, &foffIn));
383707218a29SMatthew G. Knepley       for (PetscInt i = 0; i < fdofIn; ++i) ul[foffIn + i] = xf[foff + i];
383874a0c561SMatthew G. Knepley       if (locX_t)
383974a0c561SMatthew G. Knepley         for (PetscInt i = 0; i < fdofIn; ++i) ul_t[foffIn + i] = xf_t[foff + i];
384007218a29SMatthew G. Knepley       Nx += fdofIn;
384107218a29SMatthew G. Knepley     }
384207218a29SMatthew G. Knepley     PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, &Nxf, &xf));
384374a0c561SMatthew G. Knepley     if (locX_t) PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &xf_t));
384407218a29SMatthew G. Knepley     // Loop over sides of surface
384507218a29SMatthew G. Knepley     for (s = 0; s < 2; ++s) {
384607218a29SMatthew G. Knepley       const PetscInt *support;
384707218a29SMatthew G. Knepley       const PetscInt  face = cone[s];
384807218a29SMatthew G. Knepley       PetscInt        ssize, ncell, Nxc;
384907218a29SMatthew G. Knepley 
385007218a29SMatthew G. Knepley       // I don't think I need the face to have 0 orientation in the hybrid cell
385107218a29SMatthew G. Knepley       //PetscCheck(!ornt[s], PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", face, cell, ornt[s]);
385207218a29SMatthew G. Knepley       PetscCall(DMPlexGetSupport(dm, face, &support));
385307218a29SMatthew G. Knepley       PetscCall(DMPlexGetSupportSize(dm, face, &ssize));
385407218a29SMatthew G. Knepley       if (support[0] == cell) ncell = support[1];
385507218a29SMatthew G. Knepley       else if (support[1] == cell) ncell = support[0];
385607218a29SMatthew G. Knepley       else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", face, cell);
385707218a29SMatthew G. Knepley       // Get closure of both face and cell, stick in cell for normal fields and face for cohesive fields
385807218a29SMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(plex, section, locX, ncell, &Nxc, &xc));
385974a0c561SMatthew G. Knepley       if (locX_t) PetscCall(DMPlexVecGetClosure(plex, section, locX_t, ncell, NULL, &xc_t));
386007218a29SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
386107218a29SMatthew G. Knepley         PetscInt  fdofIn, foffIn;
386207218a29SMatthew G. Knepley         PetscBool cohesive;
386307218a29SMatthew G. Knepley 
386407218a29SMatthew G. Knepley         PetscCall(PetscDSGetCohesive(dsIn, f, &cohesive));
386507218a29SMatthew G. Knepley         if (cohesive) continue;
386607218a29SMatthew G. Knepley         PetscCall(PetscDSGetFieldSize(dsIn, f, &fdofIn));
386707218a29SMatthew G. Knepley         PetscCall(PetscDSGetFieldOffsetCohesive(dsIn, f, &foffIn));
386807218a29SMatthew G. Knepley         for (PetscInt i = 0; i < fdofIn; ++i) ul[foffIn + s * fdofIn + i] = xc[foffIn + i];
386974a0c561SMatthew G. Knepley         if (locX_t)
387074a0c561SMatthew G. Knepley           for (PetscInt i = 0; i < fdofIn; ++i) ul_t[foffIn + s * fdofIn + i] = xc_t[foffIn + i];
387107218a29SMatthew G. Knepley         Nx += fdofIn;
387207218a29SMatthew G. Knepley       }
387307218a29SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, ncell, &Nxc, &xc));
387474a0c561SMatthew G. Knepley       if (locX_t) PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, ncell, NULL, &xc_t));
387507218a29SMatthew G. Knepley     }
387607218a29SMatthew G. Knepley     PetscCheck(Nx == totDim, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for cell %" PetscInt_FMT " does not match DS size %" PetscInt_FMT, Nx, cell, totDim);
387707218a29SMatthew G. Knepley 
387807218a29SMatthew G. Knepley     if (locA) {
387907218a29SMatthew G. Knepley       PetscScalar *al = &(*a)[cind * totDimAux];
388007218a29SMatthew G. Knepley       PetscInt     subcell;
388107218a29SMatthew G. Knepley 
388207218a29SMatthew G. Knepley       PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell));
388307218a29SMatthew G. Knepley       PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, &Nx, &x));
388407218a29SMatthew G. Knepley       PetscCheck(Nx == totDimAux, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for subcell %" PetscInt_FMT "does not match DS size %" PetscInt_FMT, Nx, subcell, totDimAux);
388507218a29SMatthew G. Knepley       for (PetscInt i = 0; i < totDimAux; ++i) al[i] = x[i];
388607218a29SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, &Nx, &x));
388707218a29SMatthew G. Knepley     }
388807218a29SMatthew G. Knepley   }
388907218a29SMatthew G. Knepley   PetscCall(DMDestroy(&plex));
389007218a29SMatthew G. Knepley   PetscCall(DMDestroy(&plexA));
389107218a29SMatthew G. Knepley   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
389207218a29SMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
389307218a29SMatthew G. Knepley }
389407218a29SMatthew G. Knepley 
389507218a29SMatthew G. Knepley /*
38963e2b0218SMatthew G. Knepley   DMPlexGetHybridFields - Get the field values for the negative side (s = 0) and positive side (s = 1) of the interface
389707218a29SMatthew G. Knepley 
389807218a29SMatthew G. Knepley   Input Parameters:
389907218a29SMatthew G. Knepley + dm      - The full domain DM
390007218a29SMatthew G. Knepley . dmX     - An array of DM for the field, say an auxiliary DM, indexed by s
390107218a29SMatthew G. Knepley . dsX     - An array of PetscDS for the field, indexed by s
390207218a29SMatthew G. Knepley . cellIS  - The interface cells for which we want values
390307218a29SMatthew G. Knepley . locX    - An array of local vectors with the field values, indexed by s
390407218a29SMatthew G. Knepley - useCell - Flag to have values come from neighboring cell rather than endcap face
390507218a29SMatthew G. Knepley 
390607218a29SMatthew G. Knepley   Output Parameter:
390707218a29SMatthew G. Knepley . x       - An array of field values, indexed by s
390807218a29SMatthew G. Knepley 
390907218a29SMatthew G. Knepley   Note:
391076fbde31SPierre Jolivet   The arrays in `x` will be allocated using `DMGetWorkArray()`, and must be returned using `DMPlexRestoreHybridFields()`.
391107218a29SMatthew G. Knepley 
391207218a29SMatthew G. Knepley   Level: advanced
391307218a29SMatthew G. Knepley 
391476fbde31SPierre Jolivet .seealso: `DMPlexRestoreHybridFields()`, `DMGetWorkArray()`
391507218a29SMatthew G. Knepley */
391607218a29SMatthew G. Knepley static PetscErrorCode DMPlexGetHybridFields(DM dm, DM dmX[], PetscDS dsX[], IS cellIS, Vec locX[], PetscBool useCell, PetscScalar *x[])
391707218a29SMatthew G. Knepley {
391807218a29SMatthew G. Knepley   DM              plexX[2];
391907218a29SMatthew G. Knepley   DMEnclosureType encX[2];
392007218a29SMatthew G. Knepley   PetscSection    sectionX[2];
392107218a29SMatthew G. Knepley   const PetscInt *cells;
392207218a29SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, c, s, totDimX[2];
392307218a29SMatthew G. Knepley 
392407218a29SMatthew G. Knepley   PetscFunctionBegin;
39254f572ea9SToby Isaac   PetscAssertPointer(locX, 5);
392607218a29SMatthew G. Knepley   if (!locX[0] || !locX[1]) PetscFunctionReturn(PETSC_SUCCESS);
39274f572ea9SToby Isaac   PetscAssertPointer(dmX, 2);
39284f572ea9SToby Isaac   PetscAssertPointer(dsX, 3);
392907218a29SMatthew G. Knepley   PetscValidHeaderSpecific(cellIS, IS_CLASSID, 4);
39304f572ea9SToby Isaac   PetscAssertPointer(x, 7);
39319566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
39326528b96dSMatthew G. Knepley   numCells = cEnd - cStart;
3933148442b3SMatthew G. Knepley   for (s = 0; s < 2; ++s) {
393407218a29SMatthew G. Knepley     PetscValidHeaderSpecific(dmX[s], DM_CLASSID, 2);
393507218a29SMatthew G. Knepley     PetscValidHeaderSpecific(dsX[s], PETSCDS_CLASSID, 3);
393607218a29SMatthew G. Knepley     PetscValidHeaderSpecific(locX[s], VEC_CLASSID, 5);
393707218a29SMatthew G. Knepley     PetscCall(DMPlexConvertPlex(dmX[s], &plexX[s], PETSC_FALSE));
393807218a29SMatthew G. Knepley     PetscCall(DMGetEnclosureRelation(dmX[s], dm, &encX[s]));
393907218a29SMatthew G. Knepley     PetscCall(DMGetLocalSection(dmX[s], &sectionX[s]));
394007218a29SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(dsX[s], &totDimX[s]));
394107218a29SMatthew G. Knepley     PetscCall(DMGetWorkArray(dmX[s], numCells * totDimX[s], MPIU_SCALAR, &x[s]));
394204c51a94SMatthew G. Knepley   }
3943148442b3SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
39446528b96dSMatthew G. Knepley     const PetscInt  cell = cells ? cells[c] : c;
39456528b96dSMatthew G. Knepley     const PetscInt  cind = c - cStart;
39466528b96dSMatthew G. Knepley     const PetscInt *cone, *ornt;
39476528b96dSMatthew G. Knepley 
39489566063dSJacob Faibussowitsch     PetscCall(DMPlexGetCone(dm, cell, &cone));
39499566063dSJacob Faibussowitsch     PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
395007218a29SMatthew G. Knepley     //PetscCheck(!ornt[0], PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", cone[0], cell, ornt[0]);
3951148442b3SMatthew G. Knepley     for (s = 0; s < 2; ++s) {
395207218a29SMatthew G. Knepley       const PetscInt tdX     = totDimX[s];
395307218a29SMatthew G. Knepley       PetscScalar   *closure = NULL, *xl = &x[s][cind * tdX];
395407218a29SMatthew G. Knepley       PetscInt       face = cone[s], point = face, subpoint, Nx, i;
395507218a29SMatthew G. Knepley 
395607218a29SMatthew G. Knepley       if (useCell) {
39575fedec97SMatthew G. Knepley         const PetscInt *support;
395807218a29SMatthew G. Knepley         PetscInt        ssize;
39596528b96dSMatthew G. Knepley 
396007218a29SMatthew G. Knepley         PetscCall(DMPlexGetSupport(dm, face, &support));
396107218a29SMatthew G. Knepley         PetscCall(DMPlexGetSupportSize(dm, face, &ssize));
396207218a29SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", face, cell, ssize);
396307218a29SMatthew G. Knepley         if (support[0] == cell) point = support[1];
396407218a29SMatthew G. Knepley         else if (support[1] == cell) point = support[0];
396507218a29SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", face, cell);
396607218a29SMatthew G. Knepley       }
396707218a29SMatthew G. Knepley       PetscCall(DMGetEnclosurePoint(plexX[s], dm, encX[s], point, &subpoint));
3968e8e188d2SZach Atkins       PetscCall(DMPlexVecGetOrientedClosure_Internal(plexX[s], sectionX[s], PETSC_FALSE, locX[s], subpoint, ornt[s], &Nx, &closure));
396907218a29SMatthew G. Knepley       PetscCheck(Nx == tdX, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for subpoint %" PetscInt_FMT " does not match DS size %" PetscInt_FMT, Nx, subpoint, tdX);
397007218a29SMatthew G. Knepley       for (i = 0; i < Nx; ++i) xl[i] = closure[i];
397107218a29SMatthew G. Knepley       PetscCall(DMPlexVecRestoreClosure(plexX[s], sectionX[s], locX[s], subpoint, &Nx, &closure));
39726528b96dSMatthew G. Knepley     }
39736528b96dSMatthew G. Knepley   }
397407218a29SMatthew G. Knepley   for (s = 0; s < 2; ++s) PetscCall(DMDestroy(&plexX[s]));
39759566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
39763ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
39776528b96dSMatthew G. Knepley }
39786528b96dSMatthew G. Knepley 
397907218a29SMatthew G. Knepley static PetscErrorCode DMPlexRestoreHybridFields(DM dm, DM dmX[], PetscDS dsX[], IS cellIS, Vec locX[], PetscBool useCell, PetscScalar *x[])
3980d71ae5a4SJacob Faibussowitsch {
39816528b96dSMatthew G. Knepley   PetscFunctionBegin;
398207218a29SMatthew G. Knepley   if (!locX[0] || !locX[1]) PetscFunctionReturn(PETSC_SUCCESS);
398307218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dmX[0], 0, MPIU_SCALAR, &x[0]));
398407218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dmX[1], 0, MPIU_SCALAR, &x[1]));
39853ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
39866528b96dSMatthew G. Knepley }
39876528b96dSMatthew G. Knepley 
39882f856554SMatthew G. Knepley /*@C
39892f856554SMatthew G. Knepley   DMPlexGetFaceFields - Retrieve the field values values for a chunk of faces
39902f856554SMatthew G. Knepley 
39912f856554SMatthew G. Knepley   Input Parameters:
3992a1cb98faSBarry Smith + dm           - The `DM`
39932f856554SMatthew G. Knepley . fStart       - The first face to include
39942f856554SMatthew G. Knepley . fEnd         - The first face to exclude
39952f856554SMatthew G. Knepley . locX         - A local vector with the solution fields
39962f856554SMatthew G. Knepley . locX_t       - A local vector with solution field time derivatives, or NULL
39972f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
39982f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
399960225df5SJacob Faibussowitsch - locGrad      - A local vector with field gradients, or NULL
40002f856554SMatthew G. Knepley 
40012f856554SMatthew G. Knepley   Output Parameters:
40022f856554SMatthew G. Knepley + Nface - The number of faces with field values
40032f856554SMatthew G. Knepley . uL    - The field values at the left side of the face
40042f856554SMatthew G. Knepley - uR    - The field values at the right side of the face
40052f856554SMatthew G. Knepley 
40062f856554SMatthew G. Knepley   Level: developer
40072f856554SMatthew G. Knepley 
40081cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellFields()`
40092f856554SMatthew G. Knepley @*/
4010d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
4011d71ae5a4SJacob Faibussowitsch {
40122f856554SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad = NULL;
40132f856554SMatthew G. Knepley   PetscSection       section;
40142f856554SMatthew G. Knepley   PetscDS            prob;
40152f856554SMatthew G. Knepley   DMLabel            ghostLabel;
40162f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom, *x, *lgrad;
40172f856554SMatthew G. Knepley   PetscBool         *isFE;
40182f856554SMatthew G. Knepley   PetscInt           dim, Nf, f, Nc, numFaces = fEnd - fStart, iface, face;
40192f856554SMatthew G. Knepley 
40202f856554SMatthew G. Knepley   PetscFunctionBegin;
40212f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
40222f856554SMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 4);
4023ad540459SPierre Jolivet   if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 5);
40242f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 6);
40252f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 7);
4026ad540459SPierre Jolivet   if (locGrad) PetscValidHeaderSpecific(locGrad, VEC_CLASSID, 8);
40274f572ea9SToby Isaac   PetscAssertPointer(uL, 10);
40284f572ea9SToby Isaac   PetscAssertPointer(uR, 11);
40299566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
40309566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
40319566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
40329566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
40339566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalComponents(prob, &Nc));
40349566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(Nf, &isFE));
40352f856554SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
40362f856554SMatthew G. Knepley     PetscObject  obj;
40372f856554SMatthew G. Knepley     PetscClassId id;
40382f856554SMatthew G. Knepley 
40399566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
40409566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
40419371c9d4SSatish Balay     if (id == PETSCFE_CLASSID) {
40429371c9d4SSatish Balay       isFE[f] = PETSC_TRUE;
40439371c9d4SSatish Balay     } else if (id == PETSCFV_CLASSID) {
40449371c9d4SSatish Balay       isFE[f] = PETSC_FALSE;
40459371c9d4SSatish Balay     } else {
40469371c9d4SSatish Balay       isFE[f] = PETSC_FALSE;
40479371c9d4SSatish Balay     }
40482f856554SMatthew G. Knepley   }
40499566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
40509566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(locX, &x));
40519566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
40529566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
40539566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
40549566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
40552f856554SMatthew G. Knepley   if (locGrad) {
40569566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locGrad, &dmGrad));
40579566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
40582f856554SMatthew G. Knepley   }
40599566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uL));
40609566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uR));
40612f856554SMatthew G. Knepley   /* Right now just eat the extra work for FE (could make a cell loop) */
40622f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
40632f856554SMatthew G. Knepley     const PetscInt  *cells;
40642f856554SMatthew G. Knepley     PetscFVFaceGeom *fg;
40652f856554SMatthew G. Knepley     PetscFVCellGeom *cgL, *cgR;
40662f856554SMatthew G. Knepley     PetscScalar     *xL, *xR, *gL, *gR;
40672f856554SMatthew G. Knepley     PetscScalar     *uLl = *uL, *uRl = *uR;
40682f856554SMatthew G. Knepley     PetscInt         ghost, nsupp, nchild;
40692f856554SMatthew G. Knepley 
40709566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
40719566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
40729566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
40732f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
40749566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
40759566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
40769566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
40779566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
40782f856554SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
40792f856554SMatthew G. Knepley       PetscInt off;
40802f856554SMatthew G. Knepley 
40819566063dSJacob Faibussowitsch       PetscCall(PetscDSGetComponentOffset(prob, f, &off));
40822f856554SMatthew G. Knepley       if (isFE[f]) {
40832f856554SMatthew G. Knepley         const PetscInt *cone;
40842f856554SMatthew G. Knepley         PetscInt        comp, coneSizeL, coneSizeR, faceLocL, faceLocR, ldof, rdof, d;
40852f856554SMatthew G. Knepley 
40862f856554SMatthew G. Knepley         xL = xR = NULL;
40879566063dSJacob Faibussowitsch         PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
4088835f2295SStefano Zampini         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[0], &ldof, &xL));
4089835f2295SStefano Zampini         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[1], &rdof, &xR));
40909566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[0], &cone));
40919566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[0], &coneSizeL));
40929371c9d4SSatish Balay         for (faceLocL = 0; faceLocL < coneSizeL; ++faceLocL)
40939371c9d4SSatish Balay           if (cone[faceLocL] == face) break;
40949566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[1], &cone));
40959566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[1], &coneSizeR));
40969371c9d4SSatish Balay         for (faceLocR = 0; faceLocR < coneSizeR; ++faceLocR)
40979371c9d4SSatish Balay           if (cone[faceLocR] == face) break;
40981dca8a05SBarry Smith         PetscCheck(faceLocL != coneSizeL || faceLocR != coneSizeR, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %" PetscInt_FMT " in cone of cell %" PetscInt_FMT " or cell %" PetscInt_FMT, face, cells[0], cells[1]);
40992f856554SMatthew G. Knepley         /* Check that FEM field has values in the right cell (sometimes its an FV ghost cell) */
41002f856554SMatthew G. Knepley         /* TODO: this is a hack that might not be right for nonconforming */
41012f856554SMatthew G. Knepley         if (faceLocL < coneSizeL) {
41029566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocL, xL, &uLl[iface * Nc + off]));
41039566063dSJacob Faibussowitsch           if (rdof == ldof && faceLocR < coneSizeR) PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off]));
41042f856554SMatthew G. Knepley           else {
41059371c9d4SSatish Balay             for (d = 0; d < comp; ++d) uRl[iface * Nc + off + d] = uLl[iface * Nc + off + d];
41069371c9d4SSatish Balay           }
41079371c9d4SSatish Balay         } else {
41089566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off]));
41099566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
41102f856554SMatthew G. Knepley           for (d = 0; d < comp; ++d) uLl[iface * Nc + off + d] = uRl[iface * Nc + off + d];
41112f856554SMatthew G. Knepley         }
4112835f2295SStefano Zampini         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[0], &ldof, &xL));
4113835f2295SStefano Zampini         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[1], &rdof, &xR));
41142f856554SMatthew G. Knepley       } else {
41152f856554SMatthew G. Knepley         PetscFV  fv;
41162f856554SMatthew G. Knepley         PetscInt numComp, c;
41172f856554SMatthew G. Knepley 
41189566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, (PetscObject *)&fv));
41199566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &numComp));
41209566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[0], f, x, &xL));
41219566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[1], f, x, &xR));
41222f856554SMatthew G. Knepley         if (dmGrad) {
41232f856554SMatthew G. Knepley           PetscReal dxL[3], dxR[3];
41242f856554SMatthew G. Knepley 
41259566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], lgrad, &gL));
41269566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[1], lgrad, &gR));
41272f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgL->centroid, fg->centroid, dxL);
41282f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgR->centroid, fg->centroid, dxR);
41292f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
41302f856554SMatthew G. Knepley             uLl[iface * Nc + off + c] = xL[c] + DMPlex_DotD_Internal(dim, &gL[c * dim], dxL);
41312f856554SMatthew G. Knepley             uRl[iface * Nc + off + c] = xR[c] + DMPlex_DotD_Internal(dim, &gR[c * dim], dxR);
41322f856554SMatthew G. Knepley           }
41332f856554SMatthew G. Knepley         } else {
41342f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
41352f856554SMatthew G. Knepley             uLl[iface * Nc + off + c] = xL[c];
41362f856554SMatthew G. Knepley             uRl[iface * Nc + off + c] = xR[c];
41372f856554SMatthew G. Knepley           }
41382f856554SMatthew G. Knepley         }
41392f856554SMatthew G. Knepley       }
41402f856554SMatthew G. Knepley     }
41412f856554SMatthew G. Knepley     ++iface;
41422f856554SMatthew G. Knepley   }
41432f856554SMatthew G. Knepley   *Nface = iface;
41449566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(locX, &x));
41459566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
41469566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
414748a46eb9SPierre Jolivet   if (locGrad) PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
41489566063dSJacob Faibussowitsch   PetscCall(PetscFree(isFE));
41493ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
41502f856554SMatthew G. Knepley }
41512f856554SMatthew G. Knepley 
41522f856554SMatthew G. Knepley /*@C
41532f856554SMatthew G. Knepley   DMPlexRestoreFaceFields - Restore the field values values for a chunk of faces
41542f856554SMatthew G. Knepley 
41552f856554SMatthew G. Knepley   Input Parameters:
4156a1cb98faSBarry Smith + dm           - The `DM`
41572f856554SMatthew G. Knepley . fStart       - The first face to include
41582f856554SMatthew G. Knepley . fEnd         - The first face to exclude
41592f856554SMatthew G. Knepley . locX         - A local vector with the solution fields
41602f856554SMatthew G. Knepley . locX_t       - A local vector with solution field time derivatives, or NULL
41612f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
41622f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
416360225df5SJacob Faibussowitsch - locGrad      - A local vector with field gradients, or NULL
41642f856554SMatthew G. Knepley 
41652f856554SMatthew G. Knepley   Output Parameters:
41662f856554SMatthew G. Knepley + Nface - The number of faces with field values
41672f856554SMatthew G. Knepley . uL    - The field values at the left side of the face
41682f856554SMatthew G. Knepley - uR    - The field values at the right side of the face
41692f856554SMatthew G. Knepley 
41702f856554SMatthew G. Knepley   Level: developer
41712f856554SMatthew G. Knepley 
41721cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()`
41732f856554SMatthew G. Knepley @*/
4174d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexRestoreFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
4175d71ae5a4SJacob Faibussowitsch {
41762f856554SMatthew G. Knepley   PetscFunctionBegin;
41779566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uL));
41789566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uR));
41793ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
41802f856554SMatthew G. Knepley }
41812f856554SMatthew G. Knepley 
41822f856554SMatthew G. Knepley /*@C
41832f856554SMatthew G. Knepley   DMPlexGetFaceGeometry - Retrieve the geometric values for a chunk of faces
41842f856554SMatthew G. Knepley 
41852f856554SMatthew G. Knepley   Input Parameters:
4186a1cb98faSBarry Smith + dm           - The `DM`
41872f856554SMatthew G. Knepley . fStart       - The first face to include
41882f856554SMatthew G. Knepley . fEnd         - The first face to exclude
41892f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
41902f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
41912f856554SMatthew G. Knepley 
41922f856554SMatthew G. Knepley   Output Parameters:
41932f856554SMatthew G. Knepley + Nface - The number of faces with field values
41942f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
41952f856554SMatthew G. Knepley - vol   - The cell volume
41962f856554SMatthew G. Knepley 
41972f856554SMatthew G. Knepley   Level: developer
41982f856554SMatthew G. Knepley 
41991cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellFields()`
42002f856554SMatthew G. Knepley @*/
4201d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
4202d71ae5a4SJacob Faibussowitsch {
42032f856554SMatthew G. Knepley   DM                 dmFace, dmCell;
42042f856554SMatthew G. Knepley   DMLabel            ghostLabel;
42052f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom;
42062f856554SMatthew G. Knepley   PetscInt           dim, numFaces = fEnd - fStart, iface, face;
42072f856554SMatthew G. Knepley 
42082f856554SMatthew G. Knepley   PetscFunctionBegin;
42092f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42102f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 4);
42112f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 5);
42124f572ea9SToby Isaac   PetscAssertPointer(fgeom, 7);
42134f572ea9SToby Isaac   PetscAssertPointer(vol, 8);
42149566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
42159566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
42169566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
42179566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
42189566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
42199566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
42209566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(numFaces, fgeom));
42219566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces * 2, MPIU_SCALAR, vol));
42222f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
42232f856554SMatthew G. Knepley     const PetscInt  *cells;
42242f856554SMatthew G. Knepley     PetscFVFaceGeom *fg;
42252f856554SMatthew G. Knepley     PetscFVCellGeom *cgL, *cgR;
42262f856554SMatthew G. Knepley     PetscFVFaceGeom *fgeoml = *fgeom;
42272f856554SMatthew G. Knepley     PetscReal       *voll   = *vol;
42282f856554SMatthew G. Knepley     PetscInt         ghost, d, nchild, nsupp;
42292f856554SMatthew G. Knepley 
42309566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
42319566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
42329566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
42332f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
42349566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
42359566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
42369566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
42379566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
42382f856554SMatthew G. Knepley     for (d = 0; d < dim; ++d) {
42392f856554SMatthew G. Knepley       fgeoml[iface].centroid[d] = fg->centroid[d];
42402f856554SMatthew G. Knepley       fgeoml[iface].normal[d]   = fg->normal[d];
42412f856554SMatthew G. Knepley     }
42422f856554SMatthew G. Knepley     voll[iface * 2 + 0] = cgL->volume;
42432f856554SMatthew G. Knepley     voll[iface * 2 + 1] = cgR->volume;
42442f856554SMatthew G. Knepley     ++iface;
42452f856554SMatthew G. Knepley   }
42462f856554SMatthew G. Knepley   *Nface = iface;
42479566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
42489566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
42493ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
42502f856554SMatthew G. Knepley }
42512f856554SMatthew G. Knepley 
42522f856554SMatthew G. Knepley /*@C
42532f856554SMatthew G. Knepley   DMPlexRestoreFaceGeometry - Restore the field values values for a chunk of faces
42542f856554SMatthew G. Knepley 
42552f856554SMatthew G. Knepley   Input Parameters:
4256a1cb98faSBarry Smith + dm           - The `DM`
42572f856554SMatthew G. Knepley . fStart       - The first face to include
42582f856554SMatthew G. Knepley . fEnd         - The first face to exclude
42592f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
42602f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
42612f856554SMatthew G. Knepley 
42622f856554SMatthew G. Knepley   Output Parameters:
42632f856554SMatthew G. Knepley + Nface - The number of faces with field values
42642f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
42652f856554SMatthew G. Knepley - vol   - The cell volume
42662f856554SMatthew G. Knepley 
42672f856554SMatthew G. Knepley   Level: developer
42682f856554SMatthew G. Knepley 
42691cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()`
42702f856554SMatthew G. Knepley @*/
4271d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexRestoreFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
4272d71ae5a4SJacob Faibussowitsch {
42732f856554SMatthew G. Knepley   PetscFunctionBegin;
42749566063dSJacob Faibussowitsch   PetscCall(PetscFree(*fgeom));
42759566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_REAL, vol));
42763ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
42772f856554SMatthew G. Knepley }
42782f856554SMatthew G. Knepley 
4279d71ae5a4SJacob Faibussowitsch PetscErrorCode DMSNESGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
4280d71ae5a4SJacob Faibussowitsch {
4281a1cf66bbSMatthew G. Knepley   char           composeStr[33] = {0};
4282a1cf66bbSMatthew G. Knepley   PetscObjectId  id;
4283a1cf66bbSMatthew G. Knepley   PetscContainer container;
4284a1cf66bbSMatthew G. Knepley 
4285a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
42869566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad, &id));
428763a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMSNESGetFEGeom_%" PetscInt64_FMT "\n", id));
42889566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container));
4289a1cf66bbSMatthew G. Knepley   if (container) {
42909566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **)geom));
4291a1cf66bbSMatthew G. Knepley   } else {
42929566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom));
42939566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container));
42949566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *)*geom));
429549abdd8aSBarry Smith     PetscCall(PetscContainerSetCtxDestroy(container, PetscContainerCtxDestroy_PetscFEGeom));
42969566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container));
42979566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
4298a1cf66bbSMatthew G. Knepley   }
42993ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
4300a1cf66bbSMatthew G. Knepley }
4301a1cf66bbSMatthew G. Knepley 
4302d71ae5a4SJacob Faibussowitsch PetscErrorCode DMSNESRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
4303d71ae5a4SJacob Faibussowitsch {
4304a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
4305a1cf66bbSMatthew G. Knepley   *geom = NULL;
43063ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
4307a1cf66bbSMatthew G. Knepley }
4308a1cf66bbSMatthew G. Knepley 
4309d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeResidual_Patch_Internal(DM dm, PetscSection section, IS cellIS, PetscReal t, Vec locX, Vec locX_t, Vec locF, void *user)
4310d71ae5a4SJacob Faibussowitsch {
431192d50984SMatthew G. Knepley   DM_Plex        *mesh       = (DM_Plex *)dm->data;
431292d50984SMatthew G. Knepley   const char     *name       = "Residual";
431392d50984SMatthew G. Knepley   DM              dmAux      = NULL;
431492d50984SMatthew G. Knepley   DMLabel         ghostLabel = NULL;
431592d50984SMatthew G. Knepley   PetscDS         prob       = NULL;
431692d50984SMatthew G. Knepley   PetscDS         probAux    = NULL;
431792d50984SMatthew G. Knepley   PetscBool       useFEM     = PETSC_FALSE;
431892d50984SMatthew G. Knepley   PetscBool       isImplicit = (locX_t || t == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
431992d50984SMatthew G. Knepley   DMField         coordField = NULL;
4320c0006e53SPatrick Farrell   Vec             locA;
4321c0006e53SPatrick Farrell   PetscScalar    *u = NULL, *u_t, *a, *uL = NULL, *uR = NULL;
432292d50984SMatthew G. Knepley   IS              chunkIS;
432392d50984SMatthew G. Knepley   const PetscInt *cells;
432492d50984SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
4325364207b6SKarl Rupp   PetscInt        Nf, f, totDim, totDimAux, numChunks, cellChunkSize, chunk, fStart, fEnd;
43261690c2aeSBarry Smith   PetscInt        maxDegree = PETSC_INT_MAX;
432706ad1575SMatthew G. Knepley   PetscFormKey    key;
432892d50984SMatthew G. Knepley   PetscQuadrature affineQuad = NULL, *quads = NULL;
432992d50984SMatthew G. Knepley   PetscFEGeom    *affineGeom = NULL, **geoms = NULL;
433092d50984SMatthew G. Knepley 
433192d50984SMatthew G. Knepley   PetscFunctionBegin;
43329566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0));
433392d50984SMatthew G. Knepley   /* FEM+FVM */
433492d50984SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
43359566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
43369566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
43379566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
43389566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
43399566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
434092d50984SMatthew G. Knepley   if (locA) {
43419566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
43429566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
43439566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
434492d50984SMatthew G. Knepley   }
434592d50984SMatthew G. Knepley   /* 2: Get geometric data */
434692d50984SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
434792d50984SMatthew G. Knepley     PetscObject  obj;
434892d50984SMatthew G. Knepley     PetscClassId id;
434992d50984SMatthew G. Knepley     PetscBool    fimp;
435092d50984SMatthew G. Knepley 
43519566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(prob, f, &fimp));
435292d50984SMatthew G. Knepley     if (isImplicit != fimp) continue;
43539566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
43549566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
4355ad540459SPierre Jolivet     if (id == PETSCFE_CLASSID) useFEM = PETSC_TRUE;
43565f80ce2aSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Use of FVM with PCPATCH not yet implemented");
435792d50984SMatthew G. Knepley   }
435892d50984SMatthew G. Knepley   if (useFEM) {
43599566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
43609566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
436192d50984SMatthew G. Knepley     if (maxDegree <= 1) {
43629566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad));
436348a46eb9SPierre Jolivet       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
436492d50984SMatthew G. Knepley     } else {
43659566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
436692d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
436792d50984SMatthew G. Knepley         PetscObject  obj;
436892d50984SMatthew G. Knepley         PetscClassId id;
436992d50984SMatthew G. Knepley         PetscBool    fimp;
437092d50984SMatthew G. Knepley 
43719566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(prob, f, &fimp));
437292d50984SMatthew G. Knepley         if (isImplicit != fimp) continue;
43739566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
43749566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
437592d50984SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
437692d50984SMatthew G. Knepley           PetscFE fe = (PetscFE)obj;
437792d50984SMatthew G. Knepley 
43789566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
43799566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
43809566063dSJacob Faibussowitsch           PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
438192d50984SMatthew G. Knepley         }
438292d50984SMatthew G. Knepley       }
438392d50984SMatthew G. Knepley     }
438492d50984SMatthew G. Knepley   }
438592d50984SMatthew G. Knepley   /* Loop over chunks */
43869566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
43879566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
43889566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
438992d50984SMatthew G. Knepley   numCells      = cEnd - cStart;
439092d50984SMatthew G. Knepley   numChunks     = 1;
439192d50984SMatthew G. Knepley   cellChunkSize = numCells / numChunks;
439292d50984SMatthew G. Knepley   numChunks     = PetscMin(1, numCells);
43936528b96dSMatthew G. Knepley   key.label     = NULL;
43946528b96dSMatthew G. Knepley   key.value     = 0;
439506ad1575SMatthew G. Knepley   key.part      = 0;
439692d50984SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
4397c0006e53SPatrick Farrell     PetscScalar     *elemVec, *fluxL = NULL, *fluxR = NULL;
4398c0006e53SPatrick Farrell     PetscReal       *vol   = NULL;
4399c0006e53SPatrick Farrell     PetscFVFaceGeom *fgeom = NULL;
440092d50984SMatthew G. Knepley     PetscInt         cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
4401c0006e53SPatrick Farrell     PetscInt         numFaces = 0;
440292d50984SMatthew G. Knepley 
440392d50984SMatthew G. Knepley     /* Extract field coefficients */
440492d50984SMatthew G. Knepley     if (useFEM) {
44059566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
44069566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
44079566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
44089566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells * totDim));
440992d50984SMatthew G. Knepley     }
441092d50984SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
441192d50984SMatthew G. Knepley     /* Loop over fields */
441292d50984SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
441392d50984SMatthew G. Knepley       PetscObject  obj;
441492d50984SMatthew G. Knepley       PetscClassId id;
441592d50984SMatthew G. Knepley       PetscBool    fimp;
441692d50984SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
441792d50984SMatthew G. Knepley 
44186528b96dSMatthew G. Knepley       key.field = f;
44199566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(prob, f, &fimp));
442092d50984SMatthew G. Knepley       if (isImplicit != fimp) continue;
44219566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
44229566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
442392d50984SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
442492d50984SMatthew G. Knepley         PetscFE         fe        = (PetscFE)obj;
442592d50984SMatthew G. Knepley         PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[f];
442692d50984SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
442792d50984SMatthew G. Knepley         PetscQuadrature quad      = affineQuad ? affineQuad : quads[f];
442892d50984SMatthew G. Knepley         PetscInt        Nq, Nb;
442992d50984SMatthew G. Knepley 
44309566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
44319566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
44329566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
443392d50984SMatthew G. Knepley         blockSize = Nb;
443492d50984SMatthew G. Knepley         batchSize = numBlocks * blockSize;
44359566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
443692d50984SMatthew G. Knepley         numChunks = numCells / (numBatches * batchSize);
443792d50984SMatthew G. Knepley         Ne        = numChunks * numBatches * batchSize;
443892d50984SMatthew G. Knepley         Nr        = numCells % (numBatches * batchSize);
443992d50984SMatthew G. Knepley         offset    = numCells - Nr;
444092d50984SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
444192d50984SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
44429566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom));
44439566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(prob, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
44449566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom));
44458e3a54c0SPierre Jolivet         PetscCall(PetscFEIntegrateResidual(prob, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, &a[offset * totDimAux], t, &elemVec[offset * totDim]));
44469566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom));
444792d50984SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
444892d50984SMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
444992d50984SMatthew G. Knepley 
445092d50984SMatthew G. Knepley         Ne = numFaces;
445192d50984SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
445292d50984SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
44539566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
445463a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
445592d50984SMatthew G. Knepley     }
445692d50984SMatthew G. Knepley     /* Loop over domain */
445792d50984SMatthew G. Knepley     if (useFEM) {
445892d50984SMatthew G. Knepley       /* Add elemVec to locX */
445992d50984SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
446092d50984SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
446192d50984SMatthew G. Knepley         const PetscInt cind = c - cStart;
446292d50984SMatthew G. Knepley 
44639566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim]));
446492d50984SMatthew G. Knepley         if (ghostLabel) {
446592d50984SMatthew G. Knepley           PetscInt ghostVal;
446692d50984SMatthew G. Knepley 
44679566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
446892d50984SMatthew G. Knepley           if (ghostVal > 0) continue;
446992d50984SMatthew G. Knepley         }
44709566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES));
447192d50984SMatthew G. Knepley       }
447292d50984SMatthew G. Knepley     }
447392d50984SMatthew G. Knepley     /* Handle time derivative */
447492d50984SMatthew G. Knepley     if (locX_t) {
447592d50984SMatthew G. Knepley       PetscScalar *x_t, *fa;
447692d50984SMatthew G. Knepley 
44779566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
44789566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
447992d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
448092d50984SMatthew G. Knepley         PetscFV      fv;
448192d50984SMatthew G. Knepley         PetscObject  obj;
448292d50984SMatthew G. Knepley         PetscClassId id;
448392d50984SMatthew G. Knepley         PetscInt     pdim, d;
448492d50984SMatthew G. Knepley 
44859566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
44869566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
448792d50984SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
448892d50984SMatthew G. Knepley         fv = (PetscFV)obj;
44899566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
449092d50984SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
449192d50984SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
449292d50984SMatthew G. Knepley           PetscScalar   *u_t, *r;
449392d50984SMatthew G. Knepley 
449492d50984SMatthew G. Knepley           if (ghostLabel) {
449592d50984SMatthew G. Knepley             PetscInt ghostVal;
449692d50984SMatthew G. Knepley 
44979566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
449892d50984SMatthew G. Knepley             if (ghostVal > 0) continue;
449992d50984SMatthew G. Knepley           }
45009566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
45019566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
450292d50984SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
450392d50984SMatthew G. Knepley         }
450492d50984SMatthew G. Knepley       }
45059566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
45069566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
450792d50984SMatthew G. Knepley     }
450892d50984SMatthew G. Knepley     if (useFEM) {
45099566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
45109566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
451192d50984SMatthew G. Knepley     }
451292d50984SMatthew G. Knepley   }
45139566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
45149566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
451592d50984SMatthew G. Knepley   /* TODO Could include boundary residual here (see DMPlexComputeResidual_Internal) */
451692d50984SMatthew G. Knepley   if (useFEM) {
451792d50984SMatthew G. Knepley     if (maxDegree <= 1) {
45189566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
45199566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
452092d50984SMatthew G. Knepley     } else {
452192d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
45229566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
45239566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
452492d50984SMatthew G. Knepley       }
45259566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads, geoms));
452692d50984SMatthew G. Knepley     }
452792d50984SMatthew G. Knepley   }
45289566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0));
45293ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
453092d50984SMatthew G. Knepley }
453192d50984SMatthew G. Knepley 
4532a1cf66bbSMatthew G. Knepley /*
4533a1cf66bbSMatthew G. Knepley   We always assemble JacP, and if the matrix is different from Jac and two different sets of point functions are provided, we also assemble Jac
4534a1cf66bbSMatthew G. Knepley 
4535a1cf66bbSMatthew G. Knepley   X   - The local solution vector
4536a5b23f4aSJose E. Roman   X_t - The local solution time derivative vector, or NULL
4537a1cf66bbSMatthew G. Knepley */
4538d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeJacobian_Patch_Internal(DM dm, PetscSection section, PetscSection globalSection, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *ctx)
4539d71ae5a4SJacob Faibussowitsch {
4540a1cf66bbSMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *)dm->data;
4541a1cf66bbSMatthew G. Knepley   const char     *name = "Jacobian", *nameP = "JacobianPre";
4542a1cf66bbSMatthew G. Knepley   DM              dmAux = NULL;
4543a1cf66bbSMatthew G. Knepley   PetscDS         prob, probAux = NULL;
4544a1cf66bbSMatthew G. Knepley   PetscSection    sectionAux = NULL;
4545a1cf66bbSMatthew G. Knepley   Vec             A;
4546a1cf66bbSMatthew G. Knepley   DMField         coordField;
4547a1cf66bbSMatthew G. Knepley   PetscFEGeom    *cgeomFEM;
4548a1cf66bbSMatthew G. Knepley   PetscQuadrature qGeom = NULL;
4549a1cf66bbSMatthew G. Knepley   Mat             J = Jac, JP = JacP;
4550a1cf66bbSMatthew G. Knepley   PetscScalar    *work, *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL, *elemMatD = NULL;
4551e432b41dSStefano Zampini   PetscBool       hasJac, hasPrec, hasDyn, assembleJac, *isFE, hasFV = PETSC_FALSE;
4552a1cf66bbSMatthew G. Knepley   const PetscInt *cells;
455306ad1575SMatthew G. Knepley   PetscFormKey    key;
45549b2fc754SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ, maxDegree, numCells, cStart, cEnd, numChunks, chunkSize, chunk, totDim, totDimAux = 0, sz, wsz, off = 0, offCell = 0;
4555a1cf66bbSMatthew G. Knepley 
4556a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
45579566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
45589566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
45599566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
45609566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
45619566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &A));
45629a2a23afSMatthew G. Knepley   if (A) {
45639566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
45649566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
45659566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
4566a1cf66bbSMatthew G. Knepley   }
4567a1cf66bbSMatthew G. Knepley   /* Get flags */
45689566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
45699566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, Nf, MPIU_BOOL, &isFE));
4570a1cf66bbSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
4571a1cf66bbSMatthew G. Knepley     PetscObject  disc;
4572a1cf66bbSMatthew G. Knepley     PetscClassId id;
45739566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &disc));
45749566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(disc, &id));
45759371c9d4SSatish Balay     if (id == PETSCFE_CLASSID) {
45769371c9d4SSatish Balay       isFE[fieldI] = PETSC_TRUE;
45779371c9d4SSatish Balay     } else if (id == PETSCFV_CLASSID) {
45789371c9d4SSatish Balay       hasFV        = PETSC_TRUE;
45799371c9d4SSatish Balay       isFE[fieldI] = PETSC_FALSE;
45809371c9d4SSatish Balay     }
4581a1cf66bbSMatthew G. Knepley   }
45829566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
45839566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
45849566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
4585a1cf66bbSMatthew G. Knepley   assembleJac = hasJac && hasPrec && (Jac != JacP) ? PETSC_TRUE : PETSC_FALSE;
4586a1cf66bbSMatthew G. Knepley   hasDyn      = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
45879566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE)); /* No allocated space for FV stuff, so ignore the zero entries */
45889566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
45899566063dSJacob Faibussowitsch   if (probAux) PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
4590a1cf66bbSMatthew G. Knepley   /* Compute batch sizes */
4591a1cf66bbSMatthew G. Knepley   if (isFE[0]) {
4592a1cf66bbSMatthew G. Knepley     PetscFE         fe;
4593a1cf66bbSMatthew G. Knepley     PetscQuadrature q;
4594a1cf66bbSMatthew G. Knepley     PetscInt        numQuadPoints, numBatches, batchSize, numBlocks, blockSize, Nb;
4595a1cf66bbSMatthew G. Knepley 
45969566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe));
45979566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &q));
45989566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(q, NULL, NULL, &numQuadPoints, NULL, NULL));
45999566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
46009566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
4601a1cf66bbSMatthew G. Knepley     blockSize = Nb * numQuadPoints;
4602a1cf66bbSMatthew G. Knepley     batchSize = numBlocks * blockSize;
4603a1cf66bbSMatthew G. Knepley     chunkSize = numBatches * batchSize;
4604a1cf66bbSMatthew G. Knepley     numChunks = numCells / chunkSize + numCells % chunkSize;
46059566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
4606a1cf66bbSMatthew G. Knepley   } else {
4607a1cf66bbSMatthew G. Knepley     chunkSize = numCells;
4608a1cf66bbSMatthew G. Knepley     numChunks = 1;
4609a1cf66bbSMatthew G. Knepley   }
4610a1cf66bbSMatthew G. Knepley   /* Get work space */
4611a1cf66bbSMatthew G. Knepley   wsz = (((X ? 1 : 0) + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize;
46129566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, wsz, MPIU_SCALAR, &work));
46139566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(work, wsz));
4614a1cf66bbSMatthew G. Knepley   off      = 0;
4615a1cf66bbSMatthew G. Knepley   u        = X ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL;
4616a1cf66bbSMatthew G. Knepley   u_t      = X_t ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL;
4617a1cf66bbSMatthew G. Knepley   a        = dmAux ? (sz = chunkSize * totDimAux, off += sz, work + off - sz) : NULL;
4618a1cf66bbSMatthew G. Knepley   elemMat  = hasJac ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL;
4619a1cf66bbSMatthew G. Knepley   elemMatP = hasPrec ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL;
4620a1cf66bbSMatthew G. Knepley   elemMatD = hasDyn ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL;
462163a3b9bcSJacob Faibussowitsch   PetscCheck(off == wsz, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error is workspace size %" PetscInt_FMT " should be %" PetscInt_FMT, off, wsz);
4622a1cf66bbSMatthew G. Knepley   /* Setup geometry */
46239566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
46249566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
46259566063dSJacob Faibussowitsch   if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
4626a1cf66bbSMatthew G. Knepley   if (!qGeom) {
4627a1cf66bbSMatthew G. Knepley     PetscFE fe;
4628a1cf66bbSMatthew G. Knepley 
46299566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe));
46309566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &qGeom));
46319566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)qGeom));
4632a1cf66bbSMatthew G. Knepley   }
46339566063dSJacob Faibussowitsch   PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
4634a1cf66bbSMatthew G. Knepley   /* Compute volume integrals */
46359566063dSJacob Faibussowitsch   if (assembleJac) PetscCall(MatZeroEntries(J));
46369566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(JP));
46376528b96dSMatthew G. Knepley   key.label = NULL;
46386528b96dSMatthew G. Knepley   key.value = 0;
463906ad1575SMatthew G. Knepley   key.part  = 0;
4640a1cf66bbSMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk, offCell += chunkSize) {
4641a1cf66bbSMatthew G. Knepley     const PetscInt Ncell = PetscMin(chunkSize, numCells - offCell);
4642a1cf66bbSMatthew G. Knepley     PetscInt       c;
4643a1cf66bbSMatthew G. Knepley 
4644a1cf66bbSMatthew G. Knepley     /* Extract values */
4645a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4646a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c + offCell] : c + offCell;
4647a1cf66bbSMatthew G. Knepley       PetscScalar   *x = NULL, *x_t = NULL;
4648a1cf66bbSMatthew G. Knepley       PetscInt       i;
4649a1cf66bbSMatthew G. Knepley 
4650a1cf66bbSMatthew G. Knepley       if (X) {
46519566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x));
4652a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i];
46539566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x));
4654a1cf66bbSMatthew G. Knepley       }
4655a1cf66bbSMatthew G. Knepley       if (X_t) {
46569566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t));
4657a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[c * totDim + i] = x_t[i];
46589566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t));
4659a1cf66bbSMatthew G. Knepley       }
4660a1cf66bbSMatthew G. Knepley       if (dmAux) {
46619566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, A, cell, NULL, &x));
4662a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i];
46639566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, A, cell, NULL, &x));
4664a1cf66bbSMatthew G. Knepley       }
4665a1cf66bbSMatthew G. Knepley     }
4666a1cf66bbSMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
4667a1cf66bbSMatthew G. Knepley       PetscFE fe;
46689566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
4669a1cf66bbSMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
46706528b96dSMatthew G. Knepley         key.field = fieldI * Nf + fieldJ;
46719566063dSJacob Faibussowitsch         if (hasJac) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMat));
46729566063dSJacob Faibussowitsch         if (hasPrec) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatP));
46739566063dSJacob Faibussowitsch         if (hasDyn) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatD));
4674a1cf66bbSMatthew G. Knepley       }
4675a1cf66bbSMatthew G. Knepley       /* For finite volume, add the identity */
4676a1cf66bbSMatthew G. Knepley       if (!isFE[fieldI]) {
4677a1cf66bbSMatthew G. Knepley         PetscFV  fv;
4678a1cf66bbSMatthew G. Knepley         PetscInt eOffset = 0, Nc, fc, foff;
4679a1cf66bbSMatthew G. Knepley 
46809566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(prob, fieldI, &foff));
46819566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv));
46829566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &Nc));
4683a1cf66bbSMatthew G. Knepley         for (c = 0; c < chunkSize; ++c, eOffset += totDim * totDim) {
4684a1cf66bbSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
4685a1cf66bbSMatthew G. Knepley             const PetscInt i = foff + fc;
4686ad540459SPierre Jolivet             if (hasJac) elemMat[eOffset + i * totDim + i] = 1.0;
4687ad540459SPierre Jolivet             if (hasPrec) elemMatP[eOffset + i * totDim + i] = 1.0;
4688a1cf66bbSMatthew G. Knepley           }
4689a1cf66bbSMatthew G. Knepley         }
4690a1cf66bbSMatthew G. Knepley       }
4691a1cf66bbSMatthew G. Knepley     }
4692a1cf66bbSMatthew G. Knepley     /*   Add contribution from X_t */
46939371c9d4SSatish Balay     if (hasDyn) {
46949371c9d4SSatish Balay       for (c = 0; c < chunkSize * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
46959371c9d4SSatish Balay     }
4696a1cf66bbSMatthew G. Knepley     /* Insert values into matrix */
4697a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4698a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c + offCell] : c + offCell;
4699a1cf66bbSMatthew G. Knepley       if (mesh->printFEM > 1) {
47009566063dSJacob Faibussowitsch         if (hasJac) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[(c - cStart) * totDim * totDim]));
47019566063dSJacob Faibussowitsch         if (hasPrec) PetscCall(DMPrintCellMatrix(cell, nameP, totDim, totDim, &elemMatP[(c - cStart) * totDim * totDim]));
4702a1cf66bbSMatthew G. Knepley       }
4703e8e188d2SZach Atkins       if (assembleJac) PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, Jac, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES));
4704e8e188d2SZach Atkins       PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, JP, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES));
4705a1cf66bbSMatthew G. Knepley     }
4706a1cf66bbSMatthew G. Knepley   }
4707a1cf66bbSMatthew G. Knepley   /* Cleanup */
47089566063dSJacob Faibussowitsch   PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
47099566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&qGeom));
47109566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
47119566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, Nf, MPIU_BOOL, &isFE));
47129566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, ((1 + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize, MPIU_SCALAR, &work));
4713a1cf66bbSMatthew G. Knepley   /* Compute boundary integrals */
47149566063dSJacob Faibussowitsch   /* PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, ctx)); */
4715a1cf66bbSMatthew G. Knepley   /* Assemble matrix */
47169371c9d4SSatish Balay   if (assembleJac) {
47179371c9d4SSatish Balay     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
47189371c9d4SSatish Balay     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
47199371c9d4SSatish Balay   }
47209371c9d4SSatish Balay   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
47219371c9d4SSatish Balay   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
47229566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
47233ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
4724a1cf66bbSMatthew G. Knepley }
47253e9753d6SMatthew G. Knepley 
47264ee01570SBarry Smith /* FEM Assembly Function */
47273e9753d6SMatthew G. Knepley 
4728d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMConvertPlex_Internal(DM dm, DM *plex, PetscBool copy)
4729d71ae5a4SJacob Faibussowitsch {
47303e9753d6SMatthew G. Knepley   PetscBool isPlex;
47313e9753d6SMatthew G. Knepley 
47323e9753d6SMatthew G. Knepley   PetscFunctionBegin;
47339566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex));
47343e9753d6SMatthew G. Knepley   if (isPlex) {
47353e9753d6SMatthew G. Knepley     *plex = dm;
47369566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)dm));
47373e9753d6SMatthew G. Knepley   } else {
47389566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex));
47393e9753d6SMatthew G. Knepley     if (!*plex) {
47409566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm, DMPLEX, plex));
47419566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex));
47423e9753d6SMatthew G. Knepley     } else {
47439566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)*plex));
47443e9753d6SMatthew G. Knepley     }
4745cbf8eb3cSStefano Zampini     if (copy) PetscCall(DMCopyAuxiliaryVec(dm, *plex));
47463e9753d6SMatthew G. Knepley   }
47473ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
47483e9753d6SMatthew G. Knepley }
47493e9753d6SMatthew G. Knepley 
47503e9753d6SMatthew G. Knepley /*@
47513e9753d6SMatthew G. Knepley   DMPlexGetGeometryFVM - Return precomputed geometric data
47523e9753d6SMatthew G. Knepley 
475320f4b53cSBarry Smith   Collective
47543e9753d6SMatthew G. Knepley 
47553e9753d6SMatthew G. Knepley   Input Parameter:
4756a1cb98faSBarry Smith . dm - The `DM`
47573e9753d6SMatthew G. Knepley 
47583e9753d6SMatthew G. Knepley   Output Parameters:
47593e9753d6SMatthew G. Knepley + facegeom  - The values precomputed from face geometry
47603e9753d6SMatthew G. Knepley . cellgeom  - The values precomputed from cell geometry
47613e9753d6SMatthew G. Knepley - minRadius - The minimum radius over the mesh of an inscribed sphere in a cell
47623e9753d6SMatthew G. Knepley 
47633e9753d6SMatthew G. Knepley   Level: developer
47643e9753d6SMatthew G. Knepley 
47651cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMTSSetRHSFunctionLocal()`
47663e9753d6SMatthew G. Knepley @*/
4767d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetGeometryFVM(DM dm, Vec *facegeom, Vec *cellgeom, PetscReal *minRadius)
4768d71ae5a4SJacob Faibussowitsch {
47693e9753d6SMatthew G. Knepley   DM plex;
47703e9753d6SMatthew G. Knepley 
47713e9753d6SMatthew G. Knepley   PetscFunctionBegin;
47723e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
47739566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE));
47749566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, NULL, cellgeom, facegeom, NULL));
47759566063dSJacob Faibussowitsch   if (minRadius) PetscCall(DMPlexGetMinRadius(plex, minRadius));
47769566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
47773ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
47783e9753d6SMatthew G. Knepley }
47793e9753d6SMatthew G. Knepley 
47803e9753d6SMatthew G. Knepley /*@
47813e9753d6SMatthew G. Knepley   DMPlexGetGradientDM - Return gradient data layout
47823e9753d6SMatthew G. Knepley 
478320f4b53cSBarry Smith   Collective
47843e9753d6SMatthew G. Knepley 
47853e9753d6SMatthew G. Knepley   Input Parameters:
4786a1cb98faSBarry Smith + dm - The `DM`
478720f4b53cSBarry Smith - fv - The `PetscFV`
47883e9753d6SMatthew G. Knepley 
47893e9753d6SMatthew G. Knepley   Output Parameter:
47903e9753d6SMatthew G. Knepley . dmGrad - The layout for gradient values
47913e9753d6SMatthew G. Knepley 
47923e9753d6SMatthew G. Knepley   Level: developer
47933e9753d6SMatthew G. Knepley 
47941cc06b55SBarry Smith .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetGeometryFVM()`
47953e9753d6SMatthew G. Knepley @*/
4796d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexGetGradientDM(DM dm, PetscFV fv, DM *dmGrad)
4797d71ae5a4SJacob Faibussowitsch {
47983e9753d6SMatthew G. Knepley   DM        plex;
47993e9753d6SMatthew G. Knepley   PetscBool computeGradients;
48003e9753d6SMatthew G. Knepley 
48013e9753d6SMatthew G. Knepley   PetscFunctionBegin;
48023e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
48033e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);
48044f572ea9SToby Isaac   PetscAssertPointer(dmGrad, 3);
48059566063dSJacob Faibussowitsch   PetscCall(PetscFVGetComputeGradients(fv, &computeGradients));
48069371c9d4SSatish Balay   if (!computeGradients) {
48079371c9d4SSatish Balay     *dmGrad = NULL;
48083ba16761SJacob Faibussowitsch     PetscFunctionReturn(PETSC_SUCCESS);
48099371c9d4SSatish Balay   }
48109566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE));
48119566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, fv, NULL, NULL, dmGrad));
48129566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
48133ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
48143e9753d6SMatthew G. Knepley }
48153e9753d6SMatthew G. Knepley 
4816d71ae5a4SJacob Faibussowitsch static PetscErrorCode DMPlexComputeBdResidual_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF, DMField coordField, IS facetIS)
4817d71ae5a4SJacob Faibussowitsch {
48183e9753d6SMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *)dm->data;
48193e9753d6SMatthew G. Knepley   DM              plex = NULL, plexA = NULL;
4820673019faSMatthew G. Knepley   const char     *name = "BdResidual";
48213e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
48223e9753d6SMatthew G. Knepley   PetscDS         prob, probAux       = NULL;
48233e9753d6SMatthew G. Knepley   PetscSection    section, sectionAux = NULL;
48243e9753d6SMatthew G. Knepley   Vec             locA = NULL;
48253e9753d6SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t = NULL, *a = NULL, *elemVec = NULL;
48263e9753d6SMatthew G. Knepley   PetscInt        totDim, totDimAux = 0;
48273e9753d6SMatthew G. Knepley 
48283e9753d6SMatthew G. Knepley   PetscFunctionBegin;
48299566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
48309566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
48319566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
48329566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
48339566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
48343e9753d6SMatthew G. Knepley   if (locA) {
48353e9753d6SMatthew G. Knepley     DM dmAux;
48363e9753d6SMatthew G. Knepley 
48379566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
48389566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
48399566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
48409566063dSJacob Faibussowitsch     PetscCall(DMGetDS(plexA, &probAux));
48419566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
48429566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
48433e9753d6SMatthew G. Knepley   }
48440c290148SMatthew G. Knepley   {
48453e9753d6SMatthew G. Knepley     PetscFEGeom    *fgeom;
48463e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
48473e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
48483e9753d6SMatthew G. Knepley     IS              pointIS;
48493e9753d6SMatthew G. Knepley     const PetscInt *points;
48503e9753d6SMatthew G. Knepley     PetscInt        numFaces, face, Nq;
48513e9753d6SMatthew G. Knepley 
48529566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(key.label, key.value, &pointIS));
48530c290148SMatthew G. Knepley     if (!pointIS) goto end; /* No points with that id on this process */
48543e9753d6SMatthew G. Knepley     {
48553e9753d6SMatthew G. Knepley       IS isectIS;
48563e9753d6SMatthew G. Knepley 
48573e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
48589566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
48599566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
48603e9753d6SMatthew G. Knepley       pointIS = isectIS;
48613e9753d6SMatthew G. Knepley     }
48629566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
48639566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
486432603206SJames Wright     PetscCall(PetscMalloc4(numFaces * totDim, &u, (locX_t ? (size_t)numFaces * totDim : 0), &u_t, numFaces * totDim, &elemVec, (locA ? (size_t)numFaces * totDimAux : 0), &a));
48659566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
486648a46eb9SPierre Jolivet     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
48673e9753d6SMatthew G. Knepley     if (!qGeom) {
48683e9753d6SMatthew G. Knepley       PetscFE fe;
48693e9753d6SMatthew G. Knepley 
48709566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe));
48719566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
48729566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
48733e9753d6SMatthew G. Knepley     }
48749566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
48759566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
48763e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
4877f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
48783e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
4879f15274beSMatthew Knepley       PetscInt       i;
48803e9753d6SMatthew G. Knepley 
48819566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
48829566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
48833e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i];
48849566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
48853e9753d6SMatthew G. Knepley       if (locX_t) {
48869566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
48873e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i];
48889566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
48893e9753d6SMatthew G. Knepley       }
48903e9753d6SMatthew G. Knepley       if (locA) {
48913e9753d6SMatthew G. Knepley         PetscInt subp;
48923e9753d6SMatthew G. Knepley 
48939566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
48949566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
48953e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i];
48969566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
48973e9753d6SMatthew G. Knepley       }
48983e9753d6SMatthew G. Knepley     }
48999566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(elemVec, numFaces * totDim));
49003e9753d6SMatthew G. Knepley     {
49013e9753d6SMatthew G. Knepley       PetscFE      fe;
49023e9753d6SMatthew G. Knepley       PetscInt     Nb;
49033e9753d6SMatthew G. Knepley       PetscFEGeom *chunkGeom = NULL;
49043e9753d6SMatthew G. Knepley       /* Conforming batches */
49053e9753d6SMatthew G. Knepley       PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
49063e9753d6SMatthew G. Knepley       /* Remainder */
49073e9753d6SMatthew G. Knepley       PetscInt Nr, offset;
49083e9753d6SMatthew G. Knepley 
49099566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe));
49109566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
49119566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
49123e9753d6SMatthew G. Knepley       /* TODO: documentation is unclear about what is going on with these numbers: how should Nb / Nq factor in ? */
49133e9753d6SMatthew G. Knepley       blockSize = Nb;
49143e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
49159566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
49163e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches * batchSize);
49173e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
49183e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches * batchSize);
49193e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
49209566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom));
49219566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
49229566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
49239566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
49248e3a54c0SPierre Jolivet       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, &elemVec[offset * totDim]));
49259566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
49263e9753d6SMatthew G. Knepley     }
49273e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
49283e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
49293e9753d6SMatthew G. Knepley 
4930673019faSMatthew G. Knepley       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(point, name, totDim, &elemVec[face * totDim]));
49319566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
49329566063dSJacob Faibussowitsch       PetscCall(DMPlexVecSetClosure(plex, NULL, locF, support[0], &elemVec[face * totDim], ADD_ALL_VALUES));
49333e9753d6SMatthew G. Knepley     }
49349566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
49359566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
49369566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
49379566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
49389566063dSJacob Faibussowitsch     PetscCall(PetscFree4(u, u_t, elemVec, a));
49393e9753d6SMatthew G. Knepley   }
49400c290148SMatthew G. Knepley end:
4941673019faSMatthew G. Knepley   if (mesh->printFEM) {
4942673019faSMatthew G. Knepley     PetscSection s;
4943673019faSMatthew G. Knepley     Vec          locFbc;
4944673019faSMatthew G. Knepley     PetscInt     pStart, pEnd, maxDof;
4945673019faSMatthew G. Knepley     PetscScalar *zeroes;
4946673019faSMatthew G. Knepley 
4947673019faSMatthew G. Knepley     PetscCall(DMGetLocalSection(dm, &s));
4948673019faSMatthew G. Knepley     PetscCall(VecDuplicate(locF, &locFbc));
4949673019faSMatthew G. Knepley     PetscCall(VecCopy(locF, locFbc));
4950673019faSMatthew G. Knepley     PetscCall(PetscSectionGetChart(s, &pStart, &pEnd));
4951673019faSMatthew G. Knepley     PetscCall(PetscSectionGetMaxDof(s, &maxDof));
4952673019faSMatthew G. Knepley     PetscCall(PetscCalloc1(maxDof, &zeroes));
4953673019faSMatthew G. Knepley     for (PetscInt p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, s, p, zeroes, INSERT_BC_VALUES));
4954673019faSMatthew G. Knepley     PetscCall(PetscFree(zeroes));
4955673019faSMatthew G. Knepley     PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc));
4956673019faSMatthew G. Knepley     PetscCall(VecDestroy(&locFbc));
4957673019faSMatthew G. Knepley   }
49589566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
49599566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexA));
49603ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
49613e9753d6SMatthew G. Knepley }
49623e9753d6SMatthew G. Knepley 
4963d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeBdResidualSingle(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF)
4964d71ae5a4SJacob Faibussowitsch {
49653e9753d6SMatthew G. Knepley   DMField  coordField;
49663e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
49673e9753d6SMatthew G. Knepley   IS       facetIS;
49683e9753d6SMatthew G. Knepley   PetscInt dim;
49693e9753d6SMatthew G. Knepley 
49703e9753d6SMatthew G. Knepley   PetscFunctionBegin;
49719566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
49729566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
49739566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
49749566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
49759566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS));
49769566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
49773ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
49783e9753d6SMatthew G. Knepley }
49793e9753d6SMatthew G. Knepley 
4980a4e35b19SJacob Faibussowitsch static PetscErrorCode DMPlexComputeBdResidual_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
4981d71ae5a4SJacob Faibussowitsch {
49823e9753d6SMatthew G. Knepley   PetscDS  prob;
49833e9753d6SMatthew G. Knepley   PetscInt numBd, bd;
49843e9753d6SMatthew G. Knepley   DMField  coordField = NULL;
49853e9753d6SMatthew G. Knepley   IS       facetIS    = NULL;
49863e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
49873e9753d6SMatthew G. Knepley   PetscInt dim;
49883e9753d6SMatthew G. Knepley 
49893e9753d6SMatthew G. Knepley   PetscFunctionBegin;
49909566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
49919566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
49929566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
49939566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
49949566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
49953e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
499645480ffeSMatthew G. Knepley     PetscWeakForm           wf;
49973e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
49983e9753d6SMatthew G. Knepley     DMLabel                 label;
49993e9753d6SMatthew G. Knepley     const PetscInt         *values;
50000c290148SMatthew G. Knepley     PetscInt                field, numValues, v;
50013e9753d6SMatthew G. Knepley     PetscObject             obj;
50023e9753d6SMatthew G. Knepley     PetscClassId            id;
50030c290148SMatthew G. Knepley     PetscFormKey            key;
50043e9753d6SMatthew G. Knepley 
50059566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &field, NULL, NULL, NULL, NULL, NULL));
50063d3e5d66SMatthew G. Knepley     if (type & DM_BC_ESSENTIAL) continue;
50079566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
50089566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
50093d3e5d66SMatthew G. Knepley     if (id != PETSCFE_CLASSID) continue;
50103e9753d6SMatthew G. Knepley     if (!facetIS) {
50113e9753d6SMatthew G. Knepley       DMLabel  depthLabel;
50123e9753d6SMatthew G. Knepley       PetscInt dim;
50133e9753d6SMatthew G. Knepley 
50149566063dSJacob Faibussowitsch       PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
50159566063dSJacob Faibussowitsch       PetscCall(DMGetDimension(dm, &dim));
50169566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
50173e9753d6SMatthew G. Knepley     }
50189566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
50190c290148SMatthew G. Knepley     for (v = 0; v < numValues; ++v) {
50200c290148SMatthew G. Knepley       key.label = label;
50210c290148SMatthew G. Knepley       key.value = values[v];
50220c290148SMatthew G. Knepley       key.field = field;
50230c290148SMatthew G. Knepley       key.part  = 0;
50249566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS));
50250c290148SMatthew G. Knepley     }
50263e9753d6SMatthew G. Knepley   }
50279566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
50283ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
50293e9753d6SMatthew G. Knepley }
50303e9753d6SMatthew G. Knepley 
5031d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeResidual_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
5032d71ae5a4SJacob Faibussowitsch {
50333e9753d6SMatthew G. Knepley   DM_Plex        *mesh       = (DM_Plex *)dm->data;
50343e9753d6SMatthew G. Knepley   const char     *name       = "Residual";
50353e9753d6SMatthew G. Knepley   DM              dmAux      = NULL;
50363e9753d6SMatthew G. Knepley   DM              dmGrad     = NULL;
50373e9753d6SMatthew G. Knepley   DMLabel         ghostLabel = NULL;
50386528b96dSMatthew G. Knepley   PetscDS         ds         = NULL;
50396528b96dSMatthew G. Knepley   PetscDS         dsAux      = NULL;
50403e9753d6SMatthew G. Knepley   PetscSection    section    = NULL;
50413e9753d6SMatthew G. Knepley   PetscBool       useFEM     = PETSC_FALSE;
50423e9753d6SMatthew G. Knepley   PetscBool       useFVM     = PETSC_FALSE;
50433e9753d6SMatthew G. Knepley   PetscBool       isImplicit = (locX_t || time == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
50443e9753d6SMatthew G. Knepley   PetscFV         fvm        = NULL;
50453e9753d6SMatthew G. Knepley   DMField         coordField = NULL;
50465962854dSMatthew G. Knepley   Vec             locA, cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL;
50473e9753d6SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t, *a, *uL, *uR;
50483e9753d6SMatthew G. Knepley   IS              chunkIS;
50493e9753d6SMatthew G. Knepley   const PetscInt *cells;
50503e9753d6SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
50513e9753d6SMatthew G. Knepley   PetscInt        Nf, f, totDim, totDimAux, numChunks, cellChunkSize, faceChunkSize, chunk, fStart, fEnd;
50521690c2aeSBarry Smith   PetscInt        maxDegree  = PETSC_INT_MAX;
50533e9753d6SMatthew G. Knepley   PetscQuadrature affineQuad = NULL, *quads = NULL;
50543e9753d6SMatthew G. Knepley   PetscFEGeom    *affineGeom = NULL, **geoms = NULL;
50553e9753d6SMatthew G. Knepley 
50563e9753d6SMatthew G. Knepley   PetscFunctionBegin;
50579566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0));
50583cc88e6aSStefano Zampini   if (!cellIS) goto end;
50593cc88e6aSStefano Zampini   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
50603cc88e6aSStefano Zampini   if (cStart >= cEnd) goto end;
50613e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
50623e9753d6SMatthew G. Knepley   /* TODO The FVM geometry is over-manipulated. Make the precalc functions return exactly what we need */
50633e9753d6SMatthew G. Knepley   /* FEM+FVM */
50649566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
50653e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
50669566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
50679566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
506807218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, NULL));
50699566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
50709566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
50719566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
50723e9753d6SMatthew G. Knepley   if (locA) {
50733e9753d6SMatthew G. Knepley     PetscInt subcell;
50749566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
50751059d808SMatthew G. Knepley     PetscCall(DMGetEnclosurePoint(dmAux, dm, DM_ENC_UNKNOWN, cells ? cells[cStart] : cStart, &subcell));
507607218a29SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux, subcell, &dsAux, NULL));
50779566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux));
50783e9753d6SMatthew G. Knepley   }
50793e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
50803e9753d6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
50813e9753d6SMatthew G. Knepley     PetscObject  obj;
50823e9753d6SMatthew G. Knepley     PetscClassId id;
50833e9753d6SMatthew G. Knepley     PetscBool    fimp;
50843e9753d6SMatthew G. Knepley 
50859566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(ds, f, &fimp));
50863e9753d6SMatthew G. Knepley     if (isImplicit != fimp) continue;
50879566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(ds, f, &obj));
50889566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
5089ad540459SPierre Jolivet     if (id == PETSCFE_CLASSID) useFEM = PETSC_TRUE;
50909371c9d4SSatish Balay     if (id == PETSCFV_CLASSID) {
50919371c9d4SSatish Balay       useFVM = PETSC_TRUE;
50929371c9d4SSatish Balay       fvm    = (PetscFV)obj;
50939371c9d4SSatish Balay     }
50943e9753d6SMatthew G. Knepley   }
50953e9753d6SMatthew G. Knepley   if (useFEM) {
50969566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
50979566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
50983e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
50999566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad));
510048a46eb9SPierre Jolivet       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
51013e9753d6SMatthew G. Knepley     } else {
51029566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
51033e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
51043e9753d6SMatthew G. Knepley         PetscObject  obj;
51053e9753d6SMatthew G. Knepley         PetscClassId id;
51063e9753d6SMatthew G. Knepley         PetscBool    fimp;
51073e9753d6SMatthew G. Knepley 
51089566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(ds, f, &fimp));
51093e9753d6SMatthew G. Knepley         if (isImplicit != fimp) continue;
51109566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
51119566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
51123e9753d6SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
51133e9753d6SMatthew G. Knepley           PetscFE fe = (PetscFE)obj;
51143e9753d6SMatthew G. Knepley 
51159566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
51169566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
51179566063dSJacob Faibussowitsch           PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
51183e9753d6SMatthew G. Knepley         }
51193e9753d6SMatthew G. Knepley       }
51203e9753d6SMatthew G. Knepley     }
51213e9753d6SMatthew G. Knepley   }
51225962854dSMatthew G. Knepley   // Handle non-essential (e.g. outflow) boundary values
51233e9753d6SMatthew G. Knepley   if (useFVM) {
51245962854dSMatthew G. Knepley     PetscCall(DMPlexInsertBoundaryValuesFVM(dm, fvm, locX, time, &locGrad));
51259566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL));
51269566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGradientDM(dm, fvm, &dmGrad));
51273e9753d6SMatthew G. Knepley   }
51283e9753d6SMatthew G. Knepley   /* Loop over chunks */
51299566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
51303e9753d6SMatthew G. Knepley   numCells      = cEnd - cStart;
51313e9753d6SMatthew G. Knepley   numChunks     = 1;
51323e9753d6SMatthew G. Knepley   cellChunkSize = numCells / numChunks;
51333e9753d6SMatthew G. Knepley   faceChunkSize = (fEnd - fStart) / numChunks;
51343e9753d6SMatthew G. Knepley   numChunks     = PetscMin(1, numCells);
51353e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
51363e9753d6SMatthew G. Knepley     PetscScalar     *elemVec, *fluxL, *fluxR;
51373e9753d6SMatthew G. Knepley     PetscReal       *vol;
51383e9753d6SMatthew G. Knepley     PetscFVFaceGeom *fgeom;
51393e9753d6SMatthew G. Knepley     PetscInt         cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
51403e9753d6SMatthew G. Knepley     PetscInt         fS = fStart + chunk * faceChunkSize, fE = PetscMin(fS + faceChunkSize, fEnd), numFaces = 0, face;
51413e9753d6SMatthew G. Knepley 
51423e9753d6SMatthew G. Knepley     /* Extract field coefficients */
51433e9753d6SMatthew G. Knepley     if (useFEM) {
51449566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
51459566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
51469566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
51479566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells * totDim));
51483e9753d6SMatthew G. Knepley     }
51493e9753d6SMatthew G. Knepley     if (useFVM) {
51509566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
51519566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
51529566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL));
51539566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR));
51549566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxL, numFaces * totDim));
51559566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxR, numFaces * totDim));
51563e9753d6SMatthew G. Knepley     }
51573e9753d6SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
51583e9753d6SMatthew G. Knepley     /* Loop over fields */
51593e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
51603e9753d6SMatthew G. Knepley       PetscObject  obj;
51613e9753d6SMatthew G. Knepley       PetscClassId id;
51623e9753d6SMatthew G. Knepley       PetscBool    fimp;
51633e9753d6SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
51643e9753d6SMatthew G. Knepley 
51656528b96dSMatthew G. Knepley       key.field = f;
51669566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(ds, f, &fimp));
51673e9753d6SMatthew G. Knepley       if (isImplicit != fimp) continue;
51689566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, &obj));
51699566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
51703e9753d6SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
51713e9753d6SMatthew G. Knepley         PetscFE         fe        = (PetscFE)obj;
51723e9753d6SMatthew G. Knepley         PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[f];
51733e9753d6SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
51743e9753d6SMatthew G. Knepley         PetscQuadrature quad      = affineQuad ? affineQuad : quads[f];
51753e9753d6SMatthew G. Knepley         PetscInt        Nq, Nb;
51763e9753d6SMatthew G. Knepley 
51779566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
51789566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
51799566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
51803e9753d6SMatthew G. Knepley         blockSize = Nb;
51813e9753d6SMatthew G. Knepley         batchSize = numBlocks * blockSize;
51829566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
51833e9753d6SMatthew G. Knepley         numChunks = numCells / (numBatches * batchSize);
51843e9753d6SMatthew G. Knepley         Ne        = numChunks * numBatches * batchSize;
51853e9753d6SMatthew G. Knepley         Nr        = numCells % (numBatches * batchSize);
51863e9753d6SMatthew G. Knepley         offset    = numCells - Nr;
51873e9753d6SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
51883e9753d6SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
51899566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom));
51909566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(ds, key, Ne, chunkGeom, u, u_t, dsAux, a, t, elemVec));
51919566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom));
51928e3a54c0SPierre Jolivet         PetscCall(PetscFEIntegrateResidual(ds, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, &elemVec[offset * totDim]));
51939566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom));
51943e9753d6SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
51953e9753d6SMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
51963e9753d6SMatthew G. Knepley 
51973e9753d6SMatthew G. Knepley         Ne = numFaces;
51983e9753d6SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
51993e9753d6SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
52009566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, ds, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
520163a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
52023e9753d6SMatthew G. Knepley     }
52033e9753d6SMatthew G. Knepley     /* Loop over domain */
52043e9753d6SMatthew G. Knepley     if (useFEM) {
52053e9753d6SMatthew G. Knepley       /* Add elemVec to locX */
52063e9753d6SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
52073e9753d6SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
52083e9753d6SMatthew G. Knepley         const PetscInt cind = c - cStart;
52093e9753d6SMatthew G. Knepley 
52109566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim]));
52113e9753d6SMatthew G. Knepley         if (ghostLabel) {
52123e9753d6SMatthew G. Knepley           PetscInt ghostVal;
52133e9753d6SMatthew G. Knepley 
52149566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
52153e9753d6SMatthew G. Knepley           if (ghostVal > 0) continue;
52163e9753d6SMatthew G. Knepley         }
52179566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES));
52183e9753d6SMatthew G. Knepley       }
52193e9753d6SMatthew G. Knepley     }
52203e9753d6SMatthew G. Knepley     if (useFVM) {
52213e9753d6SMatthew G. Knepley       PetscScalar *fa;
52223e9753d6SMatthew G. Knepley       PetscInt     iface;
52233e9753d6SMatthew G. Knepley 
52249566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
52253e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
52263e9753d6SMatthew G. Knepley         PetscFV      fv;
52273e9753d6SMatthew G. Knepley         PetscObject  obj;
52283e9753d6SMatthew G. Knepley         PetscClassId id;
52295962854dSMatthew G. Knepley         PetscInt     cdim, foff, pdim;
52303e9753d6SMatthew G. Knepley 
52315962854dSMatthew G. Knepley         PetscCall(DMGetCoordinateDim(dm, &cdim));
52329566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
52339566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(ds, f, &foff));
52349566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
52353e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
52363e9753d6SMatthew G. Knepley         fv = (PetscFV)obj;
52379566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
52383e9753d6SMatthew G. Knepley         /* Accumulate fluxes to cells */
52393e9753d6SMatthew G. Knepley         for (face = fS, iface = 0; face < fE; ++face) {
52403e9753d6SMatthew G. Knepley           const PetscInt *scells;
52413e9753d6SMatthew G. Knepley           PetscScalar    *fL = NULL, *fR = NULL;
52423e9753d6SMatthew G. Knepley           PetscInt        ghost, d, nsupp, nchild;
52433e9753d6SMatthew G. Knepley 
52449566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
52459566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
52469566063dSJacob Faibussowitsch           PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
52473e9753d6SMatthew G. Knepley           if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
52489566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupport(dm, face, &scells));
52499566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, scells[0], &ghost));
52509566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[0], f, fa, &fL));
52519566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, scells[1], &ghost));
52529566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[1], f, fa, &fR));
52535962854dSMatthew G. Knepley           if (mesh->printFVM > 1) {
52545962854dSMatthew G. Knepley             PetscCall(DMPrintCellVectorReal(face, "Residual: normal", cdim, fgeom[iface].normal));
52555962854dSMatthew G. Knepley             PetscCall(DMPrintCellVector(face, "Residual: left state", pdim, &uL[iface * totDim + foff]));
52565962854dSMatthew G. Knepley             PetscCall(DMPrintCellVector(face, "Residual: right state", pdim, &uR[iface * totDim + foff]));
52575962854dSMatthew G. Knepley             PetscCall(DMPrintCellVector(face, "Residual: left flux", pdim, &fluxL[iface * totDim + foff]));
52585962854dSMatthew G. Knepley             PetscCall(DMPrintCellVector(face, "Residual: right flux", pdim, &fluxR[iface * totDim + foff]));
52595962854dSMatthew G. Knepley           }
52603e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) {
52613e9753d6SMatthew G. Knepley             if (fL) fL[d] -= fluxL[iface * totDim + foff + d];
52623e9753d6SMatthew G. Knepley             if (fR) fR[d] += fluxR[iface * totDim + foff + d];
52633e9753d6SMatthew G. Knepley           }
52643e9753d6SMatthew G. Knepley           ++iface;
52653e9753d6SMatthew G. Knepley         }
52663e9753d6SMatthew G. Knepley       }
52679566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
52683e9753d6SMatthew G. Knepley     }
52693e9753d6SMatthew G. Knepley     /* Handle time derivative */
52703e9753d6SMatthew G. Knepley     if (locX_t) {
52713e9753d6SMatthew G. Knepley       PetscScalar *x_t, *fa;
52723e9753d6SMatthew G. Knepley 
52739566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
52749566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
52753e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
52763e9753d6SMatthew G. Knepley         PetscFV      fv;
52773e9753d6SMatthew G. Knepley         PetscObject  obj;
52783e9753d6SMatthew G. Knepley         PetscClassId id;
52793e9753d6SMatthew G. Knepley         PetscInt     pdim, d;
52803e9753d6SMatthew G. Knepley 
52819566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
52829566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
52833e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
52843e9753d6SMatthew G. Knepley         fv = (PetscFV)obj;
52859566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
52863e9753d6SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
52873e9753d6SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
52883e9753d6SMatthew G. Knepley           PetscScalar   *u_t, *r;
52893e9753d6SMatthew G. Knepley 
52903e9753d6SMatthew G. Knepley           if (ghostLabel) {
52913e9753d6SMatthew G. Knepley             PetscInt ghostVal;
52923e9753d6SMatthew G. Knepley 
52939566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
52943e9753d6SMatthew G. Knepley             if (ghostVal > 0) continue;
52953e9753d6SMatthew G. Knepley           }
52969566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
52979566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
52983e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
52993e9753d6SMatthew G. Knepley         }
53003e9753d6SMatthew G. Knepley       }
53019566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
53029566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
53033e9753d6SMatthew G. Knepley     }
53043e9753d6SMatthew G. Knepley     if (useFEM) {
53059566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
53069566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
53073e9753d6SMatthew G. Knepley     }
53083e9753d6SMatthew G. Knepley     if (useFVM) {
53099566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
53109566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
53119566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL));
53129566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR));
53139566063dSJacob Faibussowitsch       if (dmGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
53143e9753d6SMatthew G. Knepley     }
53153e9753d6SMatthew G. Knepley   }
53169566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
53179566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
53183e9753d6SMatthew G. Knepley 
53193e9753d6SMatthew G. Knepley   if (useFEM) {
53209566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdResidual_Internal(dm, locX, locX_t, t, locF, user));
53213e9753d6SMatthew G. Knepley 
53223e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
53239566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
53249566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
53253e9753d6SMatthew G. Knepley     } else {
53263e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
53279566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
53289566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
53293e9753d6SMatthew G. Knepley       }
53309566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads, geoms));
53313e9753d6SMatthew G. Knepley     }
53323e9753d6SMatthew G. Knepley   }
53333e9753d6SMatthew G. Knepley 
53343e9753d6SMatthew G. Knepley   /* FEM */
53353e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
53363e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
53373e9753d6SMatthew G. Knepley   /* 3: Handle boundary values */
53383e9753d6SMatthew G. Knepley   /* 4: Loop over domain */
53393e9753d6SMatthew G. Knepley   /*   Extract coefficients */
53403e9753d6SMatthew G. Knepley   /* Loop over fields */
53413e9753d6SMatthew G. Knepley   /*   Set tiling for FE*/
53423e9753d6SMatthew G. Knepley   /*   Integrate FE residual to get elemVec */
53433e9753d6SMatthew G. Knepley   /*     Loop over subdomain */
53443e9753d6SMatthew G. Knepley   /*       Loop over quad points */
53453e9753d6SMatthew G. Knepley   /*         Transform coords to real space */
53463e9753d6SMatthew G. Knepley   /*         Evaluate field and aux fields at point */
53473e9753d6SMatthew G. Knepley   /*         Evaluate residual at point */
53483e9753d6SMatthew G. Knepley   /*         Transform residual to real space */
53493e9753d6SMatthew G. Knepley   /*       Add residual to elemVec */
53503e9753d6SMatthew G. Knepley   /* Loop over domain */
53513e9753d6SMatthew G. Knepley   /*   Add elemVec to locX */
53523e9753d6SMatthew G. Knepley 
53533e9753d6SMatthew G. Knepley   /* FVM */
53543e9753d6SMatthew G. Knepley   /* Get geometric data */
53553e9753d6SMatthew G. Knepley   /* If using gradients */
53563e9753d6SMatthew G. Knepley   /*   Compute gradient data */
53573e9753d6SMatthew G. Knepley   /*   Loop over domain faces */
53583e9753d6SMatthew G. Knepley   /*     Count computational faces */
53593e9753d6SMatthew G. Knepley   /*     Reconstruct cell gradient */
53603e9753d6SMatthew G. Knepley   /*   Loop over domain cells */
53613e9753d6SMatthew G. Knepley   /*     Limit cell gradients */
53623e9753d6SMatthew G. Knepley   /* Handle boundary values */
53633e9753d6SMatthew G. Knepley   /* Loop over domain faces */
53643e9753d6SMatthew G. Knepley   /*   Read out field, centroid, normal, volume for each side of face */
53653e9753d6SMatthew G. Knepley   /* Riemann solve over faces */
53663e9753d6SMatthew G. Knepley   /* Loop over domain faces */
53673e9753d6SMatthew G. Knepley   /*   Accumulate fluxes to cells */
53683e9753d6SMatthew G. Knepley   /* TODO Change printFEM to printDisc here */
53693e9753d6SMatthew G. Knepley   if (mesh->printFEM) {
53703e9753d6SMatthew G. Knepley     Vec          locFbc;
53713e9753d6SMatthew G. Knepley     PetscInt     pStart, pEnd, p, maxDof;
53723e9753d6SMatthew G. Knepley     PetscScalar *zeroes;
53733e9753d6SMatthew G. Knepley 
53749566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(locF, &locFbc));
53759566063dSJacob Faibussowitsch     PetscCall(VecCopy(locF, locFbc));
53769566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
53779566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetMaxDof(section, &maxDof));
53789566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(maxDof, &zeroes));
537948a46eb9SPierre Jolivet     for (p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, section, p, zeroes, INSERT_BC_VALUES));
53809566063dSJacob Faibussowitsch     PetscCall(PetscFree(zeroes));
53819566063dSJacob Faibussowitsch     PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc));
53829566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&locFbc));
53833e9753d6SMatthew G. Knepley   }
53843cc88e6aSStefano Zampini end:
53859566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0));
53863ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
53873e9753d6SMatthew G. Knepley }
53883e9753d6SMatthew G. Knepley 
53896528b96dSMatthew G. Knepley /*
53906528b96dSMatthew G. Knepley   1) Allow multiple kernels for BdResidual for hybrid DS
53916528b96dSMatthew G. Knepley 
53926528b96dSMatthew G. Knepley   DONE 2) Get out dsAux for either side at the same time as cohesive cell dsAux
53936528b96dSMatthew G. Knepley 
53946528b96dSMatthew G. Knepley   DONE 3) Change DMGetCellFields() to get different aux data a[] for each side
53956528b96dSMatthew G. Knepley      - I think I just need to replace a[] with the closure from each face
53966528b96dSMatthew G. Knepley 
53976528b96dSMatthew G. Knepley   4) Run both kernels for each non-hybrid field with correct dsAux, and then hybrid field as before
53986528b96dSMatthew G. Knepley */
5399d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeResidual_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
5400d71ae5a4SJacob Faibussowitsch {
54013e9753d6SMatthew G. Knepley   DM_Plex        *mesh       = (DM_Plex *)dm->data;
54023e9753d6SMatthew G. Knepley   const char     *name       = "Hybrid Residual";
540304c51a94SMatthew G. Knepley   DM              dmAux[3]   = {NULL, NULL, NULL};
54043e9753d6SMatthew G. Knepley   DMLabel         ghostLabel = NULL;
54056528b96dSMatthew G. Knepley   PetscDS         ds         = NULL;
540607218a29SMatthew G. Knepley   PetscDS         dsIn       = NULL;
54076528b96dSMatthew G. Knepley   PetscDS         dsAux[3]   = {NULL, NULL, NULL};
540804c51a94SMatthew G. Knepley   Vec             locA[3]    = {NULL, NULL, NULL};
540907218a29SMatthew G. Knepley   DM              dmScale[3] = {NULL, NULL, NULL};
541007218a29SMatthew G. Knepley   PetscDS         dsScale[3] = {NULL, NULL, NULL};
541107218a29SMatthew G. Knepley   Vec             locS[3]    = {NULL, NULL, NULL};
54123e9753d6SMatthew G. Knepley   PetscSection    section    = NULL;
54133e9753d6SMatthew G. Knepley   DMField         coordField = NULL;
541407218a29SMatthew G. Knepley   PetscScalar    *a[3]       = {NULL, NULL, NULL};
541507218a29SMatthew G. Knepley   PetscScalar    *s[3]       = {NULL, NULL, NULL};
5416b2ab40e6SMatthew G. Knepley   PetscScalar    *u          = NULL, *u_t;
541707218a29SMatthew G. Knepley   PetscScalar    *elemVecNeg, *elemVecPos, *elemVecCoh;
54183e9753d6SMatthew G. Knepley   IS              chunkIS;
54193e9753d6SMatthew G. Knepley   const PetscInt *cells;
54203e9753d6SMatthew G. Knepley   PetscInt       *faces;
54213e9753d6SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
54223e2b0218SMatthew G. Knepley   PetscInt        Nf, f, totDim, totDimIn, totDimAux[3], totDimScale[3], numChunks, cellChunkSize, chunk;
54231690c2aeSBarry Smith   PetscInt        maxDegree  = PETSC_INT_MAX;
54243e9753d6SMatthew G. Knepley   PetscQuadrature affineQuad = NULL, *quads = NULL;
54253e9753d6SMatthew G. Knepley   PetscFEGeom    *affineGeom = NULL, **geoms = NULL;
54263e9753d6SMatthew G. Knepley 
54273e9753d6SMatthew G. Knepley   PetscFunctionBegin;
54283cc88e6aSStefano Zampini   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0));
54293cc88e6aSStefano Zampini   if (!cellIS) goto end;
5430437e83fbSMatthew G. Knepley   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
5431437e83fbSMatthew G. Knepley   PetscCall(ISGetLocalSize(cellIS, &numCells));
54323cc88e6aSStefano Zampini   if (cStart >= cEnd) goto end;
54335fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
54345fedec97SMatthew G. Knepley     const char *name;
54359566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name));
543663a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
54375fedec97SMatthew G. Knepley   }
54383e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
54393e9753d6SMatthew G. Knepley   /* FEM */
54403e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
5441*b98a7184SJames Wright   PetscCall(DMGetLocalSection(dm, &section));
54429566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
544307218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn));
54449566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
54459566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
544607218a29SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(dsIn, &totDimIn));
54479566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
544804c51a94SMatthew G. Knepley   if (locA[2]) {
54491059d808SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
54501059d808SMatthew G. Knepley 
54519566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
545207218a29SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2], NULL));
54539566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
54546528b96dSMatthew G. Knepley     {
54556528b96dSMatthew G. Knepley       const PetscInt *cone;
54566528b96dSMatthew G. Knepley       PetscInt        c;
54576528b96dSMatthew G. Knepley 
54581059d808SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
54596528b96dSMatthew G. Knepley       for (c = 0; c < 2; ++c) {
54606528b96dSMatthew G. Knepley         const PetscInt *support;
54616528b96dSMatthew G. Knepley         PetscInt        ssize, s;
54626528b96dSMatthew G. Knepley 
54639566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
54649566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
54651059d808SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
54661059d808SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
54671059d808SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
54681059d808SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
54699566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
5470c75bfeddSPierre Jolivet         PetscCheck(locA[c], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Must have auxiliary vector for (%p, %" PetscInt_FMT ", %" PetscInt_FMT ")", (void *)key[c].label, key[c].value, key[c].part);
54719566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
5472ad540459SPierre Jolivet         else dmAux[c] = dmAux[2];
547307218a29SMatthew G. Knepley         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c], NULL));
54749566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
54756528b96dSMatthew G. Knepley       }
54766528b96dSMatthew G. Knepley     }
54773e9753d6SMatthew G. Knepley   }
547807218a29SMatthew G. Knepley   /* Handle mass matrix scaling
547907218a29SMatthew G. Knepley        The field in key[2] is the field to be scaled, and the scaling field is the first in the dsScale */
548007218a29SMatthew G. Knepley   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, -key[2].value, key[2].part, &locS[2]));
548107218a29SMatthew G. Knepley   if (locS[2]) {
54823e2b0218SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
548307218a29SMatthew G. Knepley     PetscInt       Nb, Nbs;
548407218a29SMatthew G. Knepley 
548507218a29SMatthew G. Knepley     PetscCall(VecGetDM(locS[2], &dmScale[2]));
54863e2b0218SMatthew G. Knepley     PetscCall(DMGetCellDS(dmScale[2], cellStart, &dsScale[2], NULL));
54873e2b0218SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(dsScale[2], &totDimScale[2]));
548807218a29SMatthew G. Knepley     // BRAD: This is not set correctly
548907218a29SMatthew G. Knepley     key[2].field = 2;
549007218a29SMatthew G. Knepley     PetscCall(PetscDSGetFieldSize(ds, key[2].field, &Nb));
549107218a29SMatthew G. Knepley     PetscCall(PetscDSGetFieldSize(dsScale[2], 0, &Nbs));
549207218a29SMatthew G. Knepley     PetscCheck(Nb == Nbs, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Field %" PetscInt_FMT " of size %" PetscInt_FMT " cannot be scaled by field of size %" PetscInt_FMT, key[2].field, Nb, Nbs);
54933e2b0218SMatthew G. Knepley     {
54943e2b0218SMatthew G. Knepley       const PetscInt *cone;
54953e2b0218SMatthew G. Knepley       PetscInt        c;
54963e2b0218SMatthew G. Knepley 
54973e2b0218SMatthew G. Knepley       locS[1] = locS[0] = locS[2];
54983e2b0218SMatthew G. Knepley       dmScale[1] = dmScale[0] = dmScale[2];
54993e2b0218SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
55003e2b0218SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
55013e2b0218SMatthew G. Knepley         const PetscInt *support;
55023e2b0218SMatthew G. Knepley         PetscInt        ssize, s;
55033e2b0218SMatthew G. Knepley 
55043e2b0218SMatthew G. Knepley         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
55053e2b0218SMatthew G. Knepley         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
55063e2b0218SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
55073e2b0218SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
55083e2b0218SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
55093e2b0218SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
55103e2b0218SMatthew G. Knepley         PetscCall(DMGetCellDS(dmScale[c], support[s], &dsScale[c], NULL));
55113e2b0218SMatthew G. Knepley         PetscCall(PetscDSGetTotalDimension(dsScale[c], &totDimScale[c]));
55123e2b0218SMatthew G. Knepley       }
55133e2b0218SMatthew G. Knepley     }
551407218a29SMatthew G. Knepley   }
55153e9753d6SMatthew G. Knepley   /* 2: Setup geometric data */
55169566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
55179566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
55183e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
55199566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
55203e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
55213e9753d6SMatthew G. Knepley       PetscFE fe;
55223e9753d6SMatthew G. Knepley 
55239566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
55243e9753d6SMatthew G. Knepley       if (fe) {
55259566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
55269566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)quads[f]));
55273e9753d6SMatthew G. Knepley       }
55283e9753d6SMatthew G. Knepley     }
55293e9753d6SMatthew G. Knepley   }
55303e9753d6SMatthew G. Knepley   /* Loop over chunks */
55313e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
55323e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize);
553307218a29SMatthew G. Knepley   PetscCall(PetscCalloc1(2 * cellChunkSize, &faces));
553407218a29SMatthew G. Knepley   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS));
55353e9753d6SMatthew G. Knepley   /* Extract field coefficients */
55363e9753d6SMatthew G. Knepley   /* NOTE This needs the end cap faces to have identical orientations */
553707218a29SMatthew G. Knepley   PetscCall(DMPlexGetHybridCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
553807218a29SMatthew G. Knepley   PetscCall(DMPlexGetHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a));
55393e2b0218SMatthew G. Knepley   PetscCall(DMPlexGetHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s));
554007218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecNeg));
554107218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecPos));
554207218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecCoh));
55433e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
55443e9753d6SMatthew G. Knepley     PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
55453e9753d6SMatthew G. Knepley 
554607218a29SMatthew G. Knepley     PetscCall(PetscArrayzero(elemVecNeg, cellChunkSize * totDim));
554707218a29SMatthew G. Knepley     PetscCall(PetscArrayzero(elemVecPos, cellChunkSize * totDim));
554807218a29SMatthew G. Knepley     PetscCall(PetscArrayzero(elemVecCoh, cellChunkSize * totDim));
55493e9753d6SMatthew G. Knepley     /* Get faces */
55503e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
55513e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
55523e9753d6SMatthew G. Knepley       const PetscInt *cone;
55539566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(dm, cell, &cone));
555407218a29SMatthew G. Knepley       faces[(c - cS) * 2 + 0] = cone[0];
555507218a29SMatthew G. Knepley       faces[(c - cS) * 2 + 1] = cone[1];
55563e9753d6SMatthew G. Knepley     }
555707218a29SMatthew G. Knepley     PetscCall(ISGeneralSetIndices(chunkIS, 2 * cellChunkSize, faces, PETSC_USE_POINTER));
55583e9753d6SMatthew G. Knepley     /* Get geometric data */
55593e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
55609566063dSJacob Faibussowitsch       if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad));
55619566063dSJacob Faibussowitsch       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom));
55623e9753d6SMatthew G. Knepley     } else {
55633e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
55649566063dSJacob Faibussowitsch         if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]));
55653e9753d6SMatthew G. Knepley       }
55663e9753d6SMatthew G. Knepley     }
55673e9753d6SMatthew G. Knepley     /* Loop over fields */
55683e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
55693e9753d6SMatthew G. Knepley       PetscFE         fe;
55703e9753d6SMatthew G. Knepley       PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[f];
5571148442b3SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
55723e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
55733e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
55745fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
55753e9753d6SMatthew G. Knepley 
55769566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
55773e9753d6SMatthew G. Knepley       if (!fe) continue;
55789566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
55799566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
55809566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
55813e9753d6SMatthew G. Knepley       blockSize = Nb;
55823e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
55839566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
55843e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches * batchSize);
55853e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
55863e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches * batchSize);
55873e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
558807218a29SMatthew G. Knepley       PetscCall(PetscFEGeomGetChunk(geom, 0, offset * 2, &chunkGeom));
558907218a29SMatthew G. Knepley       PetscCall(PetscFEGeomGetChunk(geom, offset * 2, numCells * 2, &remGeom));
55909566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, f, &isCohesiveField));
55915fedec97SMatthew G. Knepley       chunkGeom->isCohesive = remGeom->isCohesive = PETSC_TRUE;
55926528b96dSMatthew G. Knepley       key[0].field                                = f;
55936528b96dSMatthew G. Knepley       key[1].field                                = f;
55945fedec97SMatthew G. Knepley       key[2].field                                = f;
559507218a29SMatthew G. Knepley       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, elemVecNeg));
55968e3a54c0SPierre Jolivet       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[0], 0, Nr, remGeom, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[0], PetscSafePointerPlusOffset(a[0], offset * totDimAux[0]), t, &elemVecNeg[offset * totDim]));
559707218a29SMatthew G. Knepley       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, elemVecPos));
55988e3a54c0SPierre Jolivet       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[1], 1, Nr, remGeom, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[1], PetscSafePointerPlusOffset(a[1], offset * totDimAux[1]), t, &elemVecPos[offset * totDim]));
559907218a29SMatthew G. Knepley       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, elemVecCoh));
56008e3a54c0SPierre Jolivet       PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[2], 2, Nr, remGeom, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[2], PetscSafePointerPlusOffset(a[2], offset * totDimAux[2]), t, &elemVecCoh[offset * totDim]));
56019566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &remGeom));
56029566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom, 0, offset, &chunkGeom));
56033e9753d6SMatthew G. Knepley     }
56043e9753d6SMatthew G. Knepley     /* Add elemVec to locX */
56053e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
56063e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
56073e9753d6SMatthew G. Knepley       const PetscInt cind = c - cStart;
560807218a29SMatthew G. Knepley       PetscInt       i;
56093e9753d6SMatthew G. Knepley 
561007218a29SMatthew G. Knepley       /* Scale element values */
561107218a29SMatthew G. Knepley       if (locS[0]) {
56123e2b0218SMatthew G. Knepley         PetscInt  Nb, off = cind * totDim, soff = cind * totDimScale[0];
561307218a29SMatthew G. Knepley         PetscBool cohesive;
561407218a29SMatthew G. Knepley 
561507218a29SMatthew G. Knepley         for (f = 0; f < Nf; ++f) {
561607218a29SMatthew G. Knepley           PetscCall(PetscDSGetFieldSize(ds, f, &Nb));
561707218a29SMatthew G. Knepley           PetscCall(PetscDSGetCohesive(ds, f, &cohesive));
561807218a29SMatthew G. Knepley           if (f == key[2].field) {
561907218a29SMatthew G. Knepley             PetscCheck(cohesive, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Scaling should not happen for face fields");
562007218a29SMatthew G. Knepley             // No cohesive scaling field is currently input
562107218a29SMatthew G. Knepley             for (i = 0; i < Nb; ++i) elemVecCoh[off + i] += s[0][soff + i] * elemVecNeg[off + i] + s[1][soff + i] * elemVecPos[off + i];
562207218a29SMatthew G. Knepley             off += Nb;
562307218a29SMatthew G. Knepley           } else {
562407218a29SMatthew G. Knepley             const PetscInt N = cohesive ? Nb : Nb * 2;
562507218a29SMatthew G. Knepley 
562607218a29SMatthew G. Knepley             for (i = 0; i < N; ++i) elemVecCoh[off + i] += elemVecNeg[off + i] + elemVecPos[off + i];
562707218a29SMatthew G. Knepley             off += N;
562807218a29SMatthew G. Knepley           }
562907218a29SMatthew G. Knepley         }
563007218a29SMatthew G. Knepley       } else {
563107218a29SMatthew G. Knepley         for (i = cind * totDim; i < (cind + 1) * totDim; ++i) elemVecCoh[i] += elemVecNeg[i] + elemVecPos[i];
563207218a29SMatthew G. Knepley       }
563307218a29SMatthew G. Knepley       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVecCoh[cind * totDim]));
56343e9753d6SMatthew G. Knepley       if (ghostLabel) {
56353e9753d6SMatthew G. Knepley         PetscInt ghostVal;
56363e9753d6SMatthew G. Knepley 
56379566063dSJacob Faibussowitsch         PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
56383e9753d6SMatthew G. Knepley         if (ghostVal > 0) continue;
56393e9753d6SMatthew G. Knepley       }
564007218a29SMatthew G. Knepley       PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVecCoh[cind * totDim], ADD_ALL_VALUES));
56413e9753d6SMatthew G. Knepley     }
56423e9753d6SMatthew G. Knepley   }
56439566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
564407218a29SMatthew G. Knepley   PetscCall(DMPlexRestoreHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a));
56453e2b0218SMatthew G. Knepley   PetscCall(DMPlexRestoreHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s));
564607218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecNeg));
564707218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecPos));
564807218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecCoh));
56499566063dSJacob Faibussowitsch   PetscCall(PetscFree(faces));
56509566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&chunkIS));
56519566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
56523e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
56539566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
56549566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&affineQuad));
56553e9753d6SMatthew G. Knepley   } else {
56563e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
56579566063dSJacob Faibussowitsch       if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
56589566063dSJacob Faibussowitsch       if (quads) PetscCall(PetscQuadratureDestroy(&quads[f]));
56593e9753d6SMatthew G. Knepley     }
56609566063dSJacob Faibussowitsch     PetscCall(PetscFree2(quads, geoms));
56613e9753d6SMatthew G. Knepley   }
566285cc2951SMatthew G. Knepley   if (mesh->printFEM) {
566385cc2951SMatthew G. Knepley     Vec          locFbc;
566485cc2951SMatthew G. Knepley     PetscInt     pStart, pEnd, p, maxDof;
566585cc2951SMatthew G. Knepley     PetscScalar *zeroes;
566685cc2951SMatthew G. Knepley 
566785cc2951SMatthew G. Knepley     PetscCall(VecDuplicate(locF, &locFbc));
566885cc2951SMatthew G. Knepley     PetscCall(VecCopy(locF, locFbc));
566985cc2951SMatthew G. Knepley     PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
567085cc2951SMatthew G. Knepley     PetscCall(PetscSectionGetMaxDof(section, &maxDof));
567185cc2951SMatthew G. Knepley     PetscCall(PetscCalloc1(maxDof, &zeroes));
567285cc2951SMatthew G. Knepley     for (p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, section, p, zeroes, INSERT_BC_VALUES));
567385cc2951SMatthew G. Knepley     PetscCall(PetscFree(zeroes));
567485cc2951SMatthew G. Knepley     PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc));
567585cc2951SMatthew G. Knepley     PetscCall(VecDestroy(&locFbc));
567685cc2951SMatthew G. Knepley   }
56773cc88e6aSStefano Zampini end:
56789566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0));
56793ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
56803e9753d6SMatthew G. Knepley }
56813e9753d6SMatthew G. Knepley 
5682a4e35b19SJacob Faibussowitsch static PetscErrorCode DMPlexComputeBdJacobian_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP, DMField coordField, IS facetIS)
5683d71ae5a4SJacob Faibussowitsch {
56843e9753d6SMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *)dm->data;
56853e9753d6SMatthew G. Knepley   DM              plex = NULL, plexA = NULL, tdm;
56863e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
5687be62b2b4SMatthew G. Knepley   PetscDS         ds, dsAux           = NULL;
56883e9753d6SMatthew G. Knepley   PetscSection    section, sectionAux = NULL;
5689e432b41dSStefano Zampini   PetscSection    globalSection;
56903e9753d6SMatthew G. Knepley   Vec             locA = NULL, tv;
5691be62b2b4SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL;
56923e9753d6SMatthew G. Knepley   PetscInt        v;
56933e9753d6SMatthew G. Knepley   PetscInt        Nf, totDim, totDimAux = 0;
5694be62b2b4SMatthew G. Knepley   PetscBool       hasJac = PETSC_FALSE, hasPrec = PETSC_FALSE, transform;
56953e9753d6SMatthew G. Knepley 
56963e9753d6SMatthew G. Knepley   PetscFunctionBegin;
56979566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
56989566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
56999566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
57009566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
5701be62b2b4SMatthew G. Knepley   PetscCall(DMGetDS(dm, &ds));
5702be62b2b4SMatthew G. Knepley   PetscCall(PetscDSGetNumFields(ds, &Nf));
5703be62b2b4SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
570421a4fad8SMatthew G. Knepley   PetscCall(PetscWeakFormHasBdJacobian(wf, &hasJac));
570521a4fad8SMatthew G. Knepley   PetscCall(PetscWeakFormHasBdJacobianPreconditioner(wf, &hasPrec));
570621a4fad8SMatthew G. Knepley   if (!hasJac && !hasPrec) PetscFunctionReturn(PETSC_SUCCESS);
570721a4fad8SMatthew G. Knepley   PetscCall(DMConvert(dm, DMPLEX, &plex));
57089566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, label, values[0], 0, &locA));
57093e9753d6SMatthew G. Knepley   if (locA) {
57103e9753d6SMatthew G. Knepley     DM dmAux;
57113e9753d6SMatthew G. Knepley 
57129566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
57139566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
57149566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
5715be62b2b4SMatthew G. Knepley     PetscCall(DMGetDS(plexA, &dsAux));
5716be62b2b4SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux));
57179566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
57183e9753d6SMatthew G. Knepley   }
57193e9753d6SMatthew G. Knepley 
57209566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
57213e9753d6SMatthew G. Knepley   for (v = 0; v < numValues; ++v) {
57223e9753d6SMatthew G. Knepley     PetscFEGeom    *fgeom;
57233e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
57243e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
57253e9753d6SMatthew G. Knepley     IS              pointIS;
57263e9753d6SMatthew G. Knepley     const PetscInt *points;
572706ad1575SMatthew G. Knepley     PetscFormKey    key;
57283e9753d6SMatthew G. Knepley     PetscInt        numFaces, face, Nq;
57293e9753d6SMatthew G. Knepley 
573045480ffeSMatthew G. Knepley     key.label = label;
573145480ffeSMatthew G. Knepley     key.value = values[v];
573206ad1575SMatthew G. Knepley     key.part  = 0;
57339566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
57343e9753d6SMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
57353e9753d6SMatthew G. Knepley     {
57363e9753d6SMatthew G. Knepley       IS isectIS;
57373e9753d6SMatthew G. Knepley 
57383e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a prior if one is a superset of the other */
57399566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
57409566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
57413e9753d6SMatthew G. Knepley       pointIS = isectIS;
57423e9753d6SMatthew G. Knepley     }
57439566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
57449566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
574532603206SJames Wright     PetscCall(PetscMalloc5(numFaces * totDim, &u, (locX_t ? (size_t)numFaces * totDim : 0), &u_t, (hasJac ? (size_t)numFaces * totDim * totDim : 0), &elemMat, (hasPrec ? (size_t)numFaces * totDim * totDim : 0), &elemMatP, (locA ? (size_t)numFaces * totDimAux : 0), &a));
57469566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
574748a46eb9SPierre Jolivet     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
57483e9753d6SMatthew G. Knepley     if (!qGeom) {
57493e9753d6SMatthew G. Knepley       PetscFE fe;
57503e9753d6SMatthew G. Knepley 
5751be62b2b4SMatthew G. Knepley       PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&fe));
57529566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
57539566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
57543e9753d6SMatthew G. Knepley     }
57559566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
57569566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
57573e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
5758f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
57593e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
5760f15274beSMatthew Knepley       PetscInt       i;
57613e9753d6SMatthew G. Knepley 
57629566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
57639566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
57643e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i];
57659566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
57663e9753d6SMatthew G. Knepley       if (locX_t) {
57679566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
57683e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i];
57699566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
57703e9753d6SMatthew G. Knepley       }
57713e9753d6SMatthew G. Knepley       if (locA) {
57723e9753d6SMatthew G. Knepley         PetscInt subp;
57739566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
57749566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
57753e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i];
57769566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
57773e9753d6SMatthew G. Knepley       }
57783e9753d6SMatthew G. Knepley     }
577921a4fad8SMatthew G. Knepley     if (elemMat) PetscCall(PetscArrayzero(elemMat, numFaces * totDim * totDim));
578021a4fad8SMatthew G. Knepley     if (elemMatP) PetscCall(PetscArrayzero(elemMatP, numFaces * totDim * totDim));
57813e9753d6SMatthew G. Knepley     {
57823e9753d6SMatthew G. Knepley       PetscFE  fe;
57833e9753d6SMatthew G. Knepley       PetscInt Nb;
57843e9753d6SMatthew G. Knepley       /* Conforming batches */
57853e9753d6SMatthew G. Knepley       PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
57863e9753d6SMatthew G. Knepley       /* Remainder */
57873e9753d6SMatthew G. Knepley       PetscFEGeom *chunkGeom = NULL;
57883e9753d6SMatthew G. Knepley       PetscInt     fieldJ, Nr, offset;
57893e9753d6SMatthew G. Knepley 
5790be62b2b4SMatthew G. Knepley       PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&fe));
57919566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
57929566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
57933e9753d6SMatthew G. Knepley       blockSize = Nb;
57943e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
57959566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
57963e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches * batchSize);
57973e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
57983e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches * batchSize);
57993e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
58009566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom));
58013e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
580245480ffeSMatthew G. Knepley         key.field = fieldI * Nf + fieldJ;
5803be62b2b4SMatthew G. Knepley         if (hasJac) PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, dsAux, a, t, X_tShift, elemMat));
5804be62b2b4SMatthew G. Knepley         if (hasPrec) PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, dsAux, a, t, X_tShift, elemMatP));
58053e9753d6SMatthew G. Knepley       }
58069566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
58073e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
580845480ffeSMatthew G. Knepley         key.field = fieldI * Nf + fieldJ;
5809be62b2b4SMatthew G. Knepley         if (hasJac)
58108e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * totDim]));
5811be62b2b4SMatthew G. Knepley         if (hasPrec)
58128e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN_PRE, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatP[offset * totDim * totDim]));
58133e9753d6SMatthew G. Knepley       }
58149566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
58153e9753d6SMatthew G. Knepley     }
58163e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
58173e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
58183e9753d6SMatthew G. Knepley 
58193e9753d6SMatthew G. Knepley       /* Transform to global basis before insertion in Jacobian */
58209566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
582121a4fad8SMatthew G. Knepley       if (hasJac && transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMat[face * totDim * totDim]));
582221a4fad8SMatthew G. Knepley       if (hasPrec && transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMatP[face * totDim * totDim]));
5823be62b2b4SMatthew G. Knepley       if (hasPrec) {
5824be62b2b4SMatthew G. Knepley         if (hasJac) {
58259566063dSJacob Faibussowitsch           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face * totDim * totDim]));
582621a4fad8SMatthew G. Knepley           PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, Jac, support[0], &elemMat[face * totDim * totDim], ADD_VALUES));
58273e9753d6SMatthew G. Knepley         }
5828be62b2b4SMatthew G. Knepley         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMatP[face * totDim * totDim]));
5829be62b2b4SMatthew G. Knepley         PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, JacP, support[0], &elemMatP[face * totDim * totDim], ADD_VALUES));
5830be62b2b4SMatthew G. Knepley       } else {
5831be62b2b4SMatthew G. Knepley         if (hasJac) {
5832be62b2b4SMatthew G. Knepley           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face * totDim * totDim]));
583321a4fad8SMatthew G. Knepley           PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, Jac, support[0], &elemMat[face * totDim * totDim], ADD_VALUES));
5834be62b2b4SMatthew G. Knepley         }
5835be62b2b4SMatthew G. Knepley       }
5836be62b2b4SMatthew G. Knepley     }
58379566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
58389566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
58399566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
58409566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
5841be62b2b4SMatthew G. Knepley     PetscCall(PetscFree5(u, u_t, elemMat, elemMatP, a));
58423e9753d6SMatthew G. Knepley   }
58439566063dSJacob Faibussowitsch   if (plex) PetscCall(DMDestroy(&plex));
58449566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
58453ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
58463e9753d6SMatthew G. Knepley }
58473e9753d6SMatthew G. Knepley 
5848d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeBdJacobianSingle(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP)
5849d71ae5a4SJacob Faibussowitsch {
58503e9753d6SMatthew G. Knepley   DMField  coordField;
58513e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
58523e9753d6SMatthew G. Knepley   IS       facetIS;
58533e9753d6SMatthew G. Knepley   PetscInt dim;
58543e9753d6SMatthew G. Knepley 
58553e9753d6SMatthew G. Knepley   PetscFunctionBegin;
58569566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
58579566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
58589566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
58599566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
58609566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, field, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS));
58619566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
58623ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
58633e9753d6SMatthew G. Knepley }
58643e9753d6SMatthew G. Knepley 
5865a4e35b19SJacob Faibussowitsch static PetscErrorCode DMPlexComputeBdJacobian_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP, void *user)
5866d71ae5a4SJacob Faibussowitsch {
58673e9753d6SMatthew G. Knepley   PetscDS  prob;
58683e9753d6SMatthew G. Knepley   PetscInt dim, numBd, bd;
58693e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
58703e9753d6SMatthew G. Knepley   DMField  coordField = NULL;
58713e9753d6SMatthew G. Knepley   IS       facetIS;
58723e9753d6SMatthew G. Knepley 
58733e9753d6SMatthew G. Knepley   PetscFunctionBegin;
58749566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
58759566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
58769566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
58779566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
58789566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
58799566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
58803e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
588145480ffeSMatthew G. Knepley     PetscWeakForm           wf;
58823e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
58833e9753d6SMatthew G. Knepley     DMLabel                 label;
58843e9753d6SMatthew G. Knepley     const PetscInt         *values;
58853e9753d6SMatthew G. Knepley     PetscInt                fieldI, numValues;
58863e9753d6SMatthew G. Knepley     PetscObject             obj;
58873e9753d6SMatthew G. Knepley     PetscClassId            id;
58883e9753d6SMatthew G. Knepley 
58899566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &fieldI, NULL, NULL, NULL, NULL, NULL));
58903d3e5d66SMatthew G. Knepley     if (type & DM_BC_ESSENTIAL) continue;
58919566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &obj));
58929566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
58933d3e5d66SMatthew G. Knepley     if (id != PETSCFE_CLASSID) continue;
58949566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, fieldI, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS));
58953e9753d6SMatthew G. Knepley   }
58969566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
58973ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
58983e9753d6SMatthew G. Knepley }
58993e9753d6SMatthew G. Knepley 
5900d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeJacobian_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *user)
5901d71ae5a4SJacob Faibussowitsch {
59023e9753d6SMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *)dm->data;
59033e9753d6SMatthew G. Knepley   const char     *name  = "Jacobian";
59049a2a23afSMatthew G. Knepley   DM              dmAux = NULL, plex, tdm;
59053e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
59063e9753d6SMatthew G. Knepley   Vec             A, tv;
59073e9753d6SMatthew G. Knepley   DMField         coordField;
59083e9753d6SMatthew G. Knepley   PetscDS         prob, probAux = NULL;
5909e432b41dSStefano Zampini   PetscSection    section, globalSection, sectionAux;
59103e9753d6SMatthew G. Knepley   PetscScalar    *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL;
59113e9753d6SMatthew G. Knepley   const PetscInt *cells;
59123e9753d6SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
591328351e22SJed Brown   PetscInt        totDim, totDimAux = 0, cStart, cEnd, numCells, c;
5914e04ae0b4SMatthew G. Knepley   PetscBool       hasJac = PETSC_FALSE, hasPrec = PETSC_FALSE, hasDyn, hasFV = PETSC_FALSE, transform;
59153e9753d6SMatthew G. Knepley 
59163e9753d6SMatthew G. Knepley   PetscFunctionBegin;
5917af18b51aSmarkadams4   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
5918ae3cf2c1SStefano Zampini   PetscCall(DMGetLocalSection(dm, &section));
5919ae3cf2c1SStefano Zampini   PetscCall(DMGetGlobalSection(dm, &globalSection));
5920ae3cf2c1SStefano Zampini   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
5921ae3cf2c1SStefano Zampini   if (A) {
5922ae3cf2c1SStefano Zampini     PetscCall(VecGetDM(A, &dmAux));
5923ae3cf2c1SStefano Zampini     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
5924ae3cf2c1SStefano Zampini     PetscCall(DMConvert(dmAux, DMPLEX, &plex));
5925ae3cf2c1SStefano Zampini     PetscCall(DMGetLocalSection(plex, &sectionAux));
5926ae3cf2c1SStefano Zampini     PetscCall(DMGetDS(dmAux, &probAux));
5927ae3cf2c1SStefano Zampini     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
5928ae3cf2c1SStefano Zampini   }
5929ae3cf2c1SStefano Zampini   PetscCall(DMGetCoordinateField(dm, &coordField));
5930e04ae0b4SMatthew G. Knepley   if (!cellIS) goto end;
59319566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
5932437e83fbSMatthew G. Knepley   PetscCall(ISGetLocalSize(cellIS, &numCells));
5933437e83fbSMatthew G. Knepley   if (cStart >= cEnd) goto end;
59349566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
59359566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
59369566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
593707218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL));
59389566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
59399566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
59409566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
59419566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
59423e9753d6SMatthew G. Knepley   /* user passed in the same matrix, avoid double contributions and
59433e9753d6SMatthew G. Knepley      only assemble the Jacobian */
59443e9753d6SMatthew G. Knepley   if (hasJac && Jac == JacP) hasPrec = PETSC_FALSE;
59459566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
59463e9753d6SMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
594732603206SJames Wright   PetscCall(PetscMalloc5(numCells * totDim, &u, (X_t ? (size_t)numCells * totDim : 0), &u_t, (hasJac ? (size_t)numCells * totDim * totDim : 0), &elemMat, (hasPrec ? (size_t)numCells * totDim * totDim : 0), &elemMatP, (hasDyn ? (size_t)numCells * totDim * totDim : 0), &elemMatD));
59489566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
59493e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
59503e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
59513e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
59523e9753d6SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL;
59533e9753d6SMatthew G. Knepley     PetscInt       i;
59543e9753d6SMatthew G. Knepley 
59559566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x));
59563e9753d6SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i];
59579566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x));
59583e9753d6SMatthew G. Knepley     if (X_t) {
59599566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t));
59603e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i];
59619566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t));
59623e9753d6SMatthew G. Knepley     }
59633e9753d6SMatthew G. Knepley     if (dmAux) {
59643e9753d6SMatthew G. Knepley       PetscInt subcell;
59659566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
59669566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x));
59673e9753d6SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i];
59689566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x));
59693e9753d6SMatthew G. Knepley     }
59703e9753d6SMatthew G. Knepley   }
59719566063dSJacob Faibussowitsch   if (hasJac) PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim));
59729566063dSJacob Faibussowitsch   if (hasPrec) PetscCall(PetscArrayzero(elemMatP, numCells * totDim * totDim));
59739566063dSJacob Faibussowitsch   if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim));
59743e9753d6SMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
59753e9753d6SMatthew G. Knepley     PetscClassId    id;
59763e9753d6SMatthew G. Knepley     PetscFE         fe;
59773e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
59783e9753d6SMatthew G. Knepley     PetscInt        Nb;
59793e9753d6SMatthew G. Knepley     /* Conforming batches */
59803e9753d6SMatthew G. Knepley     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
59813e9753d6SMatthew G. Knepley     /* Remainder */
59823e9753d6SMatthew G. Knepley     PetscInt     Nr, offset, Nq;
59833e9753d6SMatthew G. Knepley     PetscInt     maxDegree;
59843e9753d6SMatthew G. Knepley     PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
59853e9753d6SMatthew G. Knepley 
59869566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
59879566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
59889371c9d4SSatish Balay     if (id == PETSCFV_CLASSID) {
59899371c9d4SSatish Balay       hasFV = PETSC_TRUE;
59909371c9d4SSatish Balay       continue;
59919371c9d4SSatish Balay     }
59929566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
59939566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
59949566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
599548a46eb9SPierre Jolivet     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
59963e9753d6SMatthew G. Knepley     if (!qGeom) {
59979566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &qGeom));
59989566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
59993e9753d6SMatthew G. Knepley     }
60009566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
60019566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
60023e9753d6SMatthew G. Knepley     blockSize = Nb;
60033e9753d6SMatthew G. Knepley     batchSize = numBlocks * blockSize;
60049566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
60053e9753d6SMatthew G. Knepley     numChunks = numCells / (numBatches * batchSize);
60063e9753d6SMatthew G. Knepley     Ne        = numChunks * numBatches * batchSize;
60073e9753d6SMatthew G. Knepley     Nr        = numCells % (numBatches * batchSize);
60083e9753d6SMatthew G. Knepley     offset    = numCells - Nr;
60099566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
60109566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom));
60113e9753d6SMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
60126528b96dSMatthew G. Knepley       key.field = fieldI * Nf + fieldJ;
60133e9753d6SMatthew G. Knepley       if (hasJac) {
60149566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
60158e3a54c0SPierre Jolivet         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * totDim]));
60163e9753d6SMatthew G. Knepley       }
60173e9753d6SMatthew G. Knepley       if (hasPrec) {
60189566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatP));
60198e3a54c0SPierre Jolivet         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatP[offset * totDim * totDim]));
60203e9753d6SMatthew G. Knepley       }
60213e9753d6SMatthew G. Knepley       if (hasDyn) {
60229566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
60238e3a54c0SPierre Jolivet         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatD[offset * totDim * totDim]));
60243e9753d6SMatthew G. Knepley       }
60253e9753d6SMatthew G. Knepley     }
60269566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom));
60279566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom));
60289566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
60299566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
60303e9753d6SMatthew G. Knepley   }
60313e9753d6SMatthew G. Knepley   /*   Add contribution from X_t */
60329371c9d4SSatish Balay   if (hasDyn) {
60339371c9d4SSatish Balay     for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
60349371c9d4SSatish Balay   }
60353e9753d6SMatthew G. Knepley   if (hasFV) {
60363e9753d6SMatthew G. Knepley     PetscClassId id;
60373e9753d6SMatthew G. Knepley     PetscFV      fv;
60383e9753d6SMatthew G. Knepley     PetscInt     offsetI, NcI, NbI = 1, fc, f;
60393e9753d6SMatthew G. Knepley 
60403e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
60419566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv));
60429566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, fieldI, &offsetI));
60439566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId((PetscObject)fv, &id));
60443e9753d6SMatthew G. Knepley       if (id != PETSCFV_CLASSID) continue;
60452f86f8c5SMatthew G. Knepley       /* Put in the weighted identity */
60469566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &NcI));
60473e9753d6SMatthew G. Knepley       for (c = cStart; c < cEnd; ++c) {
60483e9753d6SMatthew G. Knepley         const PetscInt cind    = c - cStart;
60493e9753d6SMatthew G. Knepley         const PetscInt eOffset = cind * totDim * totDim;
60502f86f8c5SMatthew G. Knepley         PetscReal      vol;
60512f86f8c5SMatthew G. Knepley 
60522f86f8c5SMatthew G. Knepley         PetscCall(DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL));
60533e9753d6SMatthew G. Knepley         for (fc = 0; fc < NcI; ++fc) {
60543e9753d6SMatthew G. Knepley           for (f = 0; f < NbI; ++f) {
60553e9753d6SMatthew G. Knepley             const PetscInt i = offsetI + f * NcI + fc;
60563e9753d6SMatthew G. Knepley             if (hasPrec) {
60572f86f8c5SMatthew G. Knepley               if (hasJac) elemMat[eOffset + i * totDim + i] = vol;
60582f86f8c5SMatthew G. Knepley               elemMatP[eOffset + i * totDim + i] = vol;
60599371c9d4SSatish Balay             } else {
60602f86f8c5SMatthew G. Knepley               elemMat[eOffset + i * totDim + i] = vol;
60619371c9d4SSatish Balay             }
60623e9753d6SMatthew G. Knepley           }
60633e9753d6SMatthew G. Knepley         }
60643e9753d6SMatthew G. Knepley       }
60653e9753d6SMatthew G. Knepley     }
60663e9753d6SMatthew G. Knepley     /* No allocated space for FV stuff, so ignore the zero entries */
60679566063dSJacob Faibussowitsch     PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE));
60683e9753d6SMatthew G. Knepley   }
60693e9753d6SMatthew G. Knepley   /* Insert values into matrix */
60703e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
60713e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
60723e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
60733e9753d6SMatthew G. Knepley 
60743e9753d6SMatthew G. Knepley     /* Transform to global basis before insertion in Jacobian */
60759566063dSJacob Faibussowitsch     if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, cell, PETSC_TRUE, totDim, &elemMat[cind * totDim * totDim]));
60763e9753d6SMatthew G. Knepley     if (hasPrec) {
60773e9753d6SMatthew G. Knepley       if (hasJac) {
60789566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
6079e8e188d2SZach Atkins         PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, Jac, cell, &elemMat[cind * totDim * totDim], ADD_VALUES));
60803e9753d6SMatthew G. Knepley       }
60819566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind * totDim * totDim]));
6082be62b2b4SMatthew G. Knepley       PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, JacP, cell, &elemMatP[cind * totDim * totDim], ADD_VALUES));
60833e9753d6SMatthew G. Knepley     } else {
60843e9753d6SMatthew G. Knepley       if (hasJac) {
60859566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
6086e8e188d2SZach Atkins         PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, JacP, cell, &elemMat[cind * totDim * totDim], ADD_VALUES));
60873e9753d6SMatthew G. Knepley       }
60883e9753d6SMatthew G. Knepley     }
60893e9753d6SMatthew G. Knepley   }
60909566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
60919566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
60929566063dSJacob Faibussowitsch   PetscCall(PetscFree5(u, u_t, elemMat, elemMatP, elemMatD));
60933e9753d6SMatthew G. Knepley   if (dmAux) {
60949566063dSJacob Faibussowitsch     PetscCall(PetscFree(a));
60959566063dSJacob Faibussowitsch     PetscCall(DMDestroy(&plex));
60963e9753d6SMatthew G. Knepley   }
60973e9753d6SMatthew G. Knepley   /* Compute boundary integrals */
60989566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, user));
60993e9753d6SMatthew G. Knepley   /* Assemble matrix */
61009371c9d4SSatish Balay end: {
6101e04ae0b4SMatthew G. Knepley   PetscBool assOp = hasJac && hasPrec ? PETSC_TRUE : PETSC_FALSE, gassOp;
6102e04ae0b4SMatthew G. Knepley 
6103462c564dSBarry Smith   PetscCallMPI(MPIU_Allreduce(&assOp, &gassOp, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
61043e9753d6SMatthew G. Knepley   if (hasJac && hasPrec) {
61059566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
61069566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
61073e9753d6SMatthew G. Knepley   }
6108e04ae0b4SMatthew G. Knepley }
61099566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
61109566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
61119566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
61123ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
61133e9753d6SMatthew G. Knepley }
61143e9753d6SMatthew G. Knepley 
6115d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeJacobian_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user)
6116d71ae5a4SJacob Faibussowitsch {
61173e9753d6SMatthew G. Knepley   DM_Plex        *mesh          = (DM_Plex *)dm->data;
61183e9753d6SMatthew G. Knepley   const char     *name          = "Hybrid Jacobian";
6119148442b3SMatthew G. Knepley   DM              dmAux[3]      = {NULL, NULL, NULL};
6120148442b3SMatthew G. Knepley   DMLabel         ghostLabel    = NULL;
61213e9753d6SMatthew G. Knepley   DM              plex          = NULL;
61223e9753d6SMatthew G. Knepley   DM              plexA         = NULL;
6123148442b3SMatthew G. Knepley   PetscDS         ds            = NULL;
612407218a29SMatthew G. Knepley   PetscDS         dsIn          = NULL;
6125148442b3SMatthew G. Knepley   PetscDS         dsAux[3]      = {NULL, NULL, NULL};
6126148442b3SMatthew G. Knepley   Vec             locA[3]       = {NULL, NULL, NULL};
612707218a29SMatthew G. Knepley   DM              dmScale[3]    = {NULL, NULL, NULL};
612807218a29SMatthew G. Knepley   PetscDS         dsScale[3]    = {NULL, NULL, NULL};
612907218a29SMatthew G. Knepley   Vec             locS[3]       = {NULL, NULL, NULL};
61303e9753d6SMatthew G. Knepley   PetscSection    section       = NULL;
6131148442b3SMatthew G. Knepley   PetscSection    sectionAux[3] = {NULL, NULL, NULL};
61323e9753d6SMatthew G. Knepley   DMField         coordField    = NULL;
613307218a29SMatthew G. Knepley   PetscScalar    *a[3]          = {NULL, NULL, NULL};
613407218a29SMatthew G. Knepley   PetscScalar    *s[3]          = {NULL, NULL, NULL};
613507218a29SMatthew G. Knepley   PetscScalar    *u             = NULL, *u_t;
613607218a29SMatthew G. Knepley   PetscScalar    *elemMatNeg, *elemMatPos, *elemMatCoh;
613707218a29SMatthew G. Knepley   PetscScalar    *elemMatNegP, *elemMatPosP, *elemMatCohP;
6138e432b41dSStefano Zampini   PetscSection    globalSection;
61393e9753d6SMatthew G. Knepley   IS              chunkIS;
61403e9753d6SMatthew G. Knepley   const PetscInt *cells;
61413e9753d6SMatthew G. Knepley   PetscInt       *faces;
61423e9753d6SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
61433e2b0218SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ, totDim, totDimIn, totDimAux[3], totDimScale[3], numChunks, cellChunkSize, chunk;
61441690c2aeSBarry Smith   PetscInt        maxDegree  = PETSC_INT_MAX;
61453e9753d6SMatthew G. Knepley   PetscQuadrature affineQuad = NULL, *quads = NULL;
61463e9753d6SMatthew G. Knepley   PetscFEGeom    *affineGeom = NULL, **geoms = NULL;
6147e432b41dSStefano Zampini   PetscBool       hasBdJac, hasBdPrec;
61483e9753d6SMatthew G. Knepley 
61493e9753d6SMatthew G. Knepley   PetscFunctionBegin;
61503cc88e6aSStefano Zampini   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
61513cc88e6aSStefano Zampini   if (!cellIS) goto end;
6152437e83fbSMatthew G. Knepley   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
6153437e83fbSMatthew G. Knepley   PetscCall(ISGetLocalSize(cellIS, &numCells));
61543cc88e6aSStefano Zampini   if (cStart >= cEnd) goto end;
61555fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
61565fedec97SMatthew G. Knepley     const char *name;
61579566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name));
615863a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
61595fedec97SMatthew G. Knepley   }
61609566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
6161*b98a7184SJames Wright   PetscCall(DMGetLocalSection(dm, &section));
61629566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
61639566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
616407218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn));
61659566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
61669566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
616707218a29SMatthew G. Knepley   PetscCall(PetscDSGetTotalDimension(dsIn, &totDimIn));
61689566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobian(ds, &hasBdJac));
61699566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobianPreconditioner(ds, &hasBdPrec));
61709566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
6171148442b3SMatthew G. Knepley   if (locA[2]) {
61721059d808SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
61731059d808SMatthew G. Knepley 
61749566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
61759566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux[2], DMPLEX, &plexA));
6176*b98a7184SJames Wright     PetscCall(DMGetLocalSection(dmAux[2], &sectionAux[2]));
617707218a29SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2], NULL));
61789566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
6179148442b3SMatthew G. Knepley     {
6180148442b3SMatthew G. Knepley       const PetscInt *cone;
6181148442b3SMatthew G. Knepley       PetscInt        c;
6182148442b3SMatthew G. Knepley 
61831059d808SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
6184148442b3SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
6185148442b3SMatthew G. Knepley         const PetscInt *support;
6186148442b3SMatthew G. Knepley         PetscInt        ssize, s;
6187148442b3SMatthew G. Knepley 
61889566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
61899566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
61901059d808SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
61911059d808SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
61921059d808SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
61931059d808SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
61949566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
61959566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
6196ad540459SPierre Jolivet         else dmAux[c] = dmAux[2];
619707218a29SMatthew G. Knepley         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c], NULL));
61989566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
6199148442b3SMatthew G. Knepley       }
6200148442b3SMatthew G. Knepley     }
62013e9753d6SMatthew G. Knepley   }
620207218a29SMatthew G. Knepley   /* Handle mass matrix scaling
620307218a29SMatthew G. Knepley        The field in key[2] is the field to be scaled, and the scaling field is the first in the dsScale */
620407218a29SMatthew G. Knepley   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, -key[2].value, key[2].part, &locS[2]));
620507218a29SMatthew G. Knepley   if (locS[2]) {
62063e2b0218SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
620707218a29SMatthew G. Knepley     PetscInt       Nb, Nbs;
620807218a29SMatthew G. Knepley 
620907218a29SMatthew G. Knepley     PetscCall(VecGetDM(locS[2], &dmScale[2]));
621007218a29SMatthew G. Knepley     PetscCall(DMGetCellDS(dmScale[2], cells ? cells[cStart] : cStart, &dsScale[2], NULL));
62113e2b0218SMatthew G. Knepley     PetscCall(PetscDSGetTotalDimension(dsScale[2], &totDimScale[2]));
621207218a29SMatthew G. Knepley     // BRAD: This is not set correctly
621307218a29SMatthew G. Knepley     key[2].field = 2;
621407218a29SMatthew G. Knepley     PetscCall(PetscDSGetFieldSize(ds, key[2].field, &Nb));
621507218a29SMatthew G. Knepley     PetscCall(PetscDSGetFieldSize(dsScale[2], 0, &Nbs));
621607218a29SMatthew G. Knepley     PetscCheck(Nb == Nbs, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Field %" PetscInt_FMT " of size %" PetscInt_FMT " cannot be scaled by field of size %" PetscInt_FMT, key[2].field, Nb, Nbs);
62173e2b0218SMatthew G. Knepley     {
62183e2b0218SMatthew G. Knepley       const PetscInt *cone;
62193e2b0218SMatthew G. Knepley       PetscInt        c;
62203e2b0218SMatthew G. Knepley 
62213e2b0218SMatthew G. Knepley       locS[1] = locS[0] = locS[2];
62223e2b0218SMatthew G. Knepley       dmScale[1] = dmScale[0] = dmScale[2];
62233e2b0218SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
62243e2b0218SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
62253e2b0218SMatthew G. Knepley         const PetscInt *support;
62263e2b0218SMatthew G. Knepley         PetscInt        ssize, s;
62273e2b0218SMatthew G. Knepley 
62283e2b0218SMatthew G. Knepley         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
62293e2b0218SMatthew G. Knepley         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
62303e2b0218SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
62313e2b0218SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
62323e2b0218SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
62333e2b0218SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
62343e2b0218SMatthew G. Knepley         PetscCall(DMGetCellDS(dmScale[c], support[s], &dsScale[c], NULL));
62353e2b0218SMatthew G. Knepley         PetscCall(PetscDSGetTotalDimension(dsScale[c], &totDimScale[c]));
62363e2b0218SMatthew G. Knepley       }
62373e2b0218SMatthew G. Knepley     }
623807218a29SMatthew G. Knepley   }
623907218a29SMatthew G. Knepley   /* 2: Setup geometric data */
62409566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
62419566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
62423e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
62433e9753d6SMatthew G. Knepley     PetscInt f;
62449566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
62453e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
62463e9753d6SMatthew G. Knepley       PetscFE fe;
62473e9753d6SMatthew G. Knepley 
62489566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
62493e9753d6SMatthew G. Knepley       if (fe) {
62509566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
62519566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)quads[f]));
62523e9753d6SMatthew G. Knepley       }
62533e9753d6SMatthew G. Knepley     }
62543e9753d6SMatthew G. Knepley   }
625507218a29SMatthew G. Knepley   /* Loop over chunks */
62563e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
62573e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize);
625807218a29SMatthew G. Knepley   PetscCall(PetscCalloc1(2 * cellChunkSize, &faces));
6259a4158a15SMatthew G. Knepley   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 1 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS));
626007218a29SMatthew G. Knepley   /* Extract field coefficients */
626107218a29SMatthew G. Knepley   /* NOTE This needs the end cap faces to have identical orientations */
626207218a29SMatthew G. Knepley   PetscCall(DMPlexGetHybridCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
626307218a29SMatthew G. Knepley   PetscCall(DMPlexGetHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a));
62648bf0a22dSMatthew G. Knepley   PetscCall(DMPlexGetHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s));
626507218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNeg));
626607218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPos));
626707218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCoh));
626807218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNegP));
626907218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPosP));
627007218a29SMatthew G. Knepley   PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCohP));
62713e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
62723e9753d6SMatthew G. Knepley     PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
62733e9753d6SMatthew G. Knepley 
627407218a29SMatthew G. Knepley     if (hasBdJac) {
627507218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatNeg, cellChunkSize * totDim * totDim));
627607218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatPos, cellChunkSize * totDim * totDim));
627707218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatCoh, cellChunkSize * totDim * totDim));
627807218a29SMatthew G. Knepley     }
627907218a29SMatthew G. Knepley     if (hasBdPrec) {
628007218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatNegP, cellChunkSize * totDim * totDim));
628107218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatPosP, cellChunkSize * totDim * totDim));
628207218a29SMatthew G. Knepley       PetscCall(PetscArrayzero(elemMatCohP, cellChunkSize * totDim * totDim));
628307218a29SMatthew G. Knepley     }
62843e9753d6SMatthew G. Knepley     /* Get faces */
62853e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
62863e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
62873e9753d6SMatthew G. Knepley       const PetscInt *cone;
62889566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(plex, cell, &cone));
628907218a29SMatthew G. Knepley       faces[(c - cS) * 2 + 0] = cone[0];
629007218a29SMatthew G. Knepley       faces[(c - cS) * 2 + 1] = cone[1];
62913e9753d6SMatthew G. Knepley     }
629207218a29SMatthew G. Knepley     PetscCall(ISGeneralSetIndices(chunkIS, 2 * cellChunkSize, faces, PETSC_USE_POINTER));
62933e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
62949566063dSJacob Faibussowitsch       if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad));
62959566063dSJacob Faibussowitsch       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom));
62963e9753d6SMatthew G. Knepley     } else {
62973e9753d6SMatthew G. Knepley       PetscInt f;
62983e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
62999566063dSJacob Faibussowitsch         if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]));
63003e9753d6SMatthew G. Knepley       }
63013e9753d6SMatthew G. Knepley     }
63023e9753d6SMatthew G. Knepley 
63033e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
63043e9753d6SMatthew G. Knepley       PetscFE         feI;
63053e9753d6SMatthew G. Knepley       PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[fieldI];
63063e9753d6SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
63073e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[fieldI];
63083e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
63095fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
63103e9753d6SMatthew G. Knepley 
63119566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&feI));
63123e9753d6SMatthew G. Knepley       if (!feI) continue;
63139566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(feI, NULL, &numBlocks, NULL, &numBatches));
63149566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
63159566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feI, &Nb));
63163e9753d6SMatthew G. Knepley       blockSize = Nb;
63173e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
63189566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(feI, blockSize, numBlocks, batchSize, numBatches));
63193e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches * batchSize);
63203e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
63213e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches * batchSize);
63223e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
632307218a29SMatthew G. Knepley       PetscCall(PetscFEGeomGetChunk(geom, 0, offset * 2, &chunkGeom));
632407218a29SMatthew G. Knepley       PetscCall(PetscFEGeomGetChunk(geom, offset * 2, numCells * 2, &remGeom));
63259566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, fieldI, &isCohesiveField));
63263e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
63273e9753d6SMatthew G. Knepley         PetscFE feJ;
63283e9753d6SMatthew G. Knepley 
63299566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, fieldJ, (PetscObject *)&feJ));
63303e9753d6SMatthew G. Knepley         if (!feJ) continue;
6331148442b3SMatthew G. Knepley         key[0].field = fieldI * Nf + fieldJ;
6332148442b3SMatthew G. Knepley         key[1].field = fieldI * Nf + fieldJ;
63335fedec97SMatthew G. Knepley         key[2].field = fieldI * Nf + fieldJ;
6334148442b3SMatthew G. Knepley         if (hasBdJac) {
633507218a29SMatthew G. Knepley           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatNeg));
63368e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[0], 0, Nr, remGeom, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[0], PetscSafePointerPlusOffset(a[0], offset * totDimAux[0]), t, X_tShift, &elemMatNeg[offset * totDim * totDim]));
633707218a29SMatthew G. Knepley           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatPos));
63388e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[1], 1, Nr, remGeom, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[1], PetscSafePointerPlusOffset(a[1], offset * totDimAux[1]), t, X_tShift, &elemMatPos[offset * totDim * totDim]));
6339148442b3SMatthew G. Knepley         }
6340148442b3SMatthew G. Knepley         if (hasBdPrec) {
634107218a29SMatthew G. Knepley           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatNegP));
63428e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[0], 0, Nr, remGeom, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[0], &a[0][offset * totDimAux[0]], t, X_tShift, &elemMatNegP[offset * totDim * totDim]));
634307218a29SMatthew G. Knepley           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatPosP));
63448e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[1], 1, Nr, remGeom, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[1], &a[1][offset * totDimAux[1]], t, X_tShift, &elemMatPosP[offset * totDim * totDim]));
6345148442b3SMatthew G. Knepley         }
63465fedec97SMatthew G. Knepley         if (hasBdJac) {
634707218a29SMatthew G. Knepley           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatCoh));
63488e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[2], 2, Nr, remGeom, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[2], PetscSafePointerPlusOffset(a[2], offset * totDimAux[2]), t, X_tShift, &elemMatCoh[offset * totDim * totDim]));
6349148442b3SMatthew G. Knepley         }
63505fedec97SMatthew G. Knepley         if (hasBdPrec) {
635107218a29SMatthew G. Knepley           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatCohP));
63528e3a54c0SPierre Jolivet           PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[2], 2, Nr, remGeom, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[2], &a[2][offset * totDimAux[2]], t, X_tShift, &elemMatCohP[offset * totDim * totDim]));
63533e9753d6SMatthew G. Knepley         }
63543e9753d6SMatthew G. Knepley       }
63559566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &remGeom));
63569566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom, 0, offset, &chunkGeom));
63573e9753d6SMatthew G. Knepley     }
63583e9753d6SMatthew G. Knepley     /* Insert values into matrix */
63593e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
63603e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
636107218a29SMatthew G. Knepley       const PetscInt cind = c - cS, coff = cind * totDim * totDim;
636207218a29SMatthew G. Knepley       PetscInt       i, j;
63633e9753d6SMatthew G. Knepley 
636407218a29SMatthew G. Knepley       /* Scale element values */
636507218a29SMatthew G. Knepley       if (locS[0]) {
63663e2b0218SMatthew G. Knepley         PetscInt  Nb, soff = cind * totDimScale[0], off = 0;
636707218a29SMatthew G. Knepley         PetscBool cohesive;
636807218a29SMatthew G. Knepley 
636907218a29SMatthew G. Knepley         for (fieldI = 0; fieldI < Nf; ++fieldI) {
637007218a29SMatthew G. Knepley           PetscCall(PetscDSGetFieldSize(ds, fieldI, &Nb));
637107218a29SMatthew G. Knepley           PetscCall(PetscDSGetCohesive(ds, fieldI, &cohesive));
637207218a29SMatthew G. Knepley 
637307218a29SMatthew G. Knepley           if (fieldI == key[2].field) {
637407218a29SMatthew G. Knepley             PetscCheck(cohesive, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Scaling should not happen for face fields");
63758bf0a22dSMatthew G. Knepley             for (i = 0; i < Nb; ++i) {
637607218a29SMatthew G. Knepley               for (j = 0; j < totDim; ++j) elemMatCoh[coff + (off + i) * totDim + j] += s[0][soff + i] * elemMatNeg[coff + (off + i) * totDim + j] + s[1][soff + i] * elemMatPos[coff + (off + i) * totDim + j];
63778bf0a22dSMatthew G. Knepley               if (hasBdPrec)
63788bf0a22dSMatthew G. Knepley                 for (j = 0; j < totDim; ++j) elemMatCohP[coff + (off + i) * totDim + j] += s[0][soff + i] * elemMatNegP[coff + (off + i) * totDim + j] + s[1][soff + i] * elemMatPosP[coff + (off + i) * totDim + j];
63798bf0a22dSMatthew G. Knepley             }
638007218a29SMatthew G. Knepley             off += Nb;
638107218a29SMatthew G. Knepley           } else {
638207218a29SMatthew G. Knepley             const PetscInt N = cohesive ? Nb : Nb * 2;
638307218a29SMatthew G. Knepley 
63848bf0a22dSMatthew G. Knepley             for (i = 0; i < N; ++i) {
638507218a29SMatthew G. Knepley               for (j = 0; j < totDim; ++j) elemMatCoh[coff + (off + i) * totDim + j] += elemMatNeg[coff + (off + i) * totDim + j] + elemMatPos[coff + (off + i) * totDim + j];
63868bf0a22dSMatthew G. Knepley               if (hasBdPrec)
63878bf0a22dSMatthew G. Knepley                 for (j = 0; j < totDim; ++j) elemMatCohP[coff + (off + i) * totDim + j] += elemMatNegP[coff + (off + i) * totDim + j] + elemMatPosP[coff + (off + i) * totDim + j];
63888bf0a22dSMatthew G. Knepley             }
638907218a29SMatthew G. Knepley             off += N;
639007218a29SMatthew G. Knepley           }
639107218a29SMatthew G. Knepley         }
639207218a29SMatthew G. Knepley       } else {
639307218a29SMatthew G. Knepley         for (i = 0; i < totDim * totDim; ++i) elemMatCoh[coff + i] += elemMatNeg[coff + i] + elemMatPos[coff + i];
63948bf0a22dSMatthew G. Knepley         if (hasBdPrec)
63958bf0a22dSMatthew G. Knepley           for (i = 0; i < totDim * totDim; ++i) elemMatCohP[coff + i] += elemMatNegP[coff + i] + elemMatPosP[coff + i];
639607218a29SMatthew G. Knepley       }
63973e9753d6SMatthew G. Knepley       if (hasBdPrec) {
63983e9753d6SMatthew G. Knepley         if (hasBdJac) {
639907218a29SMatthew G. Knepley           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCoh[cind * totDim * totDim]));
6400e8e188d2SZach Atkins           PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, Jac, cell, &elemMatCoh[cind * totDim * totDim], ADD_VALUES));
64013e9753d6SMatthew G. Knepley         }
640207218a29SMatthew G. Knepley         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCohP[cind * totDim * totDim]));
640307218a29SMatthew G. Knepley         PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMatCohP[cind * totDim * totDim], ADD_VALUES));
64043e9753d6SMatthew G. Knepley       } else if (hasBdJac) {
640507218a29SMatthew G. Knepley         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCoh[cind * totDim * totDim]));
6406e8e188d2SZach Atkins         PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, JacP, cell, &elemMatCoh[cind * totDim * totDim], ADD_VALUES));
64073e9753d6SMatthew G. Knepley       }
64083e9753d6SMatthew G. Knepley     }
64093e9753d6SMatthew G. Knepley   }
64109566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
641107218a29SMatthew G. Knepley   PetscCall(DMPlexRestoreHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a));
641207218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNeg));
641307218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPos));
641407218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCoh));
641507218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNegP));
641607218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPosP));
641707218a29SMatthew G. Knepley   PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCohP));
64189566063dSJacob Faibussowitsch   PetscCall(PetscFree(faces));
64199566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&chunkIS));
64209566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
64213e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
64229566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
64239566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&affineQuad));
64243e9753d6SMatthew G. Knepley   } else {
64253e9753d6SMatthew G. Knepley     PetscInt f;
64263e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
64279566063dSJacob Faibussowitsch       if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
64289566063dSJacob Faibussowitsch       if (quads) PetscCall(PetscQuadratureDestroy(&quads[f]));
64293e9753d6SMatthew G. Knepley     }
64309566063dSJacob Faibussowitsch     PetscCall(PetscFree2(quads, geoms));
64313e9753d6SMatthew G. Knepley   }
64329566063dSJacob Faibussowitsch   if (dmAux[2]) PetscCall(DMDestroy(&plexA));
64339566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
64343cc88e6aSStefano Zampini end:
64359566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
64363ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
64373e9753d6SMatthew G. Knepley }
64388e3a2eefSMatthew G. Knepley 
64398e3a2eefSMatthew G. Knepley /*
64408e3a2eefSMatthew G. Knepley   DMPlexComputeJacobian_Action_Internal - Form the local portion of the Jacobian action Z = J(X) Y at the local solution X using pointwise functions specified by the user.
64418e3a2eefSMatthew G. Knepley 
64428e3a2eefSMatthew G. Knepley   Input Parameters:
64438e3a2eefSMatthew G. Knepley + dm     - The mesh
6444baca6076SPierre Jolivet . key    - The PetscWeakFormKey indicating where integration should happen
644506ad1575SMatthew G. Knepley . cellIS - The cells to integrate over
64468e3a2eefSMatthew G. Knepley . t      - The time
6447145b44c9SPierre Jolivet . X_tShift - The multiplier for the Jacobian with respect to X_t
64488e3a2eefSMatthew G. Knepley . X      - Local solution vector
64498e3a2eefSMatthew G. Knepley . X_t    - Time-derivative of the local solution vector
64508e3a2eefSMatthew G. Knepley . Y      - Local input vector
645106ad1575SMatthew G. Knepley - user   - the user context
64528e3a2eefSMatthew G. Knepley 
64538e3a2eefSMatthew G. Knepley   Output Parameter:
64548e3a2eefSMatthew G. Knepley . Z - Local output vector
64558e3a2eefSMatthew G. Knepley 
64568e3a2eefSMatthew G. Knepley   Note:
64578e3a2eefSMatthew G. Knepley   We form the residual one batch of elements at a time. This allows us to offload work onto an accelerator,
64588e3a2eefSMatthew G. Knepley   like a GPU, or vectorize on a multicore machine.
64598e3a2eefSMatthew G. Knepley */
6460d71ae5a4SJacob Faibussowitsch PetscErrorCode DMPlexComputeJacobian_Action_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Vec Y, Vec Z, void *user)
6461d71ae5a4SJacob Faibussowitsch {
64628e3a2eefSMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *)dm->data;
64638e3a2eefSMatthew G. Knepley   const char     *name  = "Jacobian";
64648e3a2eefSMatthew G. Knepley   DM              dmAux = NULL, plex, plexAux = NULL;
64658e3a2eefSMatthew G. Knepley   DMEnclosureType encAux;
64668e3a2eefSMatthew G. Knepley   Vec             A;
64678e3a2eefSMatthew G. Knepley   DMField         coordField;
64688e3a2eefSMatthew G. Knepley   PetscDS         prob, probAux = NULL;
64698e3a2eefSMatthew G. Knepley   PetscQuadrature quad;
64708e3a2eefSMatthew G. Knepley   PetscSection    section, globalSection, sectionAux;
64718e3a2eefSMatthew G. Knepley   PetscScalar    *elemMat, *elemMatD, *u, *u_t, *a = NULL, *y, *z;
64728e3a2eefSMatthew G. Knepley   const PetscInt *cells;
64738e3a2eefSMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
64748e3a2eefSMatthew G. Knepley   PetscInt        totDim, totDimAux = 0, cStart, cEnd, numCells, c;
64758e3a2eefSMatthew G. Knepley   PetscBool       hasDyn;
64768e3a2eefSMatthew G. Knepley 
64778e3a2eefSMatthew G. Knepley   PetscFunctionBegin;
64789566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
64799566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
64809566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
64819566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
64829566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
64839566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
648407218a29SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL));
64859566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
64869566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
64879566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
64888e3a2eefSMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
64899566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
64908e3a2eefSMatthew G. Knepley   if (A) {
64919566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
64929566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
64939566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexAux));
64949566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexAux, &sectionAux));
64959566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
64969566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
64978e3a2eefSMatthew G. Knepley   }
64989566063dSJacob Faibussowitsch   PetscCall(VecSet(Z, 0.0));
649932603206SJames Wright   PetscCall(PetscMalloc6(numCells * totDim, &u, (X_t ? (size_t)numCells * totDim : 0), &u_t, numCells * totDim * totDim, &elemMat, (hasDyn ? (size_t)numCells * totDim * totDim : 0), &elemMatD, numCells * totDim, &y, totDim, &z));
65009566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
65019566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
65028e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
65038e3a2eefSMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
65048e3a2eefSMatthew G. Knepley     const PetscInt cind = c - cStart;
65058e3a2eefSMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL;
65068e3a2eefSMatthew G. Knepley     PetscInt       i;
65078e3a2eefSMatthew G. Knepley 
65089566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, X, cell, NULL, &x));
65098e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i];
65109566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, X, cell, NULL, &x));
65118e3a2eefSMatthew G. Knepley     if (X_t) {
65129566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, X_t, cell, NULL, &x_t));
65138e3a2eefSMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i];
65149566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, X_t, cell, NULL, &x_t));
65158e3a2eefSMatthew G. Knepley     }
65168e3a2eefSMatthew G. Knepley     if (dmAux) {
65178e3a2eefSMatthew G. Knepley       PetscInt subcell;
65189566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
65199566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexAux, sectionAux, A, subcell, NULL, &x));
65208e3a2eefSMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i];
65219566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexAux, sectionAux, A, subcell, NULL, &x));
65228e3a2eefSMatthew G. Knepley     }
65239566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, Y, cell, NULL, &x));
65248e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) y[cind * totDim + i] = x[i];
65259566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, Y, cell, NULL, &x));
65268e3a2eefSMatthew G. Knepley   }
65279566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim));
65289566063dSJacob Faibussowitsch   if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim));
65298e3a2eefSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
65308e3a2eefSMatthew G. Knepley     PetscFE  fe;
65318e3a2eefSMatthew G. Knepley     PetscInt Nb;
65328e3a2eefSMatthew G. Knepley     /* Conforming batches */
65338e3a2eefSMatthew G. Knepley     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
65348e3a2eefSMatthew G. Knepley     /* Remainder */
65358e3a2eefSMatthew G. Knepley     PetscInt        Nr, offset, Nq;
65368e3a2eefSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
65378e3a2eefSMatthew G. Knepley     PetscInt        maxDegree;
65388e3a2eefSMatthew G. Knepley     PetscFEGeom    *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
65398e3a2eefSMatthew G. Knepley 
65409566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
65419566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
65429566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
65439566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
65449566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
65459566063dSJacob Faibussowitsch     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
65468e3a2eefSMatthew G. Knepley     if (!qGeom) {
65479566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &qGeom));
65489566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
65498e3a2eefSMatthew G. Knepley     }
65509566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
65519566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
65528e3a2eefSMatthew G. Knepley     blockSize = Nb;
65538e3a2eefSMatthew G. Knepley     batchSize = numBlocks * blockSize;
65549566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
65558e3a2eefSMatthew G. Knepley     numChunks = numCells / (numBatches * batchSize);
65568e3a2eefSMatthew G. Knepley     Ne        = numChunks * numBatches * batchSize;
65578e3a2eefSMatthew G. Knepley     Nr        = numCells % (numBatches * batchSize);
65588e3a2eefSMatthew G. Knepley     offset    = numCells - Nr;
65599566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
65609566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom));
65618e3a2eefSMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
65628e3a2eefSMatthew G. Knepley       key.field = fieldI * Nf + fieldJ;
65639566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
65648e3a54c0SPierre Jolivet       PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * totDim]));
65658e3a2eefSMatthew G. Knepley       if (hasDyn) {
65669566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
65678e3a54c0SPierre Jolivet         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, &a[offset * totDimAux], t, X_tShift, &elemMatD[offset * totDim * totDim]));
65688e3a2eefSMatthew G. Knepley       }
65698e3a2eefSMatthew G. Knepley     }
65709566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom));
65719566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom));
65729566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
65739566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
65748e3a2eefSMatthew G. Knepley   }
65758e3a2eefSMatthew G. Knepley   if (hasDyn) {
65768e3a2eefSMatthew G. Knepley     for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
65778e3a2eefSMatthew G. Knepley   }
65788e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
65798e3a2eefSMatthew G. Knepley     const PetscInt     cell = cells ? cells[c] : c;
65808e3a2eefSMatthew G. Knepley     const PetscInt     cind = c - cStart;
65816497c311SBarry Smith     const PetscBLASInt one  = 1;
65826497c311SBarry Smith     PetscBLASInt       M;
65838e3a2eefSMatthew G. Knepley     const PetscScalar  a = 1.0, b = 0.0;
65848e3a2eefSMatthew G. Knepley 
65856497c311SBarry Smith     PetscCall(PetscBLASIntCast(totDim, &M));
6586792fecdfSBarry Smith     PetscCallBLAS("BLASgemv", BLASgemv_("N", &M, &M, &a, &elemMat[cind * totDim * totDim], &M, &y[cind * totDim], &one, &b, z, &one));
65878e3a2eefSMatthew G. Knepley     if (mesh->printFEM > 1) {
65889566063dSJacob Faibussowitsch       PetscCall(DMPrintCellMatrix(c, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
65899566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Y", totDim, &y[cind * totDim]));
65909566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Z", totDim, z));
65918e3a2eefSMatthew G. Knepley     }
65929566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dm, section, Z, cell, z, ADD_VALUES));
65938e3a2eefSMatthew G. Knepley   }
65949566063dSJacob Faibussowitsch   PetscCall(PetscFree6(u, u_t, elemMat, elemMatD, y, z));
65958e3a2eefSMatthew G. Knepley   if (mesh->printFEM) {
65969566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)Z), "Z:\n"));
65979566063dSJacob Faibussowitsch     PetscCall(VecView(Z, NULL));
65988e3a2eefSMatthew G. Knepley   }
65991059d808SMatthew G. Knepley   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
66009566063dSJacob Faibussowitsch   PetscCall(PetscFree(a));
66019566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexAux));
66029566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
66039566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
66043ba16761SJacob Faibussowitsch   PetscFunctionReturn(PETSC_SUCCESS);
66058e3a2eefSMatthew G. Knepley }
6606d52c2f21SMatthew G. Knepley 
6607d52c2f21SMatthew G. Knepley static void f0_1(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
6608d52c2f21SMatthew G. Knepley {
6609d52c2f21SMatthew G. Knepley   f0[0] = u[0];
6610d52c2f21SMatthew G. Knepley }
6611d52c2f21SMatthew G. Knepley 
6612d52c2f21SMatthew G. Knepley static void f0_x(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
6613d52c2f21SMatthew G. Knepley {
6614d52c2f21SMatthew G. Knepley   f0[0] = x[(int)PetscRealPart(constants[0])] * u[0];
6615d52c2f21SMatthew G. Knepley }
6616d52c2f21SMatthew G. Knepley 
6617d52c2f21SMatthew G. Knepley static void f0_x2(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
6618d52c2f21SMatthew G. Knepley {
6619d52c2f21SMatthew G. Knepley   PetscInt d;
6620d52c2f21SMatthew G. Knepley 
6621d52c2f21SMatthew G. Knepley   f0[0] = 0.0;
6622d52c2f21SMatthew G. Knepley   for (d = 0; d < dim; ++d) f0[0] += PetscSqr(x[d]) * u[0];
6623d52c2f21SMatthew G. Knepley }
6624d52c2f21SMatthew G. Knepley 
6625d52c2f21SMatthew G. Knepley /*@
6626d52c2f21SMatthew G. Knepley   DMPlexComputeMoments - Compute the first three moments for a field
6627d52c2f21SMatthew G. Knepley 
6628d52c2f21SMatthew G. Knepley   Noncollective
6629d52c2f21SMatthew G. Knepley 
6630d52c2f21SMatthew G. Knepley   Input Parameters:
6631d52c2f21SMatthew G. Knepley + dm - the `DMPLEX`
6632d52c2f21SMatthew G. Knepley - u  - the field
6633d52c2f21SMatthew G. Knepley 
6634d52c2f21SMatthew G. Knepley   Output Parameter:
6635d52c2f21SMatthew G. Knepley . moments - the field moments
6636d52c2f21SMatthew G. Knepley 
6637d52c2f21SMatthew G. Knepley   Level: intermediate
6638d52c2f21SMatthew G. Knepley 
6639d52c2f21SMatthew G. Knepley   Note:
6640d52c2f21SMatthew G. Knepley   The `moments` array should be of length Nc + 2, where Nc is the number of components for the field.
6641d52c2f21SMatthew G. Knepley 
6642d52c2f21SMatthew G. Knepley .seealso: `DM`, `DMPLEX`, `DMSwarmComputeMoments()`
6643d52c2f21SMatthew G. Knepley @*/
6644d52c2f21SMatthew G. Knepley PetscErrorCode DMPlexComputeMoments(DM dm, Vec u, PetscReal moments[])
6645d52c2f21SMatthew G. Knepley {
6646d52c2f21SMatthew G. Knepley   PetscDS            ds;
6647d52c2f21SMatthew G. Knepley   PetscScalar        mom, constants[1];
6648d52c2f21SMatthew G. Knepley   const PetscScalar *oldConstants;
6649d52c2f21SMatthew G. Knepley   PetscInt           Nf, field = 0, Ncon, *comp;
6650d52c2f21SMatthew G. Knepley   MPI_Comm           comm;
6651d52c2f21SMatthew G. Knepley   void              *user;
6652d52c2f21SMatthew G. Knepley 
6653d52c2f21SMatthew G. Knepley   PetscFunctionBeginUser;
6654d52c2f21SMatthew G. Knepley   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
6655d52c2f21SMatthew G. Knepley   PetscCall(DMGetApplicationContext(dm, &user));
6656d52c2f21SMatthew G. Knepley   PetscCall(DMGetDS(dm, &ds));
6657d52c2f21SMatthew G. Knepley   PetscCall(PetscDSGetNumFields(ds, &Nf));
6658d52c2f21SMatthew G. Knepley   PetscCall(PetscDSGetComponents(ds, &comp));
6659d52c2f21SMatthew G. Knepley   PetscCall(PetscDSGetConstants(ds, &Ncon, &oldConstants));
6660d52c2f21SMatthew G. Knepley   PetscCall(PetscDSSetConstants(ds, 1, constants));
6661d52c2f21SMatthew G. Knepley   PetscCheck(Nf == 1, comm, PETSC_ERR_ARG_WRONG, "We currently only support 1 field, not %" PetscInt_FMT, Nf);
6662d52c2f21SMatthew G. Knepley   PetscCall(PetscDSSetObjective(ds, field, &f0_1));
6663d52c2f21SMatthew G. Knepley   PetscCall(DMPlexComputeIntegralFEM(dm, u, &mom, user));
6664d52c2f21SMatthew G. Knepley   moments[0] = PetscRealPart(mom);
6665d52c2f21SMatthew G. Knepley   for (PetscInt c = 0; c < comp[0]; ++c) {
6666d52c2f21SMatthew G. Knepley     constants[0] = c;
6667d52c2f21SMatthew G. Knepley     PetscCall(PetscDSSetObjective(ds, field, &f0_x));
6668d52c2f21SMatthew G. Knepley     PetscCall(DMPlexComputeIntegralFEM(dm, u, &mom, user));
6669d52c2f21SMatthew G. Knepley     moments[c + 1] = PetscRealPart(mom);
6670d52c2f21SMatthew G. Knepley   }
6671d52c2f21SMatthew G. Knepley   PetscCall(PetscDSSetObjective(ds, field, &f0_x2));
6672d52c2f21SMatthew G. Knepley   PetscCall(DMPlexComputeIntegralFEM(dm, u, &mom, user));
6673d52c2f21SMatthew G. Knepley   moments[comp[0] + 1] = PetscRealPart(mom);
6674d52c2f21SMatthew G. Knepley   PetscCall(PetscDSSetConstants(ds, Ncon, (PetscScalar *)oldConstants));
6675d52c2f21SMatthew G. Knepley   PetscFunctionReturn(PETSC_SUCCESS);
6676d52c2f21SMatthew G. Knepley }
6677